liboxen 0.53.0

Oxen is a fast data version control system, built with machine learning training data in mind. Designed to handle terabytes of data with ease, using a workflow similar to git. Version both structured and unstructured data of any modality: text, images, video, audio, CSV, Parquet, JSONL, model checkpoints, and more. liboxen is the embeddable core library behind the oxen CLI and server, which power fine tuning and inference pipelines for multimodal LLMs, image models, and video models on Oxen.ai.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83

use crate::model::Commit;
use crate::model::merkle_tree::node::{DirNode, EMerkleTreeNode, FileNode};

use serde::{Deserialize, Serialize};
use std::hash::{Hash, Hasher};
use std::path::PathBuf;

use utoipa::ToSchema;

#[derive(Deserialize, Serialize, Debug, Clone)]
pub struct CommitPath {
    pub commit: Option<Commit>,
    pub path: PathBuf,
}

/// Represents a file or directory entry at a specific commit.
///
/// `Hash` and `Eq` are based on the content hash field, so `HashSet<CommitEntry>`
/// deduplicates by file content. This is used during fetch and push to avoid
/// transferring the same content twice.
#[derive(Deserialize, Serialize, Debug, Clone, ToSchema)]
pub struct CommitEntry {
    pub commit_id: String,
    #[schema(value_type = String)]
    pub path: PathBuf,
    pub hash: String,
    pub num_bytes: u64,
    pub last_modified_seconds: i64,
    pub last_modified_nanoseconds: u32,
}

// TODONOW - maybe rename or reorg, this isn't an "entry" as such
#[derive(Deserialize, Serialize, Debug, Clone)]
pub struct CompareEntry {
    pub commit_entry: Option<CommitEntry>,
    pub path: PathBuf,
}

impl PartialEq for CommitEntry {
    fn eq(&self, other: &CommitEntry) -> bool {
        self.hash == other.hash
    }
}

impl Eq for CommitEntry {}

impl Hash for CommitEntry {
    fn hash<H: Hasher>(&self, state: &mut H) {
        self.hash.hash(state);
    }
}

impl CommitEntry {
    pub fn from_node(node: &EMerkleTreeNode) -> CommitEntry {
        match node {
            EMerkleTreeNode::Directory(dir_node) => CommitEntry::from_dir_node(dir_node),
            EMerkleTreeNode::File(file_node) => CommitEntry::from_file_node(file_node),
            _ => panic!("Cannot convert EMerkleTreeNode to CommitEntry"),
        }
    }

    pub fn from_file_node(file_node: &FileNode) -> CommitEntry {
        CommitEntry {
            commit_id: file_node.last_commit_id().to_string(),
            path: PathBuf::from(file_node.name()),
            hash: file_node.hash().to_string(),
            num_bytes: file_node.num_bytes(),
            last_modified_seconds: file_node.last_modified_seconds(),
            last_modified_nanoseconds: file_node.last_modified_nanoseconds(),
        }
    }

    pub fn from_dir_node(dir_node: &DirNode) -> CommitEntry {
        CommitEntry {
            commit_id: dir_node.last_commit_id().to_string(),
            path: PathBuf::from(dir_node.name()),
            hash: dir_node.hash().to_string(),
            num_bytes: dir_node.num_bytes(),
            last_modified_seconds: dir_node.last_modified_seconds(),
            last_modified_nanoseconds: dir_node.last_modified_nanoseconds(),
        }
    }
}