liboxen 0.53.0

Oxen is a fast data version control system, built with machine learning training data in mind. Designed to handle terabytes of data with ease, using a workflow similar to git. Version both structured and unstructured data of any modality: text, images, video, audio, CSV, Parquet, JSONL, model checkpoints, and more. liboxen is the embeddable core library behind the oxen CLI and server, which power fine tuning and inference pipelines for multimodal LLMs, image models, and video models on Oxen.ai.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94

// use std::fmt::{Display, Formatter, Result};

pub mod commit_node;
pub mod dir_node;
pub mod dir_node_with_path;
pub mod file_chunk_node;
pub mod file_node;
pub mod file_node_types;
pub mod file_node_with_dir;
pub mod legacy_pre_025;
pub mod merkle_tree_node;
pub mod merkle_tree_node_cache;
pub mod staged_merkle_tree_node;
pub mod vnode;

pub use commit_node::CommitNode;
pub use dir_node::DirNode;
pub use dir_node_with_path::DirNodeWithPath;
pub use file_chunk_node::FileChunkNode;
pub use file_node::FileNode;
pub use file_node_types::{FileChunkType, FileStorageType};
pub use file_node_with_dir::FileNodeWithDir;
pub use merkle_tree_node::MerkleTreeNode;
pub use staged_merkle_tree_node::StagedMerkleTreeNode;
pub use vnode::VNode;

use crate::model::metadata::generic_metadata::GenericMetadata;
pub use crate::model::{MerkleTreeNodeType, TMerkleTreeNode};
use serde::{Deserialize, Serialize};

use super::MerkleHash;

#[derive(Clone, Eq, PartialEq, Debug, Deserialize, Serialize)]
pub enum EMerkleTreeNode {
    File(FileNode),
    Directory(DirNode),
    VNode(VNode),
    FileChunk(FileChunkNode),
    Commit(CommitNode),
}

impl EMerkleTreeNode {
    pub fn node_type(&self) -> MerkleTreeNodeType {
        match self {
            EMerkleTreeNode::File(_) => MerkleTreeNodeType::File,
            EMerkleTreeNode::Directory(_) => MerkleTreeNodeType::Dir,
            EMerkleTreeNode::VNode(_) => MerkleTreeNodeType::VNode,
            EMerkleTreeNode::FileChunk(_) => MerkleTreeNodeType::FileChunk,
            EMerkleTreeNode::Commit(_) => MerkleTreeNodeType::Commit,
        }
    }

    pub fn hash(&self) -> &MerkleHash {
        match self {
            EMerkleTreeNode::File(file) => file.hash(),
            EMerkleTreeNode::Directory(dir) => dir.hash(),
            EMerkleTreeNode::VNode(vnode) => vnode.hash(),
            EMerkleTreeNode::FileChunk(file_chunk) => &file_chunk.hash,
            EMerkleTreeNode::Commit(commit) => commit.hash(),
        }
    }

    pub fn metadata(&self) -> Option<GenericMetadata> {
        match self {
            EMerkleTreeNode::File(file) => file.metadata(),
            _ => None,
        }
    }

    pub fn is_leaf(&self) -> bool {
        matches!(
            &self,
            EMerkleTreeNode::File(_) | EMerkleTreeNode::FileChunk(_)
        )
    }

    /// Deserialize a Merkle tree node from its on-disk type marker and msgpack-encoded body.
    pub fn from_type_and_bytes(
        dtype: MerkleTreeNodeType,
        data: &[u8],
    ) -> Result<Self, rmp_serde::decode::Error> {
        match dtype {
            MerkleTreeNodeType::Commit => {
                Ok(EMerkleTreeNode::Commit(CommitNode::deserialize(data)?))
            }
            MerkleTreeNodeType::Dir => Ok(EMerkleTreeNode::Directory(DirNode::deserialize(data)?)),
            MerkleTreeNodeType::File => Ok(EMerkleTreeNode::File(FileNode::deserialize(data)?)),
            MerkleTreeNodeType::VNode => Ok(EMerkleTreeNode::VNode(VNode::deserialize(data)?)),
            MerkleTreeNodeType::FileChunk => Ok(EMerkleTreeNode::FileChunk(
                FileChunkNode::deserialize(data)?,
            )),
        }
    }
}