liboxen 0.53.0

Oxen is a fast data version control system, built with machine learning training data in mind. Designed to handle terabytes of data with ease, using a workflow similar to git. Version both structured and unstructured data of any modality: text, images, video, audio, CSV, Parquet, JSONL, model checkpoints, and more. liboxen is the embeddable core library behind the oxen CLI and server, which power fine tuning and inference pipelines for multimodal LLMs, image models, and video models on Oxen.ai.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100

//! Node payload shapes from before v0.25.0, kept so repositories written then stay readable.
//!
//! Harvested verbatim from `aa8f564b8^:crates/liboxen/src/core/v_old/v0_19_0/model/merkle_tree/`,
//! the commit that deleted the old read path. Only the two shapes that actually changed live here:
//! `CommitNodeData` kept the same seven fields in the same order and decodes unchanged, and
//! `FileNode` already carries its own fallback.
//!
//! The difference from the current structs is one field. Both gained `num_entries`, which these
//! payloads simply do not contain, so a conversion has to supply it. Reads do not need it — a
//! directory listing counts the entries it actually returns — so the conversions below leave it
//! zero, matching what the pre-removal code did. Deriving the real value belongs to the migration
//! that rewrites these nodes, where the child counts are being walked anyway.

use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::io::Cursor;

use crate::core::v_latest::model::merkle_tree::node::commit_node::CommitNodeData;
use crate::core::v_latest::model::merkle_tree::node::dir_node::DirNodeData;
use crate::core::v_latest::model::merkle_tree::node::file_node::FileNodeData;
use crate::core::v_latest::model::merkle_tree::node::vnode::VNodeData;
use crate::model::{MerkleHash, MerkleTreeNodeType};

/// Whether these bytes decode as the payload shape written before v0.25.0 for `dtype`.
///
/// Answers "is this a retired-format node" by decoding it as one, so bytes that merely lack the
/// current envelope are not taken for a retired format on that basis alone. Damage to a payload's
/// leading bytes destroys the envelope, which makes prefix inspection alone unable to tell a
/// legacy node from a corrupt one.
///
/// `Commit` and `File` kept their field lists across the change, so the current data struct reads
/// the older bytes. `FileChunk` never had an envelope, so the question does not apply to it.
pub(crate) fn decodes_as_pre_v025(dtype: MerkleTreeNodeType, data: &[u8]) -> bool {
    match dtype {
        MerkleTreeNodeType::Commit => decodes_fully::<CommitNodeData>(data),
        MerkleTreeNodeType::Dir => decodes_fully::<DirNodeDataPre025>(data),
        MerkleTreeNodeType::File => decodes_fully::<FileNodeData>(data),
        MerkleTreeNodeType::VNode => decodes_fully::<VNodeDataPre025>(data),
        MerkleTreeNodeType::FileChunk => false,
    }
}

/// Whether `data` decodes as `T` with nothing trailing it.
///
/// `rmp_serde::from_slice` stops after the first value and ignores whatever follows, which accepts
/// a payload whose leading bytes happen to form a `T` and whose remainder is damage. A node
/// payload is stored with an explicit length, so a legacy one ends exactly where its struct does.
fn decodes_fully<T: DeserializeOwned>(data: &[u8]) -> bool {
    let mut deserializer = rmp_serde::Deserializer::new(Cursor::new(data));
    T::deserialize(&mut deserializer).is_ok() && deserializer.position() == data.len() as u64
}

/// A vnode as written before v0.25.0: no `num_entries`.
#[derive(Deserialize, Serialize, Clone, PartialEq, Eq)]
pub struct VNodeDataPre025 {
    pub hash: MerkleHash,
    pub node_type: MerkleTreeNodeType,
}

impl From<VNodeDataPre025> for VNodeData {
    fn from(legacy: VNodeDataPre025) -> Self {
        VNodeData {
            hash: legacy.hash,
            node_type: legacy.node_type,
            num_entries: 0,
        }
    }
}

/// A directory node as written before v0.25.0: no `num_entries`.
#[derive(Deserialize, Serialize, Clone, PartialEq, Eq)]
pub struct DirNodeDataPre025 {
    pub node_type: MerkleTreeNodeType,
    pub name: String,
    pub hash: MerkleHash,
    pub num_bytes: u64,
    pub last_commit_id: MerkleHash,
    pub last_modified_seconds: i64,
    pub last_modified_nanoseconds: u32,
    pub data_type_counts: HashMap<String, u64>,
    pub data_type_sizes: HashMap<String, u64>,
}

impl From<DirNodeDataPre025> for DirNodeData {
    fn from(legacy: DirNodeDataPre025) -> Self {
        DirNodeData {
            node_type: legacy.node_type,
            name: legacy.name,
            hash: legacy.hash,
            num_entries: 0,
            num_bytes: legacy.num_bytes,
            last_commit_id: legacy.last_commit_id,
            last_modified_seconds: legacy.last_modified_seconds,
            last_modified_nanoseconds: legacy.last_modified_nanoseconds,
            data_type_counts: legacy.data_type_counts,
            data_type_sizes: legacy.data_type_sizes,
        }
    }
}