liboxen 0.53.0

Oxen is a fast data version control system, built with machine learning training data in mind. Designed to handle terabytes of data with ease, using a workflow similar to git. Version both structured and unstructured data of any modality: text, images, video, audio, CSV, Parquet, JSONL, model checkpoints, and more. liboxen is the embeddable core library behind the oxen CLI and server, which power fine tuning and inference pipelines for multimodal LLMs, image models, and video models on Oxen.ai.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83

use serde::{Deserialize, Serialize};
use utoipa::ToSchema;

use crate::{
    error::OxenError,
    model::{diff::AddRemoveModifyCounts, merkle_tree::node::DirNode},
};

#[derive(Deserialize, Serialize, Debug, Clone, ToSchema)]
pub struct DirDiffSummary {
    pub dir: DirDiffSummaryImpl,
}

// Impl is so that we can wrap the json response in the "dir" field to make summaries easier to distinguish
#[derive(Deserialize, Serialize, Debug, Clone, ToSchema)]
pub struct DirDiffSummaryImpl {
    pub file_counts: AddRemoveModifyCounts,
}

impl DirDiffSummary {
    pub fn from_dir_nodes(
        base_entry: &Option<DirNode>,
        head_entry: &Option<DirNode>,
    ) -> Result<DirDiffSummary, OxenError> {
        match (base_entry, head_entry) {
            (Some(base_entry), Some(head_entry)) => {
                let base_num_files = base_entry.num_files();
                let head_num_files = head_entry.num_files();

                let num_added_files = head_num_files.saturating_sub(base_num_files);

                let num_removed_files = base_num_files.saturating_sub(head_num_files);

                Ok(DirDiffSummary {
                    dir: DirDiffSummaryImpl {
                        file_counts: AddRemoveModifyCounts {
                            added: num_added_files as usize,
                            removed: num_removed_files as usize,
                            modified: 0,
                        },
                    },
                })
            }
            (Some(base_entry), None) => {
                let num_files = base_entry.num_files();

                Ok(DirDiffSummary {
                    dir: DirDiffSummaryImpl {
                        file_counts: AddRemoveModifyCounts {
                            added: 0,
                            removed: num_files as usize,
                            modified: 0,
                        },
                    },
                })
            }

            (None, Some(head_entry)) => {
                let num_files = head_entry.num_files();

                Ok(DirDiffSummary {
                    dir: DirDiffSummaryImpl {
                        file_counts: AddRemoveModifyCounts {
                            added: num_files as usize,
                            removed: 0,
                            modified: 0,
                        },
                    },
                })
            }

            (None, None) => Ok(DirDiffSummary {
                dir: DirDiffSummaryImpl {
                    file_counts: AddRemoveModifyCounts {
                        added: 0,
                        removed: 0,
                        modified: 0,
                    },
                },
            }),
        }
    }
}