liboxen 0.53.0

Oxen is a fast data version control system, built with machine learning training data in mind. Designed to handle terabytes of data with ease, using a workflow similar to git. Version both structured and unstructured data of any modality: text, images, video, audio, CSV, Parquet, JSONL, model checkpoints, and more. liboxen is the embeddable core library behind the oxen CLI and server, which power fine tuning and inference pipelines for multimodal LLMs, image models, and video models on Oxen.ai.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156

use crate::{
    error::OxenError,
    model::data_frame::schema::{Field, Schema},
};
use polars::frame::DataFrame;
use serde::{Deserialize, Serialize};
use utoipa::ToSchema;

use super::AddRemoveModifyCounts;

#[derive(Debug, Clone, Default, Serialize, Deserialize, ToSchema)]
pub struct TabularSchemaDiff {
    pub added: Vec<Field>,
    pub removed: Vec<Field>,
}

#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
pub struct TabularDiffMods {
    pub row_counts: AddRemoveModifyCounts,
    pub col_changes: TabularSchemaDiff,
}

#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
pub struct TabularDiffSummary {
    pub modifications: TabularDiffMods,
    pub schemas: TabularDiffSchemas,
    pub dupes: TabularDiffDupes,
}

#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
pub struct TabularDiff {
    pub filename1: Option<String>,
    pub filename2: Option<String>,
    pub summary: TabularDiffSummary,
    pub parameters: TabularDiffParameters,

    // TODO: Not sure if this is the best way to represent polars data frames
    #[schema(value_type = Object)]
    pub contents: DataFrame,
}

#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
pub struct TabularDiffSchemas {
    pub left: Schema,
    pub right: Schema,
    pub diff: Schema,
}

#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
pub struct TabularDiffParameters {
    pub keys: Vec<String>,
    pub targets: Vec<String>,
    pub display: Vec<String>,
}

// Need to serialize here because we directly write this to disk to cache compares
#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
pub struct TabularDiffDupes {
    pub left: u64,
    pub right: u64,
}
impl TabularDiffDupes {
    pub fn empty() -> Self {
        TabularDiffDupes { left: 0, right: 0 }
    }
}

impl TabularDiffMods {
    pub fn empty() -> Self {
        TabularDiffMods {
            row_counts: AddRemoveModifyCounts {
                added: 0,
                removed: 0,
                modified: 0,
            },
            col_changes: TabularSchemaDiff::empty(),
        }
    }
}

impl TabularDiffSchemas {
    pub fn empty() -> Self {
        TabularDiffSchemas {
            left: Schema::empty(),
            right: Schema::empty(),
            diff: Schema::empty(),
        }
    }
}

impl TabularDiffParameters {
    pub fn empty() -> Self {
        TabularDiffParameters {
            keys: Vec::new(),
            targets: Vec::new(),
            display: Vec::new(),
        }
    }
}

impl TabularSchemaDiff {
    pub fn from_schemas(s1: &Schema, s2: &Schema) -> Result<TabularSchemaDiff, OxenError> {
        let added = s2
            .fields
            .iter()
            .filter(|field| !s1.fields.contains(field))
            .cloned()
            .collect::<Vec<Field>>();

        let removed = s1
            .fields
            .iter()
            .filter(|field| !s2.fields.contains(field))
            .cloned()
            .collect::<Vec<Field>>();

        Ok(TabularSchemaDiff { added, removed })
    }

    pub fn empty() -> Self {
        TabularSchemaDiff {
            added: vec![],
            removed: vec![],
        }
    }
}

impl TabularDiff {
    pub fn has_changes(&self) -> bool {
        self.summary.modifications.row_counts.added > 0
            || self.summary.modifications.row_counts.removed > 0
            || self.summary.modifications.row_counts.modified > 0
            || !self.summary.modifications.col_changes.added.is_empty()
            || !self.summary.modifications.col_changes.removed.is_empty()
    }

    pub fn empty() -> Self {
        TabularDiff {
            summary: TabularDiffSummary::empty(),
            parameters: TabularDiffParameters::empty(),
            contents: DataFrame::empty(),
            filename1: None,
            filename2: None,
        }
    }
}

impl TabularDiffSummary {
    pub fn empty() -> Self {
        TabularDiffSummary {
            modifications: TabularDiffMods::empty(),
            schemas: TabularDiffSchemas::empty(),
            dupes: TabularDiffDupes::empty(),
        }
    }
}