liboxen 0.53.0

Oxen is a fast data version control system, built with machine learning training data in mind. Designed to handle terabytes of data with ease, using a workflow similar to git. Version both structured and unstructured data of any modality: text, images, video, audio, CSV, Parquet, JSONL, model checkpoints, and more. liboxen is the embeddable core library behind the oxen CLI and server, which power fine tuning and inference pipelines for multimodal LLMs, image models, and video models on Oxen.ai.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57

use polars::prelude::DataFrame;
use std::fmt;

use crate::model::Schema;

/*

summary:
* base_size 10x100
* head_size 9x100
* schema_change: true

full:
* added_rows: Option<DataFrame>,
* removed_rows: Option<DataFrame>,
* added_cols: Option<DataFrame>,
* removed_cols: Option<DataFrame>,
* base_schema: Option<Schema>
* head_schema: Option<Schema>

later:
be able to run queries on removed rows, added rows, etc. to see what changed
*/

pub struct DataFrameDiff {
    pub base_schema: Option<Schema>,
    pub head_schema: Option<Schema>,
    pub added_rows: Option<DataFrame>,
    pub removed_rows: Option<DataFrame>,
    pub added_cols: Option<DataFrame>,
    pub removed_cols: Option<DataFrame>,
}

impl fmt::Display for DataFrameDiff {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        let mut results: Vec<String> = vec![];
        if let Some(rows) = &self.added_rows
            && rows.height() > 0
            && rows.width() > 0
        {
            results.push(format!("Added Rows\n\n{rows}\n\n"));
        }

        if let Some(rows) = &self.removed_rows {
            results.push(format!("Removed Rows\n\n{rows}\n\n"));
        }

        if let Some(cols) = &self.added_cols {
            results.push(format!("Added Columns\n\n{cols}\n\n"));
        }

        if let Some(cols) = &self.removed_cols {
            results.push(format!("Removed Columns\n\n{cols}\n\n"));
        }
        write!(f, "{}", results.join("\n"))
    }
}