liboxen 0.52.0

Oxen is a fast, unstructured data version control, to help version large machine learning datasets written in Rust.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140

use polars::prelude::PlSmallStr;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use utoipa::ToSchema;

use crate::model::data_frame::schema::DataType;

#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct Field {
    pub name: String,
    pub dtype: String,
    pub metadata: Option<Value>,
    #[serde(default)]
    pub changes: Option<Changes>,
}

#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct PreviousField {
    pub name: String,
    pub dtype: String,
    pub metadata: Option<Value>,
}

#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct Changes {
    pub status: String,
    pub previous: Option<PreviousField>,
}

impl PartialEq for Field {
    fn eq(&self, other: &Field) -> bool {
        self.name == other.name && self.dtype == other.dtype && self.metadata == other.metadata
    }
}

impl Field {
    pub fn new(name: &str, dtype: &str) -> Self {
        Field {
            name: name.to_owned(),
            dtype: dtype.to_owned(),
            metadata: None,
            changes: None,
        }
    }

    pub fn to_sql(&self) -> String {
        let dtype = DataType::from_string(&self.dtype).to_sql();
        format!("{} {}", self.name, dtype)
    }

    pub fn to_polars(&self) -> polars::prelude::Field {
        polars::prelude::Field::new(
            PlSmallStr::from(self.name.to_owned()),
            DataType::from_string(&self.dtype).to_polars(),
        )
    }

    pub fn fields_to_string_with_limit<V: AsRef<Vec<Field>>>(fields: V) -> String {
        let fields = fields.as_ref();
        let max_num = 2;
        if fields.len() > max_num {
            let name_0 = fields[0].name.to_owned();
            let name_3 = fields[fields.len() - 1].name.to_owned();

            let combined_names = [name_0, String::from("..."), name_3].join(", ");
            format!("[{combined_names}]")
        } else {
            let names: Vec<String> = fields.iter().map(|f| f.name.to_owned()).collect();

            let combined_names = names.join(", ");

            format!("[{combined_names}]")
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use serde::Serialize;

    // Mirrors the on-disk `Field` layout from before `changes` was briefly
    // dropped: `changes` is the trailing element. Merkle nodes embed schema
    // fields and are serialized positionally via `rmp_serde::to_vec`, so blobs
    // written by older versions still carry (or omit) this trailing element and
    // must keep deserializing into the current `Field`.
    #[derive(Serialize)]
    struct OldField {
        name: String,
        dtype: String,
        metadata: Option<Value>,
        changes: Option<OldChanges>,
    }

    #[derive(Serialize)]
    struct OldChanges {
        status: String,
        previous: Option<Box<OldField>>,
    }

    #[test]
    fn test_deserialize_field_missing_changes_element() {
        // The interim format dropped `changes` entirely (three-element array).
        #[derive(Serialize)]
        struct FieldWithoutChanges {
            name: String,
            dtype: String,
            metadata: Option<Value>,
        }
        let bytes = rmp_serde::to_vec(&FieldWithoutChanges {
            name: "col".to_owned(),
            dtype: "str".to_owned(),
            metadata: None,
        })
        .expect("serialize field without changes");

        let field: Field =
            rmp_serde::from_slice(&bytes).expect("deserialize interim format into current Field");
        assert_eq!(field.name, "col");
        assert_eq!(field.dtype, "str");
        assert!(field.changes.is_none());
    }

    #[test]
    fn test_deserialize_field_with_null_changes() {
        let bytes = rmp_serde::to_vec(&OldField {
            name: "col".to_owned(),
            dtype: "str".to_owned(),
            metadata: None,
            changes: None,
        })
        .expect("serialize old field with null changes");

        let field: Field =
            rmp_serde::from_slice(&bytes).expect("deserialize old format into current Field");
        assert_eq!(field.name, "col");
        assert_eq!(field.dtype, "str");
        assert!(field.changes.is_none());
    }
}