liboxen 0.53.0

Oxen is a fast data version control system, built with machine learning training data in mind. Designed to handle terabytes of data with ease, using a workflow similar to git. Version both structured and unstructured data of any modality: text, images, video, audio, CSV, Parquet, JSONL, model checkpoints, and more. liboxen is the embeddable core library behind the oxen CLI and server, which power fine tuning and inference pipelines for multimodal LLMs, image models, and video models on Oxen.ai.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41

pub mod data_frame_size;
pub mod schema;
pub mod update_result;

use crate::model::data_frame::data_frame_size::DataFrameSize;
use crate::model::data_frame::schema::Schema;

use serde::{Deserialize, Serialize};
use utoipa::ToSchema;

use polars::frame::DataFrame;

#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct DataFrameSchemaSize {
    pub schema: Schema,
    pub size: DataFrameSize,
}

#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct DataFrameSliceSchemas {
    pub source: DataFrameSchemaSize,
    pub slice: DataFrameSchemaSize,
}

pub struct DataFrameSlice {
    pub schemas: DataFrameSliceSchemas,
    pub slice: DataFrame,
    pub total_entries: usize,
}

impl DataFrameSchemaSize {
    pub fn from_df(df: &DataFrame, schema: &Schema) -> DataFrameSchemaSize {
        DataFrameSchemaSize {
            schema: schema.to_owned(),
            size: DataFrameSize {
                height: df.height(),
                width: df.width(),
            },
        }
    }
}