liboxen 0.53.0

Oxen is a fast data version control system, built with machine learning training data in mind. Designed to handle terabytes of data with ease, using a workflow similar to git. Version both structured and unstructured data of any modality: text, images, video, audio, CSV, Parquet, JSONL, model checkpoints, and more. liboxen is the embeddable core library behind the oxen CLI and server, which power fine tuning and inference pipelines for multimodal LLMs, image models, and video models on Oxen.ai.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66

use crate::core;
use crate::error::OxenError;
use crate::model::{LocalRepository, Schema, Workspace};

use crate::view::JsonDataFrameViews;
use crate::view::data_frames::columns::{ColumnToDelete, ColumnToUpdate, NewColumn};

use polars::frame::DataFrame;
use std::path::{Path, PathBuf};

pub fn add(
    _repo: &LocalRepository,
    workspace: &Workspace,
    file_path: impl AsRef<Path>,
    new_column: &NewColumn,
) -> Result<DataFrame, OxenError> {
    core::v_latest::workspaces::data_frames::columns::add(workspace, file_path.as_ref(), new_column)
}

pub async fn update(
    _repo: &LocalRepository,
    workspace: &Workspace,
    file_path: impl AsRef<Path>,
    column_to_update: &ColumnToUpdate,
) -> Result<DataFrame, OxenError> {
    core::v_latest::workspaces::data_frames::columns::update(
        workspace,
        file_path.as_ref(),
        column_to_update,
    )
    .await
}

pub fn delete(
    _repo: &LocalRepository,
    workspace: &Workspace,
    file_path: impl AsRef<Path>,
    column_to_delete: &ColumnToDelete,
) -> Result<DataFrame, OxenError> {
    core::v_latest::workspaces::data_frames::columns::delete(
        workspace,
        file_path.as_ref(),
        column_to_delete,
    )
}

pub fn add_column_metadata(
    repo: &LocalRepository,
    workspace: &Workspace,
    file_path: PathBuf,
    column: String,
    metadata: &serde_json::Value,
) -> Result<Schema, OxenError> {
    core::v_latest::workspaces::data_frames::columns::add_column_metadata(
        repo, workspace, &file_path, &column, metadata,
    )
}

/// Carry schema and column metadata staged in the workspace into the response
/// schemas, which are seeded from the *committed* schema.
pub fn update_column_schemas(new_schema: Option<Schema>, df_views: &mut JsonDataFrameViews) {
    if let Some(schema) = new_schema {
        df_views.source.schema.update_metadata_from_schema(&schema);
        df_views.view.schema.update_metadata_from_schema(&schema);
    }
}