liboxen 0.53.0

Oxen is a fast data version control system, built with machine learning training data in mind. Designed to handle terabytes of data with ease, using a workflow similar to git. Version both structured and unstructured data of any modality: text, images, video, audio, CSV, Parquet, JSONL, model checkpoints, and more. liboxen is the embeddable core library behind the oxen CLI and server, which power fine tuning and inference pipelines for multimodal LLMs, image models, and video models on Oxen.ai.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57

use std::path::{Path, PathBuf};

use crate::core::db::data_frames::DataFrameError;
use crate::core::df::tabular;
use crate::core::v_latest::workspaces;
use crate::model::LocalRepository;
use crate::opts::DFOpts;
use crate::repositories;
use crate::{core::db::data_frames::df_db, error::OxenError};
use polars::frame::DataFrame;
use uuid::Uuid;

pub async fn query_df_from_repo(
    sql: String,
    repo: &LocalRepository,
    path: &PathBuf,
    opts: &DFOpts,
) -> Result<DataFrame, OxenError> {
    let commit = repositories::commits::head_commit(repo)?;

    if !repositories::workspaces::data_frames::is_queryable_data_frame_indexed(repo, path, &commit)?
    {
        // If not, proceed to create a new workspace and index the data frame.
        let workspace_id = Uuid::new_v4().to_string();
        let workspace = repositories::workspaces::create(repo, &commit, workspace_id, false)?;
        repositories::workspaces::data_frames::index(repo, &workspace, path).await?;
    }

    let workspace =
        workspaces::data_frames::get_queryable_data_frame_workspace(repo, path, &commit)?;

    let db_path = repositories::workspaces::data_frames::duckdb_path(&workspace, path);
    let df = df_db::with_hardened_query_conn(&db_path, |conn| query_df(conn, sql, Some(opts)))?;

    // If we are doing this from the CLI, we don't want to export the hidden Oxen columns
    let df = tabular::strip_excluded_cols(df)?;
    Ok(df)
}

pub fn query_df(
    conn: &duckdb::Connection,
    sql: String,
    opts: Option<&DFOpts>,
) -> Result<DataFrame, DataFrameError> {
    let df = df_db::select_str(conn, &sql, opts)?;

    Ok(df)
}

pub fn export_df(
    conn: &duckdb::Connection,
    sql: String,
    opts: Option<&DFOpts>,
    tmp_path: &Path,
) -> Result<(), DataFrameError> {
    df_db::export(conn, &sql, opts, tmp_path)
}