liboxen 0.53.0

Oxen is a fast data version control system, built with machine learning training data in mind. Designed to handle terabytes of data with ease, using a workflow similar to git. Version both structured and unstructured data of any modality: text, images, video, audio, CSV, Parquet, JSONL, model checkpoints, and more. liboxen is the embeddable core library behind the oxen CLI and server, which power fine tuning and inference pipelines for multimodal LLMs, image models, and video models on Oxen.ai.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89

use super::User;

use bytes::Bytes;
use serde::de::{self, Visitor};
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use std::fmt;
use std::path::PathBuf;
use utoipa::ToSchema;

#[derive(Debug, Clone, ToSchema)]
pub enum FileContents {
    Text(String),
    Binary(Vec<u8>),
}

impl FileContents {
    /// The contents as bytes, reusing the existing allocation rather than copying.
    pub fn into_bytes(self) -> Bytes {
        match self {
            FileContents::Text(text) => Bytes::from(text.into_bytes()),
            FileContents::Binary(bytes) => Bytes::from(bytes),
        }
    }
}

impl Serialize for FileContents {
    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
    where
        S: Serializer,
    {
        match self {
            FileContents::Text(text) => serializer.serialize_str(text),
            FileContents::Binary(bytes) => serializer.serialize_bytes(bytes),
        }
    }
}

impl<'de> Deserialize<'de> for FileContents {
    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
    where
        D: Deserializer<'de>,
    {
        struct FileContentsVisitor;

        impl Visitor<'_> for FileContentsVisitor {
            type Value = FileContents;

            fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
                formatter.write_str("a string or byte array")
            }

            fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
            where
                E: de::Error,
            {
                Ok(FileContents::Text(value.to_owned()))
            }

            fn visit_bytes<E>(self, value: &[u8]) -> Result<Self::Value, E>
            where
                E: de::Error,
            {
                Ok(FileContents::Binary(value.to_vec()))
            }
        }

        deserializer.deserialize_any(FileContentsVisitor)
    }
}

#[derive(Deserialize, Serialize, Debug, Clone, ToSchema)]
pub struct FileNew {
    #[schema(value_type = String)]
    pub path: PathBuf,
    pub contents: FileContents,
    pub user: User,
}

#[derive(Deserialize, Serialize, Debug, Clone)]
pub struct TempFileNew {
    pub path: PathBuf,
    pub contents: FileContents,
}

#[derive(Deserialize, Serialize, Debug, Clone)]
pub struct TempFilePathNew {
    pub path: PathBuf,
    pub temp_file_path: PathBuf,
}