liboxen 0.53.0

Oxen is a fast data version control system, built with machine learning training data in mind. Designed to handle terabytes of data with ease, using a workflow similar to git. Version both structured and unstructured data of any modality: text, images, video, audio, CSV, Parquet, JSONL, model checkpoints, and more. liboxen is the embeddable core library behind the oxen CLI and server, which power fine tuning and inference pipelines for multimodal LLMs, image models, and video models on Oxen.ai.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132

use serde::{Deserialize, Serialize};

use crate::model::merkle_tree::node::{DirNode, FileNode};
use crate::model::metadata::generic_metadata::GenericMetadata;
use crate::model::parsed_resource::ParsedResourceView;
use crate::model::{Commit, EntryDataType, LocalRepository, StagedEntryStatus};
use crate::repositories;

use utoipa::ToSchema;

#[derive(Deserialize, Serialize, Debug, Clone)]
pub struct CLIMetadataEntry {
    pub filename: String,
    pub last_updated: Option<Commit>,
    // Hash of the file
    pub hash: String,
    // size of the file in bytes
    pub size: u64,
    // high level type of "image", "text", "video", "audio", "tabular"
    pub data_type: EntryDataType,
    // auto detected mime type of the file (e.g. "image/png")
    pub mime_type: String,
    // auto detected extension of the file
    pub extension: String,
}

/// `deny_unknown_fields` is the discriminator for the untagged `EMetadataEntry` enum:
/// payloads with a `changes` field (i.e. `WorkspaceMetadataEntry`) fail to deserialize here
/// and fall through to the workspace variant.
#[derive(Deserialize, Serialize, Debug, Clone, ToSchema)]
#[serde(deny_unknown_fields)]
pub struct MetadataEntry {
    pub filename: String,
    pub hash: String,
    pub is_dir: bool,
    pub latest_commit: Option<Commit>,
    pub resource: Option<ParsedResourceView>,
    // size of the file in bytes
    pub size: u64,
    // high level type of "image", "text", "video", "audio", "tabular"
    pub data_type: EntryDataType,
    // auto detected mime type of the file (e.g. "image/png")
    pub mime_type: String,
    // auto detected extension of the file
    pub extension: String,
    // metadata per data type
    pub metadata: Option<GenericMetadata>,
    // If it's a tabular file, is it indexed for querying?
    pub is_queryable: Option<bool>,
    // Nested children entries when depth > 0
    #[serde(skip_serializing_if = "Option::is_none")]
    #[schema(no_recursion)]
    pub children: Option<Vec<MetadataEntry>>,
}

#[derive(Deserialize, Serialize, Debug, Clone, ToSchema)]
pub struct WorkspaceMetadataEntry {
    pub filename: String,
    pub hash: String,
    pub is_dir: bool,
    pub latest_commit: Option<Commit>,
    pub resource: Option<ParsedResourceView>,
    // size of the file in bytes
    pub size: u64,
    // high level type of "image", "text", "video", "audio", "tabular"
    pub data_type: EntryDataType,
    // auto detected mime type of the file (e.g. "image/png")
    pub mime_type: String,
    // auto detected extension of the file
    pub extension: String,
    // metadata per data type
    pub metadata: Option<GenericMetadata>,
    // If it's a tabular file, is it indexed for querying?
    pub is_queryable: Option<bool>,
    // Workspace changes if the entry is part of a workspace. Skipped when None so the wire
    // shape collapses back to a plain `MetadataEntry` — the presence of this field is the
    // discriminator the untagged `EMetadataEntry` enum uses on the client side.
    #[serde(skip_serializing_if = "Option::is_none")]
    pub changes: Option<WorkspaceChanges>,
    // Nested children entries when depth > 0
    #[serde(skip_serializing_if = "Option::is_none")]
    #[schema(no_recursion)]
    pub children: Option<Vec<WorkspaceMetadataEntry>>,
}

#[derive(Deserialize, Serialize, Debug, Clone, ToSchema)]
pub struct WorkspaceChanges {
    pub status: StagedEntryStatus,
}

impl MetadataEntry {
    pub fn from_file_node(
        repo: &LocalRepository,
        node: Option<FileNode>,
        commit: &Commit,
    ) -> Option<MetadataEntry> {
        node.as_ref()?;
        repositories::metadata::from_file_node(repo, &node.unwrap(), commit).ok()
    }

    pub fn from_dir_node(
        repo: &LocalRepository,
        node: Option<DirNode>,
        commit: &Commit,
    ) -> Option<MetadataEntry> {
        node.as_ref()?;
        repositories::metadata::from_dir_node(repo, &node.unwrap(), commit).ok()
    }
}

impl WorkspaceMetadataEntry {
    /// Converts a `MetadataEntry` into a `WorkspaceMetadataEntry` with `changes` set to `None`.
    pub fn from_metadata_entry(metadata: MetadataEntry) -> Self {
        WorkspaceMetadataEntry {
            filename: metadata.filename,
            hash: metadata.hash,
            is_dir: metadata.is_dir,
            latest_commit: metadata.latest_commit,
            resource: metadata.resource,
            size: metadata.size,
            data_type: metadata.data_type,
            mime_type: metadata.mime_type,
            extension: metadata.extension,
            metadata: metadata.metadata,
            is_queryable: metadata.is_queryable,
            changes: None,
            children: metadata
                .children
                .map(|c| c.into_iter().map(Self::from_metadata_entry).collect()),
        }
    }
}