liboxen 0.46.12

Oxen is a fast, unstructured data version control, to help version large machine learning datasets written in Rust.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126

use serde::{Deserialize, Serialize};

use crate::model::merkle_tree::node::{DirNode, FileNode};
use crate::model::metadata::generic_metadata::GenericMetadata;
use crate::model::parsed_resource::ParsedResourceView;
use crate::model::{Commit, EntryDataType, LocalRepository, ParsedResource, StagedEntryStatus};
use crate::repositories;

use utoipa::ToSchema;

#[derive(Deserialize, Serialize, Debug, Clone)]
pub struct CLIMetadataEntry {
    pub filename: String,
    pub last_updated: Option<Commit>,
    // Hash of the file
    pub hash: String,
    // size of the file in bytes
    pub size: u64,
    // high level type of "image", "text", "video", "audio", "tabular"
    pub data_type: EntryDataType,
    // auto detected mime type of the file (e.g. "image/png")
    pub mime_type: String,
    // auto detected extension of the file
    pub extension: String,
}

#[derive(Deserialize, Serialize, Debug, Clone, ToSchema)]
pub struct MetadataEntry {
    pub filename: String,
    pub hash: String,
    pub is_dir: bool,
    pub latest_commit: Option<Commit>,
    pub resource: Option<ParsedResource>,
    // size of the file in bytes
    pub size: u64,
    // high level type of "image", "text", "video", "audio", "tabular"
    pub data_type: EntryDataType,
    // auto detected mime type of the file (e.g. "image/png")
    pub mime_type: String,
    // auto detected extension of the file
    pub extension: String,
    // metadata per data type
    pub metadata: Option<GenericMetadata>,
    // If it's a tabular file, is it indexed for querying?
    pub is_queryable: Option<bool>,
    // Nested children entries when depth > 0
    #[serde(skip_serializing_if = "Option::is_none")]
    #[schema(no_recursion)]
    pub children: Option<Vec<MetadataEntry>>,
}

#[derive(Deserialize, Serialize, Debug, Clone, ToSchema)]
pub struct WorkspaceMetadataEntry {
    pub filename: String,
    pub hash: String,
    pub is_dir: bool,
    pub latest_commit: Option<Commit>,
    pub resource: Option<ParsedResourceView>,
    // size of the file in bytes
    pub size: u64,
    // high level type of "image", "text", "video", "audio", "tabular"
    pub data_type: EntryDataType,
    // auto detected mime type of the file (e.g. "image/png")
    pub mime_type: String,
    // auto detected extension of the file
    pub extension: String,
    // metadata per data type
    pub metadata: Option<GenericMetadata>,
    // If it's a tabular file, is it indexed for querying?
    pub is_queryable: Option<bool>,
    // Workspace changes if the entry is part of a workspace
    pub changes: Option<WorkspaceChanges>,
    // Nested children entries when depth > 0
    #[serde(skip_serializing_if = "Option::is_none")]
    #[schema(no_recursion)]
    pub children: Option<Vec<WorkspaceMetadataEntry>>,
}

#[derive(Deserialize, Serialize, Debug, Clone, ToSchema)]
pub struct WorkspaceChanges {
    pub status: StagedEntryStatus,
}

impl MetadataEntry {
    pub fn from_file_node(
        repo: &LocalRepository,
        node: Option<FileNode>,
        commit: &Commit,
    ) -> Option<MetadataEntry> {
        node.as_ref()?;
        repositories::metadata::from_file_node(repo, &node.unwrap(), commit).ok()
    }

    pub fn from_dir_node(
        repo: &LocalRepository,
        node: Option<DirNode>,
        commit: &Commit,
    ) -> Option<MetadataEntry> {
        node.as_ref()?;
        repositories::metadata::from_dir_node(repo, &node.unwrap(), commit).ok()
    }
}

impl WorkspaceMetadataEntry {
    /// Converts a `MetadataEntry` into a `WorkspaceMetadataEntry` with `changes` set to `None`.
    pub fn from_metadata_entry(metadata: MetadataEntry) -> Self {
        WorkspaceMetadataEntry {
            filename: metadata.filename,
            hash: metadata.hash,
            is_dir: metadata.is_dir,
            latest_commit: metadata.latest_commit,
            // Convert ParsedResource to ParsedResourceView using the From trait.
            resource: metadata.resource.map(ParsedResourceView::from),
            size: metadata.size,
            data_type: metadata.data_type,
            mime_type: metadata.mime_type,
            extension: metadata.extension,
            metadata: metadata.metadata,
            is_queryable: metadata.is_queryable,
            changes: None,
            children: metadata
                .children
                .map(|c| c.into_iter().map(Self::from_metadata_entry).collect()),
        }
    }
}