liboxen 0.7.2

Oxen is a fast, unstructured data version control, to help version large machine learning datasets written in Rust.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104

use serde::{Deserialize, Serialize};

use crate::model::{Commit, EntryDataType};
use crate::view::entry::ResourceVersion;

#[derive(Deserialize, Serialize, Debug, Clone, PartialEq, Eq)]
pub enum ImgColorSpace {
    // 8-bit
    RGB,
    RGBA,
    Grayscale,
    GrayscaleAlpha,

    // 16-bit
    Rgb16,
    Rgba16,
    Grayscale16,
    GrayscaleAlpha16,

    // 32-bit float
    Rgb32F,
    Rgba32F,

    Unknown,
}

#[derive(Deserialize, Serialize, Debug, Clone)]
pub struct MetadataText {
    pub num_lines: usize,
    // pub num_words: usize,
    // pub num_chars: usize,
    // pub num_whitespace: usize,
}

#[derive(Deserialize, Serialize, Debug, Clone)]
pub struct MetadataImage {
    pub width: usize,
    pub height: usize,
    pub color_space: ImgColorSpace, // RGB, RGBA, etc.
}

#[derive(Deserialize, Serialize, Debug, Clone)]
pub struct MetadataVideo {
    pub num_seconds: f64,
    pub width: usize,
    pub height: usize,
    pub color_space: ImgColorSpace, // RGB, RGBA, etc.
    pub format: String,             // mp4, etc.
}

#[derive(Deserialize, Serialize, Debug, Clone)]
pub struct MetadataAudio {
    pub num_seconds: f64,
    pub format: String, // mp3, etc.
    pub num_channels: usize,
    pub sample_rate: usize,
}

#[derive(Deserialize, Serialize, Debug, Clone)]
pub struct MetadataTabular {
    pub width: usize,
    pub height: usize,
}

#[derive(Deserialize, Serialize, Debug, Clone)]
pub struct MetadataItem {
    pub text: Option<MetadataText>,
    pub image: Option<MetadataImage>,
    pub video: Option<MetadataVideo>,
    pub audio: Option<MetadataAudio>,
    pub tabular: Option<MetadataTabular>,
}

#[derive(Deserialize, Serialize, Debug, Clone)]
pub struct CLIMetadataEntry {
    pub filename: String,
    pub last_updated: Option<Commit>,
    // Hash of the file
    pub hash: String,
    // size of the file in bytes
    pub size: u64,
    // high level type of "image", "text", "video", "audio", "tabular"
    pub data_type: EntryDataType,
    // auto detected mime type of the file (e.g. "image/png")
    pub mime_type: String,
    // auto detected extension of the file
    pub extension: String,
}

#[derive(Deserialize, Serialize, Debug, Clone)]
pub struct MetadataEntry {
    pub filename: String,
    pub is_dir: bool,
    pub latest_commit: Option<Commit>,
    pub resource: Option<ResourceVersion>,
    // size of the file in bytes
    pub size: u64,
    // high level type of "image", "text", "video", "audio", "tabular"
    pub data_type: EntryDataType,
    // auto detected mime type of the file (e.g. "image/png")
    pub mime_type: String,
    // auto detected extension of the file
    pub extension: String,
}