liboxen 0.46.12

Oxen is a fast, unstructured data version control, to help version large machine learning datasets written in Rust.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89

use std::{collections::HashMap, path::PathBuf, time::Duration};

use crate::model::MerkleHash;

use super::StatusMessage;
use serde::{Deserialize, Serialize};
use utoipa::ToSchema;

#[derive(Serialize, Deserialize, Debug, ToSchema)]
pub struct VersionFile {
    pub hash: String,
    pub size: u64,
}

#[derive(Serialize, Deserialize, Debug, ToSchema)]
pub struct VersionFileResponse {
    #[serde(flatten)]
    pub status: StatusMessage,
    pub version: VersionFile,
}

#[derive(Clone)]
pub enum MultipartLargeFileUploadStatus {
    Pending,
    Completed,
    Failed,
}

#[derive(Clone)]
pub struct MultipartLargeFileUpload {
    pub local_path: PathBuf,      // Path to the file on the local filesystem
    pub dst_dir: Option<PathBuf>, // Path to upload the file to on the server
    pub hash: MerkleHash,         // Unique identifier for the file
    pub size: u64,                // Size of the file in bytes
    pub status: MultipartLargeFileUploadStatus, // Status of the upload
    pub reason: Option<String>,   // Reason for the upload failure
}

#[derive(Serialize, Deserialize, Debug)]
pub struct CompletedFileUpload {
    pub hash: String,
    pub file_name: String,        // The name of the file
    pub dst_dir: Option<PathBuf>, // The destination directory for the file
    // Number of chunks that were uploaded for this file (new clients send this)
    #[serde(default)]
    pub num_chunks: Option<usize>,
    // Deprecated: old clients send this list of chunk upload headers
    #[serde(default)]
    pub upload_results: Option<Vec<HashMap<String, String>>>,
}

#[derive(Serialize, Deserialize, Debug)]
pub struct CompleteVersionUploadRequest {
    pub files: Vec<CompletedFileUpload>,
    // If the workspace_id is provided, we will add the file to the workspace
    // otherwise, we will just add the file to the versions store
    pub workspace_id: Option<String>,
    #[serde(default)]
    pub update_timestamp: bool,
}

#[derive(Serialize, Deserialize, Debug)]
pub struct CreateVersionUploadRequest {
    pub hash: String,
    pub file_name: String,
    pub size: u64,
    pub dst_dir: Option<PathBuf>,
}

#[derive(Deserialize, Serialize, Debug)]
pub struct CleanCorruptedVersionsResponse {
    #[serde(flatten)]
    pub status: StatusMessage,
    pub result: CleanCorruptedVersionsResult,
}

#[derive(Deserialize, Serialize, Default, Debug, Clone)]
pub struct CleanCorruptedVersionsResult {
    // Number of version files scanned
    pub scanned: u64,
    // Number of version files that were corrupted(hash mismatch)
    pub corrupted: u64,
    // Number of version files that were successfully deleted
    pub cleaned: u64,
    // Errors that occurred during the cleanup process
    pub errors: u64,
    // Elapsed time in seconds
    pub elapsed: Duration,
}