liboxen 0.58.0

Oxen is a fast data version control system, built with machine learning training data in mind. Designed to handle terabytes of data with ease, using a workflow similar to git. Version both structured and unstructured data of any modality: text, images, video, audio, CSV, Parquet, JSONL, model checkpoints, and more. liboxen is the embeddable core library behind the oxen CLI and server, which power fine tuning and inference pipelines for multimodal LLMs, image models, and video models on Oxen.ai.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122

use crate::error::OxenError;
use serde::{Deserialize, Serialize};
use utoipa::ToSchema;
use uuid::Uuid;
#[derive(Serialize, Deserialize, Debug, Clone, ToSchema)]
pub struct Remote {
    pub name: String,
    pub url: String,
    /// The UUID the remote addresses this repository's storage by. Absent on a remote written
    /// before clients recorded it, and on one pointing at a repository that reports no identity.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub repo_uuid: Option<Uuid>,
}

impl Remote {
    /// A remote naming where a repository lives, with no UUID recorded for it.
    pub fn new(name: &str, url: &str) -> Remote {
        Remote {
            name: name.to_string(),
            url: url.to_string(),
            repo_uuid: None,
        }
    }

    /// The same remote, taking `repo_uuid` only where it has none recorded.
    ///
    pub(crate) fn with_repo_uuid_if_absent(self, repo_uuid: Option<Uuid>) -> Remote {
        Remote {
            repo_uuid: self.repo_uuid.or(repo_uuid),
            ..self
        }
    }

    /// Errors when this remote and `reported` both carry a UUID and the two disagree.
    pub(crate) fn ensure_same_repo_uuid(&self, reported: Option<Uuid>) -> Result<(), OxenError> {
        match (self.repo_uuid, reported) {
            (Some(recorded), Some(reported)) if recorded != reported => {
                Err(OxenError::RemotePointsAtDifferentRepo {
                    name: self.name.clone(),
                    url: self.url.clone(),
                    recorded,
                    reported,
                })
            }
            _ => Ok(()),
        }
    }
}

impl std::fmt::Display for Remote {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(f, "[{}] '{}'", self.name, self.url)
    }
}

impl std::error::Error for Remote {}

#[cfg(test)]
mod tests {
    use super::*;

    /// The client config is TOML, and a remote written before clients recorded a UUID has no key
    /// for it, so it has to load as absent rather than failing the whole config.
    #[test]
    fn a_remote_without_a_uuid_loads_with_none() {
        let remote: Remote =
            toml::from_str("name = \"origin\"\nurl = \"http://localhost:3000/ox/cats\"\n")
                .expect("a remote predating the field parses");

        assert_eq!(remote.repo_uuid, None);
    }

    /// Guards the `skip_serializing_if`, so a config written for a repository with no identity
    /// carries no key an older client would have to tolerate.
    #[test]
    fn an_absent_uuid_is_not_written_at_all() {
        let toml = toml::to_string(&Remote::new("origin", "http://localhost:3000/ox/cats"))
            .expect("serialize");

        assert!(!toml.contains("repo_uuid"), "unexpected key in:\n{toml}");
    }

    #[test]
    fn a_disagreeing_reported_uuid_is_refused() {
        let remote = Remote::new("origin", "http://localhost:3000/ox/cats")
            .with_repo_uuid_if_absent(Some(Uuid::new_v4()));

        let result = remote.ensure_same_repo_uuid(Some(Uuid::new_v4()));

        assert!(
            matches!(result, Err(OxenError::RemotePointsAtDifferentRepo { .. })),
            "expected a refusal, got: {result:?}"
        );
    }

    /// Only a disagreement fails: a remote with nothing recorded takes what the server reports,
    /// and a server reporting nothing leaves a recorded UUID alone.
    #[test]
    fn an_agreeing_or_absent_uuid_is_accepted() {
        let repo_uuid = Uuid::new_v4();
        let recorded = Remote::new("origin", "http://localhost:3000/ox/cats")
            .with_repo_uuid_if_absent(Some(repo_uuid));
        let unrecorded = Remote::new("origin", "http://localhost:3000/ox/cats");

        assert!(recorded.ensure_same_repo_uuid(Some(repo_uuid)).is_ok());
        assert!(recorded.ensure_same_repo_uuid(None).is_ok());
        assert!(unrecorded.ensure_same_repo_uuid(Some(repo_uuid)).is_ok());
        assert!(unrecorded.ensure_same_repo_uuid(None).is_ok());
    }

    #[test]
    fn a_recorded_uuid_round_trips() {
        let repo_uuid = Uuid::new_v4();
        let remote = Remote::new("origin", "http://localhost:3000/ox/cats")
            .with_repo_uuid_if_absent(Some(repo_uuid));

        let loaded: Remote =
            toml::from_str(&toml::to_string(&remote).expect("serialize")).expect("deserialize");

        assert_eq!(loaded.repo_uuid, Some(repo_uuid));
    }
}