liboxen 0.53.0

Oxen is a fast data version control system, built with machine learning training data in mind. Designed to handle terabytes of data with ease, using a workflow similar to git. Version both structured and unstructured data of any modality: text, images, video, audio, CSV, Parquet, JSONL, model checkpoints, and more. liboxen is the embeddable core library behind the oxen CLI and server, which power fine tuning and inference pipelines for multimodal LLMs, image models, and video models on Oxen.ai.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81

//! # Endpoint - Helpers for creating urls for the remote API
//!

use crate::api::requests::repo_new::RepoNew;
use crate::error::OxenError;
use crate::model::{Remote, RemoteRepository};
use url::Url;

pub const API_NAMESPACE: &str = "/api/repos";

pub fn url_from_host_and_scheme(
    host: impl AsRef<str>,
    uri: impl AsRef<str>,
    scheme: impl AsRef<str>,
) -> String {
    format!(
        "{}://{}{API_NAMESPACE}{}",
        scheme.as_ref(),
        host.as_ref(),
        uri.as_ref()
    )
}

pub fn url_from_host(host: &str, uri: &str) -> String {
    format!(
        "{}://{host}{API_NAMESPACE}{uri}",
        RepoNew::scheme_default(host)
    )
}

pub fn remote_url_from_namespace_name(host: &str, namespace: &str, name: &str) -> String {
    format!(
        "{}://{host}/{namespace}/{name}",
        RepoNew::scheme_default(host)
    )
}

pub fn remote_url_from_namespace_name_scheme(
    host: &str,
    namespace: &str,
    name: &str,
    scheme: &str,
) -> String {
    format!("{scheme}://{host}/{namespace}/{name}")
}

pub fn remote_url_from_name(host: &str, name: &str) -> String {
    format!("{}://{host}/{name}", RepoNew::scheme_default(host))
}

pub fn remote_url_from_name_and_scheme(host: &str, name: &str, scheme: &str) -> String {
    format!("{scheme}://{host}/{name}")
}

pub fn url_from_remote(remote: &Remote, uri: &str) -> Result<String, OxenError> {
    // log::info!("url_from_remote creating url_from_remote {remote:?} -> {uri:?}");
    match Url::parse(&remote.url) {
        Ok(mut parsed_url) => {
            // TODO: this is a workaround because to_string was URL encoding characters that we didn't want encoded
            // log::info!("url_from_remote parsed_url: {}", parsed_url);
            let new_path = format!("{}{}{}", API_NAMESPACE, parsed_url.path(), uri);

            parsed_url.set_path("");
            // log::info!("url_from_remote parsed_url after set path: {}", parsed_url);

            let mut remote_url = parsed_url.to_string();
            remote_url.pop(); // to_string adds a trailing slash we don't want
            // log::info!("url_from_remote new_path: {}", new_path);
            // log::info!("url_from_remote remote_url: {}", remote_url);
            Ok(format!("{remote_url}{new_path}"))
        }
        Err(e) => {
            log::warn!("Invalid remote url: {:?}\n{:?}", remote.url, e);
            Err(OxenError::invalid_set_remote_url(&remote.url))
        }
    }
}

pub fn url_from_repo(repo: &RemoteRepository, uri: &str) -> Result<String, OxenError> {
    url_from_remote(&repo.remote, uri)
}