1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
use std::collections::BTreeMap;
use serde::{Deserialize, Serialize};
use url::Url;
use crate::checksum::Checksum;
use crate::error::Result;
use crate::header::{Header, RequestOptions};
use crate::path::{filename_from_url, sanitize_relative_path};
/// A parsed, backend-agnostic description of what the user wants to download.
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct Resource {
/// Protocol family and its identifying details.
pub kind: ResourceKind,
/// Display name chosen by the parser, such as the URL filename or the
/// repository name.
pub name: String,
/// Files that make up the resource. Protocols that only discover files
/// while downloading, such as BitTorrent, leave this empty.
pub files: Vec<ResourceFile>,
/// Request overrides carried over from the CLI and configuration.
#[serde(default)]
pub request: RequestOptions,
/// Checksum supplied when the task was created, preserved in the
/// sidecar and shown in CLI output.
#[serde(default)]
pub checksum: Option<Checksum>,
/// Revision requested for the task, such as a Hub branch or tag.
#[serde(default)]
pub revision: Option<String>,
/// Protocol-private key/value data persisted in the sidecar. New
/// protocol fields belong here rather than in new struct fields.
#[serde(default)]
pub metadata: BTreeMap<String, String>,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case")]
#[non_exhaustive]
/// Protocol family a [`Resource`] is fetched with.
pub enum ResourceKind {
/// A direct HTTP(S) download.
Http,
/// A repository snapshot from a model hub.
Hub {
/// Hub service hosting the repository.
provider: HubProvider,
/// Repository identifier such as `org/name`.
repo: String,
/// Whether the repository is a model or a dataset.
repo_type: HubRepoType,
/// Branch, tag, or commit to resolve.
revision: String,
},
/// A torrent, given as a magnet link or a `.torrent` file.
BitTorrent {
/// Where the torrent metadata comes from.
source: BitTorrentSource,
/// Display name taken from the torrent metadata, when available.
name: Option<String>,
},
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
#[non_exhaustive]
/// Where the metadata for a BitTorrent resource comes from.
pub enum BitTorrentSource {
/// A magnet link, or a bare v1 info hash.
Magnet,
/// A local `.torrent` file.
TorrentFile,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
#[non_exhaustive]
/// Hub service hosting a repository snapshot.
pub enum HubProvider {
/// Hugging Face, including compatible mirrors.
HuggingFace,
/// ModelScope.
ModelScope,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
#[non_exhaustive]
/// Kind of repository hosted on a hub.
pub enum HubRepoType {
/// A model repository.
Model,
/// A dataset repository.
Dataset,
}
/// One file belonging to a [`Resource`].
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct ResourceFile {
/// Relative output path. Always validated and safe to join under the
/// task destination.
pub path: String,
/// Resolved URL the backend fetches.
pub url: Url,
/// Expected size in bytes, when the protocol or a previous probe
/// reported it.
#[serde(default)]
pub size: Option<u64>,
/// Expected checksum for this file, when known.
#[serde(default)]
pub checksum: Option<Checksum>,
/// Additional headers applied only to this file's requests.
#[serde(default)]
pub headers: Vec<Header>,
/// Protocol-private key/value data preserved in the sidecar.
#[serde(default)]
pub metadata: BTreeMap<String, String>,
/// Direct links prefer the server's `Content-Disposition` filename; Hub
/// snapshots use the repository-relative path.
#[serde(default = "default_true")]
pub use_response_filename: bool,
}
fn default_true() -> bool {
true
}
impl Resource {
/// Build a single-file [`ResourceKind::Http`] resource for `url`.
///
/// The display name and output path are derived from the URL's last path
/// segment and validated with [`sanitize_relative_path`], so an unsafe
/// filename fails the parse instead of being used.
pub fn http_file(
url: Url,
request: RequestOptions,
checksum: Option<Checksum>,
) -> Result<Self> {
let name = filename_from_url(&url);
let path = sanitize_relative_path(&name)?;
Ok(Self {
kind: ResourceKind::Http,
name,
files: vec![ResourceFile {
path,
url,
size: None,
checksum: None,
headers: Vec::new(),
metadata: BTreeMap::new(),
use_response_filename: true,
}],
request,
checksum,
revision: None,
metadata: BTreeMap::new(),
})
}
/// Sum of the known file sizes, or `None` if any file size is unknown.
/// An empty file list sums to `Some(0)`.
pub fn total_size(&self) -> Option<u64> {
self.files
.iter()
.try_fold(0u64, |sum, file| file.size.map(|size| sum + size))
}
}
impl ResourceFile {
/// Validate `path` as a destination-relative path and replace the file's
/// output path. Absolute paths and `..` traversal return an error.
pub fn with_path(mut self, path: impl AsRef<str>) -> Result<Self> {
self.path = sanitize_relative_path(path.as_ref())?;
Ok(self)
}
}