shaul_core/resource.rs
1use std::collections::BTreeMap;
2
3use serde::{Deserialize, Serialize};
4use url::Url;
5
6use crate::checksum::Checksum;
7use crate::error::Result;
8use crate::header::{Header, RequestOptions};
9use crate::path::{filename_from_url, sanitize_relative_path};
10
11/// A parsed, backend-agnostic description of what the user wants to download.
12#[derive(Clone, Debug, Serialize, Deserialize)]
13pub struct Resource {
14 /// Protocol family and its identifying details.
15 pub kind: ResourceKind,
16 /// Display name chosen by the parser, such as the URL filename or the
17 /// repository name.
18 pub name: String,
19 /// Files that make up the resource. Protocols that only discover files
20 /// while downloading, such as BitTorrent, leave this empty.
21 pub files: Vec<ResourceFile>,
22 /// Request overrides carried over from the CLI and configuration.
23 #[serde(default)]
24 pub request: RequestOptions,
25 /// Checksum supplied when the task was created, preserved in the
26 /// sidecar and shown in CLI output.
27 #[serde(default)]
28 pub checksum: Option<Checksum>,
29 /// Revision requested for the task, such as a Hub branch or tag.
30 #[serde(default)]
31 pub revision: Option<String>,
32 /// Protocol-private key/value data persisted in the sidecar. New
33 /// protocol fields belong here rather than in new struct fields.
34 #[serde(default)]
35 pub metadata: BTreeMap<String, String>,
36}
37
38#[derive(Clone, Debug, Serialize, Deserialize)]
39#[serde(tag = "type", rename_all = "snake_case")]
40#[non_exhaustive]
41/// Protocol family a [`Resource`] is fetched with.
42pub enum ResourceKind {
43 /// A direct HTTP(S) download.
44 Http,
45 /// A repository snapshot from a model hub.
46 Hub {
47 /// Hub service hosting the repository.
48 provider: HubProvider,
49 /// Repository identifier such as `org/name`.
50 repo: String,
51 /// Whether the repository is a model or a dataset.
52 repo_type: HubRepoType,
53 /// Branch, tag, or commit to resolve.
54 revision: String,
55 },
56 /// A torrent, given as a magnet link or a `.torrent` file.
57 BitTorrent {
58 /// Where the torrent metadata comes from.
59 source: BitTorrentSource,
60 /// Display name taken from the torrent metadata, when available.
61 name: Option<String>,
62 },
63 /// A third-party protocol selected by a custom parser and consumed by a
64 /// custom [`DownloadBackend`](crate::backend::DownloadBackend).
65 ///
66 /// `protocol` is matched by the backend's `supports` method; protocol
67 /// data belongs in [`Resource::metadata`] and
68 /// [`ResourceFile::metadata`].
69 Custom {
70 /// Parser-defined protocol identifier, for example `demo`.
71 protocol: String,
72 },
73}
74
75#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
76#[serde(rename_all = "snake_case")]
77#[non_exhaustive]
78/// Where the metadata for a BitTorrent resource comes from.
79pub enum BitTorrentSource {
80 /// A magnet link, or a bare v1 info hash.
81 Magnet,
82 /// A local `.torrent` file.
83 TorrentFile,
84}
85
86#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
87#[serde(rename_all = "snake_case")]
88#[non_exhaustive]
89/// Hub service hosting a repository snapshot.
90pub enum HubProvider {
91 /// Hugging Face, including compatible mirrors.
92 HuggingFace,
93 /// ModelScope.
94 ModelScope,
95}
96
97#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
98#[serde(rename_all = "snake_case")]
99#[non_exhaustive]
100/// Kind of repository hosted on a hub.
101pub enum HubRepoType {
102 /// A model repository.
103 Model,
104 /// A dataset repository.
105 Dataset,
106}
107
108/// One file belonging to a [`Resource`].
109#[derive(Clone, Debug, Serialize, Deserialize)]
110pub struct ResourceFile {
111 /// Relative output path. Always validated and safe to join under the
112 /// task destination.
113 pub path: String,
114 /// Resolved URL the backend fetches.
115 pub url: Url,
116 /// Expected size in bytes, when the protocol or a previous probe
117 /// reported it.
118 #[serde(default)]
119 pub size: Option<u64>,
120 /// Expected checksum for this file, when known.
121 #[serde(default)]
122 pub checksum: Option<Checksum>,
123 /// Additional headers applied only to this file's requests.
124 #[serde(default)]
125 pub headers: Vec<Header>,
126 /// Protocol-private key/value data preserved in the sidecar.
127 #[serde(default)]
128 pub metadata: BTreeMap<String, String>,
129 /// Direct links prefer the server's `Content-Disposition` filename; Hub
130 /// snapshots use the repository-relative path.
131 #[serde(default = "default_true")]
132 pub use_response_filename: bool,
133}
134
135fn default_true() -> bool {
136 true
137}
138
139impl Resource {
140 /// Build a single-file [`ResourceKind::Http`] resource for `url`.
141 ///
142 /// The display name and output path are derived from the URL's last path
143 /// segment and validated with [`sanitize_relative_path`], so an unsafe
144 /// filename fails the parse instead of being used.
145 pub fn http_file(
146 url: Url,
147 request: RequestOptions,
148 checksum: Option<Checksum>,
149 ) -> Result<Self> {
150 let name = filename_from_url(&url);
151 let path = sanitize_relative_path(&name)?;
152 Ok(Self {
153 kind: ResourceKind::Http,
154 name,
155 files: vec![ResourceFile {
156 path,
157 url,
158 size: None,
159 checksum: None,
160 headers: Vec::new(),
161 metadata: BTreeMap::new(),
162 use_response_filename: true,
163 }],
164 request,
165 checksum,
166 revision: None,
167 metadata: BTreeMap::new(),
168 })
169 }
170
171 /// Sum of the known file sizes, or `None` if any file size is unknown.
172 /// An empty file list sums to `Some(0)`.
173 pub fn total_size(&self) -> Option<u64> {
174 self.files
175 .iter()
176 .try_fold(0u64, |sum, file| file.size.map(|size| sum + size))
177 }
178}
179
180impl ResourceFile {
181 /// Validate `path` as a destination-relative path and replace the file's
182 /// output path. Absolute paths and `..` traversal return an error.
183 pub fn with_path(mut self, path: impl AsRef<str>) -> Result<Self> {
184 self.path = sanitize_relative_path(path.as_ref())?;
185 Ok(self)
186 }
187}