Skip to main content

shaul_core/
resource.rs

1use std::collections::BTreeMap;
2
3use serde::{Deserialize, Serialize};
4use url::Url;
5
6use crate::checksum::Checksum;
7use crate::error::Result;
8use crate::header::{Header, RequestOptions};
9use crate::path::{filename_from_url, sanitize_relative_path};
10
11/// A parsed, backend-agnostic description of what the user wants to download.
12#[derive(Clone, Debug, Serialize, Deserialize)]
13pub struct Resource {
14    /// Protocol family and its identifying details.
15    pub kind: ResourceKind,
16    /// Display name chosen by the parser, such as the URL filename or the
17    /// repository name.
18    pub name: String,
19    /// Files that make up the resource. Protocols that only discover files
20    /// while downloading, such as BitTorrent, leave this empty.
21    pub files: Vec<ResourceFile>,
22    /// Request overrides carried over from the CLI and configuration.
23    #[serde(default)]
24    pub request: RequestOptions,
25    /// Checksum supplied when the task was created, preserved in the
26    /// sidecar and shown in CLI output.
27    #[serde(default)]
28    pub checksum: Option<Checksum>,
29    /// Revision requested for the task, such as a Hub branch or tag.
30    #[serde(default)]
31    pub revision: Option<String>,
32    /// Protocol-private key/value data persisted in the sidecar. New
33    /// protocol fields belong here rather than in new struct fields.
34    #[serde(default)]
35    pub metadata: BTreeMap<String, String>,
36}
37
38#[derive(Clone, Debug, Serialize, Deserialize)]
39#[serde(tag = "type", rename_all = "snake_case")]
40#[non_exhaustive]
41/// Protocol family a [`Resource`] is fetched with.
42pub enum ResourceKind {
43    /// A direct HTTP(S) download.
44    Http,
45    /// A repository snapshot from a model hub.
46    Hub {
47        /// Hub service hosting the repository.
48        provider: HubProvider,
49        /// Repository identifier such as `org/name`.
50        repo: String,
51        /// Whether the repository is a model or a dataset.
52        repo_type: HubRepoType,
53        /// Branch, tag, or commit to resolve.
54        revision: String,
55    },
56    /// A torrent, given as a magnet link or a `.torrent` file.
57    BitTorrent {
58        /// Where the torrent metadata comes from.
59        source: BitTorrentSource,
60        /// Display name taken from the torrent metadata, when available.
61        name: Option<String>,
62    },
63    /// A third-party protocol selected by a custom parser and consumed by a
64    /// custom [`DownloadBackend`](crate::backend::DownloadBackend).
65    ///
66    /// `protocol` is matched by the backend's `supports` method; protocol
67    /// data belongs in [`Resource::metadata`] and
68    /// [`ResourceFile::metadata`].
69    Custom {
70        /// Parser-defined protocol identifier, for example `demo`.
71        protocol: String,
72    },
73}
74
75#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
76#[serde(rename_all = "snake_case")]
77#[non_exhaustive]
78/// Where the metadata for a BitTorrent resource comes from.
79pub enum BitTorrentSource {
80    /// A magnet link, or a bare v1 info hash.
81    Magnet,
82    /// A local `.torrent` file.
83    TorrentFile,
84}
85
86#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
87#[serde(rename_all = "snake_case")]
88#[non_exhaustive]
89/// Hub service hosting a repository snapshot.
90pub enum HubProvider {
91    /// Hugging Face, including compatible mirrors.
92    HuggingFace,
93    /// ModelScope.
94    ModelScope,
95}
96
97#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
98#[serde(rename_all = "snake_case")]
99#[non_exhaustive]
100/// Kind of repository hosted on a hub.
101pub enum HubRepoType {
102    /// A model repository.
103    Model,
104    /// A dataset repository.
105    Dataset,
106}
107
108/// One file belonging to a [`Resource`].
109#[derive(Clone, Debug, Serialize, Deserialize)]
110pub struct ResourceFile {
111    /// Relative output path. Always validated and safe to join under the
112    /// task destination.
113    pub path: String,
114    /// Resolved URL the backend fetches.
115    pub url: Url,
116    /// Expected size in bytes, when the protocol or a previous probe
117    /// reported it.
118    #[serde(default)]
119    pub size: Option<u64>,
120    /// Expected checksum for this file, when known.
121    #[serde(default)]
122    pub checksum: Option<Checksum>,
123    /// Additional headers applied only to this file's requests.
124    #[serde(default)]
125    pub headers: Vec<Header>,
126    /// Protocol-private key/value data preserved in the sidecar.
127    #[serde(default)]
128    pub metadata: BTreeMap<String, String>,
129    /// Direct links prefer the server's `Content-Disposition` filename; Hub
130    /// snapshots use the repository-relative path.
131    #[serde(default = "default_true")]
132    pub use_response_filename: bool,
133}
134
135fn default_true() -> bool {
136    true
137}
138
139impl Resource {
140    /// Build a single-file [`ResourceKind::Http`] resource for `url`.
141    ///
142    /// The display name and output path are derived from the URL's last path
143    /// segment and validated with [`sanitize_relative_path`], so an unsafe
144    /// filename fails the parse instead of being used.
145    pub fn http_file(
146        url: Url,
147        request: RequestOptions,
148        checksum: Option<Checksum>,
149    ) -> Result<Self> {
150        let name = filename_from_url(&url);
151        let path = sanitize_relative_path(&name)?;
152        Ok(Self {
153            kind: ResourceKind::Http,
154            name,
155            files: vec![ResourceFile {
156                path,
157                url,
158                size: None,
159                checksum: None,
160                headers: Vec::new(),
161                metadata: BTreeMap::new(),
162                use_response_filename: true,
163            }],
164            request,
165            checksum,
166            revision: None,
167            metadata: BTreeMap::new(),
168        })
169    }
170
171    /// Sum of the known file sizes, or `None` if any file size is unknown.
172    /// An empty file list sums to `Some(0)`.
173    pub fn total_size(&self) -> Option<u64> {
174        self.files
175            .iter()
176            .try_fold(0u64, |sum, file| file.size.map(|size| sum + size))
177    }
178}
179
180impl ResourceFile {
181    /// Validate `path` as a destination-relative path and replace the file's
182    /// output path. Absolute paths and `..` traversal return an error.
183    pub fn with_path(mut self, path: impl AsRef<str>) -> Result<Self> {
184        self.path = sanitize_relative_path(path.as_ref())?;
185        Ok(self)
186    }
187}