Skip to main content

shaul_core/
task.rs

1use std::collections::BTreeMap;
2use std::path::{Path, PathBuf};
3
4use serde::{Deserialize, Serialize};
5use url::Url;
6use uuid::Uuid;
7
8use crate::checksum::Checksum;
9use crate::header::RequestOptions;
10use crate::link::Link;
11use crate::range::{ByteRange, completed_bytes, merge_ranges};
12use crate::resource::Resource;
13use crate::util::now_ms;
14
15/// Unique identifier for a persisted task, also used as its sidecar file name.
16pub type TaskId = Uuid;
17
18/// Version of the serialized task sidecar schema.
19pub const TASK_SCHEMA_VERSION: u32 = 1;
20
21/// A persisted download task: the requested link, its resolved resource,
22/// destination and current progress.
23#[derive(Clone, Debug, Serialize, Deserialize)]
24#[non_exhaustive]
25pub struct Task {
26    /// Sidecar schema version; see [`TASK_SCHEMA_VERSION`].
27    pub schema_version: u32,
28    /// Unique task id, also used as the sidecar file name.
29    pub id: TaskId,
30    /// Original link supplied by the user.
31    pub link: Link,
32    /// Parsed, backend-agnostic description of what to download.
33    #[serde(default)]
34    pub resource: Option<Resource>,
35    /// Name of the backend selected to execute the task.
36    #[serde(default)]
37    pub backend: Option<String>,
38    /// Root directory that relative output paths are resolved against.
39    pub destination: PathBuf,
40    /// Per-task options supplied when the task was added.
41    pub options: TaskOptions,
42    /// Queue priority; larger values run first.
43    #[serde(default)]
44    pub priority: i32,
45    /// Current lifecycle state of the task.
46    pub state: TaskState,
47    /// Per-file progress entries generated from the resource.
48    #[serde(default)]
49    pub files: Vec<FileItem>,
50    /// Per-file resume checkpoints written while downloading.
51    #[serde(default)]
52    pub resume: Option<TaskResume>,
53    /// Error message from the most recent failure, if any.
54    #[serde(default)]
55    pub error: Option<String>,
56    /// Creation timestamp in Unix milliseconds.
57    pub created_at_ms: u64,
58    /// Timestamp of the last modification in Unix milliseconds.
59    pub updated_at_ms: u64,
60}
61
62impl Task {
63    /// Create a pending task with a fresh id and current timestamps. The file
64    /// list stays empty until [`Task::set_resource`] is called.
65    pub fn new(link: Link, destination: PathBuf, options: TaskOptions) -> Self {
66        let now = now_ms();
67        Self {
68            schema_version: TASK_SCHEMA_VERSION,
69            id: Uuid::now_v7(),
70            link,
71            resource: None,
72            backend: None,
73            destination,
74            priority: options.priority.unwrap_or(0),
75            options,
76            state: TaskState::Pending,
77            files: Vec::new(),
78            resume: None,
79            error: None,
80            created_at_ms: now,
81            updated_at_ms: now,
82        }
83    }
84
85    /// Refresh `updated_at_ms` to the current time.
86    pub fn touch(&mut self) {
87        self.updated_at_ms = now_ms();
88    }
89
90    /// De-duplication key built from the canonical link and the destination
91    /// path.
92    pub fn key(&self) -> String {
93        format!(
94            "{}|{}",
95            self.link.canonical(),
96            self.destination.to_string_lossy()
97        )
98    }
99
100    /// Attach the resolved resource and rebuild the per-file progress entries
101    /// from it.
102    pub fn set_resource(&mut self, resource: Resource) {
103        self.files = resource
104            .files
105            .iter()
106            .map(|file| FileItem {
107                path: file.path.clone(),
108                url: Some(file.url.clone()),
109                size: file.size,
110                downloaded: 0,
111                state: FileState::Pending,
112                checksum: file.checksum.clone(),
113                selected: true,
114            })
115            .collect();
116        self.resource = Some(resource);
117    }
118
119    /// Pull completed byte counts and final paths back from the resume
120    /// manifest so `status`/`list` show meaningful progress.
121    pub fn refresh_from_resume(&mut self) {
122        let Some(resume) = &self.resume else {
123            return;
124        };
125        for file in &mut self.files {
126            let Some(url) = &file.url else {
127                continue;
128            };
129            if let Some(entry) = resume.files.get(url.as_str()) {
130                file.downloaded = completed_bytes(&entry.completed);
131                if let Some(size) = entry.total_size {
132                    file.size = Some(size);
133                }
134                file.path = entry.final_path.clone();
135            }
136        }
137    }
138
139    /// Total size of all files in bytes, or `None` if any file has an unknown
140    /// size.
141    pub fn total_bytes(&self) -> Option<u64> {
142        self.files
143            .iter()
144            .try_fold(0u64, |sum, file| file.size.map(|size| sum + size))
145    }
146
147    /// Sum of the bytes downloaded so far across all files.
148    pub fn downloaded_bytes(&self) -> u64 {
149        self.files.iter().map(|file| file.downloaded).sum()
150    }
151}
152
153/// Lifecycle state of a task.
154#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
155#[serde(rename_all = "snake_case")]
156#[non_exhaustive]
157pub enum TaskState {
158    /// Created but not started yet.
159    Pending,
160    /// Currently downloading.
161    Running,
162    /// Interrupted by the user; may be resumed.
163    Paused,
164    /// Finished successfully.
165    Completed,
166    /// Stopped because of an error.
167    Failed,
168    /// Abandoned by the user; it cannot be resumed.
169    Cancelled,
170}
171
172impl TaskState {
173    /// Whether the task occupies an active de-duplication slot: `Pending`,
174    /// `Running` or `Paused`.
175    pub fn is_active(&self) -> bool {
176        matches!(self, Self::Pending | Self::Running | Self::Paused)
177    }
178}
179
180/// Per-task overrides captured when a task is added. Unset fields fall back to
181/// the corresponding [`Config`](crate::config::Config) defaults.
182#[derive(Clone, Debug, Default, Serialize, Deserialize)]
183pub struct TaskOptions {
184    /// Output file or directory override for a single-file direct link.
185    #[serde(default)]
186    pub output: Option<PathBuf>,
187    /// Number of parallel connections used for this task.
188    #[serde(default)]
189    pub connections: Option<usize>,
190    /// Additional bandwidth cap for this task in bytes per second. `None` or
191    /// `Some(0)` means no task-specific cap; the global limit still applies.
192    #[serde(default)]
193    pub rate_limit: Option<u64>,
194    /// Number of files downloaded concurrently within this task.
195    #[serde(default)]
196    pub file_jobs: Option<usize>,
197    /// Alternate HTTP(S) URLs for the same file. They are tried after the
198    /// primary URL fails and must pass a size/validator compatibility check.
199    #[serde(default)]
200    pub mirrors: Vec<Url>,
201    /// Queue priority; larger values are scheduled first.
202    #[serde(default)]
203    pub priority: Option<i32>,
204    /// Whether to resume from existing partial data.
205    #[serde(default)]
206    pub resume: Option<bool>,
207    /// Checksum verification policy for this task.
208    #[serde(default)]
209    pub verify: Option<VerifyMode>,
210    /// Expected checksum for the downloaded content.
211    #[serde(default)]
212    pub checksum: Option<Checksum>,
213    /// Whether to extract archives after downloading.
214    #[serde(default)]
215    pub extract: Option<bool>,
216    /// Directory to extract archives into.
217    #[serde(default)]
218    pub extract_dir: Option<PathBuf>,
219    /// Whether to delete the archive after successful extraction.
220    #[serde(default)]
221    pub delete_archive: Option<bool>,
222    /// Use the provider cache for Hub tasks (HF-compatible layout for HF).
223    #[serde(default)]
224    pub cache: Option<bool>,
225    /// Reuse verified files through the content-addressed cache.
226    #[serde(default)]
227    pub content_cache: Option<bool>,
228    /// Per-task headers, authentication and cross-origin policy.
229    #[serde(default)]
230    pub request: RequestOptions,
231    /// Hub revision (branch, tag or commit) to download.
232    #[serde(default)]
233    pub revision: Option<String>,
234    /// Glob patterns selecting which Hub or BitTorrent files to download.
235    #[serde(default)]
236    pub include: Vec<String>,
237    /// Glob patterns excluding Hub or BitTorrent files from the download.
238    #[serde(default)]
239    pub exclude: Vec<String>,
240    /// Overwrite existing destination files.
241    #[serde(default)]
242    pub overwrite: bool,
243    /// Rename conflicting destinations (`name-1.ext`, `name-2.ext`, ...)
244    /// instead of failing.
245    #[serde(default)]
246    pub rename_on_conflict: bool,
247    /// Add-time only; persisted for diagnostics.
248    #[serde(default)]
249    pub force: bool,
250}
251
252/// Policy for checking downloaded content against an expected checksum.
253#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
254#[serde(rename_all = "snake_case")]
255#[non_exhaustive]
256pub enum VerifyMode {
257    /// Verify when a checksum is known, otherwise skip. This is the default.
258    #[default]
259    Auto,
260    /// Fail when no checksum is available.
261    Require,
262    /// Never verify.
263    Disabled,
264}
265
266/// Progress and state for one file belonging to a task.
267#[derive(Clone, Debug, Serialize, Deserialize)]
268#[non_exhaustive]
269pub struct FileItem {
270    /// Relative output path under the task destination, using `/` separators.
271    pub path: String,
272    /// Source URL, when the file was resolved from a link.
273    #[serde(default)]
274    pub url: Option<Url>,
275    /// Expected size in bytes, once known.
276    #[serde(default)]
277    pub size: Option<u64>,
278    /// Bytes downloaded so far.
279    #[serde(default)]
280    pub downloaded: u64,
281    /// Current per-file state.
282    pub state: FileState,
283    /// Expected checksum, when known.
284    #[serde(default)]
285    pub checksum: Option<Checksum>,
286    /// Whether this file is selected for download.
287    #[serde(default = "default_selected")]
288    pub selected: bool,
289}
290
291fn default_selected() -> bool {
292    true
293}
294
295/// Lifecycle state of a single file.
296#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
297#[serde(rename_all = "snake_case")]
298#[non_exhaustive]
299pub enum FileState {
300    /// Not started yet.
301    Pending,
302    /// Transfer is in progress.
303    Downloading,
304    /// Downloaded and finalized.
305    Completed,
306    /// Excluded from the download or intentionally left untouched.
307    Skipped,
308    /// Stopped because of an error.
309    Failed,
310}
311
312/// Per-file resume checkpoints persisted alongside a task.
313#[derive(Clone, Debug, Serialize, Deserialize)]
314#[non_exhaustive]
315pub struct TaskResume {
316    /// Schema version of the resume manifest.
317    pub version: u32,
318    /// Checkpoints keyed by source URL.
319    #[serde(default)]
320    pub files: BTreeMap<String, FileResume>,
321}
322
323impl Default for TaskResume {
324    fn default() -> Self {
325        Self {
326            version: 1,
327            files: BTreeMap::new(),
328        }
329    }
330}
331
332impl TaskResume {
333    /// Checkpoint for `url`, if one exists.
334    pub fn file(&self, url: &str) -> Option<&FileResume> {
335        self.files.get(url)
336    }
337
338    /// Insert or replace the checkpoint for `url`.
339    pub fn upsert(&mut self, url: String, file: FileResume) {
340        self.files.insert(url, file);
341    }
342
343    /// Total completed bytes across all checkpointed files.
344    pub fn downloaded(&self) -> u64 {
345        self.files
346            .values()
347            .map(|file| completed_bytes(&file.completed))
348            .sum()
349    }
350}
351
352/// Resume checkpoint for one partially downloaded file.
353#[derive(Clone, Debug, Serialize, Deserialize)]
354pub struct FileResume {
355    /// Source URL that identifies this checkpoint.
356    pub url: String,
357    /// Relative to the task destination, using `/` separators.
358    pub final_path: String,
359    /// Relative to the task destination, using `/` separators.
360    pub part_path: String,
361    /// Validators captured when the partial data was written.
362    #[serde(default)]
363    pub validator: Validator,
364    /// Total size of the file in bytes, when known.
365    #[serde(default)]
366    pub total_size: Option<u64>,
367    /// Segment size used by the segmented download.
368    pub segment_size: u64,
369    /// Byte ranges already written to the partial file.
370    #[serde(default)]
371    pub completed: Vec<ByteRange>,
372    /// Expected checksum for the file, when known.
373    #[serde(default)]
374    pub checksum: Option<Checksum>,
375    /// Backend-specific metadata gathered while probing the file.
376    #[serde(default)]
377    pub metadata: BTreeMap<String, String>,
378    /// Timestamp of the last checkpoint update in Unix milliseconds.
379    pub updated_at_ms: u64,
380}
381
382impl FileResume {
383    /// Total bytes covered by the completed ranges.
384    pub fn completed_bytes(&self) -> u64 {
385        completed_bytes(&self.completed)
386    }
387
388    /// Whether every byte of a file with a known size has been completed.
389    pub fn is_complete(&self) -> bool {
390        match self.total_size {
391            Some(total) => self.completed_bytes() >= total,
392            None => false,
393        }
394    }
395
396    /// Merge overlapping or adjacent ranges and refresh `updated_at_ms`.
397    pub fn normalize(&mut self) {
398        self.completed = merge_ranges(std::mem::take(&mut self.completed));
399        self.updated_at_ms = now_ms();
400    }
401}
402
403/// HTTP validators captured from a remote file and used to decide whether a
404/// stored partial download can still be resumed.
405#[derive(Clone, Debug, Default, Serialize, Deserialize)]
406pub struct Validator {
407    /// `ETag` from the last response.
408    #[serde(default)]
409    pub etag: Option<String>,
410    /// `Last-Modified` from the last response.
411    #[serde(default)]
412    pub last_modified: Option<String>,
413    /// Size of the remote file in bytes.
414    #[serde(default)]
415    pub size: Option<u64>,
416}
417
418impl Validator {
419    /// Whether no validator signal has been recorded.
420    pub fn is_empty(&self) -> bool {
421        self.etag.is_none() && self.last_modified.is_none() && self.size.is_none()
422    }
423
424    /// A stored validator is compatible with a fresh probe when the strongest
425    /// available signal still matches. Weak ETags are not used for range
426    /// validation, so they fall back to Last-Modified + size.
427    pub fn matches(&self, fresh: &Validator) -> bool {
428        if let Some(etag) = &self.etag
429            && !etag.starts_with("W/")
430        {
431            return fresh.etag.as_deref() == Some(etag.as_str());
432        }
433        if let Some(last_modified) = &self.last_modified {
434            return fresh.last_modified.as_deref() == Some(last_modified.as_str())
435                && (self.size.is_none() || self.size == fresh.size);
436        }
437        if let Some(size) = self.size {
438            return fresh.size == Some(size);
439        }
440        false
441    }
442}
443
444/// Join a manifest-style relative path (using `/` separators) under
445/// `destination`.
446pub fn destination_file(destination: &Path, relative: &str) -> PathBuf {
447    destination.join(relative.replace('/', std::path::MAIN_SEPARATOR_STR))
448}
449
450/// Partial-file path used by segmented backends.
451pub fn part_path(final_path: &Path) -> PathBuf {
452    let mut value = final_path.as_os_str().to_os_string();
453    value.push(".shaul.part");
454    PathBuf::from(value)
455}