Skip to main content

shaul_core/
engine.rs

1use std::collections::HashMap;
2use std::path::PathBuf;
3use std::sync::Arc;
4use std::time::{Duration, Instant};
5
6use async_trait::async_trait;
7use futures_util::StreamExt;
8use tokio::sync::Mutex;
9use tokio_util::sync::CancellationToken;
10
11use crate::backend::{
12    DownloadBackend, DownloadOutcome, DownloadRequest, FileOutcome, ResumeCheckpoint,
13};
14use crate::config::{Config, ResolvedOptions};
15use crate::disk::ensure_free_space;
16use crate::error::{Error, Result};
17use crate::event::{EventKind, EventSink};
18use crate::finalizer::{FinalizeContext, FinalizerPipeline};
19use crate::link::Link;
20use crate::parser::ParseOptions;
21use crate::resource::Resource;
22use crate::router::{RoutedResource, Router};
23use crate::store::TaskStore;
24use crate::task::{
25    FileItem, FileState, Task, TaskId, TaskOptions, TaskResume, TaskState, part_path,
26};
27use crate::telemetry::{NoopTelemetry, Telemetry, metrics};
28
29/// Coordinates the task lifecycle: resolving links, adding and running tasks,
30/// pausing, resuming, cancelling and requeueing tasks left behind by a crashed
31/// process.
32pub struct DownloadEngine {
33    router: Arc<Router>,
34    store: Arc<dyn TaskStore>,
35    config: Config,
36    events: EventSink,
37    running: Arc<Mutex<HashMap<TaskId, CancellationToken>>>,
38    finalizers: FinalizerPipeline,
39    telemetry: Arc<dyn Telemetry>,
40}
41
42impl DownloadEngine {
43    /// Creates an engine over the given router, task store and configuration.
44    /// Subscribe with [`DownloadEngine::subscribe`] before adding tasks to
45    /// observe their events.
46    pub fn new(router: Arc<Router>, store: Arc<dyn TaskStore>, config: Config) -> Self {
47        let (events, _) = EventSink::channel(4096);
48        Self {
49            router,
50            store,
51            config,
52            events,
53            running: Arc::new(Mutex::new(HashMap::new())),
54            finalizers: FinalizerPipeline::default(),
55            telemetry: Arc::new(NoopTelemetry),
56        }
57    }
58
59    /// Replaces the no-op telemetry sink. Builder form; use
60    /// [`DownloadEngine::set_telemetry`] to change it after construction.
61    pub fn with_telemetry(mut self, telemetry: Arc<dyn Telemetry>) -> Self {
62        self.telemetry = telemetry.clone();
63        self.events.set_telemetry(telemetry);
64        self
65    }
66
67    /// Replaces the telemetry sink used for subsequent tasks.
68    pub fn set_telemetry(&mut self, telemetry: Arc<dyn Telemetry>) {
69        self.telemetry = telemetry.clone();
70        self.events.set_telemetry(telemetry);
71    }
72
73    /// Returns the installed telemetry sink.
74    pub fn telemetry(&self) -> &Arc<dyn Telemetry> {
75        &self.telemetry
76    }
77
78    /// Replaces the post-download finalizer pipeline. The default verifies
79    /// checksums and extracts archives.
80    pub fn with_finalizers(mut self, finalizers: FinalizerPipeline) -> Self {
81        self.finalizers = finalizers;
82        self
83    }
84
85    /// Replaces the finalizer pipeline used for subsequent tasks.
86    pub fn set_finalizers(&mut self, finalizers: FinalizerPipeline) {
87        self.finalizers = finalizers;
88    }
89
90    /// Subscribes to the engine's event stream. Only events emitted after the
91    /// call are received.
92    pub fn subscribe(&self) -> tokio::sync::broadcast::Receiver<crate::event::DownloadEvent> {
93        self.events.subscribe()
94    }
95
96    /// Returns the configuration used to resolve task options.
97    pub fn config(&self) -> &Config {
98        &self.config
99    }
100
101    /// Returns the event sink backends use to report progress.
102    pub fn events(&self) -> &EventSink {
103        &self.events
104    }
105
106    /// Returns the task store backing the engine.
107    pub fn store(&self) -> &Arc<dyn TaskStore> {
108        &self.store
109    }
110
111    /// Parses `link` and selects a backend without creating a task.
112    pub async fn resolve(&self, link: &Link, options: &ParseOptions) -> Result<RoutedResource> {
113        self.router.resolve(link, options).await
114    }
115
116    /// Resolves `link` and persists a new pending task.
117    ///
118    /// Unless [`TaskOptions::force`] is set, an active (pending, running or
119    /// paused) task with the same canonical link and destination is returned
120    /// instead of creating a duplicate.
121    #[tracing::instrument(
122        name = "shaul.engine.add",
123        level = "debug",
124        skip(self, link, destination, options)
125    )]
126    pub async fn add(
127        &self,
128        link: Link,
129        destination: PathBuf,
130        options: TaskOptions,
131    ) -> Result<Task> {
132        if !options.force {
133            let key = format!("{}|{}", link.canonical(), destination.display());
134            for task in self.store.list().await? {
135                if task.state.is_active() && task.key() == key {
136                    return Ok(task);
137                }
138            }
139        }
140
141        let parse_options = ParseOptions {
142            request: options.request.clone(),
143            checksum: options.checksum.clone(),
144            revision: options.revision.clone(),
145            include: options.include.clone(),
146            exclude: options.exclude.clone(),
147        };
148        let routed = self.router.resolve(&link, &parse_options).await?;
149        let mut task = Task::new(link, destination, options);
150        task.backend = Some(routed.backend.name().to_string());
151        task.set_resource(routed.resource);
152        self.store.save(&task).await?;
153        self.events.emit(EventKind::TaskAdded {
154            task_id: task.id,
155            link: task.link.to_string(),
156        });
157        let backend = task.backend.as_deref().unwrap_or("unknown");
158        self.telemetry
159            .counter(metrics::TASKS_ADDED, 1, &[("backend", backend)]);
160        tracing::debug!(task_id = %task.id, backend, "task added");
161        Ok(task)
162    }
163
164    /// Runs a task to a terminal state without an external cancellation token.
165    pub async fn run_task(&self, id: TaskId) -> Result<Task> {
166        self.run_task_with_cancel(id, None).await
167    }
168
169    /// Runs a task, optionally linked to an external cancellation token.
170    ///
171    /// A task that is already completed is returned unchanged, while running a
172    /// cancelled task is an error. A pause or cancel requested through the
173    /// store (for example by another process) is observed while the transfer
174    /// runs and stops it.
175    #[tracing::instrument(
176        name = "shaul.engine.run_task",
177        level = "debug",
178        skip(self, external_cancel),
179        fields(task_id = %id)
180    )]
181    pub async fn run_task_with_cancel(
182        &self,
183        id: TaskId,
184        external_cancel: Option<CancellationToken>,
185    ) -> Result<Task> {
186        let task = self.store.load(id).await?;
187        match task.state {
188            TaskState::Completed => return Ok(task),
189            TaskState::Cancelled => {
190                return Err(Error::InvalidState(format!("task {id} was cancelled")));
191            }
192            _ => {}
193        }
194
195        let backend_name = task
196            .backend
197            .clone()
198            .unwrap_or_else(|| "unknown".to_string());
199        let metric_labels = [("backend", backend_name.as_str())];
200        let started = Instant::now();
201        let resource = task
202            .resource
203            .clone()
204            .ok_or_else(|| Error::InvalidState(format!("task {id} has no parsed resource")))?;
205        let backend = self.router.backend_for(&resource)?;
206        let capabilities = backend.capabilities();
207        let mut options = self.config.resolve(&task.destination, &task.options)?;
208        if !capabilities.resume {
209            options.resume = false;
210        }
211        let resume = if capabilities.resume && options.resume {
212            task.resume.clone()
213        } else {
214            None
215        };
216        let cancel = external_cancel.unwrap_or_default();
217
218        self.running.lock().await.insert(id, cancel.clone());
219        self.store
220            .update(
221                id,
222                Box::new(|task| {
223                    task.state = TaskState::Running;
224                    task.error = None;
225                    Ok(())
226                }),
227            )
228            .await?;
229        self.events.emit(EventKind::TaskStarted {
230            task_id: id,
231            link: task.link.to_string(),
232        });
233        self.telemetry
234            .counter(metrics::TASKS_STARTED, 1, &metric_labels);
235        let active = self.running.lock().await.len() as f64;
236        self.telemetry.gauge(metrics::ACTIVE_TASKS, active, &[]);
237        tracing::debug!(task_id = %id, backend = %backend_name, "task started");
238        if !capabilities.resume && task.resume.is_some() {
239            self.events.emit(EventKind::Warning {
240                task_id: Some(id),
241                message: format!(
242                    "backend {backend_name} does not support resume; starting this task from scratch"
243                ),
244            });
245        }
246
247        let checkpoint = Arc::new(StoreCheckpoint {
248            store: self.store.clone(),
249            task_id: id,
250        });
251        let request = DownloadRequest {
252            task_id: id,
253            resource,
254            destination: task.destination.clone(),
255            options: options.clone(),
256            resume,
257            cancel: cancel.clone(),
258            events: self.events.clone(),
259            checkpoint,
260        };
261
262        // A pause/cancel issued by another process only changes the sidecar;
263        // this watcher turns that state change into cancellation here.
264        let watcher = {
265            let store = self.store.clone();
266            let token = cancel.clone();
267            tokio::spawn(async move {
268                loop {
269                    tokio::select! {
270                        _ = token.cancelled() => break,
271                        _ = tokio::time::sleep(Duration::from_millis(500)) => {
272                            if let Ok(task) = store.load(id).await
273                                && matches!(
274                                    task.state,
275                                    TaskState::Paused | TaskState::Cancelled
276                                ) {
277                                    token.cancel();
278                                    break;
279                                }
280                        }
281                    }
282                }
283            })
284        };
285
286        let result = self.run_attempts(&request, backend, &options).await;
287        watcher.abort();
288        self.running.lock().await.remove(&id);
289
290        let state = match &result {
291            Ok(task) => match task.state {
292                TaskState::Completed => "completed",
293                TaskState::Paused => "paused",
294                TaskState::Cancelled => "cancelled",
295                _ => "unknown",
296            },
297            Err(_) => "failed",
298        };
299        let state_labels = [("backend", backend_name.as_str()), ("state", state)];
300        self.telemetry.histogram(
301            metrics::TASK_DURATION_SECONDS,
302            started.elapsed().as_secs_f64(),
303            &state_labels,
304        );
305        match state {
306            "completed" => {
307                self.telemetry
308                    .counter(metrics::TASKS_COMPLETED, 1, &metric_labels);
309                if let Ok(task) = &result {
310                    self.telemetry.counter(
311                        metrics::BYTES_DOWNLOADED,
312                        task.downloaded_bytes(),
313                        &metric_labels,
314                    );
315                }
316            }
317            "paused" => self
318                .telemetry
319                .counter(metrics::TASKS_PAUSED, 1, &metric_labels),
320            "cancelled" => self
321                .telemetry
322                .counter(metrics::TASKS_CANCELLED, 1, &metric_labels),
323            _ => self
324                .telemetry
325                .counter(metrics::TASKS_FAILED, 1, &metric_labels),
326        }
327        let active = self.running.lock().await.len() as f64;
328        self.telemetry.gauge(metrics::ACTIVE_TASKS, active, &[]);
329        tracing::debug!(task_id = %id, backend = %backend_name, state, "task finished");
330        result
331    }
332
333    #[tracing::instrument(
334        name = "shaul.engine.attempt",
335        level = "debug",
336        skip(self, request, backend, options),
337        fields(task_id = %request.task_id)
338    )]
339    async fn run_attempts(
340        &self,
341        request: &DownloadRequest,
342        backend: Arc<dyn DownloadBackend>,
343        options: &ResolvedOptions,
344    ) -> Result<Task> {
345        let mut verification_retries = 0u32;
346        loop {
347            if options.disk_space_check
348                && let Some(required) = required_download_bytes(request)
349                && let Err(error) = ensure_free_space(&request.destination, required)
350            {
351                let message = error.to_string();
352                self.fail(request.task_id, error).await?;
353                return Err(Error::Backend(message));
354            }
355            let outcome = backend.download(request.clone()).await;
356            match outcome {
357                Ok(DownloadOutcome::Completed(files)) => {
358                    match self
359                        .finalize_files(request.task_id, &request.destination, options, &files)
360                        .await
361                    {
362                        Ok(()) => {
363                            let total_bytes: u64 = files.iter().map(|file| file.size).sum();
364                            let file_count = files.len();
365                            let items = file_items(&files);
366                            let task = self
367                                .store
368                                .update(
369                                    request.task_id,
370                                    Box::new(move |task| {
371                                        task.state = TaskState::Completed;
372                                        task.error = None;
373                                        task.resume = None;
374                                        task.files = items;
375                                        Ok(())
376                                    }),
377                                )
378                                .await?;
379                            self.events.emit(EventKind::TaskCompleted {
380                                task_id: request.task_id,
381                                files: file_count,
382                                bytes: total_bytes,
383                            });
384                            return Ok(task);
385                        }
386                        Err(Error::Verification { .. }) if verification_retries < 1 => {
387                            verification_retries += 1;
388                            tracing::warn!(
389                                task_id = %request.task_id,
390                                "checksum mismatch; retrying once from scratch"
391                            );
392                            self.events.emit(EventKind::Warning {
393                                task_id: Some(request.task_id),
394                                message: "checksum mismatch; retrying once from scratch"
395                                    .to_string(),
396                            });
397                            self.reset_outputs(request.task_id, &files).await?;
398                            continue;
399                        }
400                        Err(error) => {
401                            let message = error.to_string();
402                            self.fail(request.task_id, error).await?;
403                            return Err(Error::Backend(message));
404                        }
405                    }
406                }
407                Ok(DownloadOutcome::Paused) => {
408                    return self.mark_paused(request.task_id).await;
409                }
410                Err(error) if error.is_cancelled() => {
411                    return self.mark_paused(request.task_id).await;
412                }
413                Err(error) => {
414                    let message = error.to_string();
415                    self.fail(request.task_id, error).await?;
416                    return Err(Error::Backend(message));
417                }
418            }
419        }
420    }
421
422    #[tracing::instrument(
423        name = "shaul.engine.finalize",
424        level = "debug",
425        skip(self, destination, options, files),
426        fields(task_id = %task_id, files = files.len())
427    )]
428    async fn finalize_files(
429        &self,
430        task_id: TaskId,
431        destination: &std::path::Path,
432        options: &ResolvedOptions,
433        files: &[FileOutcome],
434    ) -> Result<()> {
435        for file in files {
436            let context = FinalizeContext {
437                task_id,
438                destination,
439                file,
440                options,
441                events: &self.events,
442            };
443            let result = self.finalizers.run(&context).await?;
444            self.events.emit(EventKind::FileCompleted {
445                task_id,
446                file: file.relative_path.clone(),
447                path: file.path.clone(),
448                size: file.size,
449                checksum_verified: result.checksum_verified,
450            });
451        }
452        Ok(())
453    }
454
455    async fn reset_outputs(&self, id: TaskId, files: &[FileOutcome]) -> Result<()> {
456        for file in files {
457            let _ = tokio::fs::remove_file(&file.path).await;
458            let _ = tokio::fs::remove_file(part_path(&file.path)).await;
459        }
460        self.store
461            .update(
462                id,
463                Box::new(|task| {
464                    task.resume = None;
465                    for file in &mut task.files {
466                        file.downloaded = 0;
467                        file.state = FileState::Pending;
468                    }
469                    Ok(())
470                }),
471            )
472            .await?;
473        Ok(())
474    }
475
476    #[tracing::instrument(
477        name = "shaul.engine.fail",
478        level = "debug",
479        skip(self, error),
480        fields(task_id = %id)
481    )]
482    async fn fail(&self, id: TaskId, error: Error) -> Result<()> {
483        let message = error.to_string();
484        tracing::warn!(task_id = %id, error = %message, "task failed");
485        let stored = message.clone();
486        self.store
487            .update(
488                id,
489                Box::new(move |task| {
490                    if task.state == TaskState::Running {
491                        task.state = TaskState::Failed;
492                        task.error = Some(stored.clone());
493                    }
494                    Ok(())
495                }),
496            )
497            .await?;
498        self.events.emit(EventKind::TaskFailed {
499            task_id: id,
500            error: message,
501        });
502        Ok(())
503    }
504
505    async fn mark_paused(&self, id: TaskId) -> Result<Task> {
506        let task = self.store.load(id).await?;
507        match task.state {
508            TaskState::Cancelled => {
509                self.events.emit(EventKind::TaskCancelled { task_id: id });
510                Ok(task)
511            }
512            TaskState::Paused => {
513                self.events.emit(EventKind::TaskPaused { task_id: id });
514                Ok(task)
515            }
516            _ => {
517                let task = self
518                    .store
519                    .update(
520                        id,
521                        Box::new(|task| {
522                            task.state = TaskState::Paused;
523                            Ok(())
524                        }),
525                    )
526                    .await?;
527                self.events.emit(EventKind::TaskPaused { task_id: id });
528                Ok(task)
529            }
530        }
531    }
532
533    /// Runs every task currently in `Pending` state, at most `jobs` at a time
534    /// (`jobs` is clamped to at least one). Higher-priority tasks are selected
535    /// first, then older tasks. All tasks share `cancel`; tasks whose run fails
536    /// are omitted from the returned list.
537    pub async fn run_pending(&self, jobs: usize, cancel: CancellationToken) -> Result<Vec<Task>> {
538        let tasks = self.store.list().await?;
539        let pending = tasks
540            .iter()
541            .filter(|task| task.state == TaskState::Pending)
542            .count() as f64;
543        self.telemetry.gauge(metrics::QUEUE_DEPTH, pending, &[]);
544        let ids = pending_task_ids(tasks);
545        let jobs = jobs.max(1);
546        let results = futures_util::stream::iter(ids.into_iter().map(|id| {
547            let cancel = cancel.clone();
548            async move { self.run_task_with_cancel(id, Some(cancel)).await }
549        }))
550        .buffer_unordered(jobs)
551        .collect::<Vec<_>>()
552        .await;
553
554        Ok(results.into_iter().filter_map(Result::ok).collect())
555    }
556
557    /// A previous process may have left tasks in `running`. The CLI calls
558    /// this only while holding the single-worker lock.
559    pub async fn requeue_stale(&self) -> Result<Vec<TaskId>> {
560        let mut requeued = Vec::new();
561        for task in self.store.list().await? {
562            if task.state == TaskState::Running {
563                self.store
564                    .update(
565                        task.id,
566                        Box::new(|task| {
567                            task.state = TaskState::Pending;
568                            Ok(())
569                        }),
570                    )
571                    .await?;
572                requeued.push(task.id);
573                self.events.emit(EventKind::Warning {
574                    task_id: Some(task.id),
575                    message: "requeued a task left running by a previous process".to_string(),
576                });
577            }
578        }
579        Ok(requeued)
580    }
581
582    /// Marks a task paused and cancels its in-flight transfer, if any. It can
583    /// later be restarted with [`DownloadEngine::resume`] or `run_task`;
584    /// pausing a completed task is an error.
585    pub async fn pause(&self, id: TaskId) -> Result<Task> {
586        let task = self
587            .store
588            .update(
589                id,
590                Box::new(|task| {
591                    if task.state == TaskState::Completed {
592                        return Err(Error::InvalidState("task already completed".to_string()));
593                    }
594                    task.state = TaskState::Paused;
595                    Ok(())
596                }),
597            )
598            .await?;
599        if let Some(token) = self.running.lock().await.get(&id) {
600            token.cancel();
601        }
602        self.events.emit(EventKind::TaskPaused { task_id: id });
603        Ok(task)
604    }
605
606    /// Marks a task cancelled and cancels its in-flight transfer, if any. A
607    /// cancelled task is not restarted by [`DownloadEngine::run_task`]; use
608    /// [`DownloadEngine::resume`] to move it back to pending. Cancelling an
609    /// already completed task is an error.
610    pub async fn cancel(&self, id: TaskId) -> Result<Task> {
611        let task = self
612            .store
613            .update(
614                id,
615                Box::new(|task| {
616                    if task.state == TaskState::Completed {
617                        return Err(Error::InvalidState("task already completed".to_string()));
618                    }
619                    task.state = TaskState::Cancelled;
620                    Ok(())
621                }),
622            )
623            .await?;
624        if let Some(token) = self.running.lock().await.get(&id) {
625            token.cancel();
626        }
627        self.events.emit(EventKind::TaskCancelled { task_id: id });
628        Ok(task)
629    }
630
631    /// Moves a task back to `Pending` and clears its error so the next run
632    /// retries it, reusing any saved resume manifest. Resuming an already
633    /// completed task is an error.
634    pub async fn resume(&self, id: TaskId) -> Result<Task> {
635        let task = self
636            .store
637            .update(
638                id,
639                Box::new(|task| {
640                    if task.state == TaskState::Completed {
641                        return Err(Error::InvalidState("task already completed".to_string()));
642                    }
643                    task.state = TaskState::Pending;
644                    task.error = None;
645                    Ok(())
646                }),
647            )
648            .await?;
649        self.events.emit(EventKind::TaskStateChanged {
650            task_id: id,
651            state: TaskState::Pending,
652        });
653        Ok(task)
654    }
655
656    /// Returns all tasks known to the store, oldest first.
657    pub async fn list(&self) -> Result<Vec<Task>> {
658        self.store.list().await
659    }
660
661    /// Loads a single task by id.
662    pub async fn status(&self, id: TaskId) -> Result<Task> {
663        self.store.load(id).await
664    }
665}
666
667fn required_download_bytes(request: &DownloadRequest) -> Option<u64> {
668    let total = request
669        .resource
670        .files
671        .iter()
672        .try_fold(0u64, |sum, file| file.size.map(|size| sum + size))?;
673    let completed = request
674        .resume
675        .as_ref()
676        .map(|resume| {
677            resume
678                .files
679                .values()
680                .map(crate::task::FileResume::completed_bytes)
681                .sum::<u64>()
682        })
683        .unwrap_or(0);
684    Some(total.saturating_sub(completed))
685}
686
687fn pending_task_ids(mut tasks: Vec<Task>) -> Vec<TaskId> {
688    tasks.retain(|task| task.state == TaskState::Pending);
689    tasks.sort_by(|left, right| {
690        right
691            .priority
692            .cmp(&left.priority)
693            .then_with(|| left.created_at_ms.cmp(&right.created_at_ms))
694            .then_with(|| left.id.cmp(&right.id))
695    });
696    tasks.into_iter().map(|task| task.id).collect()
697}
698
699struct StoreCheckpoint {
700    store: Arc<dyn TaskStore>,
701    task_id: TaskId,
702}
703
704#[async_trait]
705impl ResumeCheckpoint for StoreCheckpoint {
706    async fn save(&self, resume: &TaskResume) -> Result<()> {
707        let resume = resume.clone();
708        self.store
709            .update(
710                self.task_id,
711                Box::new(move |task| {
712                    task.resume = Some(resume.clone());
713                    task.refresh_from_resume();
714                    Ok(())
715                }),
716            )
717            .await?;
718        Ok(())
719    }
720}
721
722fn file_items(files: &[FileOutcome]) -> Vec<FileItem> {
723    files
724        .iter()
725        .map(|file| FileItem {
726            path: file.relative_path.clone(),
727            url: None,
728            size: Some(file.size),
729            downloaded: file.size,
730            state: FileState::Completed,
731            checksum: file.checksum.clone(),
732            selected: true,
733        })
734        .collect()
735}
736
737#[allow(dead_code)]
738fn _assert_resource(_: &Resource) {}
739
740#[cfg(test)]
741mod tests {
742    use super::*;
743    use crate::link::Link;
744
745    fn task(link: &str, priority: Option<i32>, created_at_ms: u64) -> Task {
746        let mut task = Task::new(
747            Link::new(link).unwrap(),
748            PathBuf::from("."),
749            TaskOptions {
750                priority,
751                ..TaskOptions::default()
752            },
753        );
754        task.created_at_ms = created_at_ms;
755        task
756    }
757
758    #[test]
759    fn orders_pending_tasks_by_priority_then_age() {
760        let low = task("low", Some(-1), 1);
761        let high_old = task("high-old", Some(10), 2);
762        let high_new = task("high-new", Some(10), 3);
763        let normal = task("normal", None, 4);
764        let mut paused = task("paused", Some(100), 0);
765        paused.state = TaskState::Paused;
766
767        let ordered = pending_task_ids(vec![
768            normal.clone(),
769            high_new.clone(),
770            paused,
771            low.clone(),
772            high_old.clone(),
773        ]);
774        assert_eq!(ordered, vec![high_old.id, high_new.id, normal.id, low.id]);
775    }
776}