1use std::collections::HashMap;
2use std::path::PathBuf;
3use std::sync::Arc;
4use std::time::{Duration, Instant};
5
6use async_trait::async_trait;
7use futures_util::StreamExt;
8use tokio::sync::Mutex;
9use tokio_util::sync::CancellationToken;
10
11use crate::backend::{
12 DownloadBackend, DownloadOutcome, DownloadRequest, FileOutcome, ResumeCheckpoint,
13};
14use crate::config::{Config, ResolvedOptions};
15use crate::disk::ensure_free_space;
16use crate::error::{Error, Result};
17use crate::event::{EventKind, EventSink};
18use crate::finalizer::{FinalizeContext, FinalizerPipeline};
19use crate::link::Link;
20use crate::parser::ParseOptions;
21use crate::resource::Resource;
22use crate::router::{RoutedResource, Router};
23use crate::store::TaskStore;
24use crate::task::{
25 FileItem, FileState, Task, TaskId, TaskOptions, TaskResume, TaskState, part_path,
26};
27use crate::telemetry::{NoopTelemetry, Telemetry, metrics};
28
29pub struct DownloadEngine {
33 router: Arc<Router>,
34 store: Arc<dyn TaskStore>,
35 config: Config,
36 events: EventSink,
37 running: Arc<Mutex<HashMap<TaskId, CancellationToken>>>,
38 finalizers: FinalizerPipeline,
39 telemetry: Arc<dyn Telemetry>,
40}
41
42impl DownloadEngine {
43 pub fn new(router: Arc<Router>, store: Arc<dyn TaskStore>, config: Config) -> Self {
47 let (events, _) = EventSink::channel(4096);
48 Self {
49 router,
50 store,
51 config,
52 events,
53 running: Arc::new(Mutex::new(HashMap::new())),
54 finalizers: FinalizerPipeline::default(),
55 telemetry: Arc::new(NoopTelemetry),
56 }
57 }
58
59 pub fn with_telemetry(mut self, telemetry: Arc<dyn Telemetry>) -> Self {
62 self.telemetry = telemetry.clone();
63 self.events.set_telemetry(telemetry);
64 self
65 }
66
67 pub fn set_telemetry(&mut self, telemetry: Arc<dyn Telemetry>) {
69 self.telemetry = telemetry.clone();
70 self.events.set_telemetry(telemetry);
71 }
72
73 pub fn telemetry(&self) -> &Arc<dyn Telemetry> {
75 &self.telemetry
76 }
77
78 pub fn with_finalizers(mut self, finalizers: FinalizerPipeline) -> Self {
81 self.finalizers = finalizers;
82 self
83 }
84
85 pub fn set_finalizers(&mut self, finalizers: FinalizerPipeline) {
87 self.finalizers = finalizers;
88 }
89
90 pub fn subscribe(&self) -> tokio::sync::broadcast::Receiver<crate::event::DownloadEvent> {
93 self.events.subscribe()
94 }
95
96 pub fn config(&self) -> &Config {
98 &self.config
99 }
100
101 pub fn events(&self) -> &EventSink {
103 &self.events
104 }
105
106 pub fn store(&self) -> &Arc<dyn TaskStore> {
108 &self.store
109 }
110
111 pub async fn resolve(&self, link: &Link, options: &ParseOptions) -> Result<RoutedResource> {
113 self.router.resolve(link, options).await
114 }
115
116 #[tracing::instrument(
122 name = "shaul.engine.add",
123 level = "debug",
124 skip(self, link, destination, options)
125 )]
126 pub async fn add(
127 &self,
128 link: Link,
129 destination: PathBuf,
130 options: TaskOptions,
131 ) -> Result<Task> {
132 if !options.force {
133 let key = format!("{}|{}", link.canonical(), destination.display());
134 for task in self.store.list().await? {
135 if task.state.is_active() && task.key() == key {
136 return Ok(task);
137 }
138 }
139 }
140
141 let parse_options = ParseOptions {
142 request: options.request.clone(),
143 checksum: options.checksum.clone(),
144 revision: options.revision.clone(),
145 include: options.include.clone(),
146 exclude: options.exclude.clone(),
147 };
148 let routed = self.router.resolve(&link, &parse_options).await?;
149 let mut task = Task::new(link, destination, options);
150 task.backend = Some(routed.backend.name().to_string());
151 task.set_resource(routed.resource);
152 self.store.save(&task).await?;
153 self.events.emit(EventKind::TaskAdded {
154 task_id: task.id,
155 link: task.link.to_string(),
156 });
157 let backend = task.backend.as_deref().unwrap_or("unknown");
158 self.telemetry
159 .counter(metrics::TASKS_ADDED, 1, &[("backend", backend)]);
160 tracing::debug!(task_id = %task.id, backend, "task added");
161 Ok(task)
162 }
163
164 pub async fn run_task(&self, id: TaskId) -> Result<Task> {
166 self.run_task_with_cancel(id, None).await
167 }
168
169 #[tracing::instrument(
176 name = "shaul.engine.run_task",
177 level = "debug",
178 skip(self, external_cancel),
179 fields(task_id = %id)
180 )]
181 pub async fn run_task_with_cancel(
182 &self,
183 id: TaskId,
184 external_cancel: Option<CancellationToken>,
185 ) -> Result<Task> {
186 let task = self.store.load(id).await?;
187 match task.state {
188 TaskState::Completed => return Ok(task),
189 TaskState::Cancelled => {
190 return Err(Error::InvalidState(format!("task {id} was cancelled")));
191 }
192 _ => {}
193 }
194
195 let backend_name = task
196 .backend
197 .clone()
198 .unwrap_or_else(|| "unknown".to_string());
199 let metric_labels = [("backend", backend_name.as_str())];
200 let started = Instant::now();
201 let resource = task
202 .resource
203 .clone()
204 .ok_or_else(|| Error::InvalidState(format!("task {id} has no parsed resource")))?;
205 let backend = self.router.backend_for(&resource)?;
206 let capabilities = backend.capabilities();
207 let mut options = self.config.resolve(&task.destination, &task.options)?;
208 if !capabilities.resume {
209 options.resume = false;
210 }
211 let resume = if capabilities.resume && options.resume {
212 task.resume.clone()
213 } else {
214 None
215 };
216 let cancel = external_cancel.unwrap_or_default();
217
218 self.running.lock().await.insert(id, cancel.clone());
219 self.store
220 .update(
221 id,
222 Box::new(|task| {
223 task.state = TaskState::Running;
224 task.error = None;
225 Ok(())
226 }),
227 )
228 .await?;
229 self.events.emit(EventKind::TaskStarted {
230 task_id: id,
231 link: task.link.to_string(),
232 });
233 self.telemetry
234 .counter(metrics::TASKS_STARTED, 1, &metric_labels);
235 let active = self.running.lock().await.len() as f64;
236 self.telemetry.gauge(metrics::ACTIVE_TASKS, active, &[]);
237 tracing::debug!(task_id = %id, backend = %backend_name, "task started");
238 if !capabilities.resume && task.resume.is_some() {
239 self.events.emit(EventKind::Warning {
240 task_id: Some(id),
241 message: format!(
242 "backend {backend_name} does not support resume; starting this task from scratch"
243 ),
244 });
245 }
246
247 let checkpoint = Arc::new(StoreCheckpoint {
248 store: self.store.clone(),
249 task_id: id,
250 });
251 let request = DownloadRequest {
252 task_id: id,
253 resource,
254 destination: task.destination.clone(),
255 options: options.clone(),
256 resume,
257 cancel: cancel.clone(),
258 events: self.events.clone(),
259 checkpoint,
260 };
261
262 let watcher = {
265 let store = self.store.clone();
266 let token = cancel.clone();
267 tokio::spawn(async move {
268 loop {
269 tokio::select! {
270 _ = token.cancelled() => break,
271 _ = tokio::time::sleep(Duration::from_millis(500)) => {
272 if let Ok(task) = store.load(id).await
273 && matches!(
274 task.state,
275 TaskState::Paused | TaskState::Cancelled
276 ) {
277 token.cancel();
278 break;
279 }
280 }
281 }
282 }
283 })
284 };
285
286 let result = self.run_attempts(&request, backend, &options).await;
287 watcher.abort();
288 self.running.lock().await.remove(&id);
289
290 let state = match &result {
291 Ok(task) => match task.state {
292 TaskState::Completed => "completed",
293 TaskState::Paused => "paused",
294 TaskState::Cancelled => "cancelled",
295 _ => "unknown",
296 },
297 Err(_) => "failed",
298 };
299 let state_labels = [("backend", backend_name.as_str()), ("state", state)];
300 self.telemetry.histogram(
301 metrics::TASK_DURATION_SECONDS,
302 started.elapsed().as_secs_f64(),
303 &state_labels,
304 );
305 match state {
306 "completed" => {
307 self.telemetry
308 .counter(metrics::TASKS_COMPLETED, 1, &metric_labels);
309 if let Ok(task) = &result {
310 self.telemetry.counter(
311 metrics::BYTES_DOWNLOADED,
312 task.downloaded_bytes(),
313 &metric_labels,
314 );
315 }
316 }
317 "paused" => self
318 .telemetry
319 .counter(metrics::TASKS_PAUSED, 1, &metric_labels),
320 "cancelled" => self
321 .telemetry
322 .counter(metrics::TASKS_CANCELLED, 1, &metric_labels),
323 _ => self
324 .telemetry
325 .counter(metrics::TASKS_FAILED, 1, &metric_labels),
326 }
327 let active = self.running.lock().await.len() as f64;
328 self.telemetry.gauge(metrics::ACTIVE_TASKS, active, &[]);
329 tracing::debug!(task_id = %id, backend = %backend_name, state, "task finished");
330 result
331 }
332
333 #[tracing::instrument(
334 name = "shaul.engine.attempt",
335 level = "debug",
336 skip(self, request, backend, options),
337 fields(task_id = %request.task_id)
338 )]
339 async fn run_attempts(
340 &self,
341 request: &DownloadRequest,
342 backend: Arc<dyn DownloadBackend>,
343 options: &ResolvedOptions,
344 ) -> Result<Task> {
345 let mut verification_retries = 0u32;
346 loop {
347 if options.disk_space_check
348 && let Some(required) = required_download_bytes(request)
349 && let Err(error) = ensure_free_space(&request.destination, required)
350 {
351 let message = error.to_string();
352 self.fail(request.task_id, error).await?;
353 return Err(Error::Backend(message));
354 }
355 let outcome = backend.download(request.clone()).await;
356 match outcome {
357 Ok(DownloadOutcome::Completed(files)) => {
358 match self
359 .finalize_files(request.task_id, &request.destination, options, &files)
360 .await
361 {
362 Ok(()) => {
363 let total_bytes: u64 = files.iter().map(|file| file.size).sum();
364 let file_count = files.len();
365 let items = file_items(&files);
366 let task = self
367 .store
368 .update(
369 request.task_id,
370 Box::new(move |task| {
371 task.state = TaskState::Completed;
372 task.error = None;
373 task.resume = None;
374 task.files = items;
375 Ok(())
376 }),
377 )
378 .await?;
379 self.events.emit(EventKind::TaskCompleted {
380 task_id: request.task_id,
381 files: file_count,
382 bytes: total_bytes,
383 });
384 return Ok(task);
385 }
386 Err(Error::Verification { .. }) if verification_retries < 1 => {
387 verification_retries += 1;
388 tracing::warn!(
389 task_id = %request.task_id,
390 "checksum mismatch; retrying once from scratch"
391 );
392 self.events.emit(EventKind::Warning {
393 task_id: Some(request.task_id),
394 message: "checksum mismatch; retrying once from scratch"
395 .to_string(),
396 });
397 self.reset_outputs(request.task_id, &files).await?;
398 continue;
399 }
400 Err(error) => {
401 let message = error.to_string();
402 self.fail(request.task_id, error).await?;
403 return Err(Error::Backend(message));
404 }
405 }
406 }
407 Ok(DownloadOutcome::Paused) => {
408 return self.mark_paused(request.task_id).await;
409 }
410 Err(error) if error.is_cancelled() => {
411 return self.mark_paused(request.task_id).await;
412 }
413 Err(error) => {
414 let message = error.to_string();
415 self.fail(request.task_id, error).await?;
416 return Err(Error::Backend(message));
417 }
418 }
419 }
420 }
421
422 #[tracing::instrument(
423 name = "shaul.engine.finalize",
424 level = "debug",
425 skip(self, destination, options, files),
426 fields(task_id = %task_id, files = files.len())
427 )]
428 async fn finalize_files(
429 &self,
430 task_id: TaskId,
431 destination: &std::path::Path,
432 options: &ResolvedOptions,
433 files: &[FileOutcome],
434 ) -> Result<()> {
435 for file in files {
436 let context = FinalizeContext {
437 task_id,
438 destination,
439 file,
440 options,
441 events: &self.events,
442 };
443 let result = self.finalizers.run(&context).await?;
444 self.events.emit(EventKind::FileCompleted {
445 task_id,
446 file: file.relative_path.clone(),
447 path: file.path.clone(),
448 size: file.size,
449 checksum_verified: result.checksum_verified,
450 });
451 }
452 Ok(())
453 }
454
455 async fn reset_outputs(&self, id: TaskId, files: &[FileOutcome]) -> Result<()> {
456 for file in files {
457 let _ = tokio::fs::remove_file(&file.path).await;
458 let _ = tokio::fs::remove_file(part_path(&file.path)).await;
459 }
460 self.store
461 .update(
462 id,
463 Box::new(|task| {
464 task.resume = None;
465 for file in &mut task.files {
466 file.downloaded = 0;
467 file.state = FileState::Pending;
468 }
469 Ok(())
470 }),
471 )
472 .await?;
473 Ok(())
474 }
475
476 #[tracing::instrument(
477 name = "shaul.engine.fail",
478 level = "debug",
479 skip(self, error),
480 fields(task_id = %id)
481 )]
482 async fn fail(&self, id: TaskId, error: Error) -> Result<()> {
483 let message = error.to_string();
484 tracing::warn!(task_id = %id, error = %message, "task failed");
485 let stored = message.clone();
486 self.store
487 .update(
488 id,
489 Box::new(move |task| {
490 if task.state == TaskState::Running {
491 task.state = TaskState::Failed;
492 task.error = Some(stored.clone());
493 }
494 Ok(())
495 }),
496 )
497 .await?;
498 self.events.emit(EventKind::TaskFailed {
499 task_id: id,
500 error: message,
501 });
502 Ok(())
503 }
504
505 async fn mark_paused(&self, id: TaskId) -> Result<Task> {
506 let task = self.store.load(id).await?;
507 match task.state {
508 TaskState::Cancelled => {
509 self.events.emit(EventKind::TaskCancelled { task_id: id });
510 Ok(task)
511 }
512 TaskState::Paused => {
513 self.events.emit(EventKind::TaskPaused { task_id: id });
514 Ok(task)
515 }
516 _ => {
517 let task = self
518 .store
519 .update(
520 id,
521 Box::new(|task| {
522 task.state = TaskState::Paused;
523 Ok(())
524 }),
525 )
526 .await?;
527 self.events.emit(EventKind::TaskPaused { task_id: id });
528 Ok(task)
529 }
530 }
531 }
532
533 pub async fn run_pending(&self, jobs: usize, cancel: CancellationToken) -> Result<Vec<Task>> {
538 let tasks = self.store.list().await?;
539 let pending = tasks
540 .iter()
541 .filter(|task| task.state == TaskState::Pending)
542 .count() as f64;
543 self.telemetry.gauge(metrics::QUEUE_DEPTH, pending, &[]);
544 let ids = pending_task_ids(tasks);
545 let jobs = jobs.max(1);
546 let results = futures_util::stream::iter(ids.into_iter().map(|id| {
547 let cancel = cancel.clone();
548 async move { self.run_task_with_cancel(id, Some(cancel)).await }
549 }))
550 .buffer_unordered(jobs)
551 .collect::<Vec<_>>()
552 .await;
553
554 Ok(results.into_iter().filter_map(Result::ok).collect())
555 }
556
557 pub async fn requeue_stale(&self) -> Result<Vec<TaskId>> {
560 let mut requeued = Vec::new();
561 for task in self.store.list().await? {
562 if task.state == TaskState::Running {
563 self.store
564 .update(
565 task.id,
566 Box::new(|task| {
567 task.state = TaskState::Pending;
568 Ok(())
569 }),
570 )
571 .await?;
572 requeued.push(task.id);
573 self.events.emit(EventKind::Warning {
574 task_id: Some(task.id),
575 message: "requeued a task left running by a previous process".to_string(),
576 });
577 }
578 }
579 Ok(requeued)
580 }
581
582 pub async fn pause(&self, id: TaskId) -> Result<Task> {
586 let task = self
587 .store
588 .update(
589 id,
590 Box::new(|task| {
591 if task.state == TaskState::Completed {
592 return Err(Error::InvalidState("task already completed".to_string()));
593 }
594 task.state = TaskState::Paused;
595 Ok(())
596 }),
597 )
598 .await?;
599 if let Some(token) = self.running.lock().await.get(&id) {
600 token.cancel();
601 }
602 self.events.emit(EventKind::TaskPaused { task_id: id });
603 Ok(task)
604 }
605
606 pub async fn cancel(&self, id: TaskId) -> Result<Task> {
611 let task = self
612 .store
613 .update(
614 id,
615 Box::new(|task| {
616 if task.state == TaskState::Completed {
617 return Err(Error::InvalidState("task already completed".to_string()));
618 }
619 task.state = TaskState::Cancelled;
620 Ok(())
621 }),
622 )
623 .await?;
624 if let Some(token) = self.running.lock().await.get(&id) {
625 token.cancel();
626 }
627 self.events.emit(EventKind::TaskCancelled { task_id: id });
628 Ok(task)
629 }
630
631 pub async fn resume(&self, id: TaskId) -> Result<Task> {
635 let task = self
636 .store
637 .update(
638 id,
639 Box::new(|task| {
640 if task.state == TaskState::Completed {
641 return Err(Error::InvalidState("task already completed".to_string()));
642 }
643 task.state = TaskState::Pending;
644 task.error = None;
645 Ok(())
646 }),
647 )
648 .await?;
649 self.events.emit(EventKind::TaskStateChanged {
650 task_id: id,
651 state: TaskState::Pending,
652 });
653 Ok(task)
654 }
655
656 pub async fn list(&self) -> Result<Vec<Task>> {
658 self.store.list().await
659 }
660
661 pub async fn status(&self, id: TaskId) -> Result<Task> {
663 self.store.load(id).await
664 }
665}
666
667fn required_download_bytes(request: &DownloadRequest) -> Option<u64> {
668 let total = request
669 .resource
670 .files
671 .iter()
672 .try_fold(0u64, |sum, file| file.size.map(|size| sum + size))?;
673 let completed = request
674 .resume
675 .as_ref()
676 .map(|resume| {
677 resume
678 .files
679 .values()
680 .map(crate::task::FileResume::completed_bytes)
681 .sum::<u64>()
682 })
683 .unwrap_or(0);
684 Some(total.saturating_sub(completed))
685}
686
687fn pending_task_ids(mut tasks: Vec<Task>) -> Vec<TaskId> {
688 tasks.retain(|task| task.state == TaskState::Pending);
689 tasks.sort_by(|left, right| {
690 right
691 .priority
692 .cmp(&left.priority)
693 .then_with(|| left.created_at_ms.cmp(&right.created_at_ms))
694 .then_with(|| left.id.cmp(&right.id))
695 });
696 tasks.into_iter().map(|task| task.id).collect()
697}
698
699struct StoreCheckpoint {
700 store: Arc<dyn TaskStore>,
701 task_id: TaskId,
702}
703
704#[async_trait]
705impl ResumeCheckpoint for StoreCheckpoint {
706 async fn save(&self, resume: &TaskResume) -> Result<()> {
707 let resume = resume.clone();
708 self.store
709 .update(
710 self.task_id,
711 Box::new(move |task| {
712 task.resume = Some(resume.clone());
713 task.refresh_from_resume();
714 Ok(())
715 }),
716 )
717 .await?;
718 Ok(())
719 }
720}
721
722fn file_items(files: &[FileOutcome]) -> Vec<FileItem> {
723 files
724 .iter()
725 .map(|file| FileItem {
726 path: file.relative_path.clone(),
727 url: None,
728 size: Some(file.size),
729 downloaded: file.size,
730 state: FileState::Completed,
731 checksum: file.checksum.clone(),
732 selected: true,
733 })
734 .collect()
735}
736
737#[allow(dead_code)]
738fn _assert_resource(_: &Resource) {}
739
740#[cfg(test)]
741mod tests {
742 use super::*;
743 use crate::link::Link;
744
745 fn task(link: &str, priority: Option<i32>, created_at_ms: u64) -> Task {
746 let mut task = Task::new(
747 Link::new(link).unwrap(),
748 PathBuf::from("."),
749 TaskOptions {
750 priority,
751 ..TaskOptions::default()
752 },
753 );
754 task.created_at_ms = created_at_ms;
755 task
756 }
757
758 #[test]
759 fn orders_pending_tasks_by_priority_then_age() {
760 let low = task("low", Some(-1), 1);
761 let high_old = task("high-old", Some(10), 2);
762 let high_new = task("high-new", Some(10), 3);
763 let normal = task("normal", None, 4);
764 let mut paused = task("paused", Some(100), 0);
765 paused.state = TaskState::Paused;
766
767 let ordered = pending_task_ids(vec![
768 normal.clone(),
769 high_new.clone(),
770 paused,
771 low.clone(),
772 high_old.clone(),
773 ]);
774 assert_eq!(ordered, vec![high_old.id, high_new.id, normal.id, low.id]);
775 }
776}