rover-fetch 0.2.0

An MCP server for fetching and prepping web content for LLM agents.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61

//! Long-running task subsystem.
//!
//! See `docs/superpowers/specs/2026-05-14-rover-m6-tasks-batching-design.md`.

pub mod batch_fetch;
pub mod deps;
pub mod error;
pub mod retry;
pub mod revalidate;
pub mod scheduler;
pub mod summarize;
pub mod types;

pub use deps::WorkerDeps;
pub use error::TasksError;
pub use scheduler::{NewTaskSender, Scheduler};
pub use types::{
    BatchFetchParams, BatchFetchResult, CoreEvent, RetryParams, RevalidateParams, TaskId, TaskKind,
    TaskStatus,
};

use std::sync::Arc;
use tokio::task::JoinSet;
use tokio_util::sync::CancellationToken;

use crate::storage::Db;

/// Default production dispatch table. Routes by `TaskKind`.
pub struct DefaultSpawner {
    pub deps: WorkerDeps,
}

impl scheduler::WorkerSpawner for DefaultSpawner {
    fn spawn(
        &self,
        join_set: &mut JoinSet<()>,
        db: Db,
        task_id: TaskId,
        kind: TaskKind,
        cancel: CancellationToken,
    ) {
        match kind {
            TaskKind::Summarize => {
                join_set.spawn(summarize::run(db, task_id, cancel));
            }
            TaskKind::BatchFetch => {
                join_set.spawn(batch_fetch::run(self.deps.clone(), db, task_id, cancel));
            }
            TaskKind::Retry => {
                join_set.spawn(retry::run(self.deps.clone(), db, task_id, cancel));
            }
            TaskKind::Revalidate => {
                join_set.spawn(revalidate::run(self.deps.clone(), db, task_id, cancel));
            }
        }
    }
}

pub fn default_spawner(deps: WorkerDeps) -> Arc<dyn scheduler::WorkerSpawner> {
    Arc::new(DefaultSpawner { deps })
}