pagesift_api/jobs/
state.rs

1//! Job state definitions and lifecycle structures.
2
3use serde::{Deserialize, Serialize};
4use std::time::{Duration, SystemTime};
5use tokio::sync::watch;
6
7use crate::limits::Caller;
8use crate::metrics::PageSource;
9use crate::types::{CrawlSummary, Document, JobError};
10use crate::webhook::WebhookConfig;
11
12/// Task results retention period (24 hours).
13pub const JOB_TTL: Duration = Duration::from_secs(24 * 3600);
14
15#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
16#[serde(rename_all = "lowercase")]
17pub enum JobState {
18    Scraping,
19    Completed,
20    Cancelled,
21    Failed,
22}
23
24impl JobState {
25    pub fn as_str(self) -> &'static str {
26        match self {
27            JobState::Scraping => "scraping",
28            JobState::Completed => "completed",
29            JobState::Cancelled => "cancelled",
30            JobState::Failed => "failed",
31        }
32    }
33}
34
35#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
36#[serde(rename_all = "snake_case")]
37pub enum JobKind {
38    Crawl,
39    BatchScrape,
40}
41
42impl JobKind {
43    pub fn as_str(self) -> &'static str {
44        match self {
45            JobKind::Crawl => "crawl",
46            JobKind::BatchScrape => "batch_scrape",
47        }
48    }
49
50    pub(crate) fn pages(self) -> PageSource {
51        match self {
52            JobKind::Crawl => PageSource::Crawl,
53            JobKind::BatchScrape => PageSource::Batch,
54        }
55    }
56}
57
58#[derive(Debug)]
59pub struct Job {
60    pub kind: JobKind,
61    pub state: JobState,
62    pub total: usize,
63    pub completed: usize,
64    pub data: Vec<Document>,
65    pub errors: Vec<JobError>,
66    pub robots_blocked: Vec<String>,
67    pub created: SystemTime,
68    pub summary: Option<CrawlSummary>,
69    pub(crate) stored_docs: usize,
70    pub(crate) owner: Option<Caller>,
71    /// 新完成了文档或者状态变了就响一下，叫醒在等这个任务的流（`/stream`）。任务删掉时随之关闭。
72    pub(crate) changed: watch::Sender<()>,
73    pub(crate) previews: tokio::sync::broadcast::Sender<crate::types::MarkdownPreview>,
74}
75
76impl Job {
77    pub(crate) fn new(kind: JobKind, total: usize, owner: Option<Caller>) -> Self {
78        Self {
79            kind,
80            state: JobState::Scraping,
81            total,
82            completed: 0,
83            data: Vec::new(),
84            errors: Vec::new(),
85            robots_blocked: Vec::new(),
86            created: SystemTime::now(),
87            summary: None,
88            stored_docs: 0,
89            owner,
90            changed: watch::channel(()).0,
91            previews: tokio::sync::broadcast::channel(4).0,
92        }
93    }
94
95    /// 通知在等这个任务的流：有新文档或状态变了。没有人在等也没关系。
96    pub(crate) fn touch(&self) {
97        self.changed.send_replace(());
98    }
99
100    pub fn expires_at(&self) -> SystemTime {
101        self.created + JOB_TTL
102    }
103
104    pub(crate) fn docs(&self) -> usize {
105        self.data.len() + self.stored_docs
106    }
107}
108
109pub struct StatusPage {
110    pub state: JobState,
111    pub total: usize,
112    pub completed: usize,
113    pub expires_at: SystemTime,
114    pub docs: usize,
115    pub data: Vec<Document>,
116    pub summary: Option<CrawlSummary>,
117}
118
119#[derive(Default)]
120pub struct Start {
121    pub owner: Option<Caller>,
122    pub webhook: Option<WebhookConfig>,
123}
pagesift_api/jobs/state.rs

pagesift_api/jobs/
state.rs
