pagesift_api/jobs/
state.rs

1//! Job state definitions and lifecycle structures.
2
3use serde::{Deserialize, Serialize};
4use std::time::{Duration, SystemTime};
5use tokio::sync::watch;
6
7use crate::limits::Caller;
8use crate::metrics::PageSource;
9use crate::types::{CrawlSummary, Document, JobError};
10use crate::webhook::WebhookConfig;
11
12/// Task results retention period (24 hours).
13pub const JOB_TTL: Duration = Duration::from_secs(24 * 3600);
14
15#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
16#[serde(rename_all = "lowercase")]
17pub enum JobState {
18    Scraping,
19    Completed,
20    Cancelled,
21    Failed,
22}
23
24impl JobState {
25    pub fn as_str(self) -> &'static str {
26        match self {
27            JobState::Scraping => "scraping",
28            JobState::Completed => "completed",
29            JobState::Cancelled => "cancelled",
30            JobState::Failed => "failed",
31        }
32    }
33}
34
35#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
36#[serde(rename_all = "snake_case")]
37pub enum JobKind {
38    Crawl,
39    BatchScrape,
40}
41
42impl JobKind {
43    pub fn as_str(self) -> &'static str {
44        match self {
45            JobKind::Crawl => "crawl",
46            JobKind::BatchScrape => "batch_scrape",
47        }
48    }
49
50    pub(crate) fn pages(self) -> PageSource {
51        match self {
52            JobKind::Crawl => PageSource::Crawl,
53            JobKind::BatchScrape => PageSource::Batch,
54        }
55    }
56}
57
58#[derive(Debug)]
59pub struct Job {
60    pub kind: JobKind,
61    pub state: JobState,
62    pub total: usize,
63    pub completed: usize,
64    pub data: Vec<Document>,
65    pub errors: Vec<JobError>,
66    pub robots_blocked: Vec<String>,
67    pub created: SystemTime,
68    pub summary: Option<CrawlSummary>,
69    pub(crate) stored_docs: usize,
70    pub(crate) owner: Option<Caller>,
71    /// 新完成了文档或者状态变了就响一下，叫醒在等这个任务的流（`/stream`）。任务删掉时随之关闭。
72    pub(crate) changed: watch::Sender<()>,
73}
74
75impl Job {
76    pub(crate) fn new(kind: JobKind, total: usize, owner: Option<Caller>) -> Self {
77        Self {
78            kind,
79            state: JobState::Scraping,
80            total,
81            completed: 0,
82            data: Vec::new(),
83            errors: Vec::new(),
84            robots_blocked: Vec::new(),
85            created: SystemTime::now(),
86            summary: None,
87            stored_docs: 0,
88            owner,
89            changed: watch::channel(()).0,
90        }
91    }
92
93    /// 通知在等这个任务的流：有新文档或状态变了。没有人在等也没关系。
94    pub(crate) fn touch(&self) {
95        self.changed.send_replace(());
96    }
97
98    pub fn expires_at(&self) -> SystemTime {
99        self.created + JOB_TTL
100    }
101
102    pub(crate) fn docs(&self) -> usize {
103        self.data.len() + self.stored_docs
104    }
105}
106
107pub struct StatusPage {
108    pub state: JobState,
109    pub total: usize,
110    pub completed: usize,
111    pub expires_at: SystemTime,
112    pub docs: usize,
113    pub data: Vec<Document>,
114    pub summary: Option<CrawlSummary>,
115}
116
117#[derive(Default)]
118pub struct Start {
119    pub owner: Option<Caller>,
120    pub webhook: Option<WebhookConfig>,
121}
pagesift_api/jobs/state.rs

pagesift_api/jobs/
state.rs
