pagesift_api/jobs/
state.rs

1//! Job state definitions and lifecycle structures.
2
3use std::time::{Duration, SystemTime};
4use serde::{Deserialize, Serialize};
5
6use crate::limits::Caller;
7use crate::metrics::PageSource;
8use crate::types::{CrawlSummary, Document, JobError};
9use crate::webhook::WebhookConfig;
10
11/// Task results retention period (24 hours).
12pub const JOB_TTL: Duration = Duration::from_secs(24 * 3600);
13
14#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
15#[serde(rename_all = "lowercase")]
16pub enum JobState {
17    Scraping,
18    Completed,
19    Cancelled,
20    Failed,
21}
22
23impl JobState {
24    pub fn as_str(self) -> &'static str {
25        match self {
26            JobState::Scraping => "scraping",
27            JobState::Completed => "completed",
28            JobState::Cancelled => "cancelled",
29            JobState::Failed => "failed",
30        }
31    }
32}
33
34#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
35#[serde(rename_all = "snake_case")]
36pub enum JobKind {
37    Crawl,
38    BatchScrape,
39}
40
41impl JobKind {
42    pub fn as_str(self) -> &'static str {
43        match self {
44            JobKind::Crawl => "crawl",
45            JobKind::BatchScrape => "batch_scrape",
46        }
47    }
48
49    pub(crate) fn pages(self) -> PageSource {
50        match self {
51            JobKind::Crawl => PageSource::Crawl,
52            JobKind::BatchScrape => PageSource::Batch,
53        }
54    }
55}
56
57#[derive(Debug)]
58pub struct Job {
59    pub kind: JobKind,
60    pub state: JobState,
61    pub total: usize,
62    pub completed: usize,
63    pub data: Vec<Document>,
64    pub errors: Vec<JobError>,
65    pub robots_blocked: Vec<String>,
66    pub created: SystemTime,
67    pub summary: Option<CrawlSummary>,
68    pub(crate) stored_docs: usize,
69    pub(crate) owner: Option<Caller>,
70}
71
72impl Job {
73    pub(crate) fn new(kind: JobKind, total: usize, owner: Option<Caller>) -> Self {
74        Self {
75            kind,
76            state: JobState::Scraping,
77            total,
78            completed: 0,
79            data: Vec::new(),
80            errors: Vec::new(),
81            robots_blocked: Vec::new(),
82            created: SystemTime::now(),
83            summary: None,
84            stored_docs: 0,
85            owner,
86        }
87    }
88
89    pub fn expires_at(&self) -> SystemTime {
90        self.created + JOB_TTL
91    }
92
93    pub(crate) fn docs(&self) -> usize {
94        self.data.len() + self.stored_docs
95    }
96}
97
98pub struct StatusPage {
99    pub state: JobState,
100    pub total: usize,
101    pub completed: usize,
102    pub expires_at: SystemTime,
103    pub docs: usize,
104    pub data: Vec<Document>,
105    pub summary: Option<CrawlSummary>,
106}
107
108#[derive(Default)]
109pub struct Start {
110    pub owner: Option<Caller>,
111    pub webhook: Option<WebhookConfig>,
112}
pagesift_api/jobs/state.rs

pagesift_api/jobs/
state.rs
