argus-common 0.1.0

Common types and utilities for the Argus web crawler
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22

#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct CrawlJob {
    pub url: String,
    pub normalized_url: String,
    pub host: String,
    pub depth: u16,
}

#[derive(Debug, Clone)]
pub struct FetchResult {
    pub url: String,
    pub final_url: String,
    pub status: u16,
    pub content_type: Option<String>,
    pub body: bytes::Bytes,
}

#[derive(Debug, Clone)]
pub struct ExtractedLink {
    pub from_url: String,
    pub to_url: String,
}