unobtanium 3.0.0

Opinioated Web search engine library with crawler and viewer companion.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44

use serde::{Deserialize,Serialize};

use crate::crawling::ExitCode;
use crate::database::id::CrawlLogEntryId;
use crate::summary::HttpSummary;
use crate::time::UtcTimestamp;
use crate::url::UrlWithoutFragment;

/// Logs a request that was made as part of a crawl.
#[derive(Debug,Serialize,Deserialize)]
pub struct Request {
	/// Numeric id of the entry in the crawl_log
	///
	/// TODO: Use a uuid here
	#[serde(skip)]
	pub crawl_log_entry: CrawlLogEntryId,

	/// If the request was approved by the robots.txt file
	///
	/// Unapproved request may be mede to see if a link is still alive.
	pub robotstxt_approved: bool,
	
	/// The url requested
	pub url: UrlWithoutFragment,
	
	/// When the request was sent
	pub time_sent: UtcTimestamp,

	/// How long it took until an answer came back and processing could start
	///
	/// If None, measuring failed.
	pub request_duration_ms: Option<i64>,
	
	/// When the last modification date was according to the server
	pub server_last_modified: Option<UtcTimestamp>,
	
	/// Exit code that inicates what the request resulted in 
	pub exit_code: ExitCode,
	
	/// If the crawl involved an http fetch
	/// the resultng metadata is in here.
	pub http: Option<HttpSummary>,
}