unobtanium 3.0.0

Opinioated Web search engine library with crawler and viewer companion.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47

use crate::crawling::CrawlType;
use crate::crawling::ExitCode;
use crate::summary::HttpSummary;
use crate::time::UtcTimestamp;

use serde::{Serialize,Deserialize};
use uuid::Uuid;

/// Relevant Inforamtion on a single crawl on *one* URI
///
/// Roughly equivalent to a request
/// (though the summary doesn't really care about requests).
/// A real crawl command may be split up into multiple crawl summaries
/// if it yielded results for multiple URIs.
#[derive(Debug,Clone,Serialize,Deserialize)]
pub struct CrawlSummary {

	/// Which kind of crawl was performed
	pub crawl_type: CrawlType,

	/// Which [CrawlLogEntry][crate::crawling::CrawlLogEntry] resulted in this crawl summary,
	/// uniquely identified by its crawl uuid
	pub crawl_uuid: Uuid,

	/// When the crawl was started
	pub crawl_time: UtcTimestamp,

	/// Which [Agent][crate::crawling::Agent] did the crawling
	pub agent_uuid: Uuid,

	/// The crawl end result
	pub exit_code: ExitCode,

	/// When the resource was last modified according to the server
	pub server_last_modified: Option<UtcTimestamp>,

	/// How long the request took
	pub request_duration_ms: Option<i64>,
	
	/// Wheter the request was robots.txt approved
	pub was_robotstxt_approved: bool,
	
	/// If the crawl involved an http fetch
	/// the resultng metadata is in here.
	pub http: Option<HttpSummary>,

}