unobtanium 3.0.0

Opinioated Web search engine library with crawler and viewer companion.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46


use serde::{Serialize,Deserialize};
use uuid::Uuid;

use crate::crawling::CrawlType;
use crate::crawling::ExitCode;
use crate::database::id::AgentId;
use crate::time::UtcTimestamp;
use crate::url::UrlWithoutFragment;

/// Entry in the Crawl Log.
#[derive(Clone,Debug,Serialize,Deserialize)]
pub struct CrawlLogEntry {

	/// The url that was crawled
	pub url: UrlWithoutFragment,

	/// The  type of crawl that happend
	pub crawl_type: CrawlType,

	/// Uniqe identifier for this crawl log entry across databases
	pub crawl_uuid: Uuid,

	/// The id of the agent that did the crawling.
	///
	///TODO: convert this to a uuid!
	#[serde(skip)]
	pub agent_id: AgentId,

	/// The timestamp of when the crawling started.
	pub time_started: UtcTimestamp,

	/// The time the whole crawl operation took.
	///
	/// If this is set to none it means, that the time couldn't be measured.
	pub time_taken_ms: Option<i64>,

	/// The genral result of the crawl operation.
	///
	/// Note: with the removal of logging unfinished requests this became
	///       a non Option type.
	pub exit_code: ExitCode,

	/// An optional message to store errors in for debugging later.
	pub message: Option<String>,
}