unobtanium 3.0.0

Opinioated Web search engine library with crawler and viewer companion.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130


use criterium::NumberCriterium;
use criterium::rusqlite::assembler::*;

use crate::criterium::OriginCriterium;
use crate::database::id::AgentId;
use crate::database::id::NumericDatabseId;
use crate::database::id::OriginId;
use crate::database::CrawlerSchema;
use crate::database::fields::*;

/// For matching against the lit of crawl candidates
#[derive(Clone,Debug)]
pub enum CrawlCandidateCriterium {
	/// Applies the OriginCriterium to the todo-items url
	OriginMatches(OriginCriterium),

	/// Only match todo-items where the url comes from the given origin id
	OriginId(OriginId),

	/// Only match todo-items where the urls origin is marked as ignored by
	/// the given agent.
	///
	/// Usually one wants to use this in combination with an inversion to
	/// only match items that are not ignored.
	OriginIgnoredByAgent(AgentId),
	
	/// Only match todo-items where the url  is marked as ignored by
	/// the given agent.
	///
	/// Usually one wants to use this in combination with an inversion to
	/// only match items that are not ignored.
	UrlIgnoredByAgent(AgentId),

	/// Compare against the last_crawl timestamp.
	/// Use a Unix-UTC timestamp.
	///
	/// TODO: Introduce a timestamp criterium and use it here.
	LastCrawlTime(NumberCriterium<i64>),

}

impl AssembleRusqliteQuery<CrawlerSchema, ()> for CrawlCandidateCriterium {
	fn assemble_rusqlite_query(
		&self,
		assembly_context: &AssemblyContext,
		_context: &()
	) -> InvertableRusqliteQuery<CrawlerSchema> {
		match self {
			Self::OriginMatches(c) => 
				c.assemble_rusqlite_query(&assembly_context.prefix_with("crawl_candidate_"), &())
					.inner_join(
						None,
						OriginField::OriginId.into(),
						None,
						UrlField::OriginId.into(),
					).inner_join(
						None,
						UrlField::UrlId.into(),
						Some(assembly_context.prefix()), // dock to current context
						CrawlCandidateField::UrlId.into()
					),
			Self::OriginId(id) =>
				NumberCriterium::Equals(id.number())
					.assemble_query(
						&assembly_context.prefix_with("crawl_candidate_"),
						&UrlField::OriginId.into()
					)
					.inner_join(
						None,
						UrlField::UrlId.into(),
						Some(assembly_context.prefix()), // dock to current context
						CrawlCandidateField::UrlId.into()
					),
			Self::OriginIgnoredByAgent(id) => {
				let new_context = assembly_context.prefix_with("crawl_candidate_");
				NumberCriterium::Equals(id.number())
					.assemble_query(
						&new_context.in_and_block(),
						&TempIgnoreOriginField::OriginId.into()
					)
					.get_corrected_query()
					.and(RusqliteQuery::test_if_null(
						new_context.prefix(), &TempIgnoreOriginField::OriginId.into(), true
					))
					.left_join(
						Some("ignore_origin"),
						TempIgnoreOriginField::OriginId.into(),
						None,
						UrlField::OriginId.into(),
					).inner_join(
						None,
						UrlField::UrlId.into(),
						Some(assembly_context.prefix()), // dock to current context
						CrawlCandidateField::UrlId.into(),
					).as_invertable()
			},
			Self::UrlIgnoredByAgent(id) => {
				let inner_context = assembly_context.prefix_with("crawl_candidate_");
				NumberCriterium::Equals(id.number())
					.assemble_query(
						&inner_context.in_and_block(),
						&TempIgnoreUrlField::AgentId.into()
					)
					.get_corrected_query()
					.and(RusqliteQuery::test_if_null(
						inner_context.prefix(), &TempIgnoreUrlField::UrlId.into(), true))
					.left_join(
						Some("ignore_url"),
						TempIgnoreUrlField::UrlId.into(),
						Some(assembly_context.prefix()), // dock to current context
						CrawlCandidateField::UrlId.into()
					).as_invertable()
			},
			Self::LastCrawlTime(number_criterium) => {
				number_criterium.assemble_query(
					assembly_context,
					&CrawlCandidateField::LastCrawlTimeUnixUtc.into()
				)
			}
		}
	}
}

impl From<OriginCriterium> for CrawlCandidateCriterium {
	fn from(c: OriginCriterium) -> Self {
		Self::OriginMatches(c)
	}
}