unobtanium 3.0.0

Opinioated Web search engine library with crawler and viewer companion.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135

use blake2::Blake2b512;
use blake2::Digest;
use serde::{Serialize,Deserialize};
use whatlang::Lang;
use whatlang::detect_lang;

use crate::types::Blake2b512Digest;

/// Collection of text inside a file sorted by semantic meaning
#[derive(Debug,Clone,Serialize,Deserialize,Default)]
pub struct TextPile {
	
	/// Normal paragraph text that makes up the main content of a file
	#[serde(default, skip_serializing_if="String::is_empty")]
	pub text: String,

	/// Text without any meaningful semantics attached
	#[serde(default, skip_serializing_if="String::is_empty")]
	pub secondary_text: String,

	/// Major headlines, in html this is h2 and h3 (and additional h1)
	#[serde(default, skip_serializing_if="String::is_empty")]
	pub big_headlines: String,

	/// Minor headlines
	#[serde(default, skip_serializing_if="String::is_empty")]
	pub small_headlines: String,

	/// Text that is marked as being code
	#[serde(default, skip_serializing_if="String::is_empty")]
	pub code_text: String,

	/// Text that is marked as being quoted
	#[serde(default, skip_serializing_if="String::is_empty")]
	pub quote_text: String,
	
}

impl TextPile {

	pub fn new() -> Self {
		Default::default()
	}

	pub fn is_empty(&self) -> bool {
		return self.text.is_empty()
			&& self.secondary_text.is_empty()
			&& self.big_headlines.is_empty()
			&& self.small_headlines.is_empty()
			&& self.code_text.is_empty()
			&& self.quote_text.is_empty()
		;
	}

	pub fn trim(&mut self) {
		self.text            = self.text.trim().to_string();
		self.secondary_text  = self.secondary_text.trim().to_string();
		self.big_headlines   = self.big_headlines.trim().to_string();
		self.small_headlines = self.small_headlines.trim().to_string();
		self.code_text       = self.code_text.trim().to_string();
		self.quote_text      = self.quote_text.trim().to_string();
	}

	pub fn to_opt(self) -> Option<Self> {
		if self.is_empty() {
			return None;
		} else {
			return Some(self);
		}
	}

	pub fn add_text(&mut self, new: &str) {
		if new.is_empty() { return; }
		if !self.text.is_empty() { self.text += "\n"; }
		self.text += new;
	}

	pub fn add_secondary_text(&mut self, new: &str) {
		if new.is_empty() { return; }
		if !self.secondary_text.is_empty() { self.secondary_text += "\n"; }
		self.secondary_text += new;
	}

	pub fn add_big_headline(&mut self, new: &str) {
		if new.is_empty() { return; }
		if !self.big_headlines.is_empty() { self.big_headlines += "\n"; }
		self.big_headlines += new;
	}

	pub fn add_small_headline(&mut self, new: &str) {
		if new.is_empty() { return; }
		if !self.small_headlines.is_empty() { self.small_headlines += "\n"; }
		self.small_headlines += new;
	}

	pub fn add_code_text(&mut self, new: &str) {
		if new.is_empty() { return; }
		if !self.code_text.is_empty() { self.code_text += "\n"; }
		self.code_text += new;
	}

	pub fn add_quote_text(&mut self, new: &str) {
		if new.is_empty() { return; }
		if !self.quote_text.is_empty() { self.quote_text += "\n"; }
		self.quote_text += new;
	}

	pub fn calculate_blake2b512_digest(&self) -> Blake2b512Digest {
		let mut hasher = Blake2b512::new();
		let spacer = "\n\n\n";
		
		hasher.update(&self.text);
		hasher.update(spacer);
		hasher.update(&self.secondary_text);
		hasher.update(spacer);
		hasher.update(&self.big_headlines);
		hasher.update(spacer);
		hasher.update(&self.small_headlines);
		hasher.update(spacer);
		hasher.update(&self.code_text);
		hasher.update(spacer);
		hasher.update(&self.quote_text);
		
		return hasher.into();
	}

	pub fn detect_text_main_language(&self) -> Option<Lang> {
		detect_lang(&self.text)
	}
	
	pub fn detect_secondary_text_main_language(&self) -> Option<Lang> {
		detect_lang(&self.secondary_text)
	}
}