spider_cloud_agent/params/
extraction.rs1use serde::de::{Error as DeError, SeqAccess, Unexpected, Visitor};
4use serde::{Deserialize, Deserializer, Serialize, Serializer};
5use std::collections::BTreeMap;
6use std::fmt;
7
8#[derive(
10 Debug, Clone, Copy, Default, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize,
11)]
12#[serde(rename_all = "lowercase")]
13#[non_exhaustive]
14pub enum ReturnFormat {
15 #[default]
18 Raw,
19 Markdown,
21 Commonmark,
24 Text,
26 Xml,
28 Bytes,
30 Empty,
33}
34
35impl ReturnFormat {
36 pub const fn as_str(&self) -> &'static str {
38 match self {
39 ReturnFormat::Raw => "raw",
40 ReturnFormat::Markdown => "markdown",
41 ReturnFormat::Commonmark => "commonmark",
42 ReturnFormat::Text => "text",
43 ReturnFormat::Xml => "xml",
44 ReturnFormat::Bytes => "bytes",
45 ReturnFormat::Empty => "empty",
46 }
47 }
48}
49
50impl fmt::Display for ReturnFormat {
51 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
52 f.write_str(self.as_str())
53 }
54}
55
56#[derive(Debug, Clone, PartialEq, Eq)]
62#[non_exhaustive]
63pub enum ReturnFormatHandling {
64 Single(ReturnFormat),
66 Multi(Vec<ReturnFormat>),
68}
69
70impl Default for ReturnFormatHandling {
71 fn default() -> ReturnFormatHandling {
72 ReturnFormatHandling::Single(ReturnFormat::Raw)
73 }
74}
75
76impl From<ReturnFormat> for ReturnFormatHandling {
77 fn from(value: ReturnFormat) -> ReturnFormatHandling {
78 ReturnFormatHandling::Single(value)
79 }
80}
81
82impl From<Vec<ReturnFormat>> for ReturnFormatHandling {
83 fn from(value: Vec<ReturnFormat>) -> ReturnFormatHandling {
84 ReturnFormatHandling::Multi(value)
85 }
86}
87
88impl ReturnFormatHandling {
89 pub fn formats(&self) -> &[ReturnFormat] {
91 match self {
92 ReturnFormatHandling::Single(format) => std::slice::from_ref(format),
93 ReturnFormatHandling::Multi(formats) => formats,
94 }
95 }
96
97 pub fn contains(&self, format: ReturnFormat) -> bool {
99 self.formats().contains(&format)
100 }
101}
102
103impl Serialize for ReturnFormatHandling {
104 fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
105 match self {
106 ReturnFormatHandling::Single(format) => format.serialize(serializer),
107 ReturnFormatHandling::Multi(formats) => formats.serialize(serializer),
108 }
109 }
110}
111
112impl<'de> Deserialize<'de> for ReturnFormatHandling {
113 fn deserialize<D: Deserializer<'de>>(
114 deserializer: D,
115 ) -> Result<ReturnFormatHandling, D::Error> {
116 struct HandlingVisitor;
117
118 impl<'de> Visitor<'de> for HandlingVisitor {
122 type Value = ReturnFormatHandling;
123
124 fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
125 f.write_str("a return format, or an array of them")
126 }
127
128 fn visit_str<E: DeError>(self, value: &str) -> Result<ReturnFormatHandling, E> {
129 parse_format(value).map(ReturnFormatHandling::Single)
130 }
131
132 fn visit_seq<A: SeqAccess<'de>>(
133 self,
134 mut seq: A,
135 ) -> Result<ReturnFormatHandling, A::Error> {
136 let mut formats = Vec::with_capacity(seq.size_hint().unwrap_or(2));
137
138 while let Some(format) = seq.next_element::<String>()? {
139 formats.push(parse_format::<A::Error>(&format)?);
140 }
141
142 Ok(ReturnFormatHandling::Multi(formats))
143 }
144 }
145
146 deserializer.deserialize_any(HandlingVisitor)
147 }
148}
149
150fn parse_format<E: DeError>(value: &str) -> Result<ReturnFormat, E> {
151 match value.trim().to_ascii_lowercase().as_str() {
152 "raw" => Ok(ReturnFormat::Raw),
153 "markdown" => Ok(ReturnFormat::Markdown),
154 "commonmark" => Ok(ReturnFormat::Commonmark),
155 "text" => Ok(ReturnFormat::Text),
156 "xml" => Ok(ReturnFormat::Xml),
157 "bytes" => Ok(ReturnFormat::Bytes),
158 "empty" => Ok(ReturnFormat::Empty),
159 _ => Err(E::invalid_value(
160 Unexpected::Str(value),
161 &"one of raw, markdown, commonmark, text, xml, bytes or empty",
162 )),
163 }
164}
165
166#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
172#[non_exhaustive]
173pub enum ExtractSelector {
174 Css(String),
176 XPath(String),
180}
181
182impl ExtractSelector {
183 pub fn css(expression: impl Into<String>) -> ExtractSelector {
185 ExtractSelector::Css(expression.into())
186 }
187
188 pub fn xpath(expression: impl Into<String>) -> ExtractSelector {
190 ExtractSelector::XPath(expression.into())
191 }
192
193 pub fn as_str(&self) -> &str {
195 match self {
196 ExtractSelector::Css(expression) | ExtractSelector::XPath(expression) => expression,
197 }
198 }
199
200 pub fn parse(expression: &str) -> ExtractSelector {
203 let head = expression.trim_start();
204
205 if head.starts_with('/') || head.starts_with("./") || head.starts_with("(/") {
206 ExtractSelector::XPath(expression.to_string())
207 } else {
208 ExtractSelector::Css(expression.to_string())
209 }
210 }
211}
212
213impl fmt::Display for ExtractSelector {
214 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
215 f.write_str(self.as_str())
216 }
217}
218
219impl Serialize for ExtractSelector {
220 fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
221 serializer.serialize_str(self.as_str())
222 }
223}
224
225impl<'de> Deserialize<'de> for ExtractSelector {
226 fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<ExtractSelector, D::Error> {
227 let expression = String::deserialize(deserializer)?;
228
229 Ok(ExtractSelector::parse(&expression))
230 }
231}
232
233#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
238pub struct SelectorGroup {
239 pub name: String,
241 pub selectors: Vec<ExtractSelector>,
243}
244
245impl SelectorGroup {
246 pub fn new(
248 name: impl Into<String>,
249 selectors: impl IntoIterator<Item = ExtractSelector>,
250 ) -> SelectorGroup {
251 SelectorGroup {
252 name: name.into(),
253 selectors: selectors.into_iter().collect(),
254 }
255 }
256
257 pub fn css<S: Into<String>>(
259 name: impl Into<String>,
260 selectors: impl IntoIterator<Item = S>,
261 ) -> SelectorGroup {
262 SelectorGroup::new(name, selectors.into_iter().map(ExtractSelector::css))
263 }
264
265 pub fn xpath<S: Into<String>>(
267 name: impl Into<String>,
268 selectors: impl IntoIterator<Item = S>,
269 ) -> SelectorGroup {
270 SelectorGroup::new(name, selectors.into_iter().map(ExtractSelector::xpath))
271 }
272}
273
274pub type CssExtractionMap = BTreeMap<String, Vec<SelectorGroup>>;
279
280#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
282#[serde(rename_all = "lowercase")]
283#[non_exhaustive]
284pub enum ChunkingKind {
285 #[default]
287 ByWords,
288 ByLines,
290 ByCharacterLength,
292 BySentence,
294}
295
296#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
302pub struct ChunkingAlg {
303 pub r#type: ChunkingKind,
305 pub value: i32,
307}
308
309impl ChunkingAlg {
310 pub const fn new(kind: ChunkingKind, value: i32) -> ChunkingAlg {
312 ChunkingAlg {
313 r#type: kind,
314 value,
315 }
316 }
317}