Skip to main content

spider_cloud_agent/params/
extraction.rs

1//! What comes back, and how much of it.
2
3use serde::de::{Error as DeError, SeqAccess, Unexpected, Visitor};
4use serde::{Deserialize, Deserializer, Serialize, Serializer};
5use std::collections::BTreeMap;
6use std::fmt;
7
8/// The shape the page content is returned in.
9#[derive(
10    Debug, Clone, Copy, Default, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize,
11)]
12#[serde(rename_all = "lowercase")]
13#[non_exhaustive]
14pub enum ReturnFormat {
15    /// The page as fetched, markup and all. The API's own default, and the
16    /// most expensive thing to hand to a model.
17    #[default]
18    Raw,
19    /// Markdown. Keeps headings, lists and links, and drops most of the bytes.
20    Markdown,
21    /// Markdown restricted to the CommonMark spec, for readers that reject the
22    /// extensions.
23    Commonmark,
24    /// Plain text with the markup taken out.
25    Text,
26    /// XML.
27    Xml,
28    /// The bytes exactly as received, for files that are not pages.
29    Bytes,
30    /// No content at all. Pair it with an extraction map or with metadata to
31    /// get the few fields you want and none of the page.
32    Empty,
33}
34
35impl ReturnFormat {
36    /// The value sent on the wire.
37    pub const fn as_str(&self) -> &'static str {
38        match self {
39            ReturnFormat::Raw => "raw",
40            ReturnFormat::Markdown => "markdown",
41            ReturnFormat::Commonmark => "commonmark",
42            ReturnFormat::Text => "text",
43            ReturnFormat::Xml => "xml",
44            ReturnFormat::Bytes => "bytes",
45            ReturnFormat::Empty => "empty",
46        }
47    }
48}
49
50impl fmt::Display for ReturnFormat {
51    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
52        f.write_str(self.as_str())
53    }
54}
55
56/// One format, or several at once.
57///
58/// Asking for several means the response carries the page more than once, so
59/// the second format doubles what crosses the wire. Ask for two only when both
60/// are read.
61#[derive(Debug, Clone, PartialEq, Eq)]
62#[non_exhaustive]
63pub enum ReturnFormatHandling {
64    /// Return the page in one format.
65    Single(ReturnFormat),
66    /// Return the page in each of these formats.
67    Multi(Vec<ReturnFormat>),
68}
69
70impl Default for ReturnFormatHandling {
71    fn default() -> ReturnFormatHandling {
72        ReturnFormatHandling::Single(ReturnFormat::Raw)
73    }
74}
75
76impl From<ReturnFormat> for ReturnFormatHandling {
77    fn from(value: ReturnFormat) -> ReturnFormatHandling {
78        ReturnFormatHandling::Single(value)
79    }
80}
81
82impl From<Vec<ReturnFormat>> for ReturnFormatHandling {
83    fn from(value: Vec<ReturnFormat>) -> ReturnFormatHandling {
84        ReturnFormatHandling::Multi(value)
85    }
86}
87
88impl ReturnFormatHandling {
89    /// Every format asked for, in the order they were given.
90    pub fn formats(&self) -> &[ReturnFormat] {
91        match self {
92            ReturnFormatHandling::Single(format) => std::slice::from_ref(format),
93            ReturnFormatHandling::Multi(formats) => formats,
94        }
95    }
96
97    /// True when this format asks for a format.
98    pub fn contains(&self, format: ReturnFormat) -> bool {
99        self.formats().contains(&format)
100    }
101}
102
103impl Serialize for ReturnFormatHandling {
104    fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
105        match self {
106            ReturnFormatHandling::Single(format) => format.serialize(serializer),
107            ReturnFormatHandling::Multi(formats) => formats.serialize(serializer),
108        }
109    }
110}
111
112impl<'de> Deserialize<'de> for ReturnFormatHandling {
113    fn deserialize<D: Deserializer<'de>>(
114        deserializer: D,
115    ) -> Result<ReturnFormatHandling, D::Error> {
116        struct HandlingVisitor;
117
118        // Written out rather than left to `untagged`, which reports every
119        // failure as "did not match any variant" and hides which format was
120        // wrong.
121        impl<'de> Visitor<'de> for HandlingVisitor {
122            type Value = ReturnFormatHandling;
123
124            fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
125                f.write_str("a return format, or an array of them")
126            }
127
128            fn visit_str<E: DeError>(self, value: &str) -> Result<ReturnFormatHandling, E> {
129                parse_format(value).map(ReturnFormatHandling::Single)
130            }
131
132            fn visit_seq<A: SeqAccess<'de>>(
133                self,
134                mut seq: A,
135            ) -> Result<ReturnFormatHandling, A::Error> {
136                let mut formats = Vec::with_capacity(seq.size_hint().unwrap_or(2));
137
138                while let Some(format) = seq.next_element::<String>()? {
139                    formats.push(parse_format::<A::Error>(&format)?);
140                }
141
142                Ok(ReturnFormatHandling::Multi(formats))
143            }
144        }
145
146        deserializer.deserialize_any(HandlingVisitor)
147    }
148}
149
150fn parse_format<E: DeError>(value: &str) -> Result<ReturnFormat, E> {
151    match value.trim().to_ascii_lowercase().as_str() {
152        "raw" => Ok(ReturnFormat::Raw),
153        "markdown" => Ok(ReturnFormat::Markdown),
154        "commonmark" => Ok(ReturnFormat::Commonmark),
155        "text" => Ok(ReturnFormat::Text),
156        "xml" => Ok(ReturnFormat::Xml),
157        "bytes" => Ok(ReturnFormat::Bytes),
158        "empty" => Ok(ReturnFormat::Empty),
159        _ => Err(E::invalid_value(
160            Unexpected::Str(value),
161            &"one of raw, markdown, commonmark, text, xml, bytes or empty",
162        )),
163    }
164}
165
166/// An expression that picks elements out of a page.
167///
168/// Both dialects go on the wire as the expression itself. Which one a string
169/// is gets worked out from how it starts, so `//h1` is read back as an XPath
170/// and `h1` as CSS.
171#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
172#[non_exhaustive]
173pub enum ExtractSelector {
174    /// A CSS selector, such as `div.price > span`.
175    Css(String),
176    /// An XPath expression, such as `//div[@class="price"]/span`. Reach for it
177    /// when the element is identified by its text or by its position rather
178    /// than by a class.
179    XPath(String),
180}
181
182impl ExtractSelector {
183    /// Read a CSS selector.
184    pub fn css(expression: impl Into<String>) -> ExtractSelector {
185        ExtractSelector::Css(expression.into())
186    }
187
188    /// Read an XPath expression.
189    pub fn xpath(expression: impl Into<String>) -> ExtractSelector {
190        ExtractSelector::XPath(expression.into())
191    }
192
193    /// The expression itself, which is what is sent.
194    pub fn as_str(&self) -> &str {
195        match self {
196            ExtractSelector::Css(expression) | ExtractSelector::XPath(expression) => expression,
197        }
198    }
199
200    /// Classify an expression the way the service does, by its opening
201    /// characters.
202    pub fn parse(expression: &str) -> ExtractSelector {
203        let head = expression.trim_start();
204
205        if head.starts_with('/') || head.starts_with("./") || head.starts_with("(/") {
206            ExtractSelector::XPath(expression.to_string())
207        } else {
208            ExtractSelector::Css(expression.to_string())
209        }
210    }
211}
212
213impl fmt::Display for ExtractSelector {
214    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
215        f.write_str(self.as_str())
216    }
217}
218
219impl Serialize for ExtractSelector {
220    fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
221        serializer.serialize_str(self.as_str())
222    }
223}
224
225impl<'de> Deserialize<'de> for ExtractSelector {
226    fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<ExtractSelector, D::Error> {
227        let expression = String::deserialize(deserializer)?;
228
229        Ok(ExtractSelector::parse(&expression))
230    }
231}
232
233/// One named field and the expressions that fill it.
234///
235/// The expressions are tried in order, so put the precise one first and a
236/// looser fallback after it.
237#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
238pub struct SelectorGroup {
239    /// The name this field takes in the response.
240    pub name: String,
241    /// The expressions that find it.
242    pub selectors: Vec<ExtractSelector>,
243}
244
245impl SelectorGroup {
246    /// A field named `name`, found by these expressions.
247    pub fn new(
248        name: impl Into<String>,
249        selectors: impl IntoIterator<Item = ExtractSelector>,
250    ) -> SelectorGroup {
251        SelectorGroup {
252            name: name.into(),
253            selectors: selectors.into_iter().collect(),
254        }
255    }
256
257    /// A field found by CSS selectors.
258    pub fn css<S: Into<String>>(
259        name: impl Into<String>,
260        selectors: impl IntoIterator<Item = S>,
261    ) -> SelectorGroup {
262        SelectorGroup::new(name, selectors.into_iter().map(ExtractSelector::css))
263    }
264
265    /// A field found by XPath expressions.
266    pub fn xpath<S: Into<String>>(
267        name: impl Into<String>,
268        selectors: impl IntoIterator<Item = S>,
269    ) -> SelectorGroup {
270        SelectorGroup::new(name, selectors.into_iter().map(ExtractSelector::xpath))
271    }
272}
273
274/// Fields to pull out, keyed by the path they apply to.
275///
276/// The key is a URL path, so a crawl can pull different fields from a listing
277/// page than from a product page. `"/"` applies to the site root.
278pub type CssExtractionMap = BTreeMap<String, Vec<SelectorGroup>>;
279
280/// What a chunk is measured in.
281#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
282#[serde(rename_all = "lowercase")]
283#[non_exhaustive]
284pub enum ChunkingKind {
285    /// Count words.
286    #[default]
287    ByWords,
288    /// Count lines.
289    ByLines,
290    /// Count characters.
291    ByCharacterLength,
292    /// Break on sentence ends, which keeps a chunk readable on its own.
293    BySentence,
294}
295
296/// Split the content into pieces before returning it.
297///
298/// Set this when the content is going straight into something with a context
299/// limit, so the split happens once on the service rather than again in every
300/// caller.
301#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
302pub struct ChunkingAlg {
303    /// What is being counted.
304    pub r#type: ChunkingKind,
305    /// How many of them go in a chunk.
306    pub value: i32,
307}
308
309impl ChunkingAlg {
310    /// Split every `value` units of `kind`.
311    pub const fn new(kind: ChunkingKind, value: i32) -> ChunkingAlg {
312        ChunkingAlg {
313            r#type: kind,
314            value,
315        }
316    }
317}