use crate::params::{CssExtractionMap, ExtractSelector, SelectorGroup};
pub const ROOT_PATH: &str = "/";
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum Need {
Text,
Markdown,
Html,
Links,
Metadata,
Fields(FieldSpec),
Screenshot(ShotSpec),
Raw,
}
impl Need {
pub fn fields<N: Into<String>, S: AsRef<str>>(pairs: impl IntoIterator<Item = (N, S)>) -> Need {
Need::Fields(FieldSpec::new(pairs))
}
pub fn screenshot() -> Need {
Need::Screenshot(ShotSpec::default())
}
pub fn is_raw(&self) -> bool {
matches!(self, Need::Raw)
}
pub fn label(&self) -> &'static str {
match self {
Need::Text => "text",
Need::Markdown => "markdown",
Need::Html => "html",
Need::Links => "links",
Need::Metadata => "metadata",
Need::Fields(_) => "fields",
Need::Screenshot(_) => "screenshot",
Need::Raw => "raw",
}
}
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct FieldSpec {
path: String,
groups: Vec<SelectorGroup>,
}
impl FieldSpec {
pub fn new<N: Into<String>, S: AsRef<str>>(
pairs: impl IntoIterator<Item = (N, S)>,
) -> FieldSpec {
let mut spec = FieldSpec {
path: ROOT_PATH.to_string(),
groups: Vec::new(),
};
for (name, selector) in pairs {
spec.push(name, selector.as_ref());
}
spec
}
pub fn at(mut self, path: impl Into<String>) -> FieldSpec {
self.path = path.into();
self
}
pub fn push(&mut self, name: impl Into<String>, selector: &str) {
let name = name.into();
let parsed = ExtractSelector::parse(selector);
match self.groups.iter_mut().find(|group| group.name == name) {
Some(group) => group.selectors.push(parsed),
None => self.groups.push(SelectorGroup::new(name, [parsed])),
}
}
pub fn path(&self) -> &str {
&self.path
}
pub fn groups(&self) -> &[SelectorGroup] {
&self.groups
}
pub fn names(&self) -> impl Iterator<Item = &str> {
self.groups.iter().map(|group| group.name.as_str())
}
pub fn is_empty(&self) -> bool {
self.groups.is_empty()
}
pub fn to_map(&self) -> CssExtractionMap {
let mut map = CssExtractionMap::new();
map.insert(self.path.clone(), self.groups.clone());
map
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct ShotSpec {
with_metadata: bool,
}
impl ShotSpec {
pub const fn new() -> ShotSpec {
ShotSpec {
with_metadata: false,
}
}
pub const fn with_metadata() -> ShotSpec {
ShotSpec {
with_metadata: true,
}
}
pub const fn wants_metadata(&self) -> bool {
self.with_metadata
}
}
#[cfg(test)]
mod tests {
#![allow(
clippy::unwrap_used,
clippy::expect_used,
clippy::panic,
clippy::unreachable,
clippy::indexing_slicing,
clippy::string_slice
)]
use super::*;
#[test]
fn the_headline_constructor_reads_the_way_the_readme_writes_it() {
let need = Need::fields([("price", ".price"), ("title", "h1")]);
let Need::Fields(spec) = &need else {
panic!("expected fields, got {need:?}");
};
assert_eq!(spec.names().collect::<Vec<_>>(), vec!["price", "title"]);
assert_eq!(spec.path(), ROOT_PATH);
assert_eq!(need.label(), "fields");
}
#[test]
fn a_selector_is_classified_the_way_the_service_classifies_it() {
let spec = FieldSpec::new([("price", ".price"), ("heading", "//h1")]);
let map = spec.to_map();
let groups = &map[ROOT_PATH];
assert_eq!(
groups[0].selectors[0],
ExtractSelector::Css(".price".into())
);
assert_eq!(
groups[1].selectors[0],
ExtractSelector::XPath("//h1".into())
);
}
#[test]
fn two_selectors_on_one_name_become_one_field_with_a_fallback() {
let mut spec = FieldSpec::new([("price", ".price")]);
spec.push("price", "[itemprop=price]");
assert_eq!(spec.groups().len(), 1);
assert_eq!(spec.groups()[0].selectors.len(), 2);
}
#[test]
fn a_spec_can_be_pinned_to_a_path() {
let spec = FieldSpec::new([("price", ".price")]).at("/products");
assert!(spec.to_map().contains_key("/products"));
assert!(!spec.to_map().contains_key(ROOT_PATH));
}
#[test]
fn an_empty_spec_says_so() {
assert!(FieldSpec::new(Vec::<(String, String)>::new()).is_empty());
assert!(!FieldSpec::new([("a", "b")]).is_empty());
}
#[test]
fn a_picture_carries_no_metadata_unless_it_was_asked_for() {
assert!(!ShotSpec::new().wants_metadata());
assert!(!ShotSpec::default().wants_metadata());
assert!(ShotSpec::with_metadata().wants_metadata());
assert!(Need::screenshot().label() == "screenshot");
}
#[test]
fn only_raw_opts_out() {
assert!(Need::Raw.is_raw());
for need in [
Need::Text,
Need::Markdown,
Need::Html,
Need::Links,
Need::Metadata,
Need::fields([("a", "b")]),
Need::screenshot(),
] {
assert!(!need.is_raw(), "{need:?} should not read as raw");
}
}
}