progscrape_scrapers/backends/
feed.rs

1use std::borrow::Cow;
2
3use itertools::Itertools;
4use serde::{Deserialize, Serialize};
5
6use crate::{ScrapeCore, ScrapeSource};
7
8use super::{
9    GenericScrape, ScrapeConfigSource, ScrapeSourceDef, ScrapeStory, Scraper, scrape_story,
10};
11
12pub struct Feed {}
13
14impl ScrapeSourceDef for Feed {
15    type Config = FeedConfig;
16    type Scrape = FeedStory;
17    type Scraper = FeedScraper;
18
19    fn comments_url(_id: &str, _subsource: Option<&str>) -> String {
20        "".to_string()
21    }
22
23    fn id_from_comments_url(_url: &str) -> Option<(&str, Option<&str>)> {
24        None
25    }
26
27    fn is_comments_host(_host: &str) -> bool {
28        false
29    }
30}
31
32#[derive(Clone, Default, Serialize, Deserialize)]
33pub struct FeedConfig {}
34
35impl ScrapeConfigSource for FeedConfig {
36    fn provide_urls(&self, _subsources: Vec<String>) -> Vec<String> {
37        vec![]
38    }
39
40    fn subsources(&self) -> Vec<String> {
41        vec![]
42    }
43}
44
45scrape_story! {
46    FeedStory {
47        tags: Vec<String>
48    }
49}
50
51impl ScrapeStory for FeedStory {
52    const TYPE: ScrapeSource = ScrapeSource::Feed;
53
54    fn merge(&mut self, _other: Self) {}
55}
56
57#[derive(Default)]
58pub struct FeedScraper {}
59
60impl Scraper for FeedScraper {
61    type Config = <Feed as ScrapeSourceDef>::Config;
62    type Output = <Feed as ScrapeSourceDef>::Scrape;
63
64    fn extract_core<'a>(
65        &self,
66        args: &Self::Config,
67        input: &'a super::GenericScrape<Self::Output>,
68    ) -> ScrapeCore<'a> {
69        let tags = input
70            .data
71            .tags
72            .iter()
73            .map(|tag| Cow::Borrowed(tag.as_str()))
74            .collect_vec();
75        ScrapeCore {
76            source: &input.shared.id,
77            title: Cow::Borrowed(&input.shared.raw_title),
78            url: &input.shared.url,
79            date: input.shared.date,
80            rank: None,
81            tags,
82        }
83    }
84
85    fn scrape(
86        &self,
87        _args: &Self::Config,
88        _input: &str,
89    ) -> Result<(Vec<super::GenericScrape<Self::Output>>, Vec<String>), crate::ScrapeError> {
90        unimplemented!()
91    }
92}
progscrape_scrapers/backends/feed.rs

progscrape_scrapers/backends/
feed.rs
