XPATH Scraper
Makes it easier to scrape websites with XPATH. Currently using my xpath parser which is incomplete, undocumented and used originally for teaching myself about parsing.
A Very simple example of this which is below and also in the example folder:
use std::io::Cursor;
use scraper_main::{
xpather,
ConvertFromValue,
ScraperMain,
Scraper,
};
#[derive(Debug, Scraper)]
pub struct RedditList(
#[scrape(xpath = r#"//div[contains(@class, "Post") and not(contains(@class, "promotedlink"))]"#)]
Vec<RedditListItem>
);
#[derive(Debug, Scraper)]
pub struct RedditListItem {
// URL of the post
#[scrape(xpath = r#". pub url: Option<String>,
#[scrape(xpath = r#".//a[@data-click-id="body"]/div/h3/text()"#)]
pub title: Option<String>,
#[scrape(xpath = r#".//a[@data-click-id="timestamp"]/text()"#)]
pub timestamp: Option<String>,
#[scrape(xpath = r#".//a[@data-click-id="comments"]/span/text()"#)]
pub comment_count: Option<String>,
#[scrape(xpath = r#"./div[1]/div/div/text()"#)]
pub votes: Option<String>,
}
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let resp = reqwest::get("https://www.reddit.com/r/nocontextpics/").await?;
let data = resp.text().await?;
let document = xpather::parse_doc(&mut Cursor::new(data));
let list = RedditList::scrape(&document, None)?;
println!("{:#?}", list);
Ok(())
}