extrablatt_v2 0.5.0

News, articles and text scraper
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

use std::borrow::Cow;
use std::collections::HashSet;
use url::Url;
use select::document::Document;
use select::predicate::{Name};

/// Extract the `href` attribute for all `<a>` tags of the document.
pub fn all_urls<'a>(doc: &'a Document) -> Vec<Cow<'a, str>> {
    let mut uniques = HashSet::new();
    doc.find(Name("a"))
        .filter_map(|n| n.attr("href").map(str::trim))
        .filter(|href| uniques.insert(*href))
        .map(Cow::Borrowed)
        .collect()
}

/// Extract all of the images of the document.
pub fn image_urls(doc: &Document, base_url: Option<&Url>) -> Vec<Url> {
    let options = Url::options().base_url(base_url);
    // TODO extract `picture` and source media
    doc.find(Name("img"))
        .filter_map(|n| n.attr("href").map(str::trim))
        .filter_map(|url| options.parse(url).ok())
        .collect()
}