embedder 0.3.5

Get basic website information with ease.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94

#![allow(clippy::field_reassign_with_default)]

use scraper::{ElementRef, Html, Selector};
use types::{OgType, WebData};
use url::Url;

pub mod types;
pub use types as ty;

#[cfg(feature = "driver")]
pub mod driver;
#[cfg(feature = "driver")]
pub use driver as dr;

/// Fetches the data from the given url.
pub async fn fetch(url: &str) -> Result<WebData, String> {
    let document = Html::parse_document(
        &reqwest::get(url)
            .await
            .map_err(|e| format!("Failed to fetch url: {:?}", e))?
            .text()
            .await
            .map_err(|e| format!("Failed to read response: {:?}", e))?,
    );

    let find = |id: &str| {
        document
            .select(
                &Selector::parse(id).unwrap_or_else(|_| panic!("Failed to build selector: {id}")),
            )
            .collect::<Vec<ElementRef>>()
    };

    let mut data = WebData::default();

    // <title>
    data.title = find("title").first().unwrap().text().collect();

    // <meta name="description" />
    data.description = find("meta[property=\"og:description\"]")
        .first()
        .map(|e| e.value().attr("content").unwrap().to_string());

    // <meta property="og:type" />
    data.r#type = find("meta[property=\"og:type\"]")
        .first()
        .map(|e| OgType::from_meta(e.value().attr("content").unwrap()))
        .unwrap_or_default();

    // <meta property="og:image" />
    data.image = find("meta[property=\"og:image\"]")
        .first()
        .map(|e| resolve_url(e.value().attr("content").unwrap(), url));

    // <meta property="book:author" />, <meta property="article:author" />
    data.author = find("meta[property$=\":author\"]")
        .iter()
        .map(|e| e.value().attr("content").unwrap().to_string())
        .collect();

    // <meta name="theme-color" />
    data.colour = find("meta[name=\"theme-color\"]")
        .first()
        .map(|e| e.value().attr("content").unwrap().to_string());

    Ok(data)
}

/// Resolves the given url to an absolute url.
pub fn resolve_url(url: &str, base: &str) -> String {
    if url.starts_with('/') || url.starts_with("./") {
        let base = Url::parse(base).unwrap().origin().unicode_serialization();
        return Url::parse(&base).unwrap().join(url).unwrap().to_string();
    }

    url.to_string()
}

#[cfg(test)]
pub mod test {
    use crate::fetch;

    #[tokio::test]
    async fn a() {
        let url = "https://reneweconomy.com.au/market-operator-ticks-off-one-of-major-challenges-of-meeting-100-pct-renewables/";
        println!("{:?}", fetch(url).await.unwrap());
    }

    #[tokio::test]
    async fn b() {
        let url = "https://oilprice.com/Latest-Energy-News/World-News/US-Natural-Gas-Prices-Tumble-10-on-Mild-Weather.html";
        println!("{:?}", fetch(url).await.unwrap());
    }
}