qrawl 0.6.0

Rust toolkit to crawl web data for AI agents
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69

//! Map Tools

mod tests;
mod utils;

use crate::selectors::LINK_SELECTOR;

/// Map URLs from HTML.
pub async fn map_page(html: &str, url: &str) -> Vec<String> {
    let html = html.to_string();
    let url = url.to_string();
    tokio::task::spawn_blocking(move || {
        let base = match url::Url::parse(&url) {
            Ok(u) => u,
            Err(_) => return Vec::new(),
        };

        let doc = scraper::Html::parse_document(&html);

        doc.select(&LINK_SELECTOR)
            .filter_map(|link| {
                let href = link
                    .value()
                    .attr("href")?
                    .trim()
                    .trim_matches('"')
                    .trim_matches('\'')
                    .trim();

                // Handle protocol-relative URLs (//example.com/path)
                let url = if href.starts_with("//") {
                    let full_href = format!("{}:{}", base.scheme(), href);
                    url::Url::parse(&full_href).ok()?
                } else {
                    url::Url::parse(href)
                        .ok()
                        .or_else(|| base.join(href).ok())?
                };

                // Only accept HTTP and HTTPS schemes
                if matches!(url.scheme(), "http" | "https") {
                    Some(url.to_string())
                } else {
                    None
                }
            })
            .collect()
    })
    .await
    .expect("map_page: spawn_blocking failed")
}

/// Map child URLs from HTML.
pub async fn map_children(html: &str, url: &str) -> Vec<String> {
    let html = html.to_string();
    let url = url.to_string();
    tokio::task::spawn_blocking(move || {
        let doc = scraper::Html::parse_document(&html);
        let siblings = utils::map_siblings_from_doc(&doc, &url);
        let itemlist = utils::map_itemlist_from_doc(&doc, &url);
        let mut result = crate::merge!(siblings, itemlist);
        if result.is_empty() {
            result = vec![url];
        }
        result
    })
    .await
    .expect("map_children: spawn_blocking failed")
}