qrawl 0.6.0

Rust toolkit to crawl web data for AI agents
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46

//! Scrape Tools

use crate::types::{Jsonld, Metadata};

mod utils;

/// Scrape body content from HTML.
pub async fn scrape_body(html: &str) -> String {
    let html = html.to_string();
    tokio::task::spawn_blocking(move || utils::scrape_body_content(&html))
        .await
        .expect("scrape_body: spawn_blocking failed")
}

/// Scrape JSON-LD scripts from HTML.
pub async fn scrape_jsonld(html: &str) -> Jsonld {
    let html = html.to_string();
    tokio::task::spawn_blocking(move || utils::scrape_jsonld_scripts(&html))
        .await
        .expect("scrape_jsonld: spawn_blocking failed")
}

/// Scrape metadata tags from HTML.
pub async fn scrape_metadata(html: &str) -> Metadata {
    let html = html.to_string();
    tokio::task::spawn_blocking(move || utils::scrape_metadata_tags(&html))
        .await
        .expect("scrape_metadata: spawn_blocking failed")
}

/// Scrape both JSON-LD and metadata in a single parse.
///
/// Equivalent to `tokio::join!(scrape_jsonld(html), scrape_metadata(html))` but
/// parses the HTML once instead of twice. Prefer this when you need both.
pub async fn scrape_all(html: &str) -> (Jsonld, Metadata) {
    let html = html.to_string();
    tokio::task::spawn_blocking(move || {
        let document = scraper::Html::parse_document(&html);
        (
            utils::scrape_jsonld_from_doc(&document),
            utils::scrape_metadata_from_doc(&document),
        )
    })
    .await
    .expect("scrape_all: spawn_blocking failed")
}