servo-fetch-cli 0.7.1

A browser engine in a binary — fetch, render, and extract web content powered by Servo.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124

//! Fetch and batch-fetch tool helpers.

use std::time::Duration;

use servo_fetch::extract::{self, ExtractInput};
use servo_fetch::{FetchOptions, Page};

use super::common::{fetch_semaphore, paginate};

pub(crate) async fn fetch_page(url: &str, timeout: u64, settle_ms: u64) -> Result<Page, String> {
    let _permit = fetch_semaphore()
        .acquire()
        .await
        .map_err(|e| format!("fetch semaphore closed: {e}"))?;
    let url = url.to_string();
    tokio::task::spawn_blocking(move || {
        servo_fetch::fetch(
            FetchOptions::new(&url)
                .timeout(Duration::from_secs(timeout))
                .settle(Duration::from_millis(settle_ms)),
        )
    })
    .await
    .map_err(|e| e.to_string())?
    .map_err(|e| format!("{e:#}"))
}

pub(crate) async fn fetch_js(url: &str, expression: &str, timeout: u64, settle_ms: u64) -> Result<Page, String> {
    let _permit = fetch_semaphore()
        .acquire()
        .await
        .map_err(|e| format!("fetch semaphore closed: {e}"))?;
    let url = url.to_string();
    let expression = expression.to_string();
    tokio::task::spawn_blocking(move || {
        servo_fetch::fetch(
            FetchOptions::javascript(&url, &expression)
                .timeout(Duration::from_secs(timeout))
                .settle(Duration::from_millis(settle_ms)),
        )
    })
    .await
    .map_err(|e| e.to_string())?
    .map_err(|e| format!("{e:#}"))
}

pub(crate) async fn fetch_screenshot(url: &str, full_page: bool, timeout: u64, settle_ms: u64) -> Result<Page, String> {
    let _permit = fetch_semaphore()
        .acquire()
        .await
        .map_err(|e| format!("fetch semaphore closed: {e}"))?;
    let url = url.to_string();
    tokio::task::spawn_blocking(move || {
        servo_fetch::fetch(
            FetchOptions::screenshot(&url, full_page)
                .timeout(Duration::from_secs(timeout))
                .settle(Duration::from_millis(settle_ms)),
        )
    })
    .await
    .map_err(|e| e.to_string())?
    .map_err(|e| format!("{e:#}"))
}

pub(crate) async fn batch_fetch_pages(
    urls: &[String],
    timeout: u64,
    settle_ms: u64,
    json: bool,
    selector: Option<&str>,
    max_len: usize,
) -> Vec<(String, String)> {
    let (tx, mut rx) = tokio::sync::mpsc::channel(urls.len().max(1));

    for url in urls {
        let permit = fetch_semaphore().acquire().await.ok();
        let tx = tx.clone();
        let url = url.clone();
        let selector = selector.map(String::from);
        tokio::task::spawn_blocking(move || {
            let text = fetch_and_render(&url, timeout, settle_ms, json, selector.as_deref(), max_len);
            let _ = tx.blocking_send((url, text));
            drop(permit);
        });
    }
    drop(tx);

    let mut results = Vec::with_capacity(urls.len());
    while let Some(pair) = rx.recv().await {
        results.push(pair);
    }
    results
}

fn fetch_and_render(
    url: &str,
    timeout: u64,
    settle_ms: u64,
    json: bool,
    selector: Option<&str>,
    max_len: usize,
) -> String {
    let page = match servo_fetch::fetch(
        FetchOptions::new(url)
            .timeout(Duration::from_secs(timeout))
            .settle(Duration::from_millis(settle_ms)),
    ) {
        Ok(p) => p,
        Err(e) => return format!("[error] {e:#}"),
    };

    let input = ExtractInput::new(&page.html, url)
        .with_layout_json(page.layout_json.as_deref())
        .with_inner_text(Some(&page.inner_text))
        .with_selector(selector);

    let full = if json {
        extract::extract_json(&input).unwrap_or_default()
    } else {
        extract::extract_text(&input).unwrap_or_default()
    };

    paginate(&servo_fetch::sanitize::sanitize(&full), 0, max_len)
}