servo-fetch-cli 0.12.0

A browser engine in a binary — fetch, render, and extract web content as Markdown, JSON, or screenshots. Powered by Servo.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

//! Map tool helper.

use super::error::{ToolError, ToolResult};

pub(crate) async fn discover_urls(
    url: &str,
    limit: usize,
    include_glob: &[String],
    exclude_glob: &[String],
) -> ToolResult<Vec<String>> {
    let opts = servo_fetch::MapOptions::new(url).limit(limit);
    let opts = if include_glob.is_empty() {
        opts
    } else {
        opts.include(&include_glob.iter().map(String::as_str).collect::<Vec<_>>())
    };
    let opts = if exclude_glob.is_empty() {
        opts
    } else {
        opts.exclude(&exclude_glob.iter().map(String::as_str).collect::<Vec<_>>())
    };

    let results = tokio::task::spawn_blocking(move || servo_fetch::blocking::map(&opts))
        .await
        .map_err(|e| ToolError::internal(e.to_string()))?
        .map_err(|e| ToolError::fetch(e.to_string()))?;

    Ok(results.iter().map(|entry| entry.url.clone()).collect())
}