web2llm 0.4.0

Fetch web pages and convert to clean Markdown for LLM pipelines
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

//! A crawling example demonstrating breadth-first link discovery followed by a final batch fetch.

use web2llm::{CrawlConfig, FetchMode, Web2llm, Web2llmConfig};

#[tokio::main]
async fn main() {
    let seed = "https://example.com";
    let engine = Web2llm::new(Web2llmConfig {
        fetch_mode: FetchMode::Static,
        ..Default::default()
    })
    .unwrap();

    let results = engine
        .crawl(
            seed,
            CrawlConfig {
                max_depth: 1,
                preserve_domain: true,
            },
        )
        .await;

    for (url, result) in results {
        match result {
            Ok(page) => println!("{} -> {} chunks", url, page.chunks.len()),
            Err(error) => println!("{} -> {}", url, error),
        }
    }
}