docrawl 0.1.6

Docs-focused crawler library and CLI: crawl documentation sites, extract main content, convert to Markdown, mirror paths, and save with frontmatter.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

use std::path::PathBuf;

#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
    let cfg = docrawl::CrawlConfig {
        base_url: url::Url::parse("https://example.com/docs")?,
        output_dir: PathBuf::from("./out"),
        user_agent: format!("mytool/{}", env!("CARGO_PKG_VERSION")),
        max_depth: Some(2),
        rate_limit_per_sec: 8,
        follow_sitemaps: false,
        concurrency: 8,
        timeout: None,
        resume: false,
        silence: false,
        config: docrawl::Config {
            host_only: true,
            skip_assets: true,
            ..Default::default()
        },
    };
    docrawl::crawl(cfg).await?;
    Ok(())
}