docrawl 0.1.6

Docs-focused crawler library and CLI: crawl documentation sites, extract main content, convert to Markdown, mirror paths, and save with frontmatter.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19

use serde::Deserialize;

#[derive(Debug, Clone, Deserialize, Default)]
pub struct Config {
    #[serde(default)]
    pub host_only: bool,
    #[serde(default)]
    pub external_assets: bool,
    #[serde(default)]
    pub allow_svg: bool,
    #[serde(default)]
    pub skip_assets: bool,
    #[serde(default)]
    pub max_pages: Option<usize>,
    #[serde(default)]
    pub selectors: Option<Vec<String>>, // preferred CSS selectors for main content
    #[serde(default)]
    pub exclude_patterns: Vec<String>, // regex strings
}