docrawl 0.1.6

Docs-focused crawler library and CLI: crawl documentation sites, extract main content, convert to Markdown, mirror paths, and save with frontmatter.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38

use std::fs;
use std::io::Write;
use std::path::Path;

use crate::util::{ensure_parent_dir, now_rfc3339};

pub fn write_markdown_with_frontmatter(
    path: &Path,
    title: &str,
    url: &str,
    body_md: &str,
    security_flags: &[String],
) -> std::io::Result<()> {
    ensure_parent_dir(path)?;

    let mut file = fs::File::create(path)?;
    writeln!(file, "---")?;
    writeln!(file, "title: {}", escape_yaml(title))?;
    writeln!(file, "source_url: {}", escape_yaml(url))?;
    writeln!(file, "fetched_at: {}", now_rfc3339())?;
    if !security_flags.is_empty() {
        writeln!(file, "quarantined: true")?;
        writeln!(file, "security_flags:")?;
        for f in security_flags {
            writeln!(file, "  - {}", escape_yaml(f))?;
        }
    }
    writeln!(file, "---\n")?;
    file.write_all(body_md.as_bytes())?;
    Ok(())
}

// Always double-quote to handle all YAML special cases:
// booleans (true/yes/on), nulls, numbers, colons, hashes,
// flow indicators ([]{}), anchors (&*), and reserved chars.
fn escape_yaml(s: &str) -> String {
    format!("\"{}\"", s.replace('\\', "\\\\").replace('"', "\\\""))
}