dw2md 0.2.3

Crawl a DeepWiki repository and compile all pages into a single, LLM-friendly markdown file
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108

use serde::Serialize;

use crate::wiki::Page;

#[derive(Serialize)]
pub struct JsonOutput {
    pub repo: String,
    pub url: String,
    pub generated_at: String,
    pub tool_version: String,
    pub page_count: usize,
    pub pages: Vec<JsonPage>,
}

#[derive(Serialize)]
pub struct JsonPage {
    pub slug: String,
    pub title: String,
    pub depth: usize,
    pub content: String,
}

/// Compile pages into JSON output.
pub fn compile(repo: &str, pages: &[Page]) -> String {
    let now = chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string();
    let page_count = pages.iter().filter(|p| p.content.is_some()).count();

    let output = JsonOutput {
        repo: repo.to_string(),
        url: format!("https://deepwiki.com/{}", repo),
        generated_at: now,
        tool_version: "0.2.0".to_string(),
        page_count,
        pages: pages
            .iter()
            .filter(|p| p.content.is_some())
            .map(|p| JsonPage {
                slug: p.slug.clone(),
                title: p.title.clone(),
                depth: p.depth,
                content: p.content.clone().unwrap_or_default(),
            })
            .collect(),
    };

    serde_json::to_string_pretty(&output).expect("Failed to serialize JSON output")
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_json_compile() {
        let pages = vec![
            Page {
                slug: "1-overview".into(),
                title: "Overview".into(),
                depth: 0,
                content: Some("Hello world.".into()),
                error: None,
            },
            Page {
                slug: "2-setup".into(),
                title: "Setup".into(),
                depth: 1,
                content: Some("Install it.".into()),
                error: None,
            },
        ];

        let result = compile("test/repo", &pages);
        let parsed: serde_json::Value = serde_json::from_str(&result).unwrap();

        assert_eq!(parsed["repo"], "test/repo");
        assert_eq!(parsed["url"], "https://deepwiki.com/test/repo");
        assert_eq!(parsed["tool_version"], "0.2.0");
        assert_eq!(parsed["page_count"], 2);
        assert_eq!(parsed["pages"][0]["slug"], "1-overview");
        assert_eq!(parsed["pages"][0]["content"], "Hello world.");
        assert_eq!(parsed["pages"][1]["depth"], 1);
    }

    #[test]
    fn test_json_compile_skips_failed_pages() {
        let pages = vec![
            Page {
                slug: "1-overview".into(),
                title: "Overview".into(),
                depth: 0,
                content: Some("Hello.".into()),
                error: None,
            },
            Page {
                slug: "2-broken".into(),
                title: "Broken".into(),
                depth: 0,
                content: None,
                error: Some("timeout".into()),
            },
        ];

        let result = compile("test/repo", &pages);
        let parsed: serde_json::Value = serde_json::from_str(&result).unwrap();
        assert_eq!(parsed["page_count"], 1);
        assert_eq!(parsed["pages"].as_array().unwrap().len(), 1);
    }
}