dw2md 0.2.3

Crawl a DeepWiki repository and compile all pages into a single, LLM-friendly markdown file
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130

use super::Page;

/// Filter pages based on include/exclude lists of slugs.
pub fn filter_pages(
    pages: Vec<Page>,
    include: &Option<Vec<String>>,
    exclude: &Option<Vec<String>>,
) -> Vec<Page> {
    pages
        .into_iter()
        .filter(|page| {
            // If include list is specified, page must match
            if let Some(include_slugs) = include {
                if !include_slugs.iter().any(|s| slug_matches(&page.slug, s)) {
                    return false;
                }
            }

            // If exclude list is specified, page must not match
            if let Some(exclude_slugs) = exclude {
                if exclude_slugs.iter().any(|s| slug_matches(&page.slug, s)) {
                    return false;
                }
            }

            true
        })
        .collect()
}

/// Check if a page slug matches a filter pattern.
/// Supports exact match and prefix match (e.g., "3" matches "3-architecture").
fn slug_matches(slug: &str, pattern: &str) -> bool {
    if slug == pattern {
        return true;
    }

    // Prefix match: pattern "3" matches slug "3-architecture"
    if let Some(rest) = slug.strip_prefix(pattern) {
        if rest.starts_with('-') || rest.starts_with('.') {
            return true;
        }
    }

    false
}

#[cfg(test)]
mod tests {
    use super::*;

    fn make_page(slug: &str) -> Page {
        Page {
            slug: slug.to_string(),
            title: slug.to_string(),
            depth: 0,
            content: None,
            error: None,
        }
    }

    #[test]
    fn test_no_filters() {
        let pages = vec![make_page("a"), make_page("b")];
        let result = filter_pages(pages, &None, &None);
        assert_eq!(result.len(), 2);
    }

    #[test]
    fn test_include_filter() {
        let pages = vec![
            make_page("1-overview"),
            make_page("2-setup"),
            make_page("3-arch"),
        ];
        let include = Some(vec!["1-overview".to_string(), "3-arch".to_string()]);
        let result = filter_pages(pages, &include, &None);
        assert_eq!(result.len(), 2);
        assert_eq!(result[0].slug, "1-overview");
        assert_eq!(result[1].slug, "3-arch");
    }

    #[test]
    fn test_exclude_filter() {
        let pages = vec![
            make_page("1-overview"),
            make_page("2-setup"),
            make_page("3-arch"),
        ];
        let exclude = Some(vec!["2-setup".to_string()]);
        let result = filter_pages(pages, &None, &exclude);
        assert_eq!(result.len(), 2);
        assert_eq!(result[0].slug, "1-overview");
        assert_eq!(result[1].slug, "3-arch");
    }

    #[test]
    fn test_slug_prefix_match() {
        assert!(slug_matches("3-architecture", "3"));
        assert!(slug_matches("3.1-data-pipeline", "3.1"));
        assert!(!slug_matches("3-architecture", "31"));
        assert!(!slug_matches("30-other", "3"));
    }

    #[test]
    fn test_slug_exact_match() {
        assert!(slug_matches("1-overview", "1-overview"));
        assert!(!slug_matches("1-overview", "1-over"));
    }

    #[test]
    fn test_include_and_exclude_combined() {
        let pages = vec![
            make_page("1-overview"),
            make_page("2-setup"),
            make_page("3-arch"),
            make_page("3.1-data"),
        ];
        let include = Some(vec![
            "1-overview".to_string(),
            "3-arch".to_string(),
            "3.1-data".to_string(),
        ]);
        let exclude = Some(vec!["3-arch".to_string()]);
        let result = filter_pages(pages, &include, &exclude);
        assert_eq!(result.len(), 2);
        assert_eq!(result[0].slug, "1-overview");
        assert_eq!(result[1].slug, "3.1-data");
    }
}