mdlens 0.1.0

Token-efficient Markdown structure CLI for agents
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158

use crate::model::{Document, Section};

/// Render a tree view of a document for human consumption.
pub fn render_tree(doc: &Document, max_depth: Option<usize>, include_preamble: bool) -> String {
    let mut out = String::new();
    out.push_str(&format!(
        "{}  lines={}  tokens\u{2248}{}\n\n",
        doc.path, doc.line_count, doc.token_estimate
    ));

    for section in &doc.sections {
        if section.title == "<preamble>" && !include_preamble {
            continue;
        }
        let depth = 0;
        render_section(&mut out, section, depth, max_depth);
    }

    out
}

fn render_section(out: &mut String, section: &Section, depth: usize, max_depth: Option<usize>) {
    if let Some(max) = max_depth {
        if depth >= max {
            return;
        }
    }

    let indent = "  ".repeat(depth);
    out.push_str(&format!(
        "{}{} {} lines {}-{}  tokens\u{2248}{}\n",
        indent,
        section.id,
        section.title,
        section.line_start,
        section.line_end,
        section.token_estimate
    ));

    for child in &section.children {
        render_section(out, child, depth + 1, max_depth);
    }
}

/// Render a section read output for human consumption.
pub fn render_read(section: &Section, content: &str, truncated: bool) -> String {
    let mut out = String::new();
    out.push_str(&format!("{}\n", section.path.join(" > ")));
    out.push_str(&format!(
        "id={}  lines={}-{}  tokens\u{2248}{}\n\n",
        section.id, section.line_start, section.line_end, section.token_estimate
    ));
    out.push_str(content);
    if truncated && !content.contains("<!-- mdlens: truncated at token budget -->") {
        out.push_str("\n\n<!-- mdlens: truncated at token budget -->\n");
    }
    out
}

/// Render search results for human consumption.
pub struct SearchResult {
    pub path: String,
    pub section_id: String,
    pub section_title: String,
    pub section_path: Vec<String>,
    pub line_start: usize,
    pub line_end: usize,
    pub token_estimate: usize,
    pub match_count: usize,
    pub snippets: Vec<SearchSnippet>,
}

pub struct SearchSnippet {
    pub line_start: usize,
    pub line_end: usize,
    pub text: String,
}

pub fn render_search(results: &[SearchResult]) -> String {
    let mut out = String::new();
    for (i, result) in results.iter().enumerate() {
        if i > 0 {
            out.push_str("\n---\n\n");
        }
        out.push_str(&format!(
            "{} > {}\nid={}  lines={}-{}  tokens\u{2248}{}  matches={}\n\n",
            result.path,
            result.section_path.join(" > "),
            result.section_id,
            result.line_start,
            result.line_end,
            result.token_estimate,
            result.match_count,
        ));
        for snippet in &result.snippets {
            out.push_str(&format!("{}:{}\n", snippet.line_start, snippet.text));
        }
    }
    out
}

/// Render stats table for human consumption.
pub fn render_stats(entries: &[StatsEntry]) -> String {
    let mut out = String::new();
    out.push_str(&format!(
        "{:<30} {:>8} {:>8} {:>8}\n",
        "path", "lines", "words", "tokens\u{2248}"
    ));

    for entry in entries {
        out.push_str(&format!(
            "{:<30} {:>8} {:>8} {:>8}\n",
            entry.path, entry.lines, entry.words, entry.tokens
        ));
    }
    out
}

pub struct StatsEntry {
    pub path: String,
    pub lines: usize,
    pub words: usize,
    pub tokens: usize,
}

/// Render pack output for human consumption.
pub fn render_pack(
    source: &str,
    budget: usize,
    included: &[PackIncluded],
    content: &str,
    truncated: bool,
) -> String {
    let mut out = String::new();
    out.push_str("<!-- mdlens pack\n");
    out.push_str(&format!("source: {}\n", source));
    out.push_str(&format!("budget: tokens\u{2248}{}\n", budget));
    out.push_str("included:\n");
    for inc in included {
        out.push_str(&format!(
            "- {} {} lines={}  tokens\u{2248}{}\n",
            inc.section_id, inc.section_title, inc.line_range, inc.token_estimate
        ));
    }
    if truncated {
        out.push_str("truncated: true\n");
    }
    out.push_str("-->\n\n");
    out.push_str(content);
    out
}

pub struct PackIncluded {
    pub section_id: String,
    pub section_title: String,
    pub line_range: String,
    pub token_estimate: usize,
}