Skip to main content

treesearch/parser/
plaintext.rs

1//! Parses plain text paragraphs with UTF-8-safe titles and exact line ranges.
2
3use crate::document::{Document, Node, SourceType};
4use anyhow::Result;
5use std::path::Path;
6
7/// Parser for plain-text files (`.txt`, `.text`, `.log`, `.csv`).
8///
9/// Splits content by double newlines (paragraphs). Each paragraph becomes a flat
10/// node with the first line (truncated to 80 chars) as the title and the full
11/// paragraph text as the body.
12pub struct PlainTextParser;
13
14/// Maximum character length for a paragraph title.
15const MAX_TITLE_LEN: usize = 80;
16
17impl super::Parser for PlainTextParser {
18    fn extensions(&self) -> &[&str] {
19        &["txt", "text", "log", "csv"]
20    }
21
22    fn source_type(&self) -> SourceType {
23        SourceType::Text
24    }
25
26    fn parse(&self, path: &Path, content: &str) -> Result<Document> {
27        let file_name = path
28            .file_name()
29            .map(|n| n.to_string_lossy().to_string())
30            .unwrap_or_default();
31        let doc_id = path.to_string_lossy().to_string();
32        let mut doc = Document::new(&doc_id, &file_name, SourceType::Text);
33
34        if content.trim().is_empty() {
35            doc.assign_node_ids();
36            return Ok(doc);
37        }
38
39        let paragraphs = split_paragraphs(content);
40
41        for paragraph in paragraphs {
42            let title = make_title(&paragraph.text);
43
44            let mut node = Node::new("", &title);
45            node.text = paragraph.text;
46            node.line_start = Some(paragraph.line_start);
47            node.line_end = Some(paragraph.line_end);
48
49            doc.structure.push(node);
50        }
51
52        doc.assign_node_ids();
53        Ok(doc)
54    }
55}
56
57/// One normalized paragraph with exact one-based source line bounds.
58struct Paragraph {
59    text: String,
60    line_start: u32,
61    line_end: u32,
62}
63
64/// Splits content on blank lines while preserving exact source line bounds.
65fn split_paragraphs(content: &str) -> Vec<Paragraph> {
66    /// Flushes an accumulated non-empty paragraph into the result list.
67    fn flush(
68        lines: &mut Vec<String>,
69        line_start: &mut Option<u32>,
70        line_end: u32,
71        result: &mut Vec<Paragraph>,
72    ) {
73        let Some(start) = line_start.take() else {
74            return;
75        };
76        result.push(Paragraph {
77            text: lines.join("\n").trim().to_string(),
78            line_start: start,
79            line_end,
80        });
81        lines.clear();
82    }
83
84    let mut result = Vec::new();
85    let mut paragraph_lines = Vec::new();
86    let mut paragraph_start = None;
87    let mut previous_content_line = 0;
88    for (index, raw_line) in content.lines().enumerate() {
89        let line_number = u32::try_from(index.saturating_add(1)).unwrap_or(u32::MAX);
90        let line = raw_line.trim_end_matches('\r');
91        if line.trim().is_empty() {
92            flush(
93                &mut paragraph_lines,
94                &mut paragraph_start,
95                previous_content_line,
96                &mut result,
97            );
98            continue;
99        }
100        paragraph_start.get_or_insert(line_number);
101        previous_content_line = line_number;
102        paragraph_lines.push(line.to_string());
103    }
104    flush(
105        &mut paragraph_lines,
106        &mut paragraph_start,
107        previous_content_line,
108        &mut result,
109    );
110    result
111}
112
113/// Create a title from the first line of a paragraph, truncated to `MAX_TITLE_LEN`.
114fn make_title(text: &str) -> String {
115    let first_line = text.lines().next().unwrap_or("");
116    let trimmed = first_line.trim();
117    if trimmed.chars().count() <= MAX_TITLE_LEN {
118        trimmed.to_string()
119    } else {
120        format!(
121            "{}...",
122            trimmed.chars().take(MAX_TITLE_LEN).collect::<String>()
123        )
124    }
125}
126
127#[cfg(test)]
128mod tests {
129    use super::*;
130    use crate::parser::Parser;
131
132    fn parse(content: &str) -> Document {
133        let parser = PlainTextParser;
134        parser
135            .parse(Path::new("notes.txt"), content)
136            .expect("parse failed")
137    }
138
139    #[test]
140    fn test_empty() {
141        let doc = parse("");
142        assert!(doc.structure.is_empty());
143    }
144
145    #[test]
146    fn test_whitespace_only() {
147        let doc = parse("   \n\n  \n  ");
148        assert!(doc.structure.is_empty());
149    }
150
151    #[test]
152    fn test_single_paragraph() {
153        let doc = parse("Hello world.\nThis is a test.");
154        assert_eq!(doc.structure.len(), 1);
155        assert_eq!(doc.structure[0].title, "Hello world.");
156        assert!(doc.structure[0].text.contains("This is a test"));
157    }
158
159    #[test]
160    fn test_multiple_paragraphs() {
161        let content = "First paragraph.\n\nSecond paragraph.\n\nThird paragraph.";
162        let doc = parse(content);
163        assert_eq!(doc.structure.len(), 3);
164        assert_eq!(doc.structure[0].title, "First paragraph.");
165        assert_eq!(doc.structure[1].title, "Second paragraph.");
166        assert_eq!(doc.structure[2].title, "Third paragraph.");
167    }
168
169    #[test]
170    fn test_title_truncation() {
171        let long_line = "A".repeat(120);
172        let doc = parse(&long_line);
173        assert_eq!(doc.structure.len(), 1);
174        assert!(doc.structure[0].title.len() <= MAX_TITLE_LEN + 3);
175        assert!(doc.structure[0].title.ends_with("..."));
176    }
177
178    #[test]
179    fn test_line_numbers() {
180        let content = "Para one line1\nPara one line2\n\nPara two.";
181        let doc = parse(content);
182        assert_eq!(doc.structure[0].line_start, Some(1));
183        assert_eq!(doc.structure[0].line_end, Some(2));
184    }
185
186    #[test]
187    fn test_line_numbers_account_for_leading_and_repeated_blank_lines() {
188        let content = "\n\nFirst\nline\n\n\n\nSecond\r\nthird\r\n";
189        let doc = parse(content);
190        assert_eq!(doc.structure[0].line_start, Some(3));
191        assert_eq!(doc.structure[0].line_end, Some(4));
192        assert_eq!(doc.structure[1].line_start, Some(8));
193        assert_eq!(doc.structure[1].line_end, Some(9));
194    }
195
196    #[test]
197    fn test_node_ids_assigned() {
198        let content = "A\n\nB\n\nC";
199        let doc = parse(content);
200        assert_eq!(doc.structure[0].node_id, "0");
201        assert_eq!(doc.structure[1].node_id, "1");
202        assert_eq!(doc.structure[2].node_id, "2");
203    }
204
205    #[test]
206    fn test_source_type() {
207        let doc = parse("hello");
208        assert_eq!(doc.source_type, SourceType::Text);
209        assert_eq!(doc.doc_id, "notes.txt");
210    }
211
212    #[test]
213    fn test_multiple_blank_lines() {
214        let content = "First\n\n\n\nSecond";
215        let doc = parse(content);
216        assert_eq!(doc.structure.len(), 2);
217    }
218
219    #[test]
220    fn test_flat_structure() {
221        let content = "A\n\nB\n\nC";
222        let doc = parse(content);
223        for node in &doc.structure {
224            assert!(node.children.is_empty());
225        }
226    }
227
228    #[test]
229    fn test_unicode_title_truncation() {
230        // Each CJK character is 3 bytes in UTF-8.
231        let long_cjk = "你".repeat(100);
232        let doc = parse(&long_cjk);
233        // Should truncate without panicking (char boundary safe).
234        assert!(doc.structure[0].title.ends_with("..."));
235        assert_eq!(doc.structure[0].title.chars().count(), MAX_TITLE_LEN + 3);
236    }
237}