treesearch/parser/
plaintext.rs1use crate::document::{Document, Node, SourceType};
4use anyhow::Result;
5use std::path::Path;
6
7pub struct PlainTextParser;
13
14const MAX_TITLE_LEN: usize = 80;
16
17impl super::Parser for PlainTextParser {
18 fn extensions(&self) -> &[&str] {
19 &["txt", "text", "log", "csv"]
20 }
21
22 fn source_type(&self) -> SourceType {
23 SourceType::Text
24 }
25
26 fn parse(&self, path: &Path, content: &str) -> Result<Document> {
27 let file_name = path
28 .file_name()
29 .map(|n| n.to_string_lossy().to_string())
30 .unwrap_or_default();
31 let doc_id = path.to_string_lossy().to_string();
32 let mut doc = Document::new(&doc_id, &file_name, SourceType::Text);
33
34 if content.trim().is_empty() {
35 doc.assign_node_ids();
36 return Ok(doc);
37 }
38
39 let paragraphs = split_paragraphs(content);
40
41 for paragraph in paragraphs {
42 let title = make_title(¶graph.text);
43
44 let mut node = Node::new("", &title);
45 node.text = paragraph.text;
46 node.line_start = Some(paragraph.line_start);
47 node.line_end = Some(paragraph.line_end);
48
49 doc.structure.push(node);
50 }
51
52 doc.assign_node_ids();
53 Ok(doc)
54 }
55}
56
57struct Paragraph {
59 text: String,
60 line_start: u32,
61 line_end: u32,
62}
63
64fn split_paragraphs(content: &str) -> Vec<Paragraph> {
66 fn flush(
68 lines: &mut Vec<String>,
69 line_start: &mut Option<u32>,
70 line_end: u32,
71 result: &mut Vec<Paragraph>,
72 ) {
73 let Some(start) = line_start.take() else {
74 return;
75 };
76 result.push(Paragraph {
77 text: lines.join("\n").trim().to_string(),
78 line_start: start,
79 line_end,
80 });
81 lines.clear();
82 }
83
84 let mut result = Vec::new();
85 let mut paragraph_lines = Vec::new();
86 let mut paragraph_start = None;
87 let mut previous_content_line = 0;
88 for (index, raw_line) in content.lines().enumerate() {
89 let line_number = u32::try_from(index.saturating_add(1)).unwrap_or(u32::MAX);
90 let line = raw_line.trim_end_matches('\r');
91 if line.trim().is_empty() {
92 flush(
93 &mut paragraph_lines,
94 &mut paragraph_start,
95 previous_content_line,
96 &mut result,
97 );
98 continue;
99 }
100 paragraph_start.get_or_insert(line_number);
101 previous_content_line = line_number;
102 paragraph_lines.push(line.to_string());
103 }
104 flush(
105 &mut paragraph_lines,
106 &mut paragraph_start,
107 previous_content_line,
108 &mut result,
109 );
110 result
111}
112
113fn make_title(text: &str) -> String {
115 let first_line = text.lines().next().unwrap_or("");
116 let trimmed = first_line.trim();
117 if trimmed.chars().count() <= MAX_TITLE_LEN {
118 trimmed.to_string()
119 } else {
120 format!(
121 "{}...",
122 trimmed.chars().take(MAX_TITLE_LEN).collect::<String>()
123 )
124 }
125}
126
127#[cfg(test)]
128mod tests {
129 use super::*;
130 use crate::parser::Parser;
131
132 fn parse(content: &str) -> Document {
133 let parser = PlainTextParser;
134 parser
135 .parse(Path::new("notes.txt"), content)
136 .expect("parse failed")
137 }
138
139 #[test]
140 fn test_empty() {
141 let doc = parse("");
142 assert!(doc.structure.is_empty());
143 }
144
145 #[test]
146 fn test_whitespace_only() {
147 let doc = parse(" \n\n \n ");
148 assert!(doc.structure.is_empty());
149 }
150
151 #[test]
152 fn test_single_paragraph() {
153 let doc = parse("Hello world.\nThis is a test.");
154 assert_eq!(doc.structure.len(), 1);
155 assert_eq!(doc.structure[0].title, "Hello world.");
156 assert!(doc.structure[0].text.contains("This is a test"));
157 }
158
159 #[test]
160 fn test_multiple_paragraphs() {
161 let content = "First paragraph.\n\nSecond paragraph.\n\nThird paragraph.";
162 let doc = parse(content);
163 assert_eq!(doc.structure.len(), 3);
164 assert_eq!(doc.structure[0].title, "First paragraph.");
165 assert_eq!(doc.structure[1].title, "Second paragraph.");
166 assert_eq!(doc.structure[2].title, "Third paragraph.");
167 }
168
169 #[test]
170 fn test_title_truncation() {
171 let long_line = "A".repeat(120);
172 let doc = parse(&long_line);
173 assert_eq!(doc.structure.len(), 1);
174 assert!(doc.structure[0].title.len() <= MAX_TITLE_LEN + 3);
175 assert!(doc.structure[0].title.ends_with("..."));
176 }
177
178 #[test]
179 fn test_line_numbers() {
180 let content = "Para one line1\nPara one line2\n\nPara two.";
181 let doc = parse(content);
182 assert_eq!(doc.structure[0].line_start, Some(1));
183 assert_eq!(doc.structure[0].line_end, Some(2));
184 }
185
186 #[test]
187 fn test_line_numbers_account_for_leading_and_repeated_blank_lines() {
188 let content = "\n\nFirst\nline\n\n\n\nSecond\r\nthird\r\n";
189 let doc = parse(content);
190 assert_eq!(doc.structure[0].line_start, Some(3));
191 assert_eq!(doc.structure[0].line_end, Some(4));
192 assert_eq!(doc.structure[1].line_start, Some(8));
193 assert_eq!(doc.structure[1].line_end, Some(9));
194 }
195
196 #[test]
197 fn test_node_ids_assigned() {
198 let content = "A\n\nB\n\nC";
199 let doc = parse(content);
200 assert_eq!(doc.structure[0].node_id, "0");
201 assert_eq!(doc.structure[1].node_id, "1");
202 assert_eq!(doc.structure[2].node_id, "2");
203 }
204
205 #[test]
206 fn test_source_type() {
207 let doc = parse("hello");
208 assert_eq!(doc.source_type, SourceType::Text);
209 assert_eq!(doc.doc_id, "notes.txt");
210 }
211
212 #[test]
213 fn test_multiple_blank_lines() {
214 let content = "First\n\n\n\nSecond";
215 let doc = parse(content);
216 assert_eq!(doc.structure.len(), 2);
217 }
218
219 #[test]
220 fn test_flat_structure() {
221 let content = "A\n\nB\n\nC";
222 let doc = parse(content);
223 for node in &doc.structure {
224 assert!(node.children.is_empty());
225 }
226 }
227
228 #[test]
229 fn test_unicode_title_truncation() {
230 let long_cjk = "你".repeat(100);
232 let doc = parse(&long_cjk);
233 assert!(doc.structure[0].title.ends_with("..."));
235 assert_eq!(doc.structure[0].title.chars().count(), MAX_TITLE_LEN + 3);
236 }
237}