semtree_parse/
text.rs

1use std::path::Path;
2
3use semtree_core::{Chunk, ChunkKind, Language, Span};
4
5use crate::lang::shared::chunk_hash;
6
7/// File types we handle as plain text (no AST)
8pub fn is_text_file(path: &Path) -> bool {
9    matches!(
10        path.extension().and_then(|e| e.to_str()).unwrap_or(""),
11        "md" | "txt"
12            | "rst"
13            | "json"
14            | "yaml"
15            | "yml"
16            | "toml"
17            | "env"
18            | "csv"
19            | "html"
20            | "xml"
21            | "graphql"
22    )
23}
24
25/// Split plain text into overlapping chunks of ~`chunk_lines` lines.
26pub fn chunk_text(path: &Path, source: &str, chunk_lines: usize, overlap: usize) -> Vec<Chunk> {
27    let lines: Vec<&str> = source.lines().collect();
28    if lines.is_empty() {
29        return vec![];
30    }
31
32    let step = chunk_lines.saturating_sub(overlap).max(1);
33    let mut chunks = Vec::new();
34    let mut start = 0;
35
36    while start < lines.len() {
37        let end = (start + chunk_lines).min(lines.len());
38        let content = lines[start..end].join("\n");
39
40        let span = Span::new(
41            0, // byte offsets not meaningful for text chunks
42            0,
43            start,
44            end - 1,
45        );
46
47        chunks.push(Chunk {
48            id: chunk_hash(&format!("{}:{start}", path.display())),
49            path: path.to_path_buf(),
50            language: Language::Unknown,
51            kind: ChunkKind::File,
52            name: Some(format!(
53                "{} (lines {}-{})",
54                path.file_name().and_then(|n| n.to_str()).unwrap_or("?"),
55                start + 1,
56                end
57            )),
58            content,
59            span,
60            doc: None,
61        });
62
63        if end == lines.len() {
64            break;
65        }
66        start += step;
67    }
68
69    chunks
70}
semtree_parse/text.rs

semtree_parse/
text.rs
