Skip to main content

treesearch/parser/
mod.rs

1//! Extensible parser registry whose built-in formats are selected with Cargo features.
2
3#[cfg(feature = "parser-config")]
4pub mod config_file;
5#[cfg(feature = "parser-html")]
6pub mod html;
7#[cfg(feature = "parser-markdown")]
8pub mod markdown;
9#[cfg(feature = "parser-plaintext")]
10pub mod plaintext;
11#[cfg(feature = "parser-code")]
12pub mod treesitter;
13
14use crate::document::{Document, SourceType};
15use anyhow::Result;
16use std::path::Path;
17
18/// Trait for document parsers.
19pub trait Parser: Send + Sync {
20    /// File extensions this parser handles (without dot, lowercase).
21    fn extensions(&self) -> &[&str];
22
23    /// Source type for parsed documents.
24    fn source_type(&self) -> SourceType;
25
26    /// Parse file content into a Document.
27    /// Implementations must call `doc.assign_node_ids()` before returning.
28    fn parse(&self, path: &Path, content: &str) -> Result<Document>;
29}
30
31/// Registry of all parsers. Routes files to the appropriate parser by extension.
32pub struct ParserRegistry {
33    parsers: Vec<Box<dyn Parser>>,
34}
35
36impl ParserRegistry {
37    /// Creates a registry containing every parser enabled at compile time.
38    pub fn new() -> Self {
39        let parsers: Vec<Box<dyn Parser>> = Vec::new();
40        #[cfg(feature = "parser-markdown")]
41        let parsers = {
42            let mut parsers = parsers;
43            parsers.push(Box::new(markdown::MarkdownParser) as Box<dyn Parser>);
44            parsers
45        };
46        #[cfg(feature = "parser-html")]
47        let parsers = {
48            let mut parsers = parsers;
49            parsers.push(Box::new(html::HtmlParser) as Box<dyn Parser>);
50            parsers
51        };
52        #[cfg(feature = "parser-config")]
53        let parsers = {
54            let mut parsers = parsers;
55            parsers.push(Box::new(config_file::JsonParser) as Box<dyn Parser>);
56            parsers.push(Box::new(config_file::YamlParser) as Box<dyn Parser>);
57            parsers.push(Box::new(config_file::TomlParser) as Box<dyn Parser>);
58            parsers
59        };
60        #[cfg(feature = "parser-code")]
61        let parsers = {
62            let mut parsers = parsers;
63            parsers.push(Box::new(treesitter::TreeSitterParser) as Box<dyn Parser>);
64            parsers
65        };
66        // Plaintext remains last because it is the most generic fallback.
67        #[cfg(feature = "parser-plaintext")]
68        let parsers = {
69            let mut parsers = parsers;
70            parsers.push(Box::new(plaintext::PlainTextParser) as Box<dyn Parser>);
71            parsers
72        };
73        Self { parsers }
74    }
75
76    /// Creates an empty registry for applications that supply only custom parsers.
77    pub fn empty() -> Self {
78        Self {
79            parsers: Vec::new(),
80        }
81    }
82
83    /// Registers one application-defined parser after the built-in parsers.
84    pub fn register<P>(&mut self, parser: P)
85    where
86        P: Parser + 'static,
87    {
88        self.parsers.push(Box::new(parser));
89    }
90
91    /// Find the parser for a given file extension (case-insensitive, without dot).
92    fn find_parser_by_ext(&self, ext: &str) -> Option<&dyn Parser> {
93        let ext_lower = ext.to_lowercase();
94        self.parsers
95            .iter()
96            .find(|p| p.extensions().iter().any(|e| *e == ext_lower))
97            .map(|p| p.as_ref())
98    }
99
100    /// Finds a parser from an extension or a known extensionless filename.
101    fn find_parser_for_path(&self, path: &Path) -> Option<&dyn Parser> {
102        path.extension()
103            .and_then(|extension| extension.to_str())
104            .or_else(|| path.file_name().and_then(|name| name.to_str()))
105            .and_then(|discriminator| self.find_parser_by_ext(discriminator))
106    }
107
108    /// Parse a file from disk. Returns `Ok(None)` if no parser handles the extension.
109    /// Sets `source_path` to the canonical absolute path.
110    pub fn parse_file(&self, path: &Path) -> Result<Option<Document>> {
111        let parser = match self.find_parser_for_path(path) {
112            Some(p) => p,
113            None => return Ok(None),
114        };
115
116        let content = std::fs::read_to_string(path)?;
117        let abs_path = std::fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf());
118        let mut doc = parser.parse(path, &content)?;
119        doc.source_path = abs_path.to_string_lossy().to_string();
120        Ok(Some(doc))
121    }
122
123    /// Parse file content that has already been read. Returns `Ok(None)` if no
124    /// parser handles the extension.
125    pub fn parse_content(&self, path: &Path, content: &str) -> Result<Option<Document>> {
126        let parser = match self.find_parser_for_path(path) {
127            Some(p) => p,
128            None => return Ok(None),
129        };
130
131        let mut doc = parser.parse(path, content)?;
132        doc.source_path = path.to_string_lossy().to_string();
133        Ok(Some(doc))
134    }
135
136    /// Check whether a file extension is supported by any registered parser.
137    pub fn supports(&self, path: &Path) -> bool {
138        self.find_parser_for_path(path).is_some()
139    }
140}
141
142impl Default for ParserRegistry {
143    fn default() -> Self {
144        Self::new()
145    }
146}
147
148#[cfg(test)]
149mod tests {
150    use super::*;
151    #[cfg(any(feature = "parser-config", feature = "parser-markdown"))]
152    use std::io::Write;
153    use tempfile::NamedTempFile;
154
155    #[test]
156    fn test_registry_supports() {
157        let reg = ParserRegistry::new();
158        #[cfg(feature = "parser-markdown")]
159        assert!(reg.supports(Path::new("readme.md")));
160        #[cfg(feature = "parser-html")]
161        assert!(reg.supports(Path::new("index.html")));
162        #[cfg(feature = "parser-code")]
163        assert!(reg.supports(Path::new("main.rs")));
164        #[cfg(feature = "parser-code")]
165        assert!(reg.supports(Path::new("Dockerfile")));
166        #[cfg(feature = "parser-code")]
167        assert!(reg.supports(Path::new("Makefile")));
168        #[cfg(feature = "parser-config")]
169        assert!(reg.supports(Path::new("config.json")));
170        #[cfg(feature = "parser-config")]
171        assert!(reg.supports(Path::new("data.yaml")));
172        #[cfg(feature = "parser-config")]
173        assert!(reg.supports(Path::new("data.yml")));
174        #[cfg(feature = "parser-config")]
175        assert!(reg.supports(Path::new("Cargo.toml")));
176        #[cfg(feature = "parser-plaintext")]
177        assert!(reg.supports(Path::new("notes.txt")));
178        assert!(!reg.supports(Path::new("image.png")));
179        assert!(!reg.supports(Path::new("noext")));
180    }
181
182    #[cfg(feature = "parser-markdown")]
183    #[test]
184    fn test_registry_parse_file_markdown() {
185        let mut tmp = NamedTempFile::with_suffix(".md").unwrap();
186        writeln!(tmp, "# Hello\n\nWorld").unwrap();
187        let reg = ParserRegistry::new();
188        let doc = reg.parse_file(tmp.path()).unwrap().unwrap();
189        assert_eq!(doc.source_type, SourceType::Markdown);
190        assert!(!doc.structure.is_empty());
191    }
192
193    #[cfg(feature = "parser-config")]
194    #[test]
195    fn test_registry_parse_file_json() {
196        let mut tmp = NamedTempFile::with_suffix(".json").unwrap();
197        write!(tmp, r#"{{"key": "value"}}"#).unwrap();
198        let reg = ParserRegistry::new();
199        let doc = reg.parse_file(tmp.path()).unwrap().unwrap();
200        assert_eq!(doc.source_type, SourceType::Json);
201    }
202
203    #[cfg(feature = "parser-markdown")]
204    #[test]
205    fn test_registry_parse_content() {
206        let reg = ParserRegistry::new();
207        let doc = reg
208            .parse_content(Path::new("test.md"), "# Hello\n\nWorld")
209            .unwrap()
210            .unwrap();
211        assert_eq!(doc.source_type, SourceType::Markdown);
212        assert!(!doc.structure.is_empty());
213    }
214
215    #[test]
216    fn test_registry_unsupported_extension() {
217        let tmp = NamedTempFile::with_suffix(".png").unwrap();
218        let reg = ParserRegistry::new();
219        assert!(reg.parse_file(tmp.path()).unwrap().is_none());
220    }
221
222    #[test]
223    fn test_registry_no_extension() {
224        let reg = ParserRegistry::new();
225        assert!(!reg.supports(Path::new("Makefile_no_ext")));
226    }
227
228    #[test]
229    fn test_registry_case_insensitive_ext() {
230        let reg = ParserRegistry::new();
231        #[cfg(feature = "parser-markdown")]
232        assert!(reg.supports(Path::new("README.MD")));
233        #[cfg(feature = "parser-html")]
234        assert!(reg.supports(Path::new("page.HTML")));
235        #[cfg(feature = "parser-config")]
236        assert!(reg.supports(Path::new("data.JSON")));
237    }
238}