Skip to main content

treesearch/parser/
treesitter.rs

1//! Parses supported source files into stable syntax-aware document trees.
2
3use crate::document::{Document, Node, SourceType};
4use anyhow::Result;
5use regex::Regex;
6use std::path::Path;
7
8/// Phase 1 tree-sitter stub: uses regex heuristics to detect function/class
9/// boundaries in source code files. Each detected definition becomes a node
10/// with the function/class name as title, its body as text, and line numbers.
11///
12/// Phase 2 will replace the regex approach with actual tree-sitter grammars.
13pub struct TreeSitterParser;
14
15/// File extensions handled by this parser (all source code files).
16const CODE_EXTENSIONS: &[&str] = &[
17    "rs",
18    "py",
19    "js",
20    "ts",
21    "jsx",
22    "tsx",
23    "go",
24    "java",
25    "c",
26    "cpp",
27    "h",
28    "hpp",
29    "cs",
30    "rb",
31    "php",
32    "swift",
33    "kt",
34    "scala",
35    "sh",
36    "bash",
37    "zsh",
38    "fish",
39    "lua",
40    "r",
41    "m",
42    "mm",
43    "pl",
44    "pm",
45    "ex",
46    "exs",
47    "erl",
48    "hs",
49    "ml",
50    "mli",
51    "clj",
52    "cljs",
53    "el",
54    "vim",
55    "sql",
56    "graphql",
57    "proto",
58    "tf",
59    "hcl",
60    "zig",
61    "nim",
62    "v",
63    "d",
64    "dart",
65    "cmake",
66    "makefile",
67    "dockerfile",
68    "css",
69    "scss",
70    "sass",
71    "less",
72    "vue",
73    "svelte",
74];
75
76impl super::Parser for TreeSitterParser {
77    fn extensions(&self) -> &[&str] {
78        CODE_EXTENSIONS
79    }
80
81    fn source_type(&self) -> SourceType {
82        SourceType::Code
83    }
84
85    fn parse(&self, path: &Path, content: &str) -> Result<Document> {
86        let file_name = path
87            .file_name()
88            .map(|n| n.to_string_lossy().to_string())
89            .unwrap_or_default();
90        let doc_id = path.to_string_lossy().to_string();
91        let mut doc = Document::new(&doc_id, &file_name, SourceType::Code);
92
93        if content.trim().is_empty() {
94            doc.assign_node_ids();
95            return Ok(doc);
96        }
97
98        let ext = path
99            .extension()
100            .and_then(|e| e.to_str())
101            .unwrap_or("")
102            .to_lowercase();
103
104        let boundaries = detect_boundaries(content, &ext);
105
106        if boundaries.is_empty() {
107            // No detectable boundaries — treat entire file as one node.
108            let mut node = Node::new("", &file_name);
109            node.text = content.to_string();
110            node.line_start = Some(1);
111            node.line_end = Some(content.lines().count().max(1) as u32);
112            doc.structure.push(node);
113        } else {
114            let lines: Vec<&str> = content.lines().collect();
115            let total_lines = lines.len();
116
117            // Capture preamble before the first boundary (imports, comments, etc.)
118            if boundaries[0].line > 0 {
119                let preamble_text: String = lines[..boundaries[0].line].join("\n");
120                let trimmed = preamble_text.trim();
121                if !trimmed.is_empty() {
122                    let mut node = Node::new("", "(preamble)");
123                    node.text = trimmed.to_string();
124                    node.line_start = Some(1);
125                    node.line_end = Some(boundaries[0].line as u32);
126                    doc.structure.push(node);
127                }
128            }
129
130            for (i, boundary) in boundaries.iter().enumerate() {
131                let start = boundary.line;
132                let end = if i + 1 < boundaries.len() {
133                    boundaries[i + 1].line
134                } else {
135                    total_lines
136                };
137
138                let body: String = lines[start..end].join("\n");
139                let trimmed = body.trim_end();
140
141                let mut node = Node::new("", &boundary.name);
142                node.text = trimmed.to_string();
143                node.line_start = Some(start as u32 + 1); // 1-based
144                node.line_end = Some(end as u32);
145
146                doc.structure.push(node);
147            }
148        }
149
150        doc.assign_node_ids();
151        Ok(doc)
152    }
153}
154
155// ---------------------------------------------------------------------------
156// Boundary detection
157// ---------------------------------------------------------------------------
158
159/// A detected function/class/struct boundary.
160struct Boundary {
161    /// 0-based line index in the source.
162    line: usize,
163    /// Extracted definition name (e.g. "main", "Foo", "hello").
164    name: String,
165}
166
167/// Detect function/class boundaries by matching regex patterns against each line.
168fn detect_boundaries(content: &str, ext: &str) -> Vec<Boundary> {
169    let patterns = get_patterns(ext);
170    if patterns.is_empty() {
171        return Vec::new();
172    }
173
174    let regexes: Vec<Regex> = patterns.iter().filter_map(|p| Regex::new(p).ok()).collect();
175
176    let mut boundaries = Vec::new();
177
178    for (line_idx, line) in content.lines().enumerate() {
179        let trimmed = line.trim();
180        for re in &regexes {
181            if let Some(caps) = re.captures(trimmed) {
182                let name = caps
183                    .get(1)
184                    .map(|m| m.as_str().to_string())
185                    .unwrap_or_else(|| trimmed.to_string());
186                boundaries.push(Boundary {
187                    line: line_idx,
188                    name,
189                });
190                break; // one match per line
191            }
192        }
193    }
194
195    boundaries
196}
197
198/// Return regex patterns for the given language extension.
199/// Each pattern must contain a capture group (1) for the definition name.
200fn get_patterns(ext: &str) -> Vec<&'static str> {
201    match ext {
202        // Rust
203        "rs" => vec![
204            r"^\s*(?:pub\s+)?(?:async\s+)?fn\s+(\w+)",
205            r"^\s*(?:pub\s+)?struct\s+(\w+)",
206            r"^\s*(?:pub\s+)?enum\s+(\w+)",
207            r"^\s*(?:pub\s+)?trait\s+(\w+)",
208            r"^\s*(?:pub\s+)?impl(?:<[^>]*>)?\s+(\w+)",
209            r"^\s*(?:pub\s+)?mod\s+(\w+)",
210        ],
211        // Python
212        "py" => vec![r"^(?:async\s+)?def\s+(\w+)", r"^class\s+(\w+)"],
213        // JavaScript / TypeScript
214        "js" | "jsx" | "ts" | "tsx" | "vue" | "svelte" => vec![
215            r"^\s*(?:export\s+)?(?:async\s+)?function\s+(\w+)",
216            r"^\s*(?:export\s+)?class\s+(\w+)",
217            r"^\s*(?:export\s+)?(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s+)?\(",
218            r"^\s*(?:export\s+)?(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s+)?(?:\([^)]*\)|[^=])\s*=>",
219            r"^\s*(?:export\s+)?interface\s+(\w+)",
220            r"^\s*(?:export\s+)?type\s+(\w+)",
221        ],
222        // Go
223        "go" => vec![
224            r"^func\s+(?:\([^)]+\)\s+)?(\w+)",
225            r"^type\s+(\w+)\s+struct",
226            r"^type\s+(\w+)\s+interface",
227        ],
228        // Java / Kotlin / Scala
229        "java" | "kt" | "scala" => vec![
230            r"(?:public|private|protected|static|final|abstract)?\s*(?:class|interface|enum)\s+(\w+)",
231            r"(?:public|private|protected|static|final|abstract|override)?\s*(?:fun|void|int|String|boolean|long|double|float|Object|var|val)\s+(\w+)\s*\(",
232        ],
233        // C / C++
234        "c" | "cpp" | "h" | "hpp" | "cc" | "cxx" => vec![
235            r"^\s*(?:static\s+)?(?:inline\s+)?(?:virtual\s+)?(?:const\s+)?(?:\w+[\s*&]+)+(\w+)\s*\([^;]*$",
236            r"^\s*(?:class|struct|enum)\s+(\w+)",
237            r"^\s*namespace\s+(\w+)",
238        ],
239        // C#
240        "cs" => vec![
241            r"(?:public|private|protected|internal|static|virtual|abstract|override|async)?\s*(?:class|interface|struct|enum)\s+(\w+)",
242            r"(?:public|private|protected|internal|static|virtual|abstract|override|async)?\s*(?:void|int|string|bool|Task|var|dynamic|\w+)\s+(\w+)\s*\(",
243        ],
244        // Ruby
245        "rb" => vec![
246            r"^\s*def\s+(\w+)",
247            r"^\s*class\s+(\w+)",
248            r"^\s*module\s+(\w+)",
249        ],
250        // PHP
251        "php" => vec![
252            r"^\s*(?:public|private|protected|static)?\s*function\s+(\w+)",
253            r"^\s*class\s+(\w+)",
254        ],
255        // Swift
256        "swift" => vec![
257            r"^\s*(?:public|private|internal|open|fileprivate)?\s*(?:static|class)?\s*func\s+(\w+)",
258            r"^\s*(?:public|private|internal|open|fileprivate)?\s*class\s+(\w+)",
259            r"^\s*(?:public|private|internal|open|fileprivate)?\s*struct\s+(\w+)",
260            r"^\s*(?:public|private|internal|open|fileprivate)?\s*enum\s+(\w+)",
261            r"^\s*(?:public|private|internal|open|fileprivate)?\s*protocol\s+(\w+)",
262        ],
263        // Lua
264        "lua" => vec![r"^\s*(?:local\s+)?function\s+(\w[\w.:]*)"],
265        // Shell
266        "sh" | "bash" | "zsh" | "fish" => {
267            vec![r"^\s*(?:function\s+)?(\w+)\s*\(\)", r"^\s*function\s+(\w+)"]
268        }
269        // Elixir
270        "ex" | "exs" => vec![r"^\s*def[p]?\s+(\w+)", r"^\s*defmodule\s+(\w[\w.]*)"],
271        // Haskell
272        "hs" => vec![r"^(\w+)\s+::"],
273        // SQL
274        "sql" => vec![
275            r"(?i)^\s*CREATE\s+(?:OR\s+REPLACE\s+)?(?:TABLE|VIEW|FUNCTION|PROCEDURE|INDEX)\s+(?:IF\s+NOT\s+EXISTS\s+)?(\w+)",
276        ],
277        // Dart
278        "dart" => vec![
279            r"^\s*(?:abstract\s+)?class\s+(\w+)",
280            r"^\s*(?:\w+\s+)*(\w+)\s*\([^)]*\)\s*(?:async\s*)?\{",
281        ],
282        // Zig
283        "zig" => vec![
284            r"^\s*(?:pub\s+)?fn\s+(\w+)",
285            r"^\s*(?:pub\s+)?const\s+(\w+)\s*=\s*struct",
286        ],
287        // Nim
288        "nim" => vec![
289            r"^\s*(?:proc|func|method|template|macro)\s+(\w+)",
290            r"^\s*type\s+(\w+)",
291        ],
292        // Default: no patterns → file treated as a single node
293        _ => vec![],
294    }
295}
296
297#[cfg(test)]
298mod tests {
299    use super::*;
300    use crate::parser::Parser;
301
302    fn parse_code(content: &str, filename: &str) -> Document {
303        let parser = TreeSitterParser;
304        parser
305            .parse(Path::new(filename), content)
306            .expect("parse failed")
307    }
308
309    #[test]
310    fn test_empty() {
311        let doc = parse_code("", "main.rs");
312        assert!(doc.structure.is_empty());
313    }
314
315    #[test]
316    fn test_rust_functions() {
317        let content = "use std::io;\n\nfn hello() {\n    println!(\"hello\");\n}\n\npub fn world() {\n    println!(\"world\");\n}\n";
318        let doc = parse_code(content, "lib.rs");
319        assert_eq!(doc.source_type, SourceType::Code);
320        // preamble + two functions
321        assert_eq!(doc.structure.len(), 3);
322        assert_eq!(doc.structure[0].title, "(preamble)");
323        assert_eq!(doc.structure[1].title, "hello");
324        assert_eq!(doc.structure[2].title, "world");
325    }
326
327    #[test]
328    fn test_rust_struct_and_impl() {
329        let content = "pub struct Foo {\n    x: i32,\n}\n\nimpl Foo {\n    pub fn new() -> Self {\n        Self { x: 0 }\n    }\n}\n";
330        let doc = parse_code(content, "foo.rs");
331        assert!(doc.structure.len() >= 2);
332        assert_eq!(doc.structure[0].title, "Foo"); // struct
333    }
334
335    #[test]
336    fn test_python_functions() {
337        let content = "import os\n\ndef hello():\n    print('hello')\n\nclass Foo:\n    def bar(self):\n        pass\n";
338        let doc = parse_code(content, "app.py");
339        assert!(doc.structure.len() >= 3);
340        assert_eq!(doc.structure[1].title, "hello");
341        assert_eq!(doc.structure[2].title, "Foo");
342    }
343
344    #[test]
345    fn test_javascript_functions() {
346        let content = "\nfunction greet(name) {\n    return `Hello ${name}`;\n}\n\nclass App {\n    constructor() {}\n}\n";
347        let doc = parse_code(content, "app.js");
348        let titles: Vec<&str> = doc.structure.iter().map(|n| n.title.as_str()).collect();
349        assert!(titles.contains(&"greet"));
350        assert!(titles.contains(&"App"));
351    }
352
353    #[test]
354    fn test_go_functions() {
355        let content = "package main\n\nfunc main() {\n    fmt.Println(\"hello\")\n}\n\nfunc (s *Server) Start() error {\n    return nil\n}\n\ntype Config struct {\n    Port int\n}\n";
356        let doc = parse_code(content, "main.go");
357        let titles: Vec<&str> = doc.structure.iter().map(|n| n.title.as_str()).collect();
358        assert!(titles.contains(&"main"));
359        assert!(titles.contains(&"Config"));
360    }
361
362    #[test]
363    fn test_line_numbers() {
364        let content = "fn foo() {\n    1\n}\n\nfn bar() {\n    2\n}\n";
365        let doc = parse_code(content, "test.rs");
366        let foo = &doc.structure[0];
367        assert_eq!(foo.title, "foo");
368        assert_eq!(foo.line_start, Some(1));
369        let bar = &doc.structure[1];
370        assert_eq!(bar.title, "bar");
371        assert_eq!(bar.line_start, Some(5));
372    }
373
374    #[test]
375    fn test_no_boundaries_fallback() {
376        let content = "some random content\nwith lines\n";
377        let doc = parse_code(content, "data.xyz");
378        // No patterns for .xyz → single-node fallback
379        assert_eq!(doc.structure.len(), 1);
380        assert!(doc.structure[0].text.contains("some random content"));
381    }
382
383    #[test]
384    fn test_node_ids_assigned() {
385        let content = "fn a() {}\n\nfn b() {}\n";
386        let doc = parse_code(content, "test.rs");
387        for (i, node) in doc.structure.iter().enumerate() {
388            assert_eq!(node.node_id, i.to_string());
389        }
390    }
391
392    #[test]
393    fn test_source_type() {
394        let doc = parse_code("fn main() {}", "main.rs");
395        assert_eq!(doc.source_type, SourceType::Code);
396        assert_eq!(doc.doc_id, "main.rs");
397    }
398
399    #[test]
400    fn test_ruby() {
401        let content = "class Foo\n  def bar\n    puts 'hi'\n  end\nend\n";
402        let doc = parse_code(content, "app.rb");
403        let titles: Vec<&str> = doc.structure.iter().map(|n| n.title.as_str()).collect();
404        assert!(titles.contains(&"Foo"));
405    }
406
407    #[test]
408    fn test_shell() {
409        let content =
410            "#!/bin/bash\n\nhello() {\n    echo 'hi'\n}\n\nfunction world {\n    echo 'world'\n}\n";
411        let doc = parse_code(content, "script.sh");
412        let titles: Vec<&str> = doc.structure.iter().map(|n| n.title.as_str()).collect();
413        assert!(titles.contains(&"hello"));
414        assert!(titles.contains(&"world"));
415    }
416
417    #[test]
418    fn test_sql() {
419        let content = "CREATE TABLE users (\n  id INT PRIMARY KEY\n);\n\nCREATE VIEW active_users AS\nSELECT * FROM users;\n";
420        let doc = parse_code(content, "schema.sql");
421        let titles: Vec<&str> = doc.structure.iter().map(|n| n.title.as_str()).collect();
422        assert!(titles.contains(&"users"));
423        assert!(titles.contains(&"active_users"));
424    }
425
426    #[test]
427    fn test_preamble_included() {
428        let content = "// Copyright 2024\n// License: MIT\n\nfn main() {}\n";
429        let doc = parse_code(content, "main.rs");
430        assert_eq!(doc.structure[0].title, "(preamble)");
431        assert!(doc.structure[0].text.contains("Copyright"));
432    }
433
434    #[test]
435    fn test_rust_enum_and_trait() {
436        let content = "pub enum Color {\n    Red,\n    Blue,\n}\n\npub trait Drawable {\n    fn draw(&self);\n}\n";
437        let doc = parse_code(content, "lib.rs");
438        let titles: Vec<&str> = doc.structure.iter().map(|n| n.title.as_str()).collect();
439        assert!(titles.contains(&"Color"));
440        assert!(titles.contains(&"Drawable"));
441    }
442
443    #[test]
444    fn test_python_async_def() {
445        let content = "async def fetch_data():\n    pass\n\ndef sync_func():\n    pass\n";
446        let doc = parse_code(content, "app.py");
447        let titles: Vec<&str> = doc.structure.iter().map(|n| n.title.as_str()).collect();
448        assert!(titles.contains(&"fetch_data"));
449        assert!(titles.contains(&"sync_func"));
450    }
451}