1use crate::document::{Document, Node, SourceType};
4use anyhow::Result;
5use regex::Regex;
6use std::path::Path;
7
8pub struct TreeSitterParser;
14
15const CODE_EXTENSIONS: &[&str] = &[
17 "rs",
18 "py",
19 "js",
20 "ts",
21 "jsx",
22 "tsx",
23 "go",
24 "java",
25 "c",
26 "cpp",
27 "h",
28 "hpp",
29 "cs",
30 "rb",
31 "php",
32 "swift",
33 "kt",
34 "scala",
35 "sh",
36 "bash",
37 "zsh",
38 "fish",
39 "lua",
40 "r",
41 "m",
42 "mm",
43 "pl",
44 "pm",
45 "ex",
46 "exs",
47 "erl",
48 "hs",
49 "ml",
50 "mli",
51 "clj",
52 "cljs",
53 "el",
54 "vim",
55 "sql",
56 "graphql",
57 "proto",
58 "tf",
59 "hcl",
60 "zig",
61 "nim",
62 "v",
63 "d",
64 "dart",
65 "cmake",
66 "makefile",
67 "dockerfile",
68 "css",
69 "scss",
70 "sass",
71 "less",
72 "vue",
73 "svelte",
74];
75
76impl super::Parser for TreeSitterParser {
77 fn extensions(&self) -> &[&str] {
78 CODE_EXTENSIONS
79 }
80
81 fn source_type(&self) -> SourceType {
82 SourceType::Code
83 }
84
85 fn parse(&self, path: &Path, content: &str) -> Result<Document> {
86 let file_name = path
87 .file_name()
88 .map(|n| n.to_string_lossy().to_string())
89 .unwrap_or_default();
90 let doc_id = path.to_string_lossy().to_string();
91 let mut doc = Document::new(&doc_id, &file_name, SourceType::Code);
92
93 if content.trim().is_empty() {
94 doc.assign_node_ids();
95 return Ok(doc);
96 }
97
98 let ext = path
99 .extension()
100 .and_then(|e| e.to_str())
101 .unwrap_or("")
102 .to_lowercase();
103
104 let boundaries = detect_boundaries(content, &ext);
105
106 if boundaries.is_empty() {
107 let mut node = Node::new("", &file_name);
109 node.text = content.to_string();
110 node.line_start = Some(1);
111 node.line_end = Some(content.lines().count().max(1) as u32);
112 doc.structure.push(node);
113 } else {
114 let lines: Vec<&str> = content.lines().collect();
115 let total_lines = lines.len();
116
117 if boundaries[0].line > 0 {
119 let preamble_text: String = lines[..boundaries[0].line].join("\n");
120 let trimmed = preamble_text.trim();
121 if !trimmed.is_empty() {
122 let mut node = Node::new("", "(preamble)");
123 node.text = trimmed.to_string();
124 node.line_start = Some(1);
125 node.line_end = Some(boundaries[0].line as u32);
126 doc.structure.push(node);
127 }
128 }
129
130 for (i, boundary) in boundaries.iter().enumerate() {
131 let start = boundary.line;
132 let end = if i + 1 < boundaries.len() {
133 boundaries[i + 1].line
134 } else {
135 total_lines
136 };
137
138 let body: String = lines[start..end].join("\n");
139 let trimmed = body.trim_end();
140
141 let mut node = Node::new("", &boundary.name);
142 node.text = trimmed.to_string();
143 node.line_start = Some(start as u32 + 1); node.line_end = Some(end as u32);
145
146 doc.structure.push(node);
147 }
148 }
149
150 doc.assign_node_ids();
151 Ok(doc)
152 }
153}
154
155struct Boundary {
161 line: usize,
163 name: String,
165}
166
167fn detect_boundaries(content: &str, ext: &str) -> Vec<Boundary> {
169 let patterns = get_patterns(ext);
170 if patterns.is_empty() {
171 return Vec::new();
172 }
173
174 let regexes: Vec<Regex> = patterns.iter().filter_map(|p| Regex::new(p).ok()).collect();
175
176 let mut boundaries = Vec::new();
177
178 for (line_idx, line) in content.lines().enumerate() {
179 let trimmed = line.trim();
180 for re in ®exes {
181 if let Some(caps) = re.captures(trimmed) {
182 let name = caps
183 .get(1)
184 .map(|m| m.as_str().to_string())
185 .unwrap_or_else(|| trimmed.to_string());
186 boundaries.push(Boundary {
187 line: line_idx,
188 name,
189 });
190 break; }
192 }
193 }
194
195 boundaries
196}
197
198fn get_patterns(ext: &str) -> Vec<&'static str> {
201 match ext {
202 "rs" => vec![
204 r"^\s*(?:pub\s+)?(?:async\s+)?fn\s+(\w+)",
205 r"^\s*(?:pub\s+)?struct\s+(\w+)",
206 r"^\s*(?:pub\s+)?enum\s+(\w+)",
207 r"^\s*(?:pub\s+)?trait\s+(\w+)",
208 r"^\s*(?:pub\s+)?impl(?:<[^>]*>)?\s+(\w+)",
209 r"^\s*(?:pub\s+)?mod\s+(\w+)",
210 ],
211 "py" => vec![r"^(?:async\s+)?def\s+(\w+)", r"^class\s+(\w+)"],
213 "js" | "jsx" | "ts" | "tsx" | "vue" | "svelte" => vec![
215 r"^\s*(?:export\s+)?(?:async\s+)?function\s+(\w+)",
216 r"^\s*(?:export\s+)?class\s+(\w+)",
217 r"^\s*(?:export\s+)?(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s+)?\(",
218 r"^\s*(?:export\s+)?(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s+)?(?:\([^)]*\)|[^=])\s*=>",
219 r"^\s*(?:export\s+)?interface\s+(\w+)",
220 r"^\s*(?:export\s+)?type\s+(\w+)",
221 ],
222 "go" => vec![
224 r"^func\s+(?:\([^)]+\)\s+)?(\w+)",
225 r"^type\s+(\w+)\s+struct",
226 r"^type\s+(\w+)\s+interface",
227 ],
228 "java" | "kt" | "scala" => vec![
230 r"(?:public|private|protected|static|final|abstract)?\s*(?:class|interface|enum)\s+(\w+)",
231 r"(?:public|private|protected|static|final|abstract|override)?\s*(?:fun|void|int|String|boolean|long|double|float|Object|var|val)\s+(\w+)\s*\(",
232 ],
233 "c" | "cpp" | "h" | "hpp" | "cc" | "cxx" => vec![
235 r"^\s*(?:static\s+)?(?:inline\s+)?(?:virtual\s+)?(?:const\s+)?(?:\w+[\s*&]+)+(\w+)\s*\([^;]*$",
236 r"^\s*(?:class|struct|enum)\s+(\w+)",
237 r"^\s*namespace\s+(\w+)",
238 ],
239 "cs" => vec![
241 r"(?:public|private|protected|internal|static|virtual|abstract|override|async)?\s*(?:class|interface|struct|enum)\s+(\w+)",
242 r"(?:public|private|protected|internal|static|virtual|abstract|override|async)?\s*(?:void|int|string|bool|Task|var|dynamic|\w+)\s+(\w+)\s*\(",
243 ],
244 "rb" => vec![
246 r"^\s*def\s+(\w+)",
247 r"^\s*class\s+(\w+)",
248 r"^\s*module\s+(\w+)",
249 ],
250 "php" => vec![
252 r"^\s*(?:public|private|protected|static)?\s*function\s+(\w+)",
253 r"^\s*class\s+(\w+)",
254 ],
255 "swift" => vec![
257 r"^\s*(?:public|private|internal|open|fileprivate)?\s*(?:static|class)?\s*func\s+(\w+)",
258 r"^\s*(?:public|private|internal|open|fileprivate)?\s*class\s+(\w+)",
259 r"^\s*(?:public|private|internal|open|fileprivate)?\s*struct\s+(\w+)",
260 r"^\s*(?:public|private|internal|open|fileprivate)?\s*enum\s+(\w+)",
261 r"^\s*(?:public|private|internal|open|fileprivate)?\s*protocol\s+(\w+)",
262 ],
263 "lua" => vec![r"^\s*(?:local\s+)?function\s+(\w[\w.:]*)"],
265 "sh" | "bash" | "zsh" | "fish" => {
267 vec![r"^\s*(?:function\s+)?(\w+)\s*\(\)", r"^\s*function\s+(\w+)"]
268 }
269 "ex" | "exs" => vec![r"^\s*def[p]?\s+(\w+)", r"^\s*defmodule\s+(\w[\w.]*)"],
271 "hs" => vec![r"^(\w+)\s+::"],
273 "sql" => vec![
275 r"(?i)^\s*CREATE\s+(?:OR\s+REPLACE\s+)?(?:TABLE|VIEW|FUNCTION|PROCEDURE|INDEX)\s+(?:IF\s+NOT\s+EXISTS\s+)?(\w+)",
276 ],
277 "dart" => vec![
279 r"^\s*(?:abstract\s+)?class\s+(\w+)",
280 r"^\s*(?:\w+\s+)*(\w+)\s*\([^)]*\)\s*(?:async\s*)?\{",
281 ],
282 "zig" => vec![
284 r"^\s*(?:pub\s+)?fn\s+(\w+)",
285 r"^\s*(?:pub\s+)?const\s+(\w+)\s*=\s*struct",
286 ],
287 "nim" => vec![
289 r"^\s*(?:proc|func|method|template|macro)\s+(\w+)",
290 r"^\s*type\s+(\w+)",
291 ],
292 _ => vec![],
294 }
295}
296
297#[cfg(test)]
298mod tests {
299 use super::*;
300 use crate::parser::Parser;
301
302 fn parse_code(content: &str, filename: &str) -> Document {
303 let parser = TreeSitterParser;
304 parser
305 .parse(Path::new(filename), content)
306 .expect("parse failed")
307 }
308
309 #[test]
310 fn test_empty() {
311 let doc = parse_code("", "main.rs");
312 assert!(doc.structure.is_empty());
313 }
314
315 #[test]
316 fn test_rust_functions() {
317 let content = "use std::io;\n\nfn hello() {\n println!(\"hello\");\n}\n\npub fn world() {\n println!(\"world\");\n}\n";
318 let doc = parse_code(content, "lib.rs");
319 assert_eq!(doc.source_type, SourceType::Code);
320 assert_eq!(doc.structure.len(), 3);
322 assert_eq!(doc.structure[0].title, "(preamble)");
323 assert_eq!(doc.structure[1].title, "hello");
324 assert_eq!(doc.structure[2].title, "world");
325 }
326
327 #[test]
328 fn test_rust_struct_and_impl() {
329 let content = "pub struct Foo {\n x: i32,\n}\n\nimpl Foo {\n pub fn new() -> Self {\n Self { x: 0 }\n }\n}\n";
330 let doc = parse_code(content, "foo.rs");
331 assert!(doc.structure.len() >= 2);
332 assert_eq!(doc.structure[0].title, "Foo"); }
334
335 #[test]
336 fn test_python_functions() {
337 let content = "import os\n\ndef hello():\n print('hello')\n\nclass Foo:\n def bar(self):\n pass\n";
338 let doc = parse_code(content, "app.py");
339 assert!(doc.structure.len() >= 3);
340 assert_eq!(doc.structure[1].title, "hello");
341 assert_eq!(doc.structure[2].title, "Foo");
342 }
343
344 #[test]
345 fn test_javascript_functions() {
346 let content = "\nfunction greet(name) {\n return `Hello ${name}`;\n}\n\nclass App {\n constructor() {}\n}\n";
347 let doc = parse_code(content, "app.js");
348 let titles: Vec<&str> = doc.structure.iter().map(|n| n.title.as_str()).collect();
349 assert!(titles.contains(&"greet"));
350 assert!(titles.contains(&"App"));
351 }
352
353 #[test]
354 fn test_go_functions() {
355 let content = "package main\n\nfunc main() {\n fmt.Println(\"hello\")\n}\n\nfunc (s *Server) Start() error {\n return nil\n}\n\ntype Config struct {\n Port int\n}\n";
356 let doc = parse_code(content, "main.go");
357 let titles: Vec<&str> = doc.structure.iter().map(|n| n.title.as_str()).collect();
358 assert!(titles.contains(&"main"));
359 assert!(titles.contains(&"Config"));
360 }
361
362 #[test]
363 fn test_line_numbers() {
364 let content = "fn foo() {\n 1\n}\n\nfn bar() {\n 2\n}\n";
365 let doc = parse_code(content, "test.rs");
366 let foo = &doc.structure[0];
367 assert_eq!(foo.title, "foo");
368 assert_eq!(foo.line_start, Some(1));
369 let bar = &doc.structure[1];
370 assert_eq!(bar.title, "bar");
371 assert_eq!(bar.line_start, Some(5));
372 }
373
374 #[test]
375 fn test_no_boundaries_fallback() {
376 let content = "some random content\nwith lines\n";
377 let doc = parse_code(content, "data.xyz");
378 assert_eq!(doc.structure.len(), 1);
380 assert!(doc.structure[0].text.contains("some random content"));
381 }
382
383 #[test]
384 fn test_node_ids_assigned() {
385 let content = "fn a() {}\n\nfn b() {}\n";
386 let doc = parse_code(content, "test.rs");
387 for (i, node) in doc.structure.iter().enumerate() {
388 assert_eq!(node.node_id, i.to_string());
389 }
390 }
391
392 #[test]
393 fn test_source_type() {
394 let doc = parse_code("fn main() {}", "main.rs");
395 assert_eq!(doc.source_type, SourceType::Code);
396 assert_eq!(doc.doc_id, "main.rs");
397 }
398
399 #[test]
400 fn test_ruby() {
401 let content = "class Foo\n def bar\n puts 'hi'\n end\nend\n";
402 let doc = parse_code(content, "app.rb");
403 let titles: Vec<&str> = doc.structure.iter().map(|n| n.title.as_str()).collect();
404 assert!(titles.contains(&"Foo"));
405 }
406
407 #[test]
408 fn test_shell() {
409 let content =
410 "#!/bin/bash\n\nhello() {\n echo 'hi'\n}\n\nfunction world {\n echo 'world'\n}\n";
411 let doc = parse_code(content, "script.sh");
412 let titles: Vec<&str> = doc.structure.iter().map(|n| n.title.as_str()).collect();
413 assert!(titles.contains(&"hello"));
414 assert!(titles.contains(&"world"));
415 }
416
417 #[test]
418 fn test_sql() {
419 let content = "CREATE TABLE users (\n id INT PRIMARY KEY\n);\n\nCREATE VIEW active_users AS\nSELECT * FROM users;\n";
420 let doc = parse_code(content, "schema.sql");
421 let titles: Vec<&str> = doc.structure.iter().map(|n| n.title.as_str()).collect();
422 assert!(titles.contains(&"users"));
423 assert!(titles.contains(&"active_users"));
424 }
425
426 #[test]
427 fn test_preamble_included() {
428 let content = "// Copyright 2024\n// License: MIT\n\nfn main() {}\n";
429 let doc = parse_code(content, "main.rs");
430 assert_eq!(doc.structure[0].title, "(preamble)");
431 assert!(doc.structure[0].text.contains("Copyright"));
432 }
433
434 #[test]
435 fn test_rust_enum_and_trait() {
436 let content = "pub enum Color {\n Red,\n Blue,\n}\n\npub trait Drawable {\n fn draw(&self);\n}\n";
437 let doc = parse_code(content, "lib.rs");
438 let titles: Vec<&str> = doc.structure.iter().map(|n| n.title.as_str()).collect();
439 assert!(titles.contains(&"Color"));
440 assert!(titles.contains(&"Drawable"));
441 }
442
443 #[test]
444 fn test_python_async_def() {
445 let content = "async def fetch_data():\n pass\n\ndef sync_func():\n pass\n";
446 let doc = parse_code(content, "app.py");
447 let titles: Vec<&str> = doc.structure.iter().map(|n| n.title.as_str()).collect();
448 assert!(titles.contains(&"fetch_data"));
449 assert!(titles.contains(&"sync_func"));
450 }
451}