1use std::ops::ControlFlow;
2
3use thiserror::Error;
4use tree_sitter::{Language, ParseOptions, Parser};
5
6use crate::{ExtractionLimitExceeded, ExtractionTracker};
7
8#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10pub enum SourceSyntaxLanguage {
11 JavaScript,
13 TypeScript,
15 Rust,
17 Python,
19 Go,
21 Java,
23}
24
25#[derive(Debug, Clone, Copy, PartialEq, Eq)]
27pub struct SourceSyntaxInspection {
28 pub has_error: bool,
30 pub boundary_candidate_count: usize,
32}
33
34#[derive(Debug, Error)]
36pub enum SourceSyntaxError {
37 #[error("failed to configure Tree-sitter grammar: {0}")]
39 Grammar(#[from] tree_sitter::LanguageError),
40 #[error("Tree-sitter did not return a syntax tree")]
42 MissingTree,
43 #[error(transparent)]
45 LimitExceeded(#[from] ExtractionLimitExceeded),
46}
47
48pub fn inspect_source_syntax(
57 language: SourceSyntaxLanguage,
58 source_path: &str,
59 source: &str,
60 tracker: &mut ExtractionTracker,
61) -> Result<SourceSyntaxInspection, SourceSyntaxError> {
62 tracker.check_input_bytes(u64::try_from(source.len()).unwrap_or(u64::MAX))?;
63 tracker.check_tree_sitter_time()?;
64 let grammar = grammar(language, source_path);
65 let mut parser = Parser::new();
66 parser.set_language(&grammar)?;
67 let mut timeout = None;
68 let tree = {
69 let mut progress = |_: &tree_sitter::ParseState| match tracker.check_tree_sitter_time() {
70 Ok(()) => ControlFlow::Continue(()),
71 Err(error) => {
72 timeout = Some(error);
73 ControlFlow::Break(())
74 }
75 };
76 let options = ParseOptions::new().progress_callback(&mut progress);
77 let bytes = source.as_bytes();
78 parser.parse_with_options(
79 &mut |offset, _| bytes.get(offset..).unwrap_or_default(),
80 None,
81 Some(options),
82 )
83 };
84 if let Some(error) = timeout {
85 return Err(error.into());
86 }
87 let tree = tree.ok_or(SourceSyntaxError::MissingTree)?;
88 let root = tree.root_node();
89 let mut boundary_candidate_count = 0;
90 count_candidates(language, root, &mut boundary_candidate_count, tracker)?;
91 tracker.check_tree_sitter_time()?;
92 Ok(SourceSyntaxInspection {
93 has_error: root.has_error(),
94 boundary_candidate_count,
95 })
96}
97
98fn grammar(language: SourceSyntaxLanguage, source_path: &str) -> Language {
99 match language {
100 SourceSyntaxLanguage::JavaScript => tree_sitter_javascript::LANGUAGE.into(),
101 SourceSyntaxLanguage::TypeScript
102 if std::path::Path::new(source_path)
103 .extension()
104 .is_some_and(|extension| extension.eq_ignore_ascii_case("tsx")) =>
105 {
106 tree_sitter_typescript::LANGUAGE_TSX.into()
107 }
108 SourceSyntaxLanguage::TypeScript => tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into(),
109 SourceSyntaxLanguage::Rust => tree_sitter_rust::LANGUAGE.into(),
110 SourceSyntaxLanguage::Python => tree_sitter_python::LANGUAGE.into(),
111 SourceSyntaxLanguage::Go => tree_sitter_go::LANGUAGE.into(),
112 SourceSyntaxLanguage::Java => tree_sitter_java::LANGUAGE.into(),
113 }
114}
115
116fn count_candidates(
117 language: SourceSyntaxLanguage,
118 root: tree_sitter::Node<'_>,
119 count: &mut usize,
120 tracker: &mut ExtractionTracker,
121) -> Result<(), ExtractionLimitExceeded> {
122 tracker.charge_tree_sitter_node(0)?;
123 let mut pending = vec![(root, 0_u64)];
124 while let Some((node, depth)) = pending.pop() {
125 if candidate_kind(language, node.kind()) {
126 *count = count.saturating_add(1);
127 }
128 let child_depth = depth.saturating_add(1);
129 let mut cursor = node.walk();
130 for child in node.children(&mut cursor) {
131 tracker.charge_tree_sitter_node(child_depth)?;
132 pending.push((child, child_depth));
133 }
134 }
135 Ok(())
136}
137
138fn candidate_kind(language: SourceSyntaxLanguage, kind: &str) -> bool {
139 match language {
140 SourceSyntaxLanguage::JavaScript | SourceSyntaxLanguage::TypeScript => {
141 matches!(
142 kind,
143 "call_expression"
144 | "decorator"
145 | "method_definition"
146 | "function_declaration"
147 | "lexical_declaration"
148 )
149 }
150 SourceSyntaxLanguage::Rust => matches!(
151 kind,
152 "call_expression" | "attribute_item" | "macro_definition" | "macro_invocation"
153 ),
154 SourceSyntaxLanguage::Python => {
155 matches!(
156 kind,
157 "call" | "decorator" | "dictionary" | "class_definition"
158 )
159 }
160 SourceSyntaxLanguage::Go => kind == "call_expression",
161 SourceSyntaxLanguage::Java => {
162 matches!(
163 kind,
164 "method_invocation" | "annotation" | "marker_annotation"
165 )
166 }
167 }
168}
169
170#[cfg(test)]
171mod tests {
172 use super::{SourceSyntaxError, SourceSyntaxLanguage, inspect_source_syntax};
173 use crate::{ExtractionBudgets, ExtractionResource, ExtractionTracker};
174
175 #[test]
176 fn mandatory_grammars_should_find_boundary_candidates() {
177 let fixtures = [
178 (
179 SourceSyntaxLanguage::JavaScript,
180 "client.js",
181 "fetch('/orders');",
182 ),
183 (
184 SourceSyntaxLanguage::TypeScript,
185 "router.ts",
186 "app.post('/orders', handler);",
187 ),
188 (
189 SourceSyntaxLanguage::JavaScript,
190 "src/app/api/orders/route.js",
191 "export async function POST() {}",
192 ),
193 (
194 SourceSyntaxLanguage::Rust,
195 "lib.rs",
196 "#[tokio::test]\nasync fn test_order() {}",
197 ),
198 (
199 SourceSyntaxLanguage::Python,
200 "test_api.py",
201 "@app.post('/orders')\ndef create(): pass",
202 ),
203 (
204 SourceSyntaxLanguage::Python,
205 "methods_config.py",
206 "METHODS = {'get_order': ['GET', '/orders/{id}']}",
207 ),
208 (
209 SourceSyntaxLanguage::Python,
210 "factories.py",
211 "class OrderFactory(factory.Factory):\n class Meta:\n model = Order",
212 ),
213 (
214 SourceSyntaxLanguage::Go,
215 "main.go",
216 "package main\nfunc main() { http.Get(\"/orders\") }",
217 ),
218 (
219 SourceSyntaxLanguage::Java,
220 "Client.java",
221 "class Client { void run() { client.get(); } }",
222 ),
223 ];
224
225 for (language, path, source) in fixtures {
226 let mut tracker =
227 ExtractionTracker::new(path, "tree-sitter", &ExtractionBudgets::default());
228 let result = inspect_source_syntax(language, path, source, &mut tracker);
229 assert!(
230 matches!(result, Ok(report) if report.boundary_candidate_count > 0),
231 "{path}: {result:?}"
232 );
233 }
234 }
235
236 #[test]
237 fn javascript_ast_depth_should_accept_256_and_reject_257() {
238 fn nested_arrays(count: usize) -> String {
239 format!("{}0{};", "[".repeat(count), "]".repeat(count))
240 }
241
242 let budgets = ExtractionBudgets {
243 max_ast_depth_per_artifact: 256,
244 ..ExtractionBudgets::default()
245 };
246 let mut exact = ExtractionTracker::new("exact.js", "tree-sitter", &budgets);
247 let mut above = ExtractionTracker::new("above.js", "tree-sitter", &budgets);
248
249 assert!(
250 inspect_source_syntax(
251 SourceSyntaxLanguage::JavaScript,
252 "exact.js",
253 &nested_arrays(254),
254 &mut exact,
255 )
256 .is_ok()
257 );
258 assert!(matches!(
259 inspect_source_syntax(
260 SourceSyntaxLanguage::JavaScript,
261 "above.js",
262 &nested_arrays(255),
263 &mut above,
264 ),
265 Err(SourceSyntaxError::LimitExceeded(error))
266 if error.resource == ExtractionResource::AstDepth
267 && error.observed == 257
268 && error.maximum == 256
269 ));
270 }
271
272 #[test]
273 fn javascript_node_budget_should_accept_500000_and_reject_500001() {
274 let budgets = ExtractionBudgets {
275 max_tree_sitter_nodes_per_artifact: 500_000,
276 ..ExtractionBudgets::default()
277 };
278 let mut exact = ExtractionTracker::new("exact.js", "tree-sitter", &budgets);
279 let mut above = ExtractionTracker::new("above.js", "tree-sitter", &budgets);
280 let exact_source = format!("{}// one", ";".repeat(249_999));
281 let above_source = ";".repeat(250_000);
282
283 assert!(
284 inspect_source_syntax(
285 SourceSyntaxLanguage::JavaScript,
286 "exact.js",
287 &exact_source,
288 &mut exact,
289 )
290 .is_ok()
291 );
292 assert!(matches!(
293 inspect_source_syntax(
294 SourceSyntaxLanguage::JavaScript,
295 "above.js",
296 &above_source,
297 &mut above,
298 ),
299 Err(SourceSyntaxError::LimitExceeded(error))
300 if error.resource == ExtractionResource::TreeSitterNodes
301 && error.observed == 500_001
302 && error.maximum == 500_000
303 ));
304 }
305
306 #[test]
307 fn parser_should_cancel_on_a_real_wall_time_deadline() {
308 let budgets = ExtractionBudgets {
309 max_tree_sitter_wall_time_ms_per_artifact: 1,
310 max_tree_sitter_nodes_per_artifact: 1_000_000,
311 ..ExtractionBudgets::default()
312 };
313 let mut tracker = ExtractionTracker::new("slow.js", "tree-sitter", &budgets);
314 let source = ";".repeat(750_000);
315
316 assert!(matches!(
317 inspect_source_syntax(
318 SourceSyntaxLanguage::JavaScript,
319 "slow.js",
320 &source,
321 &mut tracker,
322 ),
323 Err(SourceSyntaxError::LimitExceeded(error))
324 if error.resource == ExtractionResource::TreeSitterWallTimeMs
325 ));
326 }
327}