meta_ast/language/
mod.rs

1//! Language system: compile-time enum dispatch over 9 language packs.
2//!
3//! `LangId` enum selects a `LanguageSpec` via exhaustive `match`.
4//! Each `LanguageSpec` bundles a grammar constructor, tree-sitter query
5//! constructors (symbols, imports, references), and language-specific
6//! extraction heuristics.
7
8pub(crate) mod c;
9pub(crate) mod common;
10pub(crate) mod cpp;
11#[cfg(feature = "dataflow")]
12pub(crate) mod dataflow;
13pub(crate) mod go;
14pub mod import_resolver;
15pub(crate) mod javascript;
16pub(crate) mod pack;
17pub(crate) mod python;
18pub(crate) mod ruby;
19pub(crate) mod rust;
20pub(crate) mod tsx;
21pub(crate) mod typescript;
22
23use serde::{Deserialize, Serialize};
24use tree_sitter::Query;
25
26use crate::model::Visibility;
27
28/// Configuration for extracting doc comments from source code.
29///
30/// Used for languages where doc comments are tree-sitter extras
31/// (siblings of declarations, not children) and cannot be captured
32/// inline in queries.
33#[derive(Debug, Clone)]
34pub struct DocCommentConfig {
35    /// Line comment prefixes that indicate doc comments (e.g., `["///", "//!"]` for Rust).
36    pub line_prefixes: &'static [&'static str],
37    /// Block comment opening (e.g., `Some("/**")` for doxygen/JSDoc).
38    pub block_open: Option<&'static str>,
39    /// Block comment closing (e.g., `"*/"`).
40    pub block_close: &'static str,
41    /// Whether to strip leading `*` continuation markers in block comments.
42    pub strip_continuation_marker: bool,
43}
44
45/// Default visibility assumed when a symbol declares no explicit modifier.
46#[derive(Debug, Clone, Copy, PartialEq, Eq)]
47#[non_exhaustive]
48pub enum DefaultVisibility {
49    /// When visibility is None, treat the symbol as public (Python, C functions).
50    PublicByDefault,
51    /// When visibility is None, treat the symbol as private (Rust, JS, TS, Go, C++).
52    PrivateByDefault,
53}
54
55/// Shared JSDoc/C-style doc comment config.
56///
57/// Used by JS, TS, TSX, C, C++ specs. Only Rust, Go, Ruby, Python differ.
58pub const C_LIKE_DOC_COMMENT: DocCommentConfig = DocCommentConfig {
59    line_prefixes: &["//"],
60    block_open: Some("/**"),
61    block_close: "*/",
62    strip_continuation_marker: true,
63};
64
65#[derive(
66    Debug,
67    Clone,
68    Copy,
69    PartialEq,
70    Eq,
71    Hash,
72    Serialize,
73    Deserialize,
74    strum::Display,
75    strum::AsRefStr,
76    strum::EnumString,
77)]
78#[non_exhaustive]
79#[serde(rename_all = "snake_case")]
80#[strum(serialize_all = "snake_case")]
81#[repr(usize)]
82pub enum LangId {
83    Python,
84    #[strum(serialize = "javascript")]
85    #[serde(rename = "javascript")]
86    JavaScript,
87    #[strum(serialize = "typescript")]
88    #[serde(rename = "typescript")]
89    TypeScript,
90    Tsx,
91    C,
92    Cpp,
93    Rust,
94    Go,
95    Ruby,
96}
97
98impl LangId {
99    pub const COUNT: usize = 9;
100
101    pub fn all() -> [LangId; Self::COUNT] {
102        [
103            LangId::Python,
104            LangId::JavaScript,
105            LangId::TypeScript,
106            LangId::Tsx,
107            LangId::C,
108            LangId::Cpp,
109            LangId::Rust,
110            LangId::Go,
111            LangId::Ruby,
112        ]
113    }
114
115    pub fn spec(self) -> &'static LanguageSpec {
116        spec_for(self)
117    }
118
119    #[cfg(feature = "metacall-deploy")]
120    pub fn metacall_tag(self) -> &'static str {
121        crate::deploy::tags::metacall_tag(self)
122    }
123}
124
125#[derive(Debug, Clone, Serialize)]
126pub struct RawSymbol<'a> {
127    pub name: std::borrow::Cow<'a, str>,
128    pub kind: crate::model::SymbolKind,
129    pub source_range: crate::model::SourceRange,
130    /// Range of the captured name node, when the query has a `@name` capture.
131    pub name_range: Option<crate::model::SourceRange>,
132    pub visibility: Option<crate::model::Visibility>,
133    pub signature: Option<std::borrow::Cow<'a, str>>,
134    pub docstring: Option<std::borrow::Cow<'a, str>>,
135    pub is_async: bool,
136}
137
138pub struct LanguageSpec {
139    pub extensions: &'static [&'static str],
140    pub grammar_fn: fn() -> tree_sitter::Language,
141    pub query_fn: fn() -> Result<&'static Query, crate::error::Error>,
142    pub import_path_resolver: fn(
143        raw: &str,
144        source_dir: &std::path::Path,
145        project_root: &std::path::Path,
146    ) -> Option<std::path::PathBuf>,
147    pub import_ref_query_fn: fn() -> Result<&'static Query, crate::error::Error>,
148    pub class_like_parents: &'static [&'static str],
149    pub ancestor_visibility_rules: &'static [(&'static str, Visibility)],
150    pub visibility_from_name: Option<fn(&str) -> Option<Visibility>>,
151    pub import_statement_kinds: &'static [&'static str],
152    pub default_visibility: DefaultVisibility,
153    pub doc_comment_config: Option<DocCommentConfig>,
154}
155
156pub fn spec_for(id: LangId) -> &'static LanguageSpec {
157    match id {
158        LangId::Python => &python::PYTHON_SPEC,
159        LangId::JavaScript => &javascript::JS_SPEC,
160        LangId::TypeScript => &typescript::TS_SPEC,
161        LangId::Tsx => &tsx::TSX_SPEC,
162        LangId::C => &c::C_SPEC,
163        LangId::Cpp => &cpp::CPP_SPEC,
164        LangId::Rust => &rust::RUST_SPEC,
165        LangId::Go => &go::GO_SPEC,
166        LangId::Ruby => &ruby::RUBY_SPEC,
167    }
168}
169
170pub fn grammar_for(id: LangId) -> tree_sitter::Language {
171    (spec_for(id).grammar_fn)()
172}
173
174/// Eagerly initialize all language query statics.
175///
176/// Call at startup so a broken query is reported before the first file. A
177/// failure is logged, not fatal: the per-file path turns it into a diagnostic.
178pub fn validate_queries() {
179    for id in LangId::all() {
180        let spec = spec_for(id);
181        for (label, outcome) in [
182            ("symbols", (spec.query_fn)()),
183            ("imports and references", (spec.import_ref_query_fn)()),
184        ] {
185            if let Err(error) = outcome {
186                tracing::error!(language = ?id, query = label, %error, "query failed to compile");
187            }
188        }
189    }
190}
191
192/// Extract symbols, reporting a query failure as an empty result.
193///
194/// The analysis path uses [`extract_symbols_for_checked`] so the failure
195/// reaches the file diagnostics; this wrapper serves callers that only need
196/// the symbols.
197pub fn extract_symbols_for<'a>(
198    id: LangId,
199    tree: &'a tree_sitter::Tree,
200    source: &'a [u8],
201) -> Vec<RawSymbol<'a>> {
202    match extract_symbols_for_checked(id, tree, source) {
203        Ok(symbols) => symbols,
204        Err(error) => {
205            tracing::error!(language = ?id, %error, "symbol extraction skipped");
206            Vec::new()
207        }
208    }
209}
210
211pub fn extract_symbols_for_checked<'a>(
212    id: LangId,
213    tree: &'a tree_sitter::Tree,
214    source: &'a [u8],
215) -> Result<Vec<RawSymbol<'a>>, crate::error::Error> {
216    common::extract_with_spec(tree, source, spec_for(id))
217}
218
219/// Extract imports and references, reporting a query failure as empty results.
220pub fn extract_imports_and_references_for<'a>(
221    id: LangId,
222    tree: &'a tree_sitter::Tree,
223    source: &'a [u8],
224    file_path: &std::path::Path,
225) -> (
226    Vec<crate::model::UnresolvedImport>,
227    Vec<crate::model::UnresolvedReference>,
228    Vec<crate::error::Diagnostic>,
229) {
230    match extract_imports_and_references_for_checked(id, tree, source, file_path) {
231        Ok(extracted) => extracted,
232        Err(error) => {
233            tracing::error!(language = ?id, %error, "import and reference extraction skipped");
234            (Vec::new(), Vec::new(), Vec::new())
235        }
236    }
237}
238
239pub fn extract_imports_and_references_for_checked<'a>(
240    id: LangId,
241    tree: &'a tree_sitter::Tree,
242    source: &'a [u8],
243    file_path: &std::path::Path,
244) -> Result<common::ImportExtraction, crate::error::Error> {
245    common::extract_imports_and_references_with_spec(tree, source, spec_for(id), file_path)
246}
247
248#[cfg(test)]
249mod tests {
250    use std::str::FromStr;
251
252    use super::*;
253
254    #[test]
255    fn lang_id_all_variants_exist() {
256        let variants = LangId::all();
257        for i in 0..variants.len() {
258            for j in (i + 1)..variants.len() {
259                assert_ne!(variants[i], variants[j]);
260            }
261        }
262    }
263
264    #[test]
265    fn lang_id_display() {
266        assert_eq!(format!("{}", LangId::Python), "python");
267        assert_eq!(format!("{}", LangId::Ruby), "ruby");
268        assert_eq!(format!("{}", LangId::JavaScript), "javascript");
269        assert_eq!(format!("{}", LangId::TypeScript), "typescript");
270    }
271
272    #[test]
273    fn lang_id_vocabulary_is_canonical() {
274        let expected = [
275            "python",
276            "javascript",
277            "typescript",
278            "tsx",
279            "c",
280            "cpp",
281            "rust",
282            "go",
283            "ruby",
284        ];
285        let all = LangId::all();
286        let actual: Vec<&str> = all.iter().map(|l| l.as_ref()).collect();
287        assert_eq!(actual, expected);
288    }
289
290    #[test]
291    fn lang_id_from_str_round_trip() {
292        for id in LangId::all() {
293            let name: &str = id.as_ref();
294            let parsed = LangId::from_str(name)
295                .unwrap_or_else(|e| panic!("from_str failed for documented name {name:?}: {e}"));
296            assert_eq!(parsed, id);
297        }
298        assert_eq!(LangId::from_str("typescript"), Ok(LangId::TypeScript));
299        assert_eq!(LangId::from_str("javascript"), Ok(LangId::JavaScript));
300    }
301
302    #[test]
303    fn lang_id_serde_snake_case() {
304        let json = serde_json::to_string(&LangId::Python).unwrap();
305        assert_eq!(json, "\"python\"");
306
307        let json = serde_json::to_string(&LangId::Ruby).unwrap();
308        assert_eq!(json, "\"ruby\"");
309
310        let json = serde_json::to_string(&LangId::JavaScript).unwrap();
311        assert_eq!(json, "\"javascript\"");
312
313        let json = serde_json::to_string(&LangId::TypeScript).unwrap();
314        assert_eq!(json, "\"typescript\"");
315    }
316
317    #[test]
318    fn grammar_for_all_variants() {
319        for id in LangId::all() {
320            let _lang = grammar_for(id);
321        }
322    }
323
324    #[test]
325    fn extract_symbols_for_python() {
326        let mut parser = tree_sitter::Parser::new();
327        parser.set_language(&grammar_for(LangId::Python)).unwrap();
328        let tree = parser.parse(b"def hello(): pass", None).unwrap();
329        let symbols = extract_symbols_for(LangId::Python, &tree, b"def hello(): pass");
330        assert!(!symbols.is_empty());
331    }
332
333    #[test]
334    fn spec_for_returns_spec_with_matching_extensions() {
335        let python_spec = spec_for(LangId::Python);
336        assert!(python_spec.extensions.contains(&"py"));
337        assert!(python_spec.extensions.contains(&"pyi"));
338
339        let js_spec = spec_for(LangId::JavaScript);
340        assert!(js_spec.extensions.contains(&"js"));
341    }
342
343    #[test]
344    fn lang_id_count_matches_variant_count() {
345        assert_eq!(LangId::COUNT, 9);
346        assert_eq!(LangId::all().len(), LangId::COUNT);
347    }
348
349    #[test]
350    fn all_specs_have_non_empty_extensions() {
351        for id in LangId::all() {
352            let spec = spec_for(id);
353            assert!(
354                !spec.extensions.is_empty(),
355                "{id:?} spec has empty extensions"
356            );
357        }
358    }
359
360    #[test]
361    fn no_duplicate_extensions_across_specs() {
362        use std::collections::HashSet;
363        let mut seen: HashSet<&str> = HashSet::new();
364        for id in LangId::all() {
365            let spec = spec_for(id);
366            for &ext in spec.extensions {
367                assert!(
368                    seen.insert(ext),
369                    "extension {ext:?} appears in more than one language spec"
370                );
371            }
372        }
373    }
374
375    #[test]
376    fn grammar_fn_smoke_test_all_variants() {
377        for id in LangId::all() {
378            let spec = spec_for(id);
379            let grammar = (spec.grammar_fn)();
380            let mut parser = tree_sitter::Parser::new();
381            assert!(
382                parser.set_language(&grammar).is_ok(),
383                "grammar_fn failed for {id:?}"
384            );
385        }
386    }
387
388    #[test]
389    fn query_fn_smoke_test_all_variants() {
390        for id in LangId::all() {
391            let spec = spec_for(id);
392            let _query = (spec.query_fn)();
393        }
394    }
395
396    #[test]
397    #[cfg(feature = "metacall-deploy")]
398    fn test_lang_id_metacall_tag() {
399        assert_eq!(LangId::Python.metacall_tag(), "py");
400        assert_eq!(LangId::JavaScript.metacall_tag(), "node");
401        assert_eq!(LangId::TypeScript.metacall_tag(), "ts");
402        assert_eq!(LangId::Tsx.metacall_tag(), "ts");
403        assert_eq!(LangId::C.metacall_tag(), "c");
404        assert_eq!(LangId::Cpp.metacall_tag(), "c");
405        assert_eq!(LangId::Rust.metacall_tag(), "rs");
406        assert_eq!(LangId::Go.metacall_tag(), "go");
407        assert_eq!(LangId::Ruby.metacall_tag(), "rb");
408    }
409
410    /// The pack declarations keep this surface when a macro owns them:
411    /// extensions, visibility defaults, doc comment rules and the node kinds
412    /// each extraction step keys on.
413    #[test]
414    fn specs_keep_their_documented_surface() {
415        struct Expected {
416            extensions: &'static [&'static str],
417            default_visibility: DefaultVisibility,
418            doc_prefixes: Option<&'static [&'static str]>,
419            doc_block_open: Option<&'static str>,
420            class_like_parents: &'static [&'static str],
421            ancestor_rules: usize,
422            import_kinds: &'static [&'static str],
423            visibility_from_name: bool,
424        }
425
426        let public = DefaultVisibility::PublicByDefault;
427        let private = DefaultVisibility::PrivateByDefault;
428        let cases: [(LangId, Expected); 9] = [
429            (
430                LangId::Python,
431                Expected {
432                    extensions: &["py", "pyi"],
433                    default_visibility: public,
434                    doc_prefixes: None,
435                    doc_block_open: None,
436                    class_like_parents: &["class_definition"],
437                    ancestor_rules: 0,
438                    import_kinds: &["import_statement", "import_from_statement"],
439                    visibility_from_name: false,
440                },
441            ),
442            (
443                LangId::JavaScript,
444                Expected {
445                    extensions: &["js", "mjs", "cjs"],
446                    default_visibility: private,
447                    doc_prefixes: Some(&["//"]),
448                    doc_block_open: Some("/**"),
449                    class_like_parents: &["class_declaration", "class"],
450                    ancestor_rules: 1,
451                    import_kinds: &["import_statement"],
452                    visibility_from_name: false,
453                },
454            ),
455            (
456                LangId::TypeScript,
457                Expected {
458                    extensions: &["ts", "cts", "mts"],
459                    default_visibility: private,
460                    doc_prefixes: Some(&["//"]),
461                    doc_block_open: Some("/**"),
462                    class_like_parents: &["class_declaration", "class"],
463                    ancestor_rules: 1,
464                    import_kinds: &["import_statement"],
465                    visibility_from_name: false,
466                },
467            ),
468            (
469                LangId::Tsx,
470                Expected {
471                    extensions: &["tsx"],
472                    default_visibility: private,
473                    doc_prefixes: Some(&["//"]),
474                    doc_block_open: Some("/**"),
475                    class_like_parents: &["class_declaration", "class"],
476                    ancestor_rules: 1,
477                    import_kinds: &["import_statement"],
478                    visibility_from_name: false,
479                },
480            ),
481            (
482                LangId::C,
483                Expected {
484                    extensions: &["c", "h"],
485                    default_visibility: public,
486                    doc_prefixes: Some(&["//"]),
487                    doc_block_open: Some("/**"),
488                    class_like_parents: &[],
489                    ancestor_rules: 0,
490                    import_kinds: &["preproc_include"],
491                    visibility_from_name: false,
492                },
493            ),
494            (
495                LangId::Cpp,
496                Expected {
497                    extensions: &["cc", "cpp", "cxx", "hpp"],
498                    default_visibility: private,
499                    doc_prefixes: Some(&["//"]),
500                    doc_block_open: Some("/**"),
501                    class_like_parents: &["class_specifier", "struct_specifier"],
502                    ancestor_rules: 0,
503                    import_kinds: &["preproc_include"],
504                    visibility_from_name: false,
505                },
506            ),
507            (
508                LangId::Rust,
509                Expected {
510                    extensions: &["rs"],
511                    default_visibility: private,
512                    doc_prefixes: Some(&["///", "//!"]),
513                    doc_block_open: Some("/**"),
514                    class_like_parents: &["impl_item"],
515                    ancestor_rules: 0,
516                    import_kinds: &["use_declaration"],
517                    visibility_from_name: false,
518                },
519            ),
520            (
521                LangId::Go,
522                Expected {
523                    extensions: &["go"],
524                    default_visibility: private,
525                    doc_prefixes: Some(&["//"]),
526                    doc_block_open: None,
527                    class_like_parents: &[],
528                    ancestor_rules: 0,
529                    import_kinds: &["import_declaration"],
530                    visibility_from_name: true,
531                },
532            ),
533            (
534                LangId::Ruby,
535                Expected {
536                    extensions: &["rb", "gemspec"],
537                    default_visibility: public,
538                    doc_prefixes: Some(&["#"]),
539                    doc_block_open: None,
540                    class_like_parents: &["class", "module"],
541                    ancestor_rules: 0,
542                    import_kinds: &[],
543                    visibility_from_name: false,
544                },
545            ),
546        ];
547
548        for (lang, want) in cases {
549            let spec = spec_for(lang);
550            assert_eq!(spec.extensions, want.extensions, "{lang:?} extensions");
551            assert_eq!(
552                spec.default_visibility, want.default_visibility,
553                "{lang:?} default visibility"
554            );
555            assert_eq!(
556                spec.class_like_parents, want.class_like_parents,
557                "{lang:?} class-like parents"
558            );
559            assert_eq!(
560                spec.ancestor_visibility_rules.len(),
561                want.ancestor_rules,
562                "{lang:?} ancestor visibility rules"
563            );
564            assert_eq!(
565                spec.import_statement_kinds, want.import_kinds,
566                "{lang:?} import statement kinds"
567            );
568            assert_eq!(
569                spec.visibility_from_name.is_some(),
570                want.visibility_from_name,
571                "{lang:?} name-based visibility"
572            );
573            match (spec.doc_comment_config.as_ref(), want.doc_prefixes) {
574                (None, None) => {}
575                (Some(config), Some(prefixes)) => {
576                    assert_eq!(config.line_prefixes, prefixes, "{lang:?} doc prefixes");
577                    assert_eq!(
578                        config.block_open, want.doc_block_open,
579                        "{lang:?} doc block opener"
580                    );
581                }
582                (config, prefixes) => panic!(
583                    "{lang:?} doc comment configuration mismatch: {config:?} against {prefixes:?}"
584                ),
585            }
586        }
587
588        let rule = spec_for(LangId::Go).visibility_from_name;
589        assert!(rule.is_some(), "Go derives visibility from the name");
590        let rule = rule.unwrap();
591        assert_eq!(rule("Exported"), Some(Visibility::Public));
592        assert_eq!(rule("hidden"), None);
593    }
594}
meta_ast/language/mod.rs

meta_ast/language/
mod.rs
