ast_grep_language/
lib.rs

1//! This module defines the supported programming languages for ast-grep.
2//!
3//! It provides a set of customized languages with expando_char / pre_process_pattern,
4//! and a set of stub languages without preprocessing.
5//! A rule of thumb: if your language does not accept identifiers like `$VAR`.
6//! You need use `impl_lang_expando!` macro and a standalone file for testing.
7//! Otherwise, you can define it as a stub language using `impl_lang!`.
8//! To see the full list of languages, visit `<https://ast-grep.github.io/reference/languages.html>`
9//!
10//! ```
11//! use ast_grep_language::{LanguageExt, SupportLang};
12//!
13//! let lang: SupportLang = "rs".parse().unwrap();
14//! let src = "fn foo() {}";
15//! let root = lang.ast_grep(src);
16//! let found = root.root().find_all("fn $FNAME() {}").next().unwrap();
17//! assert_eq!(found.start_pos().line(), 0);
18//! assert_eq!(found.text(), "fn foo() {}");
19//! ```
20
21mod bash;
22mod cpp;
23mod csharp;
24mod css;
25mod dart;
26mod elixir;
27mod go;
28mod haskell;
29mod hcl;
30mod html;
31mod json;
32mod kotlin;
33mod lua;
34mod markdown;
35mod nix;
36mod parsers;
37mod php;
38mod python;
39mod ruby;
40mod rust;
41mod scala;
42mod solidity;
43mod swift;
44mod yaml;
45
46use ast_grep_core::matcher::{Pattern, PatternBuilder, PatternError};
47pub use html::Html;
48
49use ast_grep_core::Node;
50use ast_grep_core::meta_var::MetaVariable;
51use ast_grep_core::tree_sitter::{StrDoc, TSLanguage, TSRange};
52use ignore::types::{Types, TypesBuilder};
53use serde::de::Visitor;
54use serde::{Deserialize, Deserializer, Serialize, de};
55use std::borrow::Cow;
56use std::fmt;
57use std::fmt::{Display, Formatter};
58use std::path::Path;
59use std::str::FromStr;
60
61pub use ast_grep_core::language::Language;
62pub use ast_grep_core::tree_sitter::LanguageExt;
63
64/// this macro implements bare-bone methods for a language
65macro_rules! impl_lang {
66  ($lang: ident, $func: ident) => {
67    #[derive(Clone, Copy, Debug)]
68    pub struct $lang;
69    impl Language for $lang {
70      fn kind_to_id(&self, kind: &str) -> u16 {
71        self
72          .get_ts_language()
73          .id_for_node_kind(kind, /*named*/ true)
74      }
75      fn field_to_id(&self, field: &str) -> Option<u16> {
76        self
77          .get_ts_language()
78          .field_id_for_name(field)
79          .map(|f| f.get())
80      }
81      fn build_pattern(&self, builder: &PatternBuilder) -> Result<Pattern, PatternError> {
82        builder.build(|src| StrDoc::try_new(src, self.clone()))
83      }
84    }
85    impl LanguageExt for $lang {
86      fn get_ts_language(&self) -> TSLanguage {
87        parsers::$func().into()
88      }
89    }
90  };
91}
92
93fn pre_process_pattern(expando: char, query: &str) -> std::borrow::Cow<'_, str> {
94  let mut ret = Vec::with_capacity(query.len());
95  let mut dollar_count = 0;
96  for c in query.chars() {
97    if c == '$' {
98      dollar_count += 1;
99      continue;
100    }
101    let need_replace = matches!(c, 'A'..='Z' | '_') // $A or $$A or $$$A
102      || dollar_count == 3; // anonymous multiple
103    let sigil = if need_replace { expando } else { '$' };
104    ret.extend(std::iter::repeat_n(sigil, dollar_count));
105    dollar_count = 0;
106    ret.push(c);
107  }
108  // trailing anonymous multiple
109  let sigil = if dollar_count == 3 { expando } else { '$' };
110  ret.extend(std::iter::repeat_n(sigil, dollar_count));
111  std::borrow::Cow::Owned(ret.into_iter().collect())
112}
113
114/// this macro will implement expando_char and pre_process_pattern
115/// use this if your language does not accept $ as valid identifier char
116macro_rules! impl_lang_expando {
117  ($lang: ident, $func: ident, $char: expr) => {
118    #[derive(Clone, Copy, Debug)]
119    pub struct $lang;
120    impl Language for $lang {
121      fn kind_to_id(&self, kind: &str) -> u16 {
122        self
123          .get_ts_language()
124          .id_for_node_kind(kind, /*named*/ true)
125      }
126      fn field_to_id(&self, field: &str) -> Option<u16> {
127        self
128          .get_ts_language()
129          .field_id_for_name(field)
130          .map(|f| f.get())
131      }
132      fn expando_char(&self) -> char {
133        $char
134      }
135      fn pre_process_pattern<'q>(&self, query: &'q str) -> std::borrow::Cow<'q, str> {
136        pre_process_pattern(self.expando_char(), query)
137      }
138      fn build_pattern(&self, builder: &PatternBuilder) -> Result<Pattern, PatternError> {
139        builder.build(|src| StrDoc::try_new(src, self.clone()))
140      }
141    }
142    impl LanguageExt for $lang {
143      fn get_ts_language(&self) -> TSLanguage {
144        $crate::parsers::$func().into()
145      }
146    }
147  };
148}
149
150pub trait Alias: Display {
151  const ALIAS: &'static [&'static str];
152}
153
154/// Implements the `ALIAS` associated constant for the given lang, which is
155/// then used to define the `alias` const fn and a `Deserialize` impl.
156macro_rules! impl_alias {
157  ($lang:ident => $as:expr) => {
158    impl Alias for $lang {
159      const ALIAS: &'static [&'static str] = $as;
160    }
161
162    impl fmt::Display for $lang {
163      fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
164        write!(f, "{:?}", self)
165      }
166    }
167
168    impl<'de> Deserialize<'de> for $lang {
169      fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
170      where
171        D: Deserializer<'de>,
172      {
173        let vis = AliasVisitor {
174          aliases: Self::ALIAS,
175        };
176        deserializer.deserialize_str(vis)?;
177        Ok($lang)
178      }
179    }
180
181    impl From<$lang> for SupportLang {
182      fn from(_: $lang) -> Self {
183        Self::$lang
184      }
185    }
186  };
187}
188/// Generates as convenience conversions between the lang types
189/// and `SupportedType`.
190macro_rules! impl_aliases {
191  ($($lang:ident => $as:expr),* $(,)?) => {
192    $(impl_alias!($lang => $as);)*
193    const fn alias(lang: SupportLang) -> &'static [&'static str] {
194      match lang {
195        $(SupportLang::$lang => $lang::ALIAS),*
196      }
197    }
198  };
199}
200
201/* Customized Language with expando_char / pre_process_pattern */
202// https://en.cppreference.com/w/cpp/language/identifiers
203impl_lang_expando!(C, language_c, '𐀀');
204impl_lang_expando!(Cpp, language_cpp, '𐀀');
205// https://docs.microsoft.com/en-us/dotnet/csharp/language-reference/language-specification/lexical-structure#643-identifiers
206// all letter number is accepted
207// https://www.compart.com/en/unicode/category/Nl
208impl_lang_expando!(CSharp, language_c_sharp, 'µ');
209// https://www.w3.org/TR/CSS21/grammar.html#scanner
210impl_lang_expando!(Css, language_css, '_');
211// https://github.com/elixir-lang/tree-sitter-elixir/blob/a2861e88a730287a60c11ea9299c033c7d076e30/grammar.js#L245
212impl_lang_expando!(Elixir, language_elixir, 'µ');
213// we can use any Unicode code point categorized as "Letter"
214// https://go.dev/ref/spec#letter
215impl_lang_expando!(Go, language_go, 'µ');
216// GHC supports Unicode syntax per
217// https://ghc.gitlab.haskell.org/ghc/doc/users_guide/exts/unicode_syntax.html
218// and the tree-sitter-haskell grammar parses it too.
219impl_lang_expando!(Haskell, language_haskell, 'µ');
220// https://developer.hashicorp.com/terraform/language/syntax/configuration#identifiers
221impl_lang_expando!(Hcl, language_hcl, 'µ');
222// https://github.com/fwcd/tree-sitter-kotlin/pull/93
223impl_lang_expando!(Kotlin, language_kotlin, 'µ');
224// Nix uses $ for string interpolation (e.g., "${pkgs.hello}")
225impl_lang_expando!(Nix, language_nix, '_');
226// PHP accepts unicode to be used as some name not var name though
227impl_lang_expando!(Php, language_php, 'µ');
228// we can use any char in unicode range [:XID_Start:]
229// https://docs.python.org/3/reference/lexical_analysis.html#identifiers
230// see also [PEP 3131](https://peps.python.org/pep-3131/) for further details.
231impl_lang_expando!(Python, language_python, 'µ');
232// https://github.com/tree-sitter/tree-sitter-ruby/blob/f257f3f57833d584050336921773738a3fd8ca22/grammar.js#L30C26-L30C78
233impl_lang_expando!(Ruby, language_ruby, 'µ');
234// we can use any char in unicode range [:XID_Start:]
235// https://doc.rust-lang.org/reference/identifiers.html
236impl_lang_expando!(Rust, language_rust, 'µ');
237//https://docs.swift.org/swift-book/documentation/the-swift-programming-language/lexicalstructure/#Identifiers
238impl_lang_expando!(Swift, language_swift, 'µ');
239
240// Stub Language without preprocessing
241// Language Name, tree-sitter-name, alias, extension
242impl_lang!(Bash, language_bash);
243impl_lang!(Java, language_java);
244impl_lang!(JavaScript, language_javascript);
245impl_lang!(Json, language_json);
246impl_lang!(Lua, language_lua);
247impl_lang!(Markdown, language_markdown);
248impl_lang!(Scala, language_scala);
249impl_lang!(Solidity, language_solidity);
250impl_lang!(Tsx, language_tsx);
251impl_lang!(TypeScript, language_typescript);
252impl_lang!(Dart, language_dart);
253impl_lang!(Yaml, language_yaml);
254// See ripgrep for extensions
255// https://github.com/BurntSushi/ripgrep/blob/master/crates/ignore/src/default_types.rs
256
257/// Represents all built-in languages.
258#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Hash)]
259pub enum SupportLang {
260  Bash,
261  C,
262  Cpp,
263  CSharp,
264  Css,
265  Dart,
266  Go,
267  Elixir,
268  Haskell,
269  Hcl,
270  Html,
271  Java,
272  JavaScript,
273  Json,
274  Kotlin,
275  Lua,
276  Markdown,
277  Nix,
278  Php,
279  Python,
280  Ruby,
281  Rust,
282  Scala,
283  Solidity,
284  Swift,
285  Tsx,
286  TypeScript,
287  Yaml,
288}
289
290impl SupportLang {
291  pub const fn all_langs() -> &'static [SupportLang] {
292    use SupportLang::*;
293    &[
294      Bash, C, Cpp, CSharp, Css, Dart, Elixir, Go, Haskell, Hcl, Html, Java, JavaScript, Json,
295      Kotlin, Lua, Markdown, Nix, Php, Python, Ruby, Rust, Scala, Solidity, Swift, Tsx, TypeScript,
296      Yaml,
297    ]
298  }
299
300  pub fn file_types(&self) -> Types {
301    file_types(*self)
302  }
303}
304
305impl fmt::Display for SupportLang {
306  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
307    write!(f, "{self:?}")
308  }
309}
310
311#[derive(Debug)]
312pub enum SupportLangErr {
313  LanguageNotSupported(String),
314}
315
316impl Display for SupportLangErr {
317  fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), std::fmt::Error> {
318    use SupportLangErr::*;
319    match self {
320      LanguageNotSupported(lang) => write!(f, "{lang} is not supported!"),
321    }
322  }
323}
324
325impl std::error::Error for SupportLangErr {}
326
327impl<'de> Deserialize<'de> for SupportLang {
328  fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
329  where
330    D: Deserializer<'de>,
331  {
332    deserializer.deserialize_str(SupportLangVisitor)
333  }
334}
335
336struct SupportLangVisitor;
337
338impl Visitor<'_> for SupportLangVisitor {
339  type Value = SupportLang;
340
341  fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
342    f.write_str("SupportLang")
343  }
344
345  fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
346  where
347    E: de::Error,
348  {
349    v.parse().map_err(de::Error::custom)
350  }
351}
352struct AliasVisitor {
353  aliases: &'static [&'static str],
354}
355
356impl Visitor<'_> for AliasVisitor {
357  type Value = &'static str;
358
359  fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
360    write!(f, "one of {:?}", self.aliases)
361  }
362
363  fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
364  where
365    E: de::Error,
366  {
367    self
368      .aliases
369      .iter()
370      .copied()
371      .find(|&a| v.eq_ignore_ascii_case(a))
372      .ok_or_else(|| de::Error::invalid_value(de::Unexpected::Str(v), &self))
373  }
374}
375
376impl_aliases! {
377  Bash => &["bash"],
378  C => &["c"],
379  Cpp => &["cc", "c++", "cpp", "cxx"],
380  CSharp => &["cs", "csharp"],
381  Css => &["css"],
382  Dart => &["dart"],
383  Elixir => &["ex", "elixir"],
384  Go => &["go", "golang"],
385  Haskell => &["hs", "haskell"],
386  Hcl => &["hcl"],
387  Html => &["html"],
388  Java => &["java"],
389  JavaScript => &["javascript", "js", "jsx"],
390  Json => &["json"],
391  Kotlin => &["kotlin", "kt"],
392  Lua => &["lua"],
393  Markdown => &["markdown", "md"],
394  Nix => &["nix"],
395  Php => &["php"],
396  Python => &["py", "python"],
397  Ruby => &["rb", "ruby"],
398  Rust => &["rs", "rust"],
399  Scala => &["scala"],
400  Solidity => &["sol", "solidity"],
401  Swift => &["swift"],
402  TypeScript => &["ts", "typescript"],
403  Tsx => &["tsx"],
404  Yaml => &["yaml", "yml"],
405}
406
407/// Implements the language names and aliases.
408impl FromStr for SupportLang {
409  type Err = SupportLangErr;
410  fn from_str(s: &str) -> Result<Self, Self::Err> {
411    for &lang in Self::all_langs() {
412      for moniker in alias(lang) {
413        if s.eq_ignore_ascii_case(moniker) {
414          return Ok(lang);
415        }
416      }
417    }
418    Err(SupportLangErr::LanguageNotSupported(s.to_string()))
419  }
420}
421
422macro_rules! execute_lang_method {
423  ($me: path, $method: ident, $($pname:tt),*) => {
424    use SupportLang as S;
425    match $me {
426      S::Bash => Bash.$method($($pname,)*),
427      S::C => C.$method($($pname,)*),
428      S::Cpp => Cpp.$method($($pname,)*),
429      S::CSharp => CSharp.$method($($pname,)*),
430      S::Css => Css.$method($($pname,)*),
431      S::Dart => Dart.$method($($pname,)*),
432      S::Elixir => Elixir.$method($($pname,)*),
433      S::Go => Go.$method($($pname,)*),
434      S::Haskell => Haskell.$method($($pname,)*),
435      S::Hcl => Hcl.$method($($pname,)*),
436      S::Html => Html.$method($($pname,)*),
437      S::Java => Java.$method($($pname,)*),
438      S::JavaScript => JavaScript.$method($($pname,)*),
439      S::Json => Json.$method($($pname,)*),
440      S::Kotlin => Kotlin.$method($($pname,)*),
441      S::Lua => Lua.$method($($pname,)*),
442      S::Markdown => Markdown.$method($($pname,)*),
443      S::Nix => Nix.$method($($pname,)*),
444      S::Php => Php.$method($($pname,)*),
445      S::Python => Python.$method($($pname,)*),
446      S::Ruby => Ruby.$method($($pname,)*),
447      S::Rust => Rust.$method($($pname,)*),
448      S::Scala => Scala.$method($($pname,)*),
449      S::Solidity => Solidity.$method($($pname,)*),
450      S::Swift => Swift.$method($($pname,)*),
451      S::Tsx => Tsx.$method($($pname,)*),
452      S::TypeScript => TypeScript.$method($($pname,)*),
453      S::Yaml => Yaml.$method($($pname,)*),
454    }
455  }
456}
457
458macro_rules! impl_lang_method {
459  ($method: ident, ($($pname:tt: $ptype:ty),*) => $return_type: ty) => {
460    #[inline]
461    fn $method(&self, $($pname: $ptype),*) -> $return_type {
462      execute_lang_method!{ self, $method, $($pname),* }
463    }
464  };
465}
466impl Language for SupportLang {
467  impl_lang_method!(kind_to_id, (kind: &str) => u16);
468  impl_lang_method!(field_to_id, (field: &str) => Option<u16>);
469  impl_lang_method!(meta_var_char, () => char);
470  impl_lang_method!(expando_char, () => char);
471  impl_lang_method!(extract_meta_var, (source: &str) => Option<MetaVariable>);
472  impl_lang_method!(build_pattern, (builder: &PatternBuilder) => Result<Pattern, PatternError>);
473  fn pre_process_pattern<'q>(&self, query: &'q str) -> Cow<'q, str> {
474    execute_lang_method! { self, pre_process_pattern, query }
475  }
476  fn from_path<P: AsRef<Path>>(path: P) -> Option<Self> {
477    from_extension(path.as_ref())
478  }
479}
480
481impl LanguageExt for SupportLang {
482  impl_lang_method!(get_ts_language, () => TSLanguage);
483  impl_lang_method!(injectable_languages, () => Option<&'static [&'static str]>);
484  fn extract_injections<L: LanguageExt>(
485    &self,
486    root: Node<StrDoc<L>>,
487  ) -> Vec<(String, Vec<TSRange>)> {
488    match self {
489      SupportLang::Html => Html.extract_injections(root),
490      _ => Vec::new(),
491    }
492  }
493}
494
495fn extensions(lang: SupportLang) -> &'static [&'static str] {
496  use SupportLang::*;
497  match lang {
498    Bash => &[
499      "bash", "bats", "cgi", "command", "env", "fcgi", "ksh", "sh", "tmux", "tool", "zsh",
500    ],
501    C => &["c", "h"],
502    Cpp => &["cc", "hpp", "cpp", "c++", "hh", "cxx", "cu", "ino"],
503    CSharp => &["cs"],
504    Css => &["css", "scss"],
505    Dart => &["dart"],
506    Elixir => &["ex", "exs"],
507    Go => &["go"],
508    Haskell => &["hs"],
509    Hcl => &["hcl", "nomad", "tf", "tfvars", "workflow"],
510    Html => &["html", "htm", "xhtml"],
511    Java => &["java"],
512    JavaScript => &["cjs", "js", "mjs", "jsx"],
513    Json => &["json"],
514    Kotlin => &["kt", "ktm", "kts"],
515    Lua => &["lua"],
516    Markdown => &["markdown", "md"],
517    Nix => &["nix"],
518    Php => &["php"],
519    Python => &["py", "py3", "pyi", "bzl", "bazel"],
520    Ruby => &["rb", "rbw", "gemspec"],
521    Rust => &["rs"],
522    Scala => &["scala", "sc", "sbt"],
523    Solidity => &["sol"],
524    Swift => &["swift"],
525    TypeScript => &["ts", "cts", "mts"],
526    Tsx => &["tsx"],
527    Yaml => &["yaml", "yml"],
528  }
529}
530
531/// Guess which programming language a file is written in
532/// Adapt from `<https://github.com/Wilfred/difftastic/blob/master/src/parse/guess_language.rs>`
533/// N.B do not confuse it with `FromStr` trait. This function is to guess language from file extension.
534fn from_extension(path: &Path) -> Option<SupportLang> {
535  let ext = path.extension()?.to_str()?;
536  SupportLang::all_langs()
537    .iter()
538    .copied()
539    .find(|&l| extensions(l).contains(&ext))
540}
541
542fn add_custom_file_type<'b>(
543  builder: &'b mut TypesBuilder,
544  file_type: &str,
545  suffix_list: &[&str],
546) -> &'b mut TypesBuilder {
547  for suffix in suffix_list {
548    let glob = format!("*.{suffix}");
549    builder
550      .add(file_type, &glob)
551      .expect("file pattern must compile");
552  }
553  builder.select(file_type)
554}
555
556fn file_types(lang: SupportLang) -> Types {
557  let mut builder = TypesBuilder::new();
558  let exts = extensions(lang);
559  let lang_name = lang.to_string();
560  add_custom_file_type(&mut builder, &lang_name, exts);
561  builder.build().expect("file type must be valid")
562}
563
564pub fn config_file_type() -> Types {
565  let mut builder = TypesBuilder::new();
566  let builder = add_custom_file_type(&mut builder, "yml", &["yml", "yaml"]);
567  builder.build().expect("yaml type must be valid")
568}
569
570#[cfg(test)]
571mod test {
572  use super::*;
573  use ast_grep_core::{Pattern, matcher::MatcherExt};
574
575  pub fn test_match_lang(query: &str, source: &str, lang: impl LanguageExt) {
576    let cand = lang.ast_grep(source);
577    let pattern = Pattern::new(query, lang);
578    assert!(
579      pattern.find_node(cand.root()).is_some(),
580      "goal: {pattern:?}, candidate: {}",
581      cand.root().get_inner_node().to_sexp(),
582    );
583  }
584
585  pub fn test_non_match_lang(query: &str, source: &str, lang: impl LanguageExt) {
586    let cand = lang.ast_grep(source);
587    let pattern = Pattern::new(query, lang);
588    assert!(
589      pattern.find_node(cand.root()).is_none(),
590      "goal: {pattern:?}, candidate: {}",
591      cand.root().get_inner_node().to_sexp(),
592    );
593  }
594
595  pub fn test_replace_lang(
596    src: &str,
597    pattern: &str,
598    replacer: &str,
599    lang: impl LanguageExt,
600  ) -> String {
601    let mut source = lang.ast_grep(src);
602    assert!(
603      source
604        .replace(pattern, replacer)
605        .expect("should parse successfully")
606    );
607    source.generate()
608  }
609
610  #[test]
611  fn test_js_string() {
612    test_match_lang("'a'", "'a'", JavaScript);
613    test_match_lang("\"\"", "\"\"", JavaScript);
614    test_match_lang("''", "''", JavaScript);
615  }
616
617  #[test]
618  fn test_guess_by_extension() {
619    let path = Path::new("foo.rs");
620    assert_eq!(from_extension(path), Some(SupportLang::Rust));
621    let path = Path::new("README.md");
622    assert_eq!(from_extension(path), Some(SupportLang::Markdown));
623    let path = Path::new("README.markdown");
624    assert_eq!(from_extension(path), Some(SupportLang::Markdown));
625  }
626
627  // TODO: add test for file_types
628}
ast_grep_language/lib.rs

ast_grep_language/
lib.rs
