use std::{cell::RefCell, collections::HashMap};
use tree_sitter::{Language, Parser};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum SupportedLanguage {
Rust,
Python,
TypeScript,
JavaScript,
Go,
Java,
Cpp,
C,
}
impl SupportedLanguage {
#[must_use]
pub fn from_language_name(name: &str) -> Option<Self> {
match name {
"Rust" => Some(Self::Rust),
"Python" => Some(Self::Python),
"TypeScript" | "TSX" => Some(Self::TypeScript),
"JavaScript" | "JSX" => Some(Self::JavaScript),
"Go" | "Golang" => Some(Self::Go),
"Java" => Some(Self::Java),
"C++" | "C++ Header File" => Some(Self::Cpp),
"C" | "C Header" | "C Header File" => Some(Self::C),
_ => None,
}
}
#[must_use]
pub fn from_extension(extension: &str) -> Option<Self> {
match extension {
"rs" => Some(Self::Rust),
"py" | "pyi" => Some(Self::Python),
"ts" | "tsx" => Some(Self::TypeScript),
"js" | "jsx" | "mjs" | "cjs" => Some(Self::JavaScript),
"go" => Some(Self::Go),
"java" => Some(Self::Java),
"cpp" | "cc" | "cxx" | "hpp" | "hh" => Some(Self::Cpp),
"c" | "h" => Some(Self::C),
_ => None,
}
}
#[must_use]
fn tree_sitter_language(self) -> Language {
match self {
Self::Rust => tree_sitter_rust::LANGUAGE.into(),
Self::Python => tree_sitter_python::LANGUAGE.into(),
Self::TypeScript => {
tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into()
}
Self::JavaScript => tree_sitter_javascript::LANGUAGE.into(),
Self::Go => tree_sitter_go::LANGUAGE.into(),
Self::Java => tree_sitter_java::LANGUAGE.into(),
Self::Cpp => tree_sitter_cpp::LANGUAGE.into(),
Self::C => tree_sitter_c::LANGUAGE.into(),
}
}
#[must_use]
pub const fn all() -> &'static [Self] {
&[
Self::Rust,
Self::Python,
Self::TypeScript,
Self::JavaScript,
Self::Go,
Self::Java,
Self::Cpp,
Self::C,
]
}
}
pub fn new_parser(language: SupportedLanguage) -> anyhow::Result<Parser> {
let mut parser = Parser::new();
parser
.set_language(&language.tree_sitter_language())
.map_err(|error| {
anyhow::anyhow!(
"failed to set Tree-sitter language for {language:?}: {error}",
)
})?;
Ok(parser)
}
const CACHE_AFTER_BYTES: usize = 2 * 1024 * 1024;
#[cfg(test)]
mod built {
thread_local! {
static PARSERS: std::cell::Cell<usize> =
const { std::cell::Cell::new(0) };
}
pub(super) fn count() -> usize {
PARSERS.with(std::cell::Cell::get)
}
pub(super) fn reset() {
PARSERS.with(|count| count.set(0));
}
pub(super) fn record_one_built() {
PARSERS.with(|count| count.set(count.get() + 1));
}
}
#[cfg(test)]
fn parsers_built() -> usize {
built::count()
}
#[cfg(test)]
fn reset_parsers_built() {
built::reset();
}
thread_local! {
static PARSERS: RefCell<HashMap<SupportedLanguage, Parser>> =
RefCell::new(HashMap::new());
}
pub fn with_parser<R>(
language: SupportedLanguage,
source_bytes: usize,
use_parser: impl FnOnce(&mut Parser) -> anyhow::Result<R>,
) -> anyhow::Result<R> {
let setup_failure = std::cell::Cell::new(None);
let parsed = PARSERS.with(|parsers| {
let mut parsers = parsers.borrow_mut();
let mut parser = if let Some(parser) = parsers.remove(&language) {
parser
} else {
#[cfg(test)]
built::record_one_built();
match new_parser(language) {
Ok(parser) => parser,
Err(problem) => {
setup_failure.set(Some(problem));
return None;
}
}
};
let outcome = use_parser(&mut parser);
if source_bytes <= CACHE_AFTER_BYTES {
parsers.insert(language, parser);
}
outcome.ok()
});
parsed.ok_or_else(|| {
setup_failure
.into_inner()
.unwrap_or_else(|| anyhow::anyhow!("parsing {language:?} failed"))
})
}
#[cfg(test)]
mod tests {
use super::*;
fn cached_parser_count() -> usize {
PARSERS.with(|parsers| parsers.borrow().len())
}
#[test]
fn a_reused_parser_answers_the_same_as_a_fresh_one() {
let source = b"use std::io;\n\npub fn run() -> u32 { 7 }\n";
let first =
with_parser(SupportedLanguage::Rust, source.len(), |parser| {
Ok(parser.parse(source.as_slice(), None))
})
.expect("first parse")
.expect("a tree");
let second =
with_parser(SupportedLanguage::Rust, source.len(), |parser| {
Ok(parser.parse(source.as_slice(), None))
})
.expect("second parse")
.expect("a tree");
assert_eq!(first.root_node().to_sexp(), second.root_node().to_sexp());
}
#[test]
fn a_parser_is_kept_between_calls_and_not_for_every_language() {
let source = b"fn main() {}\n";
reset_parsers_built();
for _ in 0..5 {
with_parser(SupportedLanguage::Rust, source.len(), |parser| {
Ok(parser.parse(source.as_slice(), None))
})
.expect("rust parse")
.expect("a tree");
}
assert_eq!(
parsers_built(),
1,
"five parses of one language must build one parser, not five"
);
assert_eq!(cached_parser_count(), 1, "the parser was not kept");
with_parser(SupportedLanguage::Python, source.len(), |parser| {
Ok(parser.parse(source.as_slice(), None))
})
.expect("python parse")
.expect("a tree");
assert_eq!(
cached_parser_count(),
2,
"a second language gets its own parser, because `set_language` is \
the expensive part and a parser is only worth keeping for the \
language it was configured with"
);
}
#[test]
fn an_oversized_source_is_parsed_but_not_kept() {
let source = b"fn main() {}\n";
with_parser(SupportedLanguage::Go, source.len(), |parser| {
Ok(parser.parse(source.as_slice(), None))
})
.expect("first parse")
.expect("a tree");
assert_eq!(cached_parser_count(), 1);
with_parser(SupportedLanguage::Go, CACHE_AFTER_BYTES + 1, |parser| {
Ok(parser.parse(source.as_slice(), None))
})
.expect("the oversized parse still succeeds")
.expect("a tree");
assert_eq!(
cached_parser_count(),
0,
"a parser that parsed something this large must be released rather \
than kept"
);
}
#[test]
fn a_parse_that_fails_leaves_the_cache_usable() {
let source = b"fn main() {}\n";
let failed = with_parser(SupportedLanguage::Java, source.len(), |_| {
Err::<tree_sitter::Tree, _>(anyhow::anyhow!("deliberate failure"))
});
assert!(failed.is_err(), "the error must reach the caller");
let recovered =
with_parser(SupportedLanguage::Java, source.len(), |parser| {
Ok(parser.parse(source.as_slice(), None))
})
.expect("a later parse still works")
.expect("a tree");
assert!(!recovered.root_node().has_error());
}
#[test]
fn creates_parser_for_all_languages() {
for language in SupportedLanguage::all() {
let parser = new_parser(*language);
assert!(parser.is_ok(), "failed to create parser for {language:?}");
}
}
#[test]
fn resolves_language_names() {
assert_eq!(
SupportedLanguage::from_language_name("Rust"),
Some(SupportedLanguage::Rust)
);
assert_eq!(
SupportedLanguage::from_language_name("Python"),
Some(SupportedLanguage::Python)
);
assert_eq!(
SupportedLanguage::from_language_name("TypeScript"),
Some(SupportedLanguage::TypeScript)
);
assert_eq!(
SupportedLanguage::from_language_name("JavaScript"),
Some(SupportedLanguage::JavaScript)
);
assert_eq!(
SupportedLanguage::from_language_name("Go"),
Some(SupportedLanguage::Go)
);
assert_eq!(
SupportedLanguage::from_language_name("Golang"),
Some(SupportedLanguage::Go)
);
assert_eq!(
SupportedLanguage::from_language_name("Java"),
Some(SupportedLanguage::Java)
);
assert_eq!(
SupportedLanguage::from_language_name("C++"),
Some(SupportedLanguage::Cpp)
);
assert_eq!(
SupportedLanguage::from_language_name("C"),
Some(SupportedLanguage::C)
);
}
#[test]
fn returns_none_for_unsupported_languages() {
assert_eq!(SupportedLanguage::from_language_name("TOML"), None);
assert_eq!(SupportedLanguage::from_language_name("YAML"), None);
assert_eq!(SupportedLanguage::from_language_name("Markdown"), None);
}
#[test]
fn parses_simple_rust_source() {
let mut parser = new_parser(SupportedLanguage::Rust).unwrap();
let tree = parser.parse("fn main() {}", None).unwrap();
assert!(!tree.root_node().has_error());
}
#[test]
fn parses_simple_python_source() {
let mut parser = new_parser(SupportedLanguage::Python).unwrap();
let tree = parser
.parse("def greet(name):\n print(name)\n", None)
.unwrap();
assert!(!tree.root_node().has_error());
}
}