use std::{
collections::{BTreeMap, BTreeSet},
fs,
path::Path,
};
use serde_json::Value;
use super::{
compiled_grammar_closure, decode_bundle_grammars, grammar_closure,
grammar_ir::encode_compiled_grammar,
};
use crate::grammars;
#[test]
fn dependency_walk_follows_rule_local_repository_includes() {
let sources = [
r##"{"scopeName":"source.host","patterns":[{"include":"#container"}],"repository":{
"child":{"include":"source.shadowed"},
"container":{"patterns":[{"include":"#child"}],"repository":{
"child":{"include":"source.embedded"}
}}
}}"##,
r#"{"scopeName":"source.embedded","patterns":[{"match":"x"}]}"#,
r#"{"scopeName":"source.shadowed","patterns":[{"match":"y"}]}"#,
];
let compiled: Vec<_> = sources
.iter()
.enumerate()
.map(|(index, source)| {
super::grammar::load_dev_grammar_from_str(super::state::GrammarId(index as u16), source)
.unwrap()
})
.collect();
assert_eq!(
grammar_closure::dependency_closure(&compiled, 0),
vec![0, 1]
);
}
#[test]
fn compiled_dependency_walk_matches_representative_json_contracts() {
let bundle = grammars::embedded_bundle();
let sources = reference_grammars();
for language_id in ["asciidoc", "markdown", "mdx", "php", "wikitext", "yaml"] {
let language = bundle
.languages
.iter()
.find(|language| language.canonical == language_id)
.expect("representative language is bundled");
let actual = compiled_grammar_closure(bundle, &language.scope_name)
.unwrap_or_else(|error| panic!("{}: {error}", language.canonical))
.into_iter()
.map(|grammar| grammar.scope_name)
.collect::<Vec<_>>();
let expected = reference_closure(bundle, &language.scope_name, &sources);
assert_eq!(actual, expected, "{}", language.canonical);
}
}
#[test]
fn recorded_bundle_closures_match_the_dependency_walk() {
let bundle = grammars::embedded_bundle();
let compiled = decode_bundle_grammars(bundle);
assert_eq!(bundle.grammar_graphs.len(), compiled.len());
let walked = bundle
.grammar_graphs
.iter()
.flat_map(|graph| &graph.closure)
.filter(|member| member.traits.repository_contexts)
.map(|member| member.blob as usize)
.collect::<BTreeSet<_>>();
for (root, recorded) in bundle.grammar_graphs.iter().enumerate() {
let members = grammar_closure::dependency_closure(&compiled, root);
let traits = grammar_closure::closure_member_traits(&compiled, &members);
let expected = grammars::bundle::GrammarGraph {
closure: members
.into_iter()
.zip(traits)
.map(|(blob, traits)| grammars::bundle::ClosureMember {
blob: blob as u32,
traits,
})
.collect(),
repository_walk_skeleton: walked.contains(&root).then(|| {
encode_compiled_grammar(&grammar_closure::repository_walk_skeleton(&compiled[root]))
.unwrap()
.into()
}),
top_level_availability: grammar_closure::top_level_availability_chain(&compiled[root]),
};
assert_eq!(
recorded, &expected,
"{}: regenerate the bundle with syntaxmate-bundle",
bundle.grammar_blobs[root].language
);
}
}
fn reference_grammars() -> BTreeMap<String, Value> {
let directory = Path::new(env!("CARGO_MANIFEST_DIR")).join("assets/grammars/languages");
let mut entries = fs::read_dir(directory)
.unwrap()
.collect::<Result<Vec<_>, _>>()
.unwrap();
entries.sort_by_key(|entry| entry.file_name());
let mut sources = BTreeMap::new();
for entry in entries {
let path = entry.path();
if path.extension().and_then(|extension| extension.to_str()) != Some("json") {
continue;
}
let grammar = serde_json::from_slice::<Value>(&fs::read(path).unwrap()).unwrap();
let scope = grammar
.get("scopeName")
.and_then(Value::as_str)
.expect("bundled source grammar has scopeName")
.to_owned();
sources.insert(scope, grammar);
}
sources
}
fn reference_closure(
bundle: &grammars::bundle::Bundle,
root_scope: &str,
sources: &BTreeMap<String, Value>,
) -> Vec<String> {
let mut pending = vec![(root_scope.to_owned(), None::<String>)];
let mut selected = BTreeSet::new();
let mut inspected = BTreeSet::new();
while let Some((scope, repository)) = pending.pop() {
let Some((index, blob)) = bundle
.grammar_blobs
.iter()
.enumerate()
.find(|(_, blob)| blob.scope_name == scope)
else {
continue;
};
selected.insert(index);
if !inspected.insert((index, repository.clone())) {
continue;
}
let json = sources
.get(&scope)
.unwrap_or_else(|| panic!("missing source grammar for {scope}"));
collect_external_scopes(
json,
&blob.scope_name,
root_scope,
repository.as_deref(),
bundle,
&mut pending,
);
}
selected
.into_iter()
.map(|index| bundle.grammar_blobs[index].scope_name.clone())
.collect()
}
fn collect_external_scopes(
grammar: &Value,
grammar_scope: &str,
root_scope: &str,
repository_rule: Option<&str>,
bundle: &grammars::bundle::Bundle,
pending: &mut Vec<(String, Option<String>)>,
) {
let Some(object) = grammar.as_object() else {
return;
};
let repository: BTreeMap<_, _> = object
.get("repository")
.and_then(Value::as_object)
.into_iter()
.flatten()
.map(|(name, rule)| (name.as_str(), rule))
.collect();
let mut visited_local = BTreeSet::new();
if let Some(name) = repository_rule {
if let Some(rule) = repository.get(name) {
collect_rule_dependencies(
rule,
grammar_scope,
root_scope,
&repository,
bundle,
pending,
&mut visited_local,
);
}
} else {
if let Some(patterns) = object.get("patterns") {
collect_pattern_dependencies(
patterns,
grammar_scope,
root_scope,
&repository,
bundle,
pending,
&mut visited_local,
);
}
if grammar_scope == root_scope
&& let Some(injections) = object.get("injections").and_then(Value::as_object)
{
for rule in injections.values() {
collect_rule_dependencies(
rule,
grammar_scope,
root_scope,
&repository,
bundle,
pending,
&mut visited_local,
);
}
}
}
}
fn collect_pattern_dependencies<'a>(
patterns: &'a Value,
grammar_scope: &str,
root_scope: &str,
repository: &BTreeMap<&'a str, &'a Value>,
bundle: &grammars::bundle::Bundle,
pending: &mut Vec<(String, Option<String>)>,
visited_local: &mut BTreeSet<usize>,
) {
let Some(patterns) = patterns.as_array() else {
return;
};
for rule in patterns {
collect_rule_dependencies(
rule,
grammar_scope,
root_scope,
repository,
bundle,
pending,
visited_local,
);
}
}
fn collect_rule_dependencies<'a>(
rule: &'a Value,
grammar_scope: &str,
root_scope: &str,
repository: &BTreeMap<&'a str, &'a Value>,
bundle: &grammars::bundle::Bundle,
pending: &mut Vec<(String, Option<String>)>,
visited_local: &mut BTreeSet<usize>,
) {
if !visited_local.insert(rule as *const Value as usize) {
return;
}
let Some(rule) = rule.as_object() else {
return;
};
let mut repository = repository.clone();
if let Some(local) = rule.get("repository").and_then(Value::as_object) {
repository.extend(local.iter().map(|(name, rule)| (name.as_str(), rule)));
}
if let Some(include) = rule.get("include").and_then(Value::as_str) {
if let Some(name) = include.strip_prefix('#') {
if let Some(local) = repository.get(name) {
collect_rule_dependencies(
local,
grammar_scope,
root_scope,
&repository,
bundle,
pending,
visited_local,
);
}
} else if include == "$self" {
pending.push((grammar_scope.to_owned(), None));
} else if include == "$base" {
pending.push((root_scope.to_owned(), None));
} else {
let (scope, repository) = include
.split_once('#')
.map_or((include, None), |(scope, repository)| {
(scope, Some(repository.to_owned()))
});
if bundle.grammar_blob_for_scope(scope).is_some() {
pending.push((scope.to_owned(), repository));
}
}
return;
}
if let Some(patterns) = rule.get("patterns") {
collect_pattern_dependencies(
patterns,
grammar_scope,
root_scope,
&repository,
bundle,
pending,
visited_local,
);
}
}