use std::collections::{BTreeMap, BTreeSet};
use std::fmt::Write as _;
use crate::roles::{DeclaredRow, LanguageTable, RowView};
use crate::test_support::{
control_flow_corpus_present, control_flow_sources, for_each_node_with_sibling_cache,
for_each_node_with_sibling_cache_tolerating_errors,
};
use crate::{Ancestors, Checker, LANG, Node, with_code_tag};
static CONTROL_FLOW_ABSENT: std::sync::Once = std::sync::Once::new();
struct Source {
name: String,
bytes: Vec<u8>,
}
const BYTE_GATE_FIXTURES: &[ByteGateFixture] = &[
ByteGateFixture {
table: "Java",
name: "gate/java-records",
source: b"record A(int a) { A(long a) { this((int) a); } }\nrecord B(int b) { B(int b) { this.b = b; } }\n",
},
ByteGateFixture {
table: "Groovy",
name: "gate/groovy-records",
source: b"record A(int a) { A(long a) { this(1) } }\nrecord B(int b) { B(int b) { this.b = b } }\n",
},
ByteGateFixture {
table: "Bash",
name: "gate/bash-tests",
source: b"if [ \"$a\" -eq 1 -a \"$b\" -eq 2 ]; then :; fi\n[ -n \"$a\" -o -n \"$b\" ]\n",
},
];
struct ByteGateFixture {
table: &'static str,
name: &'static str,
source: &'static [u8],
}
fn sources_for(lang: LANG) -> Vec<Source> {
let mut out: Vec<Source> = if let Some(files) = control_flow_sources(lang) {
files
.into_iter()
.map(|(name, bytes)| Source { name, bytes })
.collect()
} else {
CONTROL_FLOW_ABSENT.call_once(|| {
println!(
"control_flow/ not checked out: the census reads the documentation fixtures \
only; run `make worktree-setup` for the control-flow rows"
);
});
Vec::new()
};
out.extend(
super::doc_tests::fixtures()
.into_iter()
.filter(|f| f.lang == lang)
.map(|f| Source {
name: f.name.to_owned(),
bytes: f.source.into_owned(),
}),
);
let table = crate::roles::tables()
.iter()
.find(|t| t.lang == lang)
.map(|t| t.name);
out.extend(
BYTE_GATE_FIXTURES
.iter()
.filter(|fixture| Some(fixture.table) == table)
.map(|fixture| Source {
name: fixture.name.to_owned(),
bytes: fixture.source.to_vec(),
}),
);
out
}
#[test]
fn byte_gate_fixtures_are_keyed_by_a_table() {
let names: BTreeSet<&str> = crate::roles::tables().iter().map(|t| t.name).collect();
let stray: Vec<String> = BYTE_GATE_FIXTURES
.iter()
.filter(|fixture| !names.contains(fixture.table))
.map(|fixture| format!("`{}` is keyed by `{}`", fixture.name, fixture.table))
.collect();
assert_eq!(stray.len(), 0, "fixtures keyed by no table: {stray:#?}");
}
#[derive(Clone, Copy, Debug)]
#[allow(clippy::struct_excessive_bools)]
struct Verdict {
owns: bool,
pays_at_leaf: bool,
pays_at_token: bool,
pays_at_wrapper: bool,
is_callable: bool,
projects: bool,
}
struct Occurrence {
kind: u16,
zero_width: bool,
word: Option<&'static str>,
verdicts: Vec<(usize, Verdict)>,
}
#[derive(Default, Clone, Copy)]
struct KindCount {
seen: usize,
owned: usize,
}
#[derive(Default)]
struct RowTally {
owned: usize,
rejected_owners: usize,
kinds_seen: BTreeMap<u16, KindCount>,
}
struct Census {
lang: LANG,
name: &'static str,
targets_beyond_space_tree: Vec<u16>,
opens_space_fn: for<'a> fn(&Node<'a>, &[u8], Ancestors<'a, '_>) -> crate::SpaceKind,
kind_name_fn: fn(u16) -> &'static str,
rows: Vec<RowView>,
declared: Vec<Option<DeclaredRow>>,
by_kind: BTreeMap<u16, Vec<usize>>,
tallies: Vec<RowTally>,
occurrences: Vec<Occurrence>,
nodes: usize,
}
impl Census {
fn new(table: &LanguageTable) -> Self {
let rows = (table.rows)();
let declared: Vec<Option<DeclaredRow>> =
rows.iter().map(|r| r.declared().copied()).collect();
let mut by_kind: BTreeMap<u16, Vec<usize>> = BTreeMap::new();
for (i, row) in declared.iter().enumerate() {
let Some(row) = row else { continue };
for kind in row.candidate_kinds() {
by_kind.entry(kind).or_default().push(i);
}
}
let tallies = rows.iter().map(|_| RowTally::default()).collect();
Self {
lang: table.lang,
name: table.name,
targets_beyond_space_tree: super::doc_tests::parity::targets_beyond_the_space_tree(
table.name,
),
opens_space_fn: table.opens_space,
kind_name_fn: table.kind_name,
rows,
declared,
by_kind,
tallies,
occurrences: Vec::new(),
nodes: 0,
}
}
fn declared_rows(&self) -> impl Iterator<Item = (usize, &DeclaredRow)> {
self.declared
.iter()
.enumerate()
.filter_map(|(i, r)| r.as_ref().map(|r| (i, r)))
}
fn declared_row(&self, index: usize) -> &DeclaredRow {
self.declared[index]
.as_ref()
.expect("index came from the declared list")
}
fn row_named(&self, role: &str) -> Option<&DeclaredRow> {
assert!(
crate::roles::ROLES.contains(&role),
"`{role}` is not a role; the relation over it would hold vacuously"
);
self.declared_rows()
.map(|(_, r)| r)
.find(|r| r.role == role)
}
fn kind_name(&self, kind: u16) -> &'static str {
(self.kind_name_fn)(kind)
}
fn record<'a>(&mut self, node: &Node<'a>, code: &[u8], ancestors: Ancestors<'a, '_>) {
self.nodes += 1;
let kind = node.kind_id();
let Some(indices) = self.by_kind.get(&kind) else {
return;
};
let word = (self.lang == LANG::Elixir && kind == crate::Elixir::Call as u16)
.then(|| {
crate::lang_helpers::elixir::elixir_call_target_word(
crate::lang_helpers::elixir::elixir_call_target(node, code, ancestors),
)
})
.flatten();
let zero_width = node.start_byte() == node.end_byte();
let mut verdicts = Vec::with_capacity(indices.len());
for &i in indices {
let row = *self.declared_row(i);
let v = Verdict {
owns: (row.owns_kind)(kind, node, code, ancestors),
pays_at_leaf: (row.pays_at_leaf_kind)(kind, node, code, ancestors),
pays_at_token: (row.pays_at_token_kind)(kind, node, code, ancestors),
pays_at_wrapper: (row.pays_at_wrapper)(node, code, ancestors),
is_callable: (row.is_callable_kind)(kind, node, code, ancestors),
projects: (row.project)(node).is_some(),
};
let tally = &mut self.tallies[i];
let count = tally.kinds_seen.entry(kind).or_default();
count.seen += 1;
if v.owns {
tally.owned += 1;
count.owned += 1;
} else if row.owners.contains(&kind) {
tally.rejected_owners += 1;
}
verdicts.push((i, v));
}
self.occurrences.push(Occurrence {
kind,
zero_width,
word,
verdicts,
});
}
}
fn census_of(table: &LanguageTable) -> Census {
let mut census = Census::new(table);
let lang = table.lang;
let sources = sources_for(lang);
assert!(
!sources.is_empty(),
"{lang:?}: no source routed; the table is asked nothing"
);
for source in &sources {
let code = source.bytes.as_slice();
with_code_tag!(lang, |C| {
for_each_node_with_sibling_cache::<C>(code, |node, ancestors| {
census.record(node, code, ancestors);
});
});
}
assert!(
!census.occurrences.is_empty(),
"{lang:?}: {} sources, {} nodes, no candidate of any row",
sources.len(),
census.nodes
);
census
}
fn every_census() -> Vec<Census> {
let out: Vec<Census> = crate::roles::tables()
.iter()
.filter(|table| table.lang.is_enabled())
.map(census_of)
.collect();
assert!(
!out.is_empty(),
"at least one language feature must be enabled for this test to mean anything"
);
out
}
#[test]
fn macro_invariants_hold_on_every_candidate() {
let mut asked = 0;
let mut languages = 0;
let mut zero_width = 0;
for census in every_census() {
languages += 1;
for occ in &census.occurrences {
if occ.zero_width {
zero_width += 1;
continue;
}
for (i, v) in &occ.verdicts {
let row = census.declared_row(*i);
let at = format!(
"{} {} on `{}` (kind {})",
census.name,
row.role,
census.kind_name(occ.kind),
occ.kind
);
asked += 1;
assert!(
!v.owns || (row.lists_kind)(occ.kind),
"{at}: owns but not listed"
);
assert!(
!v.pays_at_token || row.payers.contains(&occ.kind),
"{at}: pays at the token but is no declared payer"
);
assert!(
!v.pays_at_leaf || row.leaf_kinds.contains(&occ.kind),
"{at}: pays at the leaf but is no declared leaf kind"
);
assert!(
!v.is_callable || (v.owns && row.callable.contains(&occ.kind)),
"{at}: callable but not an owning CALLABLE kind"
);
assert_eq!(
v.pays_at_wrapper,
v.owns && !row.pays_never,
"{at}: pays_at_wrapper is owns ∧ ¬PAYS_NEVER"
);
assert!(
!v.pays_at_token || v.pays_at_leaf,
"{at}: pays at the token but not at the leaf"
);
assert!(
!v.owns || !row.holds_payer.contains(&occ.kind) || v.projects,
"{at}: a holds_payer owner that owns but projects to nothing"
);
}
}
}
if zero_width > 0 {
println!("zero-width candidates skipped: {zero_width}");
}
assert!(
asked >= 2 * languages,
"{asked} verdicts over {languages} language(s); the corpus is thinner than it was"
);
}
const NO_POSITIVE_IN_CORPUS: &[(&str, &str, &str, &str)] = &[
(
"C",
"ShortCircuit",
"binary_expression",
"a `binary_expression` owns a short circuit only when it holds `&&` / `||`; none in the fixtures",
),
(
"Csharp",
"Modifier",
"modifier",
"the childless `modifier` alias owns only `ref` / `out` / `in` / `scoped` / `readonly` / `static` / `async` in a parameter, none spelled; `file` and `required` are kinds of their own",
),
(
"Csharp",
"ShortCircuit",
"binary_expression",
"a `binary_expression` owns a short circuit only when it holds `&&` / `||`; none in the fixtures",
),
(
"Elixir",
"Member",
"call",
"`defstruct` is not spelled; the row is text-gated on it",
),
(
"Go",
"ShortCircuit",
"binary_expression",
"a `binary_expression` owns a short circuit only when it holds `&&` / `||`; none in the fixtures",
),
(
"Groovy",
"Container",
"object_creation_expression",
"the one `new` has no class body; the gate admits only an anonymous class",
),
(
"Groovy",
"DeclKeyword",
"type_identifier",
"`var` as a field's type is not spelled; the row is text-gated on the identifier",
),
(
"Groovy",
"Primitive",
"identifier",
"no primitive type is spelled as a bare `identifier`; the row is text-gated on it",
),
(
"Groovy",
"ShortCircuit",
"binary_expression",
"a `binary_expression` owns a short circuit only when it holds `&&` / `||`; none in the fixtures",
),
(
"Irules",
"ShortCircuit",
"binop_expr",
"a `binop_expr` owns a short circuit only when it holds `&&` / `||` / `and` / `or`; none in the fixtures",
),
(
"Irules",
"StringLit",
"braced_word",
"every braced word is a script body or a value slot the gate rejects",
),
(
"Java",
"Container",
"object_creation_expression",
"the one `new` has no class body; the gate admits only an anonymous class",
),
(
"Javascript",
"Closure",
"arrow_function",
"the one arrow is bound (`const arrow = …`), the gate's negative",
),
(
"Javascript",
"ShortCircuit",
"binary_expression",
"a `binary_expression` owns a short circuit only when it holds `&&` / `||`; none in the fixtures",
),
(
"Kotlin",
"ShortCircuit",
"binary_expression",
"a `binary_expression` owns a short circuit only when it holds `&&` / `||`; none in the fixtures",
),
(
"Lua",
"ShortCircuit",
"binary_expression",
"a `binary_expression` owns a short circuit only when it holds `and` / `or`; none in the fixtures",
),
(
"Mozjs",
"Closure",
"arrow_function",
"the one arrow is bound (`const arrow = …`), the gate's negative",
),
(
"Mozjs",
"ShortCircuit",
"binary_expression",
"a `binary_expression` owns a short circuit only when it holds `&&` / `||`; none in the fixtures",
),
(
"Objc",
"Modifier",
"identifier",
"no property-attribute word is spelled; the row is text-gated on the identifier",
),
(
"Objc",
"ShortCircuit",
"binary_expression",
"a `binary_expression` owns a short circuit only when it holds `&&` / `||`; none in the fixtures",
),
(
"Perl",
"EagerLogical",
"binary_expression",
"a `binary_expression` owns an eager logical only when it holds `xor`; none in the fixtures",
),
(
"Perl",
"ShortCircuit",
"binary_expression",
"a `binary_expression` owns a short circuit only when it holds `&&` / `||` / `and` / `or`; none in the fixtures",
),
(
"Perl",
"ShortCircuit",
"unary_expression",
"tree-sitter-perl spells a low-precedence `and` as a two-operand `unary_expression` (#1539); none in the fixtures",
),
(
"Php",
"EagerLogical",
"binary_expression",
"a `binary_expression` owns an eager logical only when it holds `xor`; none in the fixtures",
),
(
"Php",
"ShortCircuit",
"binary_expression",
"a `binary_expression` owns a short circuit only when it holds `&&` / `||`; none in the fixtures",
),
(
"Ruby",
"ShortCircuit",
"binary",
"a `binary` owns a short circuit only when it holds `&&` / `||` / `and` / `or`; none in the fixtures",
),
(
"Rust",
"ShortCircuit",
"binary_expression",
"a `binary_expression` owns a short circuit only when it holds `&&` / `||`; none in the fixtures",
),
(
"Tcl",
"ShortCircuit",
"binop_expr",
"a `binop_expr` owns a short circuit only when it holds `&&` / `||`; none in the fixtures",
),
(
"Tsx",
"Closure",
"arrow_function",
"the two arrows are bound, the gate's negative",
),
(
"Tsx",
"ShortCircuit",
"binary_expression",
"a `binary_expression` owns a short circuit only when it holds `&&` / `||`; none in the fixtures",
),
(
"Typescript",
"Closure",
"arrow_function",
"the one arrow is bound (`const arrow = …`), the gate's negative",
),
(
"Typescript",
"ShortCircuit",
"binary_expression",
"a `binary_expression` owns a short circuit only when it holds `&&` / `||`; none in the fixtures",
),
];
struct OwnerPositives {
contradicted: Vec<String>,
unowned: Vec<String>,
unmet: Vec<String>,
report: String,
}
fn owner_positives(censuses: &[Census]) -> OwnerPositives {
let mut unowned = Vec::new();
let mut contradicted = Vec::new();
let mut unmet = Vec::new();
let mut used = std::collections::BTreeSet::new();
let mut report = String::new();
for census in censuses {
for (i, row) in census.declared_rows() {
let tally = &census.tallies[i];
for &kind in row.owners {
let Some(&KindCount { seen, owned }) = tally.kinds_seen.get(&kind) else {
continue;
};
let name = census.kind_name(kind);
let named = NO_POSITIVE_IN_CORPUS
.iter()
.any(|(l, role, k, _)| *l == census.name && *role == row.role && *k == name);
if named {
used.insert((census.name, row.role, name));
}
match (owned > 0, named) {
(false, false) => unowned.push(format!(
"{} {} `{name}` ({kind}): {seen} candidate(s), none owned",
census.name, row.role
)),
(true, true) => contradicted.push(format!(
"{} {} `{name}`: named as having no positive, but owns {owned} of {seen}",
census.name, row.role
)),
_ => {}
}
}
}
for (i, row) in census.declared_rows() {
for kind in row.candidate_kinds() {
if !census.tallies[i].kinds_seen.contains_key(&kind) {
let _ = writeln!(
report,
"no occurrence: {} {} `{}` ({kind})",
census.name,
row.role,
census.kind_name(kind)
);
}
}
}
for (i, row) in census.declared_rows() {
let tally = &census.tallies[i];
if tally.owned > 0
&& tally.rejected_owners == 0
&& (row.text_gated || row.owners.len() > 1)
{
let _ = writeln!(
report,
"no rejected owner: {} {} ({} owned)",
census.name, row.role, tally.owned
);
}
}
}
for (l, role, k, why) in NO_POSITIVE_IN_CORPUS {
let enabled = censuses.iter().any(|c| c.name == *l);
if enabled && !used.contains(&(*l, *role, *k)) {
unmet.push(format!(
"{l} {role} `{k}` ({why}): names no candidate the corpus spells"
));
}
}
OwnerPositives {
contradicted,
unowned,
unmet,
report,
}
}
#[test]
fn no_named_no_positive_entry_is_contradicted() {
let positives = owner_positives(&every_census());
print!("{}", positives.report);
assert!(
positives.contradicted.is_empty(),
"NO_POSITIVE_IN_CORPUS entries the corpus contradicts:\n{}",
positives.contradicted.join("\n")
);
}
#[test]
fn corpus_every_owner_kind_the_corpus_spells_owns_somewhere_or_is_named() {
assert!(
control_flow_corpus_present(),
"control_flow/ not checked out; run `make worktree-setup` — or filter `corpus_` tests"
);
let positives = owner_positives(&every_census());
assert!(
positives.unowned.is_empty(),
"owner kinds the corpus spells that never own — a gate or projection that rejects \
every spelling, or a construct the fixtures lack (name it in NO_POSITIVE_IN_CORPUS):\n{}",
positives.unowned.join("\n")
);
assert!(
positives.unmet.is_empty(),
"stale NO_POSITIVE_IN_CORPUS entries:\n{}",
positives.unmet.join("\n")
);
}
#[cfg(feature = "elixir")]
#[test]
fn corpus_elixir_control_rows_are_covered_per_word() {
assert!(
control_flow_corpus_present(),
"control_flow/ not checked out; run `make worktree-setup` — or filter `corpus_` tests"
);
let census = every_census()
.into_iter()
.find(|c| c.lang == LANG::Elixir)
.expect("Elixir is enabled");
let mut words: BTreeMap<(&str, &str), usize> = BTreeMap::new();
for occ in &census.occurrences {
let Some(word) = occ.word else {
continue;
};
for (i, v) in &occ.verdicts {
if v.owns {
*words
.entry((census.declared_row(*i).role, word))
.or_default() += 1;
}
}
}
for ((role, word), n) in &words {
println!("elixir word: {role} `{word}` owned {n} time(s)");
}
let control: BTreeSet<&str> = words
.keys()
.filter(|(role, _)| matches!(*role, "Branch" | "Switch" | "Loop"))
.map(|(_, w)| *w)
.collect();
assert!(
control.len() >= 6,
"Elixir's control rows own {} distinct words in the corpus: {control:?}",
control.len()
);
}
struct Forward {
name: &'static str,
answered: bool,
row: bool,
}
struct ForwardRows {
branch: Option<DeclaredRow>,
else_if: Option<DeclaredRow>,
closure: Option<DeclaredRow>,
doc_body_residual: Option<DeclaredRow>,
doc_comment: Option<DeclaredRow>,
doc_run_directive: Option<DeclaredRow>,
comment: Option<DeclaredRow>,
call: Option<DeclaredRow>,
string_lit: Option<DeclaredRow>,
func_opener: Option<DeclaredRow>,
docstring: Option<DeclaredRow>,
}
impl ForwardRows {
fn of(census: &Census) -> Self {
let row = |role| census.row_named(role).copied();
Self {
branch: row("Branch"),
else_if: row("ElseIf"),
closure: row("Closure"),
doc_body_residual: row("DocBodyResidual"),
doc_comment: row("DocComment"),
doc_run_directive: row("DocRunDirective"),
comment: row("Comment"),
call: row("Call"),
string_lit: row("StringLit"),
func_opener: row("FuncOpener"),
docstring: row("Docstring"),
}
}
}
#[allow(clippy::too_many_lines)]
fn forwards_on<'a, C: Checker + crate::Getter>(
census: &Census,
rows: &ForwardRows,
node: &Node<'a>,
code: &[u8],
ancestors: Ancestors<'a, '_>,
sink: &mut dyn FnMut(Forward),
) {
use crate::SpaceKind;
let kind = node.kind_id();
let owns = |row: &Option<DeclaredRow>| {
row.as_ref()
.is_some_and(|r| (r.owns_kind)(kind, node, code, ancestors))
};
let lists = |row: &Option<DeclaredRow>| row.as_ref().is_some_and(|r| (r.lists_kind)(kind));
let mut push = |name, answered, row| {
sink(Forward {
name,
answered,
row,
});
};
push("is_else_if", C::is_else_if(node, ancestors), {
if census.lang == LANG::Python {
owns(&rows.branch) && owns(&rows.else_if)
} else {
owns(&rows.else_if)
}
});
push(
"is_closure",
C::is_closure(node, ancestors),
owns(&rows.closure),
);
let holds_for_any_child = node
.children()
.any(|below| C::holds_doc_comment_in_body(node, code, ancestors, &below));
let composition = C::is_func_with_code(node, code, ancestors)
|| C::is_closure(node, ancestors)
|| owns(&rows.doc_body_residual);
push(
"holds_doc_comment_in_body ⇒ is_func_with_code ∨ is_closure ∨ row",
holds_for_any_child,
holds_for_any_child && composition,
);
let row_owner = owns(&rows.doc_body_residual);
push(
"DocBodyResidual row ⇒ holds_doc_comment_in_body for some child",
row_owner,
row_owner && holds_for_any_child,
);
let is_doc_comment = C::is_doc_comment(node, code);
push("is_doc_comment", is_doc_comment, owns(&rows.doc_comment));
push(
"is_doc_run_directive",
C::is_doc_run_directive(node, code),
owns(&rows.doc_run_directive),
);
let is_comment = C::is_comment(node);
push(
"is_comment",
is_comment,
lists(&rows.comment) || lists(&rows.doc_comment),
);
push(
"is_doc_comment ⇒ is_comment",
is_doc_comment,
is_doc_comment && is_comment,
);
let docstring = owns(&rows.docstring);
push(
"docstring reaches neither is_comment nor is_doc_comment",
docstring,
docstring && !is_comment && !is_doc_comment,
);
push("is_call", C::is_call(node), owns(&rows.call));
push("is_string", C::is_string(node), owns(&rows.string_lit));
let with_code = C::is_func_space_with_code(node, code, ancestors);
let opens = C::get_space_kind_with_code(node, code, ancestors);
push(
"is_call_with_code",
C::is_call_with_code(node, code, ancestors),
owns(&rows.call),
);
push(
"is_string_with_code",
C::is_string_with_code(node, code, ancestors),
owns(&rows.string_lit),
);
push(
"is_func_space_with_code",
with_code,
opens != SpaceKind::Unknown,
);
push(
"get_space_kind_with_code",
opens == (census.opens_space_fn)(node, code, ancestors),
true,
);
push("is_func_space", C::is_func_space(node), with_code);
push("get_space_kind", C::get_space_kind(node) == opens, true);
push(
"promotes_to_func_space_with_code",
C::promotes_to_func_space_with_code(node, code, ancestors),
C::is_func_with_code(node, code, ancestors) || with_code,
);
push(
"is_func_with_code",
C::is_func_with_code(node, code, ancestors),
rows.func_opener
.as_ref()
.is_some_and(|r| (r.is_callable_kind)(kind, node, code, ancestors)),
);
}
const KNOWN_DISAGREEMENTS: &[(&str, &str, &str, &str)] = &[
(
"Elixir",
"is_func_space",
"call",
"`def` / `defmodule` are `Call`s told apart by their target text; the byte-less spelling lists only `source` and `anonymous_function` (#275, #1130)",
),
(
"Elixir",
"get_space_kind",
"call",
"same: no bytes, no `defmodule` arm",
),
(
"Csharp",
"is_func_space",
"parameter_list",
"the `FuncOpener` row gates the list on a type-declaration parent; the known-empty chain has none",
),
(
"Csharp",
"get_space_kind",
"parameter_list",
"same: the byte-less spelling forwards with no chain",
),
(
"Java",
"is_func_space",
"formal_parameters",
"the `FuncOpener` row gates the list on a record parent with no canonical constructor; the known-empty chain has none",
),
(
"Java",
"get_space_kind",
"formal_parameters",
"same: the byte-less spelling forwards with no chain",
),
(
"Groovy",
"is_func_space",
"record_components",
"the `FuncOpener` row gates the list on a record parent with no canonical constructor; the known-empty chain has none",
),
(
"Groovy",
"get_space_kind",
"record_components",
"same: the byte-less spelling forwards with no chain",
),
(
"Tcl",
"is_string",
"braced_word",
"the byte-less spelling is the kind table; the row's gate reads the chain",
),
("Irules", "is_string", "braced_word", "as Tcl"),
(
"Perl",
"is_call",
"call_expression_with_bareword",
"a bareword before `=>` or as a hash key is no call; the gate reads the chain (`perl_bareword_is_a_call_only_to_the_byteless_spelling`)",
),
];
fn forward_disagreements<C: Checker + crate::Getter + crate::LanguageInfo>(
census: &Census,
rows: &ForwardRows,
source_name: &str,
code: &[u8],
asked: &mut usize,
seen: &mut BTreeSet<(&'static str, &'static str, String)>,
unnamed: &mut Vec<String>,
) -> bool {
let name = census.name;
let (_, recovered) =
for_each_node_with_sibling_cache_tolerating_errors::<C>(code, |node, ancestors| {
if node.start_byte() == node.end_byte() {
return;
}
forwards_on::<C>(census, rows, node, code, ancestors, &mut |f| {
*asked += 1;
if f.answered == f.row {
return;
}
let kind = node.kind().to_owned();
let named = KNOWN_DISAGREEMENTS
.iter()
.any(|(l, fwd, k, _)| *l == name && *fwd == f.name && *k == kind);
if named {
seen.insert((name, f.name, kind));
} else {
unnamed.push(format!(
"{name} {} on `{kind}` in {source_name}: forward {} row {}",
f.name, f.answered, f.row
));
}
});
});
recovered
}
#[test]
fn every_checker_forward_agrees_with_its_row_or_is_named() {
let mut unnamed: Vec<String> = Vec::new();
let mut seen: BTreeSet<(&str, &str, String)> = BTreeSet::new();
let mut asked = 0;
let censuses = every_census();
for census in &censuses {
let rows = ForwardRows::of(census);
for source in sources_for(census.lang) {
with_code_tag!(census.lang, |C| {
let recovered = forward_disagreements::<C>(
census,
&rows,
&source.name,
&source.bytes,
&mut asked,
&mut seen,
&mut unnamed,
);
assert!(!recovered, "{}: a fixture parses clean", source.name);
});
}
}
assert!(asked > 0);
let stale: Vec<String> = KNOWN_DISAGREEMENTS
.iter()
.filter(|(l, _, _, _)| censuses.iter().any(|c| c.name == *l))
.filter(|(l, f, k, _)| !seen.contains(&(*l, *f, (*k).to_owned())))
.map(|(l, f, k, _)| format!("{l} {f} on `{k}`"))
.collect();
assert!(
unnamed.is_empty(),
"forwards that disagree with their row on the real context, unnamed:\n{}",
unnamed.join("\n")
);
if control_flow_corpus_present() {
assert!(
stale.is_empty(),
"KNOWN_DISAGREEMENTS entries the corpus no longer shows:\n{}",
stale.join("\n")
);
}
}
#[test]
fn corpus_every_known_disagreement_is_still_met() {
assert!(
control_flow_corpus_present(),
"control_flow/ not checked out; run `make worktree-setup` — or filter `corpus_` tests"
);
let mut asked = 0;
let mut seen: BTreeSet<(&str, &str, String)> = BTreeSet::new();
let mut unnamed: Vec<String> = Vec::new();
let censuses = every_census();
for census in &censuses {
let rows = ForwardRows::of(census);
for source in sources_for(census.lang) {
with_code_tag!(census.lang, |C| {
forward_disagreements::<C>(
census,
&rows,
&source.name,
&source.bytes,
&mut asked,
&mut seen,
&mut unnamed,
);
});
}
}
let stale: Vec<String> = KNOWN_DISAGREEMENTS
.iter()
.filter(|(l, _, _, _)| censuses.iter().any(|c| c.name == *l))
.filter(|(l, f, k, _)| !seen.contains(&(*l, *f, (*k).to_owned())))
.map(|(l, f, k, _)| format!("{l} {f} on `{k}`"))
.collect();
assert!(
stale.is_empty(),
"KNOWN_DISAGREEMENTS entries the corpus no longer shows:\n{}",
stale.join("\n")
);
}
#[derive(Default)]
struct ChainProbe {
asked: usize,
disagreements: BTreeSet<String>,
climbs: BTreeSet<String>,
}
impl ChainProbe {
fn probe<'a>(
&mut self,
table: &LanguageTable,
rows: &[RowView],
unchained: &[DeclaredRow],
node: &Node<'a>,
code: &[u8],
ancestors: Ancestors<'a, '_>,
) {
let empty = Ancestors::known(&[]);
let kind = node.kind_id();
let mut owner_of_unchained = false;
let climbed = |p: usize, s: usize| {
(
crate::node::node_resolved_parent_lookups::observed() - p,
crate::node::node_resolved_sibling_lookups::observed() - s,
)
};
for row in unchained {
if !row.owners.contains(&kind) {
continue;
}
owner_of_unchained = true;
self.asked += 1;
let at = format!("{} {} on `{}`", table.name, row.role, node.kind());
let (p, s) = climbed(0, 0);
let real = (row.owns_kind)(kind, node, code, ancestors);
let rootless = (row.owns_kind)(kind, node, code, empty);
let (parents, siblings) = climbed(p, s);
if (parents, siblings) != (0, 0) {
self.climbs.insert(format!(
"{at}: {parents} parent and {siblings} sibling climb(s)"
));
}
if real != rootless {
self.disagreements.insert(format!(
"{at}: owns {real} with the chain, {rootless} without"
));
}
}
let chained_opener = rows.iter().filter_map(RowView::declared).any(|r| {
r.chain_gated
&& matches!(r.role, "FuncOpener" | "Container")
&& r.owners.contains(&kind)
});
if owner_of_unchained && !chained_opener {
let (p, s) = climbed(0, 0);
let real = (table.opens_space)(node, code, ancestors);
let rootless = (table.opens_space)(node, code, empty);
let (parents, siblings) = climbed(p, s);
if (parents, siblings) != (0, 0) {
self.climbs.insert(format!(
"{} opens_space on `{}`: {parents} parent and {siblings} sibling climb(s)",
table.name,
node.kind()
));
}
if real != rootless {
self.disagreements.insert(format!(
"{} opens_space on `{}`: {real:?} with the chain, {rootless:?} without",
table.name,
node.kind()
));
}
}
}
}
#[test]
fn no_chain_rows_answer_alike_under_the_empty_chain() {
let mut probe = ChainProbe::default();
let mut rows_asked = 0;
for table in crate::roles::tables() {
if !table.lang.is_enabled() {
continue;
}
let rows = (table.rows)();
let unchained: Vec<DeclaredRow> = rows
.iter()
.filter_map(RowView::declared)
.filter(|r| !r.chain_gated)
.copied()
.collect();
rows_asked += unchained.len();
for source in sources_for(table.lang) {
let code = source.bytes.as_slice();
with_code_tag!(table.lang, |C| {
for_each_node_with_sibling_cache::<C>(code, |node, ancestors| {
if node.start_byte() != node.end_byte() {
probe.probe(table, &rows, &unchained, node, code, ancestors);
}
});
});
}
}
assert!(
rows_asked > 0,
"no `!CHAIN_GATED` row in any enabled language"
);
assert!(
probe.asked > 0,
"no owner candidate of a `!CHAIN_GATED` row in the corpus"
);
assert!(
probe.disagreements.is_empty(),
"a `!CHAIN_GATED` row answers differently without its chain — a `no_chain` gate climbs:\n{}",
probe
.disagreements
.into_iter()
.collect::<Vec<_>>()
.join("\n")
);
assert!(
probe.climbs.is_empty(),
"a `!CHAIN_GATED` row climbed with `Node::parent` / `Node::previous_sibling` while asked:\n{}",
probe.climbs.into_iter().collect::<Vec<_>>().join("\n")
);
}
#[test]
fn no_closure_row_reads_bytes() {
for table in crate::roles::tables() {
for row in (table.rows)() {
if row.role() == "Closure"
&& let Some(r) = row.declared()
{
assert!(!r.text_gated, "{}: `is_closure` forwards `&[]`", table.lang);
}
}
}
}
struct Relation {
name: &'static str,
left: &'static str,
declared_right: fn(&Census, u16) -> bool,
gated_right: for<'a> fn(&Census, &Node<'a>, &[u8], Ancestors<'a, '_>) -> bool,
}
fn owns_any<'a>(
census: &Census,
roles: &[&str],
node: &Node<'a>,
code: &[u8],
ancestors: Ancestors<'a, '_>,
) -> bool {
let kind = node.kind_id();
roles.iter().any(|role| {
census
.row_named(role)
.is_some_and(|r| (r.owns_kind)(kind, node, code, ancestors))
})
}
fn lists_any(census: &Census, roles: &[&str], kind: u16) -> bool {
roles
.iter()
.any(|role| census.row_named(role).is_some_and(|r| (r.lists_kind)(kind)))
}
const ELSE_IF_DEDICATED: &[(&str, &str, &str)] = &[
(
"Bash",
"elif_clause",
"the `elif` clause is the chain link; no nested `if`",
),
("Irules", "elseif", "the `elseif` word is the chain link"),
(
"Lua",
"elseif_statement",
"the `elseif` statement is the chain link",
),
(
"Perl",
"elsif_clause",
"the `elsif` clause is the chain link",
),
(
"Php",
"else_if_clause",
"the one-word `elseif` clause, in its brace and `:` … `endif` aliases; the two-word `else if` is an `if_statement` the `Branch` row also owns",
),
(
"Python",
"elif_clause",
"the `elif` clause is the chain link, paid at its keyword",
),
("Ruby", "elsif", "the `elsif` clause is the chain link"),
("Tcl", "elseif", "the `elseif` word is the chain link"),
];
fn doc_target_outside_space_tree_is_named(census: &Census, kind: u16) -> bool {
lists_any(census, &["FuncOpener", "Container", "Member"], kind)
|| census.targets_beyond_space_tree.contains(&kind)
}
const RELATIONS: &[Relation] = &[
Relation {
name: "ElseIf ⊆ Branch ∪ dedicated clause",
left: "ElseIf",
declared_right: |census, kind| {
lists_any(census, &["Branch"], kind)
|| ELSE_IF_DEDICATED
.iter()
.any(|(l, k, _)| *l == census.name && census.kind_name(kind) == *k)
},
gated_right: |census, node, code, ancestors| {
owns_any(census, &["Branch"], node, code, ancestors)
|| ELSE_IF_DEDICATED
.iter()
.any(|(l, k, _)| *l == census.name && node.kind() == *k)
},
},
Relation {
name: "Closure ∩ Container = ∅",
left: "Closure",
declared_right: |census, kind| !lists_any(census, &["Container"], kind),
gated_right: |census, node, code, ancestors| {
!owns_any(census, &["Container"], node, code, ancestors)
},
},
Relation {
name: "DocTarget ⊆ FuncOpener ∪ Container ∪ Member ∪ named",
left: "DocTarget",
declared_right: doc_target_outside_space_tree_is_named,
gated_right: |census, node, code, ancestors| {
owns_any(
census,
&["FuncOpener", "Container", "Member"],
node,
code,
ancestors,
) || census.targets_beyond_space_tree.contains(&node.kind_id())
|| super::doc_tests::parity::gated_out_of_the_space_tree(census.name)
.iter()
.any(|(k, _)| *k == node.kind())
},
},
];
#[test]
fn doc_targets_the_space_tree_rejects_by_gate_are_exactly_the_named_shapes() {
let censuses = every_census();
let mut failures = Vec::new();
for census in &censuses {
let Some(target) = census.row_named("DocTarget") else {
continue;
};
let listed: BTreeSet<u16> = ["FuncOpener", "Container", "Member"]
.iter()
.filter_map(|r| census.row_named(r))
.flat_map(|r| r.owners.iter().copied())
.collect();
let mut seen: BTreeSet<(String, String)> = BTreeSet::new();
for source in sources_for(census.lang) {
let code = source.bytes.as_slice();
with_code_tag!(census.lang, |C| {
for_each_node_with_sibling_cache::<C>(code, |node, ancestors| {
let kind = node.kind_id();
if !(target.owns_kind)(kind, node, code, ancestors) || !listed.contains(&kind) {
return;
}
if owns_any(
census,
&["FuncOpener", "Container", "Member"],
node,
code,
ancestors,
) {
return;
}
let text = node
.utf8_text(code)
.unwrap_or("")
.lines()
.next()
.unwrap_or("");
seen.insert((node.kind().to_owned(), text.to_owned()));
});
});
}
let want: BTreeSet<(String, String)> =
super::doc_tests::parity::gated_out_of_the_space_tree(census.name)
.iter()
.map(|(k, t)| ((*k).to_owned(), (*t).to_owned()))
.collect();
if seen != want {
failures.push(format!(
"{}: targets the space tree lists but rejects by gate\n seen: {seen:?}\n named: {want:?}",
census.name
));
}
}
assert!(failures.is_empty(), "{}", failures.join("\n"));
}
#[test]
fn every_row_relation_holds_on_the_lists_and_the_corpus() {
let censuses = every_census();
let mut gaps_used: BTreeSet<(&str, &str)> = BTreeSet::new();
let mut vacuous = Vec::new();
for relation in RELATIONS {
let mut failures = Vec::new();
for census in &censuses {
let Some(left) = census.row_named(relation.left) else {
if let Some(RowView::Absent { why, .. }) =
census.rows.iter().find(|r| r.role() == relation.left)
{
println!(
"[{}] {}: {} absent — {why}",
relation.name, census.name, relation.left
);
}
continue;
};
let mut positives = 0;
for &kind in left.owners {
if !(relation.declared_right)(census, kind) {
failures.push(format!(
"{} [{}]: declared kind `{}` ({kind}) breaks the relation",
census.name,
relation.name,
census.kind_name(kind)
));
}
}
let sources = sources_for(census.lang);
for source in &sources {
let code = source.bytes.as_slice();
with_code_tag!(census.lang, |C| {
for_each_node_with_sibling_cache::<C>(code, |node, ancestors| {
if !(left.owns_kind)(node.kind_id(), node, code, ancestors) {
return;
}
positives += 1;
if !(relation.gated_right)(census, node, code, ancestors) {
failures.push(format!(
"{} [{}]: `{}` in {} breaks the relation",
census.name,
relation.name,
node.kind(),
source.name
));
}
});
});
}
let gap_named = RELATION_NO_POSITIVE_IN_CORPUS
.iter()
.any(|(l, r, _)| *l == census.name && *r == relation.left);
if gap_named {
gaps_used.insert((census.name, relation.left));
}
match (positives > 0, gap_named) {
(false, false) if !left.owners.is_empty() => vacuous.push(format!(
"{} [{}]: {} declares owners and the corpus spells none that own; vacuous",
census.name, relation.name, relation.left
)),
(true, true) => failures.push(format!(
"{} [{}]: named as having no positive, but has {positives}",
census.name, relation.name
)),
_ => {}
}
}
assert!(failures.is_empty(), "{}", failures.join("\n"));
}
let stale: Vec<String> = RELATION_NO_POSITIVE_IN_CORPUS
.iter()
.filter(|(l, _, _)| censuses.iter().any(|c| c.name == *l))
.filter(|(l, r, _)| !gaps_used.contains(&(*l, *r)))
.map(|(l, r, why)| {
format!("{l} {r} ({why}): no relation has that left row declared with owners here")
})
.collect();
assert!(
stale.is_empty(),
"stale RELATION_NO_POSITIVE_IN_CORPUS entries:\n{}",
stale.join("\n")
);
if control_flow_corpus_present() {
assert!(vacuous.is_empty(), "{}", vacuous.join("\n"));
}
}
#[test]
fn corpus_every_relation_has_a_positive_per_language_or_a_named_gap() {
assert!(
control_flow_corpus_present(),
"control_flow/ not checked out; run `make worktree-setup` — or filter `corpus_` tests"
);
every_row_relation_holds_on_the_lists_and_the_corpus();
}
const RELATION_NO_POSITIVE_IN_CORPUS: &[(&str, &str, &str)] = &[
(
"Csharp",
"Closure",
"no lambda or anonymous method in the fixtures",
),
(
"Javascript",
"Closure",
"the one arrow is bound, the gate's negative",
),
(
"Mozjs",
"Closure",
"the one arrow is bound, the gate's negative",
),
("Objc", "Closure", "no block literal in the fixtures"),
("Perl", "Closure", "no anonymous `sub` in the fixtures"),
("Python", "Closure", "no `lambda` in the fixtures"),
];
#[test]
fn every_named_target_beyond_the_space_tree_is_still_one() {
let mut failures = Vec::new();
for table in super::doc_tests::parity::doc_target_tables() {
let owned: std::collections::BTreeSet<u16> =
table.space.iter().copied().flatten().copied().collect();
for (kind, why) in &table.beyond {
if !table.doc_target.contains(kind) {
failures.push(format!(
"{}: named kind {kind} ({why}) is no longer a target",
table.lang
));
}
if owned.contains(kind) {
failures.push(format!(
"{}: named kind {kind} ({why}) is in the space tree; the name is stale",
table.lang
));
}
}
}
assert!(failures.is_empty(), "{}", failures.join("\n"));
}
#[test]
fn the_dedicated_else_if_clauses_are_exactly_the_owners_branch_does_not_list() {
let censuses = every_census();
let mut dedicated_failures = Vec::new();
for census in &censuses {
let (Some(else_if), branch) = (census.row_named("ElseIf"), census.row_named("Branch"))
else {
continue;
};
for &kind in else_if.owners {
let in_branch = branch.is_some_and(|b| b.owners.contains(&kind));
let name = census.kind_name(kind);
let named = ELSE_IF_DEDICATED
.iter()
.any(|(l, k, _)| *l == census.name && *k == name);
if in_branch == named {
dedicated_failures.push(format!(
"{}: ElseIf owner `{name}` ({kind}) is {} and {}",
census.name,
if in_branch {
"a Branch owner"
} else {
"no Branch owner"
},
if named {
"named dedicated"
} else {
"not named dedicated"
}
));
}
}
}
for (lang, kind_name, why) in ELSE_IF_DEDICATED {
let Some(census) = censuses.iter().find(|c| c.name == *lang) else {
continue;
};
let owned_by = |role: &str| {
census
.row_named(role)
.is_some_and(|r| r.owners.iter().any(|&k| census.kind_name(k) == *kind_name))
};
if !owned_by("ElseIf") || owned_by("Branch") {
dedicated_failures.push(format!(
"{lang}: ELSE_IF_DEDICATED names `{kind_name}` ({why}), which ElseIf {} and Branch {}",
if owned_by("ElseIf") { "owns" } else { "does not own" },
if owned_by("Branch") { "owns" } else { "does not own" }
));
}
}
assert!(
dedicated_failures.is_empty(),
"{}",
dedicated_failures.join("\n")
);
}
#[test]
#[ignore = "walks the integration corpora; run through `make role-parity-corpus`"]
fn every_checker_forward_agrees_with_its_row_across_the_integration_corpora() {
let repos = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
.parent()
.expect("the ast crate sits under the workspace root")
.join("tests")
.join("repositories");
let censuses: Vec<(Census, ForwardRows)> = every_census()
.into_iter()
.map(|c| {
let rows = ForwardRows::of(&c);
(c, rows)
})
.collect();
let mut files = 0;
let mut present: BTreeMap<&str, usize> = BTreeMap::new();
let mut per_corpus: BTreeMap<&str, usize> = BTreeMap::new();
let mut recovery_files = 0;
let mut asked = 0;
let mut seen: BTreeSet<(&str, &str, String)> = BTreeSet::new();
let mut unnamed: BTreeSet<String> = BTreeSet::new();
let mut recovery_only: BTreeSet<String> = BTreeSet::new();
for corpus in INTEGRATION_CORPORA {
let root = repos.join(corpus);
assert!(
root.is_dir(),
"{} missing; run `make worktree-setup`",
root.display()
);
let mut paths: Vec<std::path::PathBuf> = files_under(&root);
paths.sort();
present.insert(corpus, paths.len());
for path in paths {
let Some(lang) = crate::tools::get_language_for_file(&path) else {
continue;
};
let Some((census, rows)) = censuses.iter().find(|(c, _)| c.lang == lang) else {
continue;
};
let code = match crate::tools::read_file_with_eol(&path) {
Ok(Some(code)) => code,
Ok(None) => continue,
Err(e) => panic!("{}: unreadable corpus file: {e}", path.display()),
};
files += 1;
*per_corpus.entry(corpus).or_insert(0) += 1;
let name = format!(
"{corpus}/{}",
path.strip_prefix(&root).expect("under root").display()
);
let mut pending = Vec::new();
let recovered = with_code_tag!(lang, |C| {
forward_disagreements::<C>(
census,
rows,
&name,
&code,
&mut asked,
&mut seen,
&mut pending,
)
});
if recovered {
recovery_only.extend(pending);
} else {
unnamed.extend(pending);
}
recovery_files += usize::from(recovered);
}
}
println!("integration corpora: {files} files walked, {recovery_files} with error recovery");
for line in &recovery_only {
println!("recovery-only disagreement: {line}");
}
assert!(files > 0, "no corpus file routed to an enabled language");
assert!(asked > 0);
println!("named disagreements met in the corpora: {}", seen.len());
assert_every_corpus_present(&present, &per_corpus);
assert!(
unnamed.is_empty(),
"forwards that disagree with their row on a clean-parsing corpus file, unnamed:\n{}",
unnamed.into_iter().collect::<Vec<_>>().join("\n")
);
}
const INTEGRATION_CORPORA: [&str; 3] = ["DeepSpeech", "pdf.js", "serde"];
fn assert_every_corpus_present(present: &BTreeMap<&str, usize>, routed: &BTreeMap<&str, usize>) {
for corpus in INTEGRATION_CORPORA {
let files = present.get(corpus).copied().unwrap_or(0);
assert!(
files > 0,
"{corpus}: no file under it; run `make worktree-setup` (an interrupted checkout, #1171)"
);
println!(
"{corpus}: {files} files, {} routed to an enabled language",
routed.get(corpus).copied().unwrap_or(0)
);
}
}
fn files_under(root: &std::path::Path) -> Vec<std::path::PathBuf> {
let toplevel = git_output(root, &["rev-parse", "--show-toplevel"]);
let toplevel = path_from_git(toplevel.strip_suffix(b"\n").unwrap_or(&toplevel));
let canonical = |p: &std::path::Path| {
std::fs::canonicalize(p).unwrap_or_else(|e| panic!("{}: {e}", p.display()))
};
assert_eq!(
canonical(&toplevel),
canonical(root),
"{}: not the root of its own git repository; run `make worktree-setup`",
root.display()
);
let listing = git_output(root, &["ls-files", "-s", "-z"]);
let mut seen = BTreeSet::new();
let mut out = Vec::new();
for record in listing.split(|&b| b == 0).filter(|r| !r.is_empty()) {
let tab = record
.iter()
.position(|&b| b == b'\t')
.unwrap_or_else(|| panic!("{}: malformed `git ls-files` record", root.display()));
let (meta, name) = (&record[..tab], &record[tab + 1..]);
let mode = meta
.split(|&b| b == b' ')
.next()
.expect("`split` yields at least one field");
if !SAMPLED_MODES.contains(&mode) || !seen.insert(name) {
continue;
}
let path = root.join(path_from_git(name));
let on_disk = match std::fs::metadata(&path) {
Ok(meta) => meta.is_file(),
Err(e)
if matches!(
e.kind(),
std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory
) =>
{
false
}
Err(e) => panic!("{}: tracked but unreadable: {e}", path.display()),
};
assert!(
on_disk,
"{}: tracked but not on disk; {MISSING_FILE_REMEDY}",
path.display()
);
out.push(path);
}
out
}
const SAMPLED_MODES: [&[u8]; 2] = [b"100644", b"100755"];
const MISSING_FILE_REMEDY: &str = "run `make worktree-setup` (a sparse or skip-worktree corpus \
needs `git sparse-checkout disable` or `git update-index --no-skip-worktree`, and a file \
deleted on purpose `git rm`)";
const REPOSITORY_ENV: [&str; 6] = [
"GIT_DIR",
"GIT_WORK_TREE",
"GIT_INDEX_FILE",
"GIT_COMMON_DIR",
"GIT_OBJECT_DIRECTORY",
"GIT_ALTERNATE_OBJECT_DIRECTORIES",
];
fn git_output(dir: &std::path::Path, args: &[&str]) -> Vec<u8> {
let mut cmd = std::process::Command::new("git");
for var in REPOSITORY_ENV {
cmd.env_remove(var);
}
let out = cmd
.arg("-C")
.arg(dir)
.args(args)
.output()
.unwrap_or_else(|e| panic!("{}: cannot run git: {e}", dir.display()));
assert!(
out.status.success(),
"{}: `git {}` failed: {}",
dir.display(),
args.join(" "),
String::from_utf8_lossy(&out.stderr).trim()
);
out.stdout
}
#[cfg(unix)]
fn path_from_git(bytes: &[u8]) -> std::path::PathBuf {
use std::os::unix::ffi::OsStrExt;
std::ffi::OsStr::from_bytes(bytes).into()
}
#[cfg(not(unix))]
fn path_from_git(bytes: &[u8]) -> std::path::PathBuf {
std::str::from_utf8(bytes)
.unwrap_or_else(|e| panic!("git printed a non-UTF-8 path: {e}"))
.into()
}
fn fixture_git(dir: &std::path::Path, args: &[&str]) -> Vec<u8> {
let mut cmd = std::process::Command::new("git");
for (key, _) in std::env::vars_os() {
if key.as_encoded_bytes().starts_with(b"GIT_") {
cmd.env_remove(key);
}
}
let out = cmd
.arg("-C")
.arg(dir)
.args(["-c", "user.name=t", "-c", "user.email=t@x"])
.args(args)
.env("GIT_CONFIG_GLOBAL", "/dev/null")
.env("GIT_CONFIG_NOSYSTEM", "1")
.env("GIT_CONFIG_COUNT", "2")
.env("GIT_CONFIG_KEY_0", "core.excludesFile")
.env("GIT_CONFIG_VALUE_0", "/dev/null")
.env("GIT_CONFIG_KEY_1", "core.attributesFile")
.env("GIT_CONFIG_VALUE_1", "/dev/null")
.output()
.expect("spawn git");
assert!(
out.status.success(),
"git {args:?}: {}",
String::from_utf8_lossy(&out.stderr)
);
out.stdout
}
#[test]
fn the_two_enumerators_share_their_contract() {
let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
.parent()
.expect("the ast crate sits under the workspace root");
let script = r#"
import importlib.util, json, sys
spec = importlib.util.spec_from_file_location("cov", sys.argv[1])
cov = importlib.util.module_from_spec(spec)
sys.modules["cov"] = cov
spec.loader.exec_module(cov)
print(json.dumps({
"modes": [m.decode() for m in cov.SAMPLED_MODES],
"env": list(cov.REPOSITORY_ENV),
"remedy": cov.MISSING_FILE_REMEDY,
}))
"#;
let out = std::process::Command::new("python3")
.arg("-B")
.arg("-c")
.arg(script)
.arg(root.join("utils/check-role-coverage.py"))
.output()
.expect("python3 is on PATH: the role gates need it, so this test does too");
assert!(
out.status.success(),
"check-role-coverage.py did not import:\n{}",
String::from_utf8_lossy(&out.stderr)
);
let theirs: serde_json::Value =
serde_json::from_slice(&out.stdout).expect("the probe prints one JSON object");
let ours = serde_json::json!({
"modes": SAMPLED_MODES.iter().map(|m| std::str::from_utf8(m).expect("ASCII mode")).collect::<Vec<_>>(),
"env": REPOSITORY_ENV,
"remedy": MISSING_FILE_REMEDY,
});
pretty_assertions::assert_eq!(theirs, ours);
}
#[cfg(unix)]
#[test]
fn files_under_lists_the_tracked_regular_files_and_nothing_else() {
use std::os::unix::fs::PermissionsExt;
let dir = tempfile::tempdir().expect("scratch dir");
let root = dir.path();
std::fs::create_dir_all(root.join("a/b")).expect("mkdir");
std::fs::write(root.join("a/x.rs"), "fn x() {}\n").expect("write");
std::fs::write(root.join("a/b/y.rs"), "fn y() {}\n").expect("write");
std::fs::write(root.join("a/run.rs"), "fn run() {}\n").expect("write");
std::fs::set_permissions(
root.join("a/run.rs"),
std::fs::Permissions::from_mode(0o755),
)
.expect("chmod");
std::os::unix::fs::symlink(root.join("a/b"), root.join("link_to_b")).expect("symlink");
std::os::unix::fs::symlink(root.join("a/x.rs"), root.join("link_to_x.rs")).expect("symlink");
fixture_git(root, &["init", "-q"]);
fixture_git(root, &["add", "--", "a", "link_to_b", "link_to_x.rs"]);
let source = tempfile::tempdir().expect("scratch dir");
std::fs::write(source.path().join("n.rs"), "fn n() {}\n").expect("write");
fixture_git(source.path(), &["init", "-q"]);
fixture_git(source.path(), &["add", "n.rs"]);
fixture_git(source.path(), &["commit", "-q", "-m", "nested"]);
let url = source.path().to_str().expect("UTF-8 scratch path");
fixture_git(
root,
&[
"-c",
"protocol.file.allow=always",
"submodule",
"add",
"-q",
url,
"nested",
],
);
std::fs::write(root.join("stray.rs"), "fn stray() {}\n").expect("write");
let index = String::from_utf8(fixture_git(root, &["ls-files", "-s"])).expect("UTF-8");
let modes: BTreeSet<&str> = index.lines().filter_map(|l| l.split(' ').next()).collect();
assert_eq!(
modes,
BTreeSet::from(["100644", "100755", "120000", "160000"]),
"the fixture spells every mode the index can hold"
);
let mut found: Vec<String> = files_under(root)
.into_iter()
.map(|p| {
p.strip_prefix(root)
.expect("under root")
.to_str()
.expect("UTF-8")
.to_owned()
})
.collect();
found.sort();
assert_eq!(found, [".gitmodules", "a/b/y.rs", "a/run.rs", "a/x.rs"]);
}
#[cfg(unix)]
#[test]
fn files_under_and_its_fixture_ignore_an_inherited_repository() {
let victim = tempfile::tempdir().expect("scratch dir");
std::fs::write(victim.path().join("keep.rs"), "fn keep() {}\n").expect("write");
fixture_git(victim.path(), &["init", "-q"]);
fixture_git(victim.path(), &["add", "keep.rs"]);
fixture_git(victim.path(), &["commit", "-q", "-m", "victim"]);
let read =
|args: &[&str]| String::from_utf8_lossy(&fixture_git(victim.path(), args)).into_owned();
let state = || (read(&["rev-parse", "HEAD"]), read(&["ls-files", "-s"]));
let before = state();
let git_dir = victim.path().join(".git");
rerun_listing_test_with(&[
("GIT_DIR", git_dir.clone()),
("GIT_INDEX_FILE", git_dir.join("index")),
]);
assert_eq!(state(), before, "the inherited repository was written to");
}
#[cfg(unix)]
#[test]
fn files_under_ignores_an_inherited_object_directory() {
let gone = tempfile::tempdir().expect("scratch dir");
let objects = gone.path().join("quarantine");
rerun_listing_test_with(&[("GIT_OBJECT_DIRECTORY", objects)]);
}
#[cfg(unix)]
fn rerun_listing_test_with(vars: &[(&str, std::path::PathBuf)]) {
let child = format!(
"{}::files_under_lists_the_tracked_regular_files_and_nothing_else",
module_path!()
.strip_prefix(concat!(env!("CARGO_CRATE_NAME"), "::"))
.expect("the test module is nested under the crate root")
);
let output = std::process::Command::new(std::env::current_exe().expect("test binary"))
.args(["--exact", &child, "--format", "pretty"])
.envs(vars.iter().map(|(k, v)| (k, v)))
.output()
.expect("spawn the child");
let stdout = String::from_utf8_lossy(&output.stdout);
assert!(
output.status.success() && stdout.contains("test result: ok. 1 passed"),
"the listing test must pass with {vars:?} inherited; stdout:\n{stdout}\nstderr:\n{}",
String::from_utf8_lossy(&output.stderr)
);
}
#[test]
#[should_panic(expected = "not the root of its own git repository")]
fn files_under_refuses_a_directory_that_is_not_its_own_repository() {
let dir = tempfile::tempdir().expect("scratch dir");
std::fs::create_dir(dir.path().join("corpus")).expect("mkdir");
std::fs::write(dir.path().join("corpus/x.rs"), "fn x() {}\n").expect("write");
fixture_git(dir.path(), &["init", "-q"]);
fixture_git(dir.path(), &["add", "corpus/x.rs"]);
files_under(&dir.path().join("corpus"));
}
#[cfg(unix)]
#[test]
fn files_under_names_an_unreadable_tracked_file_as_unreadable() {
use std::os::unix::fs::PermissionsExt;
let dir = tempfile::tempdir().expect("scratch dir");
let locked = dir.path().join("locked");
std::fs::create_dir(&locked).expect("mkdir");
std::fs::write(locked.join("x.rs"), "fn x() {}\n").expect("write");
fixture_git(dir.path(), &["init", "-q"]);
fixture_git(dir.path(), &["add", "locked/x.rs"]);
std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o000)).expect("chmod");
let refused = std::fs::metadata(locked.join("x.rs"))
.is_err()
.then(|| std::panic::catch_unwind(|| files_under(dir.path())));
std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o755)).expect("chmod back");
let Some(refused) = refused else {
return;
};
let message = refused
.expect_err("an unreadable tracked file is refused")
.downcast::<String>()
.expect("a formatted panic message");
assert!(message.contains("tracked but unreadable"), "{message}");
}
#[test]
#[should_panic(expected = "tracked but not on disk")]
fn files_under_refuses_a_tracked_file_whose_directory_is_now_a_file() {
let dir = tempfile::tempdir().expect("scratch dir");
std::fs::create_dir(dir.path().join("a")).expect("mkdir");
std::fs::write(dir.path().join("a/x.rs"), "fn x() {}\n").expect("write");
fixture_git(dir.path(), &["init", "-q"]);
fixture_git(dir.path(), &["add", "a/x.rs"]);
std::fs::remove_dir_all(dir.path().join("a")).expect("rm");
std::fs::write(dir.path().join("a"), "").expect("write");
files_under(dir.path());
}
#[test]
#[should_panic(expected = "tracked but not on disk")]
fn files_under_refuses_a_tracked_file_missing_from_disk() {
let dir = tempfile::tempdir().expect("scratch dir");
std::fs::write(dir.path().join("x.rs"), "fn x() {}\n").expect("write");
std::fs::write(dir.path().join("gone.rs"), "fn gone() {}\n").expect("write");
fixture_git(dir.path(), &["init", "-q"]);
fixture_git(dir.path(), &["add", "x.rs", "gone.rs"]);
std::fs::remove_file(dir.path().join("gone.rs")).expect("rm");
files_under(dir.path());
}
#[test]
fn an_empty_corpus_directory_is_an_interrupted_checkout() {
let present: BTreeMap<&str, usize> = INTEGRATION_CORPORA.iter().map(|c| (*c, 3)).collect();
let routed: BTreeMap<&str, usize> = [("serde", 0)].into_iter().collect();
assert_every_corpus_present(&present, &routed);
let mut empty = present;
empty.insert("pdf.js", 0);
let failed = std::panic::catch_unwind(|| assert_every_corpus_present(&empty, &routed));
let message = failed
.expect_err("an empty corpus fails")
.downcast::<String>()
.expect("a formatted panic message");
assert!(message.contains("pdf.js: no file under it"), "{message}");
}