#![allow(clippy::wildcard_imports, clippy::enum_glob_use)]
use crate::token_role::TokenRole;
use crate::lang_helpers::tcl_family::{
ArgumentRoles, Dialect, SWITCH_COMMAND, argument_slots_are_readable, fills_script_slot,
namespace_script_slots, script_slots,
};
use crate::space_kind::SpaceKind;
use crate::traits::Search;
use generic_angles::{GenericAngleKinds, never_misparsed};
use js_family::impl_js_family_get_op_type;
use crate::*;
#[inline]
pub(crate) fn node_text<'a>(code: &'a [u8], node: &Node) -> Option<&'a str> {
code.get(node.start_byte()..node.end_byte())
.and_then(|bytes| std::str::from_utf8(bytes).ok())
}
macro_rules! get_operator {
($language:ident) => {
#[inline]
fn get_operator_id_as_str(id: u16) -> &'static str {
let typ = id.into();
match typ {
$language::LPAREN => "()",
$language::LBRACK => "[]",
$language::LBRACE => "{}",
_ => typ.into(),
}
}
};
}
macro_rules! impl_byte_less_space_kind {
($roles:ident) => {
fn get_space_kind(node: &Node) -> SpaceKind {
const _: () = assert!(
!$crate::roles::$roles::func_opener::TEXT_GATED
&& !$crate::roles::$roles::func_opener::CHAIN_GATED
&& !$crate::roles::$roles::container::TEXT_GATED
&& !$crate::roles::$roles::container::CHAIN_GATED,
"a space-opening row reads bytes or the chain; the byte-less spelling cannot forward with none"
);
Self::get_space_kind_with_code(node, &[], Ancestors::known(&[]))
}
};
($roles:ident, opener_gated) => {
fn get_space_kind(node: &Node) -> SpaceKind {
const _: () = assert!(
$crate::roles::$roles::func_opener::TEXT_GATED
|| $crate::roles::$roles::func_opener::CHAIN_GATED,
"an ungated opener row takes the plain arm"
);
const _: () = assert!(
!$crate::roles::$roles::container::TEXT_GATED
&& !$crate::roles::$roles::container::CHAIN_GATED,
"the `Container` row reads bytes or the chain; the byte-less spelling cannot forward with none"
);
Self::get_space_kind_with_code(node, &[], Ancestors::known(&[]))
}
};
}
macro_rules! keyword_roles {
($roles:ident) => {
keyword_roles!($roles, primitive: Operator);
};
($roles:ident, primitive: $primitive_role:ident) => {
#[inline]
fn get_keyword_role<'a>(
node: &Node<'a>,
code: &[u8],
ancestors: Ancestors<'a, '_>,
) -> TokenRole {
use $crate::roles::$roles::{decl_keyword, modifier, primitive};
let kind = node.kind_id();
if modifier::pays_at_leaf_kind(kind, node, code, ancestors)
|| decl_keyword::pays_at_leaf_kind(kind, node, code, ancestors)
{
TokenRole::Operator
} else if primitive::pays_at_leaf_kind(kind, node, code, ancestors) {
TokenRole::$primitive_role
} else {
TokenRole::Unknown
}
}
};
}
macro_rules! paying_roles {
($roles:ident) => {
paying_roles!(@emit $roles []);
};
($roles:ident, control_flow) => {
paying_roles!(@emit $roles [branch, switch, r#loop, catch, else_if]);
};
(@emit $roles:ident [$($more:ident),*]) => {
#[inline]
fn get_paying_role<'a>(
node: &Node<'a>,
code: &[u8],
ancestors: Ancestors<'a, '_>,
) -> TokenRole {
use $crate::roles::$roles::{
case_arm, comparison, eager_logical, guard, null_safe, short_circuit, ternary,
$($more,)*
};
let kind = node.kind_id();
if comparison::pays_at_token_kind(kind, node, code, ancestors)
|| short_circuit::pays_at_token_kind(kind, node, code, ancestors)
|| eager_logical::pays_at_token_kind(kind, node, code, ancestors)
|| ternary::pays_at_token_kind(kind, node, code, ancestors)
|| case_arm::pays_at_token_kind(kind, node, code, ancestors)
|| guard::pays_at_token_kind(kind, node, code, ancestors)
|| null_safe::pays_at_token_kind(kind, node, code, ancestors)
$(|| $more::pays_at_token_kind(kind, node, code, ancestors))*
{
TokenRole::Operator
} else {
TokenRole::Unknown
}
}
};
}
#[inline]
#[must_use]
pub(crate) fn keyword_or_op_type<'a, G: Getter + ?Sized>(
node: &Node<'a>,
code: &[u8],
ancestors: Ancestors<'a, '_>,
) -> TokenRole {
match G::get_keyword_role(node, code, ancestors) {
TokenRole::Unknown => match G::get_paying_role(node, code, ancestors) {
TokenRole::Unknown => G::get_op_type(node, ancestors),
role => role,
},
role => role,
}
}
#[must_use]
pub fn default_func_space_name<'a, 'tree>(
node: &Node<'tree>,
code: &'a [u8],
_ancestors: Ancestors<'tree, '_>,
) -> Option<&'a str> {
if let Some(name) = node.child_by_field_name("name") {
node_text(code, &name)
} else {
Some("<anonymous>")
}
}
#[must_use]
pub(crate) fn func_space_name_from_owner<'a, 'tree>(
node: &Node<'tree>,
code: &'a [u8],
ancestors: Ancestors<'tree, '_>,
owned_list: impl FnOnce(&Node<'tree>, &Node<'tree>) -> bool,
) -> Option<&'a str> {
let named = ancestors
.parent(node)
.filter(|owner| owned_list(node, owner))
.unwrap_or(*node);
default_func_space_name(&named, code, ancestors)
}
#[doc(hidden)]
pub struct BracedWordKinds {
pub value: u16,
pub script: u16,
pub comment: u16,
pub command: u16,
pub word_list: u16,
pub simple_word: u16,
pub argument: u16,
pub procedure: u16,
pub namespace: u16,
pub open_brace: u16,
}
pub trait Getter {
#[must_use]
fn get_func_name<'a, 'tree>(
node: &Node<'tree>,
code: &'a [u8],
ancestors: Ancestors<'tree, '_>,
) -> Option<&'a str> {
Self::get_func_space_name(node, code, ancestors)
}
#[must_use]
fn get_func_space_name<'a, 'tree>(
node: &Node<'tree>,
code: &'a [u8],
ancestors: Ancestors<'tree, '_>,
) -> Option<&'a str> {
default_func_space_name(node, code, ancestors)
}
#[must_use]
fn get_space_kind(node: &Node) -> SpaceKind;
#[must_use]
fn get_space_kind_with_code<'a>(
node: &Node<'a>,
code: &[u8],
ancestors: Ancestors<'a, '_>,
) -> SpaceKind;
#[must_use]
fn get_op_type<'a>(_node: &Node<'a>, _ancestors: Ancestors<'a, '_>) -> TokenRole {
TokenRole::Unknown
}
#[must_use]
fn get_keyword_role<'a>(
node: &Node<'a>,
code: &[u8],
ancestors: Ancestors<'a, '_>,
) -> TokenRole;
#[inline]
#[must_use]
fn get_paying_role<'a>(
_node: &Node<'a>,
_code: &[u8],
_ancestors: Ancestors<'a, '_>,
) -> TokenRole {
TokenRole::Unknown
}
#[inline]
#[must_use]
fn get_op_type_with_code<'a>(
node: &Node<'a>,
code: &[u8],
ancestors: Ancestors<'a, '_>,
) -> TokenRole {
keyword_or_op_type::<Self>(node, code, ancestors)
}
#[inline]
#[must_use]
fn get_operator_spelling<'a>(
_node: &Node<'a>,
_ancestors: Ancestors<'a, '_>,
) -> Option<&'static str> {
None
}
#[inline]
#[must_use]
fn get_operand_id<'a>(
node: &Node<'a>,
code: &'a [u8],
_ancestors: Ancestors<'a, '_>,
) -> &'a [u8] {
&code[node.start_byte()..node.end_byte()]
}
#[must_use]
fn string_operand_type(node: &Node, interp_kinds: &[u16]) -> TokenRole {
if node.wraps_any(interp_kinds) {
TokenRole::Unknown
} else {
TokenRole::Operand
}
}
#[must_use]
fn is_subsumed_braced_word<'a>(
node: &Node<'a>,
ancestors: Ancestors<'a, '_>,
kinds: &BracedWordKinds,
) -> bool {
ancestors.parent_has_kind(node, kinds.value)
|| (node.kind_id() == kinds.script
&& node
.children()
.any(|child| child.is_named() && child.kind_id() != kinds.comment))
}
#[must_use]
fn generic_argument_command<'tree, 'chain>(
word: &Node<'tree>,
ancestors: Ancestors<'tree, 'chain>,
kinds: &BracedWordKinds,
) -> Option<(Node<'tree>, Ancestors<'tree, 'chain>)> {
let mut chain = ancestors.iter(word);
let (parent, _) = chain.next()?;
if parent.kind_id() != kinds.word_list {
return None;
}
let (grandparent, grandparent_ancestors) = chain.next()?;
(grandparent.kind_id() == kinds.command).then_some((grandparent, grandparent_ancestors))
}
#[must_use]
fn is_value_braced_word<'a>(
word: &Node<'a>,
code: &[u8],
ancestors: Ancestors<'a, '_>,
kinds: &BracedWordKinds,
) -> bool {
if ancestors
.parent(word)
.is_some_and(|parent| [kinds.argument, kinds.command].contains(&parent.kind_id()))
{
return true;
}
let Some((command, command_ancestors)) =
Self::generic_argument_command(word, ancestors, kinds)
else {
return false;
};
if Self::command_leading_word(&command, code, kinds)
.and_then(script_slots)
.is_some()
{
return false;
}
!(Self::is_switch_arm(&command, code, command_ancestors, kinds)
&& Self::is_switch_arm_body(word, &command))
}
#[must_use]
fn is_braced_script_word<'a>(
node: &Node<'a>,
code: &[u8],
ancestors: Ancestors<'a, '_>,
kinds: &BracedWordKinds,
) -> bool {
node.kind_id() == kinds.script
&& !Self::is_value_braced_word(node, code, ancestors, kinds)
&& !Self::is_braced_literal_slot(node, code, ancestors, kinds)
}
#[must_use]
fn is_braced_literal_slot<'a>(
word: &Node<'a>,
code: &[u8],
ancestors: Ancestors<'a, '_>,
kinds: &BracedWordKinds,
) -> bool {
let mut chain = ancestors.iter(word);
let Some((parent, above_parent)) = chain.next() else {
return false;
};
if parent.kind_id() == kinds.procedure {
let is_name =
matches!(parent.child_by_field_name("name"), Some(name) if name.id() == word.id());
return is_name && !Self::is_switch_arm(&parent, code, above_parent, kinds);
}
let Some((owner, above_owner)) = chain.next() else {
return false;
};
if parent.kind_id() != kinds.word_list
|| !argument_slots_are_readable(&owner, &parent, ancestors)
|| Self::is_switch_arm(&owner, code, above_owner, kinds)
{
return false;
}
let dialect = Dialect { code, kinds };
let roles = if owner.kind_id() == kinds.namespace {
namespace_script_slots(&parent, dialect).map(|slots| ArgumentRoles { slots, first: 1 })
} else if owner.kind_id() == kinds.command {
Self::command_leading_word(&owner, code, kinds)
.and_then(script_slots)
.map(|slots| ArgumentRoles { slots, first: 0 })
} else {
None
};
roles.is_some_and(|roles| !fills_script_slot(word, &parent, roles, dialect))
}
#[must_use]
fn is_switch_arm_body(word: &Node<'_>, command: &Node<'_>) -> bool {
let Some(arguments) = command.child_by_field_name("arguments") else {
return false;
};
let mut cursor = arguments.cursor();
let index = cursor.goto_first_child_for_byte(word.start_byte());
cursor.node().id() == word.id() && matches!(index, Some(index) if index.is_multiple_of(2))
}
#[must_use]
fn command_leading_word<'c>(
command: &Node<'_>,
code: &'c [u8],
kinds: &BracedWordKinds,
) -> Option<&'c str> {
let name = command.child_by_field_name("name")?;
if name.kind_id() != kinds.simple_word {
return None;
}
let text = node_text(code, &name)?;
Some(crate::lang_helpers::strip_global_qualifier(text))
}
#[must_use]
fn is_switch_arm<'a>(
command: &Node<'a>,
code: &[u8],
ancestors: Ancestors<'a, '_>,
kinds: &BracedWordKinds,
) -> bool {
ancestors
.iter(command)
.next()
.is_some_and(|(arm_list, above)| {
arm_list.kind_id() == kinds.script
&& Self::generic_argument_command(&arm_list, above, kinds)
.and_then(|(switch, _)| Self::command_leading_word(&switch, code, kinds))
== Some(SWITCH_COMMAND)
})
}
#[must_use]
fn braced_word_op_type<'a>(
node: &Node<'a>,
code: &[u8],
ancestors: Ancestors<'a, '_>,
kinds: &BracedWordKinds,
) -> TokenRole {
let base = keyword_or_op_type::<Self>(node, code, ancestors);
if node.kind_id() != kinds.open_brace || !matches!(base, TokenRole::Operator) {
return base;
}
let quotes_a_literal = ancestors.iter(node).next().is_some_and(|(parent, above)| {
parent.kind_id() == kinds.script
&& !Self::is_braced_script_word(&parent, code, above, kinds)
});
if quotes_a_literal {
TokenRole::Unknown
} else {
base
}
}
#[must_use]
fn get_operator_id_as_str(_id: u16) -> &'static str {
""
}
}
mod bash;
mod c;
mod ccomment;
mod cpp;
mod csharp;
mod elixir;
mod generic_angles;
mod go;
mod groovy;
mod irules;
mod java;
mod javascript;
mod js_family;
mod kotlin;
mod lua;
mod mozcpp;
mod mozjs;
mod objc;
mod perl;
mod php;
mod preproc;
mod python;
mod ruby;
mod rust;
mod tcl;
mod tsx;
mod typescript;
#[cfg(test)]
mod node_text_tests {
use super::node_text;
use crate::langs::RustParser;
use crate::traits::ParserTrait;
use std::path::PathBuf;
#[cfg(feature = "rust")]
#[test]
fn in_bounds_span_returns_text() {
let src = "fn x() {}";
let parser = RustParser::new(src.as_bytes().to_vec(), &PathBuf::from("t.rs"), None);
let root = parser.root();
assert_eq!(node_text(parser.code(), &root), Some(src));
}
#[cfg(feature = "rust")]
#[test]
fn out_of_bounds_span_returns_none_not_panic() {
let src = "fn x() {}";
let parser = RustParser::new(src.as_bytes().to_vec(), &PathBuf::from("t.rs"), None);
let root = parser.root();
assert!(root.end_byte() > 2);
let truncated = &src.as_bytes()[..2];
assert_eq!(node_text(truncated, &root), None);
}
#[cfg(feature = "rust")]
#[test]
fn non_utf8_span_returns_none() {
let mut src = b"fn ".to_vec();
src.extend_from_slice(&[0xF0, 0x9F]);
src.extend_from_slice(b"() {}");
let parser = RustParser::new(src.clone(), &PathBuf::from("t.rs"), None);
let root = parser.root();
assert_eq!(root.end_byte(), src.len());
assert_eq!(node_text(parser.code(), &root), None);
}
}
#[cfg(test)]
mod ancestor_tests {
use super::Getter;
use crate::node::{Ancestors, Node};
use crate::test_support::for_each_node_with_chain;
use crate::traits::LanguageInfo;
#[cfg(any(
feature = "csharp",
feature = "elixir",
feature = "javascript",
feature = "kotlin",
feature = "mozjs",
feature = "typescript",
))]
fn assert_name_parity<L: LanguageInfo + Getter>(
label: &str,
code: &[u8],
expect_named: &[&str],
) {
let mut seen: Vec<&str> = Vec::new();
let visited = for_each_node_with_chain::<L>(code, |node: &Node<'_>, chain| {
let known = L::get_func_space_name(node, code, Ancestors::known(chain));
let climbing = L::get_func_space_name(node, code, Ancestors::unknown());
assert_eq!(
known,
climbing,
"{label}: name of {} at row {} disagrees",
node.kind(),
node.start_row()
);
if let Some(name) = known
&& expect_named.contains(&name)
&& !seen.contains(&name)
{
seen.push(name);
}
});
assert!(visited > 20, "{label}: fixture is too small to prove much");
for name in expect_named {
assert!(
seen.contains(name),
"{label}: no node resolved to {name:?}, so the fixture no longer \
exercises the ancestor-derived naming it was added for"
);
}
}
#[cfg(all(
feature = "csharp",
feature = "elixir",
feature = "javascript",
feature = "kotlin",
feature = "mozjs",
feature = "typescript",
))]
#[test]
fn func_space_name_agrees_between_known_and_climbing() {
let js_source =
b"var outer = function () { return 1; };\nvar o = { keyed: function () { return 2; } };\n";
assert_name_parity::<crate::langs::JavascriptCode>(
"javascript",
js_source,
&["outer", "keyed"],
);
assert_name_parity::<crate::langs::MozjsCode>("mozjs", js_source, &["outer", "keyed"]);
assert_name_parity::<crate::langs::TypescriptCode>(
"typescript",
js_source,
&["outer", "keyed"],
);
assert_name_parity::<crate::langs::TsxCode>("tsx", js_source, &["outer", "keyed"]);
assert_name_parity::<crate::langs::ElixirCode>(
"elixir",
b"defmodule Foo do\n defmacro multi do\n quote do\n def a, do: 1\n end\n end\nend\n",
&["Foo", "multi"],
);
assert_name_parity::<crate::langs::KotlinCode>(
"kotlin",
b"class Outer(val a: Int) {\n class Inner(val b: Int) {\n fun m(x: Int) = x\n }\n}\n",
&["Outer", "Inner", "m"],
);
assert_name_parity::<crate::langs::CsharpCode>(
"csharp",
b"class Outer(int a) {\n record Inner(int b) {\n int M(int x) => x;\n }\n}\n",
&["Outer", "Inner", "M"],
);
}
}
#[cfg(test)]
#[cfg(any(feature = "tcl", feature = "irules"))]
mod braced_slot_tests {
use super::{BracedWordKinds, Getter};
#[cfg(feature = "tcl")]
use crate::Tcl;
use crate::node::{Ancestors, Node};
use crate::test_support::for_each_node_with_chain;
use crate::traits::LanguageInfo;
fn assert_no_shallow_value_slot<L: LanguageInfo + Getter>(
label: &str,
code: &[u8],
kinds: &BracedWordKinds,
) {
let (mut roots, mut children) = (0, 0);
for_each_node_with_chain::<L>(code, |node: &Node<'_>, chain| {
if chain.len() > 1 {
return;
}
for ancestors in [Ancestors::known(chain), Ancestors::unknown()] {
assert!(
!L::is_braced_literal_slot(node, code, ancestors, kinds),
"{label}: a {} at depth {} filled a value slot",
node.kind(),
chain.len()
);
}
if chain.is_empty() {
roots += 1;
} else {
children += 1;
}
});
assert_eq!(roots, 1, "{label}: exactly one root");
assert!(
children > 1,
"{label}: the fixture must put several nodes directly under the \
root, or the depth-1 exit goes unexercised"
);
}
#[test]
#[cfg(feature = "tcl")]
fn tcl_no_node_above_an_argument_fills_a_value_slot() {
assert_no_shallow_value_slot::<crate::langs::TclCode>(
"tcl",
b"{a b}\nproc {my proc} {} {}\nnamespace export {c d}\n",
&crate::lang_helpers::tcl::BRACED_WORD_KINDS,
);
}
#[test]
#[cfg(feature = "irules")]
fn irules_no_node_above_an_argument_fills_a_value_slot() {
assert_no_shallow_value_slot::<crate::langs::IrulesCode>(
"irules",
b"{a b}\nproc {my proc} {} {}\nnamespace export {c d}\n",
&crate::lang_helpers::irules::BRACED_WORD_KINDS,
);
}
#[cfg(feature = "tcl")]
fn value_slot_texts(code: &[u8]) -> Vec<String> {
let tree = crate::node::Tree::new::<crate::langs::TclCode>(code);
tree.get_root()
.preorder()
.filter(|node| {
<crate::langs::TclCode as Getter>::is_braced_literal_slot(
node,
code,
Ancestors::unknown(),
&crate::lang_helpers::tcl::BRACED_WORD_KINDS,
)
})
.map(|node| node.utf8_text(code).unwrap_or_default().to_owned())
.collect()
}
#[test]
#[cfg(feature = "tcl")]
fn an_error_inside_a_script_argument_keeps_the_sibling_value_slots() {
let well_formed = value_slot_texts(b"trap {pat} {v} {puts j}\n");
assert_eq!(
well_formed,
["{pat}", "{v}"],
"the fixture must spell two value slots before an error is added"
);
assert_eq!(
value_slot_texts(b"trap {pat} {v} {puts ]}\n"),
well_formed,
"an error inside the script argument moves no sibling's index"
);
}
#[test]
#[cfg(feature = "tcl")]
fn an_error_occupying_a_slot_withdraws_the_layout() {
assert_eq!(
value_slot_texts(b"on {code} {v} {puts j}\n"),
["{code}", "{v}"],
"the fixture must spell two value slots before the error \
takes one of their places"
);
assert!(
value_slot_texts(b"on {code} ] {puts j}\n").is_empty(),
"an ERROR token in the argument list makes every position \
unreadable"
);
}
#[test]
#[cfg(feature = "tcl")]
fn a_handler_clause_of_the_wrong_arity_withdraws_the_layout() {
assert_eq!(
value_slot_texts(b"try {puts a} trap {p} {v} {puts b}\n"),
["{p}", "{v}"],
"a three-argument `trap` clause spells two value slots"
);
assert!(
value_slot_texts(b"try {puts a} trap {p} {v} {puts b} finally {puts c}\n").is_empty(),
"a trailing `finally` joins the same generic `trap` command, \
so no position in its argument list is trustworthy"
);
}
#[test]
#[cfg(feature = "tcl")]
fn a_command_without_arguments_has_no_switch_arm_body() {
use crate::node::Tree;
let code = b"switch\nswitch $v { p {puts a} }\n";
let tree = Tree::new::<crate::langs::TclCode>(code);
let root = tree.get_root();
let bare = root.child(0).expect("the bare `switch` command");
let name = bare
.child_by_field_name("name")
.expect("even a bare command names itself");
assert!(
bare.child_by_field_name("arguments").is_none(),
"the fixture's first command must carry no argument list"
);
assert!(
!<crate::langs::TclCode as Getter>::is_switch_arm_body(&name, &bare),
"a command with no argument list has no body position"
);
let arm = find_arm_named_p(&root, code).expect("the arm command named `p`");
let body = arm
.child_by_field_name("arguments")
.and_then(|args| args.child(0))
.expect("the arm's body");
assert!(
<crate::langs::TclCode as Getter>::is_switch_arm_body(&body, &arm),
"the arm's sole argument is its body, or the assertion above is \
satisfied by a constant"
);
}
#[cfg(feature = "tcl")]
fn find_arm_named_p<'a>(node: &Node<'a>, code: &[u8]) -> Option<Node<'a>> {
let named_p = node.kind_id() == Tcl::Command as u16
&& node
.child_by_field_name("name")
.is_some_and(|name| name.utf8_text(code) == Some("p"));
if named_p {
return Some(*node);
}
node.children()
.find_map(|child| find_arm_named_p(&child, code))
}
fn assert_word_list_owners<L: LanguageInfo>(label: &str, code: &[u8], kinds: &BracedWordKinds) {
let mut seen = 0;
for_each_node_with_chain::<L>(code, |node: &Node<'_>, chain| {
if node.kind_id() != kinds.word_list {
return;
}
let owner = chain.last().expect("a word_list is never the root");
assert!(
[kinds.command, kinds.namespace].contains(&owner.kind_id()),
"{label}: a word_list owned by {}",
owner.kind()
);
seen += 1;
});
assert!(seen > 2, "{label}: fixture reached only {seen} word_lists");
}
const WORD_LIST_OWNER_FIXTURE: &[u8] = b"proc p {x} { puts $x }\n\
namespace eval ns { puts hi }\n\
namespace export {a b}\n\
if {$x} { puts a } else { puts b }\n\
while {$x} { puts a }\n\
foreach i {1 2} { puts $i }\n\
catch { puts a } msg\n\
eval {puts hi}\n\
switch $v { p {puts a} }\n\
lappend l {c d}\n";
#[test]
#[cfg(feature = "tcl")]
fn tcl_a_word_list_is_owned_only_by_a_command_or_a_namespace() {
assert_word_list_owners::<crate::langs::TclCode>(
"tcl",
WORD_LIST_OWNER_FIXTURE,
&crate::lang_helpers::tcl::BRACED_WORD_KINDS,
);
}
#[test]
#[cfg(feature = "irules")]
fn irules_a_word_list_is_owned_only_by_a_command_or_a_namespace() {
assert_word_list_owners::<crate::langs::IrulesCode>(
"irules",
WORD_LIST_OWNER_FIXTURE,
&crate::lang_helpers::irules::BRACED_WORD_KINDS,
);
}
}
#[cfg(test)]
mod keyword_role_tests {
use super::{Getter, keyword_or_op_type};
use crate::node::{Ancestors, Node};
use crate::test_support::for_each_node_with_chain;
use crate::token_role::TokenRole;
use crate::traits::LanguageInfo;
#[cfg(any(
feature = "c",
feature = "cpp",
feature = "mozcpp",
feature = "objc",
feature = "java",
feature = "groovy",
feature = "javascript",
feature = "mozjs",
feature = "typescript",
feature = "python",
feature = "rust",
feature = "kotlin",
feature = "elixir",
))]
fn roles_of<L: LanguageInfo + Getter>(
source: &[u8],
classify: impl for<'a> Fn(&Node<'a>, &[u8], Ancestors<'a, '_>) -> TokenRole,
) -> Vec<(&'static str, TokenRole)> {
let mut out = Vec::new();
for_each_node_with_chain::<L>(source, |node, chain| {
let role = classify(node, source, Ancestors::known(chain));
if role != TokenRole::Unknown {
out.push((node.kind(), role));
}
});
out
}
#[cfg(feature = "kotlin")]
#[test]
fn kotlin_byte_less_string_arm_reads_the_rows_wraps_list() {
use crate::KotlinCode;
let source = b"val a = \"plain\"\nval b = \"x ${a} y\"\nval c = \"\"\"raw\"\"\"\n";
let byte_less = roles_of::<KotlinCode>(source, |n, _, a| KotlinCode::get_op_type(n, a))
.into_iter()
.filter(|(kind, _)| matches!(*kind, "string_literal" | "multiline_string_literal"))
.collect::<Vec<_>>();
let with_code = roles_of::<KotlinCode>(source, KotlinCode::get_op_type_with_code)
.into_iter()
.filter(|(kind, _)| matches!(*kind, "string_literal" | "multiline_string_literal"))
.collect::<Vec<_>>();
assert_eq!(
byte_less,
[
("string_literal", TokenRole::Operand),
("multiline_string_literal", TokenRole::Operand),
]
);
assert_eq!(
byte_less, with_code,
"the two spellings agree on every literal"
);
}
#[cfg(any(
feature = "c",
feature = "cpp",
feature = "mozcpp",
feature = "objc",
feature = "java",
feature = "groovy",
feature = "javascript",
feature = "mozjs",
feature = "typescript",
feature = "python",
feature = "rust",
))]
fn control_keywords_bill_through_the_rows<L: LanguageInfo + Getter>(
source: &[u8],
keywords: &[&'static str],
) {
let via_walk = roles_of::<L>(source, L::get_op_type_with_code);
let via_rows = roles_of::<L>(source, L::get_paying_role);
let via_byte_less = roles_of::<L>(source, |n, _, a| L::get_op_type(n, a));
for &kw in keywords {
let billed = (kw, TokenRole::Operator);
assert!(via_walk.contains(&billed), "{kw}: missing from the walk");
assert!(via_rows.contains(&billed), "{kw}: the rows do not bill it");
assert!(
!via_byte_less.contains(&billed),
"{kw}: still billed byte-less"
);
}
let else_kw = ("else", TokenRole::Operator);
assert!(
via_byte_less.contains(&else_kw),
"`else` is no row's and stays"
);
assert!(!via_rows.contains(&else_kw));
}
macro_rules! control_keywords_test {
($name:ident, $feature:literal, $code:ident, $source:expr, $keywords:expr) => {
#[cfg(feature = $feature)]
#[test]
fn $name() {
control_keywords_bill_through_the_rows::<crate::$code>($source, $keywords);
}
};
}
#[cfg(any(feature = "c", feature = "objc"))]
const CONTROL_KEYWORDS_C: &[&str] = &["if", "for", "while"];
#[cfg(any(
feature = "cpp",
feature = "mozcpp",
feature = "javascript",
feature = "mozjs",
feature = "typescript"
))]
const CONTROL_KEYWORDS_WITH_CATCH: &[&str] = &["if", "for", "while", "catch"];
#[cfg(any(feature = "java", feature = "groovy"))]
const CONTROL_KEYWORDS_JVM: &[&str] = &["if", "for", "while", "switch", "catch"];
#[cfg(any(feature = "cpp", feature = "mozcpp"))]
const CONTROL_KEYWORDS_CPP_SOURCE: &[u8] =
b"int f(int a) { if (a) { } else { } for (;;) { } while (a) { } try { } catch (int e) { } return 0; }\n";
#[cfg(any(feature = "javascript", feature = "mozjs", feature = "typescript"))]
const CONTROL_KEYWORDS_JS_SOURCE: &[u8] =
b"function f(a) { if (a) { } else { } for (;;) { } while (a) { } try { } catch (e) { } }\n";
control_keywords_test!(
c_control_keywords_are_billed_by_the_rows,
"c",
CCode,
b"int f(int a) { if (a) { } else { } for (;;) { } while (a) { } switch (a) { } return 0; }\n",
CONTROL_KEYWORDS_C
);
control_keywords_test!(
cpp_control_keywords_are_billed_by_the_rows,
"cpp",
CppCode,
CONTROL_KEYWORDS_CPP_SOURCE,
CONTROL_KEYWORDS_WITH_CATCH
);
control_keywords_test!(
mozcpp_control_keywords_are_billed_by_the_rows,
"mozcpp",
MozcppCode,
CONTROL_KEYWORDS_CPP_SOURCE,
CONTROL_KEYWORDS_WITH_CATCH
);
control_keywords_test!(
objc_control_keywords_are_billed_by_the_rows,
"objc",
ObjcCode,
b"@implementation A\n- (int)f:(int)a { if (a) { } else { } for (;;) { } while (a) { } return 0; }\n@end\n",
CONTROL_KEYWORDS_C
);
control_keywords_test!(
java_control_keywords_are_billed_by_the_rows,
"java",
JavaCode,
b"class A { void f(int a) { if (a > 0) { } else { } for (;;) { } while (a > 0) { } switch (a) { } try { } catch (Exception e) { } } }\n",
CONTROL_KEYWORDS_JVM
);
control_keywords_test!(
groovy_control_keywords_are_billed_by_the_rows,
"groovy",
GroovyCode,
b"class A { void f(int a) { if (a) { } else { } for (;;) { } while (a) { } switch (a) { } try { } catch (Exception e) { } } }\n",
CONTROL_KEYWORDS_JVM
);
control_keywords_test!(
javascript_control_keywords_are_billed_by_the_rows,
"javascript",
JavascriptCode,
CONTROL_KEYWORDS_JS_SOURCE,
CONTROL_KEYWORDS_WITH_CATCH
);
control_keywords_test!(
mozjs_control_keywords_are_billed_by_the_rows,
"mozjs",
MozjsCode,
CONTROL_KEYWORDS_JS_SOURCE,
CONTROL_KEYWORDS_WITH_CATCH
);
control_keywords_test!(
typescript_control_keywords_are_billed_by_the_rows,
"typescript",
TypescriptCode,
CONTROL_KEYWORDS_JS_SOURCE,
CONTROL_KEYWORDS_WITH_CATCH
);
control_keywords_test!(
tsx_control_keywords_are_billed_by_the_rows,
"typescript",
TsxCode,
CONTROL_KEYWORDS_JS_SOURCE,
CONTROL_KEYWORDS_WITH_CATCH
);
control_keywords_test!(
python_control_keywords_are_billed_by_the_rows,
"python",
PythonCode,
b"def f(a, xs):\n if a:\n pass\n elif a:\n pass\n else:\n pass\n assert a\n for x in xs:\n pass\n while a:\n break\n try:\n pass\n except E:\n pass\n",
&["elif", "assert", "for", "while", "except"]
);
control_keywords_test!(
rust_control_keywords_are_billed_by_the_rows,
"rust",
RustCode,
b"fn f(a: bool, xs: &[u8]) { if a { } else { } for _x in xs { } while a { break; } loop { break; } }\n",
&["if", "while", "loop"]
);
#[cfg(feature = "rust")]
#[test]
fn rust_type_syntax_for_still_bills_as_an_operator() {
let source = b"trait T {}\nstruct S;\nimpl T for S {}\nfn f<F: for<'a> Fn(&'a u8)>(_: F) {}\ntype P = for<'a> fn(&'a u8);\nfn a(_: impl for<'a> Fn(&'a u8)) {}\nfn r(xs: &[u8]) { for _x in xs {} }\n";
let for_operators = |roles: Vec<(&str, TokenRole)>| {
roles
.into_iter()
.filter(|&(kind, role)| kind == "for" && role == TokenRole::Operator)
.count()
};
let via_walk = for_operators(roles_of::<crate::RustCode>(
source,
crate::RustCode::get_op_type_with_code,
));
let via_rows = for_operators(roles_of::<crate::RustCode>(
source,
crate::RustCode::get_paying_role,
));
let via_byte_less = for_operators(roles_of::<crate::RustCode>(source, |n, _, a| {
crate::RustCode::get_op_type(n, a)
}));
assert_eq!((via_walk, via_rows, via_byte_less), (5, 1, 5));
}
#[cfg(feature = "elixir")]
#[test]
fn elixir_paying_role_bills_each_rows_applied_tokens() {
use crate::ElixirCode;
let source = b"x = a == b && c < d or e + f\ng = &==/2\n";
let via_rows = roles_of::<ElixirCode>(source, ElixirCode::get_paying_role);
assert_eq!(
via_rows,
[
("==", TokenRole::Operator),
("&&", TokenRole::Operator),
("<", TokenRole::Operator),
("or", TokenRole::Operator),
]
);
}
#[cfg(feature = "java")]
#[test]
fn java_byte_less_arm_bills_class_only_in_a_class_literal() {
use crate::JavaCode;
let source = b"class A { Object o = A.class; }\n";
let class_tokens = |roles: Vec<(&'static str, TokenRole)>| {
roles
.into_iter()
.filter(|&(kind, _)| kind == "class")
.map(|(_, role)| role)
.collect::<Vec<_>>()
};
let via_walk = class_tokens(roles_of::<JavaCode>(
source,
JavaCode::get_op_type_with_code,
));
let via_rows = class_tokens(roles_of::<JavaCode>(source, JavaCode::get_keyword_role));
let via_byte_less = class_tokens(roles_of::<JavaCode>(source, |n, _, a| {
JavaCode::get_op_type(n, a)
}));
assert_eq!(via_walk, [TokenRole::Operator, TokenRole::Operand]);
assert_eq!(via_rows, [TokenRole::Operator]);
assert_eq!(via_byte_less, [TokenRole::Operand]);
}
#[cfg(feature = "perl")]
#[test]
fn perl_byte_less_arm_tells_an_and_operator_from_an_and_key() {
use crate::PerlCode;
use crate::test_support::for_each_node_with_chain_tolerating_errors;
let source = b"my $c = $a and $b;\nmy %h = (and => 1);\n";
let and_tokens = |classify: fn(&Node<'_>, &[u8], Ancestors<'_, '_>) -> TokenRole| {
let mut out = Vec::new();
for_each_node_with_chain_tolerating_errors::<PerlCode>(source, |node, chain| {
if node.kind() == "and" {
out.push(classify(node, source, Ancestors::known(chain)));
}
});
out
};
let via_walk = and_tokens(PerlCode::get_op_type_with_code);
let via_rows = and_tokens(PerlCode::get_paying_role);
let via_byte_less = and_tokens(|n, _, a| PerlCode::get_op_type(n, a));
assert_eq!(via_walk, [TokenRole::Operator, TokenRole::Operand]);
assert_eq!(via_rows, [TokenRole::Operator, TokenRole::Unknown]);
assert_eq!(via_byte_less, [TokenRole::Operator, TokenRole::Operand]);
}
#[cfg(feature = "c-family-helpers")]
#[test]
fn row_less_languages_render_no_operator_name() {
use crate::{CcommentCode, PreprocCode};
for id in [0, 1, u16::MAX] {
assert_eq!(CcommentCode::get_operator_id_as_str(id), "");
assert_eq!(PreprocCode::get_operator_id_as_str(id), "");
}
}
#[cfg(feature = "c")]
#[test]
fn c_keywords_are_billed_by_the_rows_not_the_byte_less_arm() {
use crate::CCode;
let source = b"static struct S { unsigned int a; } s;\n";
let via_walk = roles_of::<CCode>(source, CCode::get_op_type_with_code);
let via_rows = roles_of::<CCode>(source, CCode::get_keyword_role);
let via_byte_less = roles_of::<CCode>(source, |n, _, a| CCode::get_op_type(n, a));
let keywords = [
("static", TokenRole::Operator),
("struct", TokenRole::Operator),
("unsigned", TokenRole::Operator),
("primitive_type", TokenRole::Operator),
];
assert_eq!(via_rows, keywords, "the rows bill exactly the keywords");
for keyword in &keywords {
assert!(
via_walk.contains(keyword),
"{keyword:?} missing from the walk: {via_walk:?}"
);
assert!(
!via_byte_less.contains(keyword),
"{keyword:?} is still billed by the byte-less arm: {via_byte_less:?}"
);
}
for pair in &via_byte_less {
assert!(
via_walk.contains(pair),
"{pair:?} lost on the walk: {via_walk:?}"
);
}
assert_eq!(via_walk.len(), via_byte_less.len() + keywords.len());
}
#[cfg(feature = "cpp")]
#[test]
fn keyword_or_op_type_merges_the_row_with_the_byte_less_arm() {
use crate::CppCode;
let source = b"const int x = 1;\n";
let merged = roles_of::<CppCode>(source, keyword_or_op_type::<CppCode>);
assert_eq!(
merged,
[
("const", TokenRole::Operator),
("primitive_type", TokenRole::Operator),
("identifier", TokenRole::Operand),
("=", TokenRole::Operator),
("number_literal", TokenRole::Operand),
(";", TokenRole::Operator),
]
);
}
struct Conflict;
macro_rules! conflicting_getter {
($name:ident, $row:ident, $arm:ident) => {
impl Getter for $name {
fn get_space_kind(_node: &Node) -> crate::SpaceKind {
crate::SpaceKind::Unknown
}
fn get_space_kind_with_code<'a>(
_node: &Node<'a>,
_code: &[u8],
_ancestors: Ancestors<'a, '_>,
) -> crate::SpaceKind {
crate::SpaceKind::Unknown
}
fn get_keyword_role<'a>(
_node: &Node<'a>,
_code: &[u8],
_ancestors: Ancestors<'a, '_>,
) -> TokenRole {
TokenRole::$row
}
fn get_op_type<'a>(_node: &Node<'a>, _ancestors: Ancestors<'a, '_>) -> TokenRole {
TokenRole::$arm
}
}
};
}
conflicting_getter!(Conflict, Operand, Operator);
#[cfg(feature = "cpp")]
#[test]
fn keyword_or_op_type_prefers_the_row_on_a_conflict() {
use crate::CppCode;
let roles = roles_of::<CppCode>(b"x;\n", keyword_or_op_type::<Conflict>);
assert_eq!(
roles,
[
("translation_unit", TokenRole::Operand),
("expression_statement", TokenRole::Operand),
("identifier", TokenRole::Operand),
(";", TokenRole::Operand),
]
);
}
#[cfg(feature = "java")]
#[test]
fn java_override_reaches_the_fallback() {
use crate::JavaCode;
let source = b"class A { int f() { return 1; } }\n";
let walk = roles_of::<JavaCode>(source, JavaCode::get_op_type_with_code);
let fallback = roles_of::<JavaCode>(source, keyword_or_op_type::<JavaCode>);
assert_eq!(walk, fallback);
assert!(walk.contains(&("class", TokenRole::Operator)), "{walk:?}");
}
#[cfg(feature = "tcl")]
#[test]
fn tcl_braced_word_path_reaches_the_fallback() {
use crate::TclCode;
let source = b"proc f {a} {\n lappend x {b c}\n return $x\n}\n";
let offsets = |classify: fn(&Node<'_>, &[u8], Ancestors<'_, '_>) -> TokenRole| {
let mut out = Vec::new();
for_each_node_with_chain::<TclCode>(source, |node, chain| {
if classify(node, source, Ancestors::known(chain)) != TokenRole::Unknown {
out.push((node.start_byte(), node.kind()));
}
});
out
};
let walk = offsets(TclCode::get_op_type_with_code);
let fallback = offsets(keyword_or_op_type::<TclCode>);
let literal = source
.windows(5)
.position(|w| w == b"{b c}")
.expect("fixture spells the literal");
let withdrawn: Vec<_> = fallback
.iter()
.filter(|pair| !walk.contains(pair))
.collect();
assert_eq!(
withdrawn,
[&(literal, "{")],
"walk {walk:?}\nfallback {fallback:?}"
);
assert_eq!(walk.len() + 1, fallback.len());
}
}