use std::collections::HashSet;
use regex::Regex;
use tree_sitter::{Node, Parser};
use crate::config::{CallKind, CallSpec};
use crate::decode::{Decoded, Lang, decode_token, unescape_js, unescape_php_double};
use crate::guard::{Guard, Segment, guards_from_segments};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SourceLang {
Php,
Js,
Jsx,
Ts,
Tsx,
Twig,
}
impl SourceLang {
pub fn from_extension(ext: &str) -> Option<SourceLang> {
match ext.to_lowercase().as_str() {
"php" => Some(SourceLang::Php),
"twig" => Some(SourceLang::Twig),
"js" | "mjs" | "cjs" => Some(SourceLang::Js),
"jsx" => Some(SourceLang::Jsx),
"ts" | "mts" | "cts" => Some(SourceLang::Ts),
"tsx" => Some(SourceLang::Tsx),
_ => None,
}
}
pub fn family_label(&self) -> &'static str {
match self {
SourceLang::Php => "php",
SourceLang::Twig => "twig",
SourceLang::Js | SourceLang::Jsx | SourceLang::Ts | SourceLang::Tsx => "js",
}
}
pub fn language(&self) -> tree_sitter::Language {
match self {
SourceLang::Php => tree_sitter_php::LANGUAGE_PHP.into(),
SourceLang::Js | SourceLang::Jsx => tree_sitter_javascript::LANGUAGE.into(),
SourceLang::Ts => tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into(),
SourceLang::Tsx => tree_sitter_typescript::LANGUAGE_TSX.into(),
SourceLang::Twig => unreachable!("Twig uses the regex path, not a grammar"),
}
}
fn family(&self) -> Family {
match self {
SourceLang::Php => Family::Php,
SourceLang::Twig => Family::Twig,
_ => Family::Js,
}
}
}
#[derive(Clone, Copy, PartialEq, Eq)]
enum Family {
Php,
Js,
Twig,
}
#[derive(Debug, Default, Clone, PartialEq, Eq)]
pub struct ExtractResult {
pub literals: HashSet<String>,
pub guards: Vec<Guard>,
pub blind: usize,
pub source_literals: HashSet<String>,
}
impl ExtractResult {
fn record(&mut self, segments: Vec<Segment>, min_guard_len: usize) {
let has_hole = segments.iter().any(|s| matches!(s, Segment::Hole));
if !has_hole {
let literal: String = segments
.iter()
.filter_map(|s| match s {
Segment::Static(t) => Some(t.as_str()),
Segment::Hole => None,
})
.collect();
self.literals.insert(literal);
} else {
let ext = guards_from_segments(&segments, min_guard_len);
self.guards.extend(ext.guards);
if ext.blind {
self.blind += 1;
}
}
}
fn record_source_literal(&mut self, segments: &[Segment]) {
if !segments.iter().all(|s| matches!(s, Segment::Static(_))) {
return;
}
let literal: String = segments
.iter()
.filter_map(|s| match s {
Segment::Static(t) => Some(t.as_str()),
Segment::Hole => None,
})
.collect();
if !literal.is_empty() {
self.source_literals.insert(literal);
}
}
}
#[derive(Default)]
pub struct ParserPool {
php: Option<Parser>,
js: Option<Parser>,
ts: Option<Parser>,
tsx: Option<Parser>,
}
impl ParserPool {
pub fn new() -> Self {
Self::default()
}
fn get(&mut self, lang: SourceLang) -> Option<&mut Parser> {
let slot = match lang {
SourceLang::Php => &mut self.php,
SourceLang::Js | SourceLang::Jsx => &mut self.js,
SourceLang::Ts => &mut self.ts,
SourceLang::Tsx => &mut self.tsx,
SourceLang::Twig => return None,
};
if slot.is_none() {
let mut parser = Parser::new();
if parser.set_language(&lang.language()).is_err() {
return None;
}
*slot = Some(parser);
}
slot.as_mut()
}
}
pub fn extract(
lang: SourceLang,
source: &str,
calls: &[CallSpec],
min_guard_len: usize,
) -> ExtractResult {
let mut pool = ParserPool::new();
extract_with_pool(&mut pool, lang, source, calls, min_guard_len)
}
pub fn extract_with_pool(
pool: &mut ParserPool,
lang: SourceLang,
source: &str,
calls: &[CallSpec],
min_guard_len: usize,
) -> ExtractResult {
let applicable: Vec<&CallSpec> = calls.iter().filter(|c| call_applies(c, lang)).collect();
match lang.family() {
Family::Twig => extract_twig(source, &applicable),
_ => match pool.get(lang) {
Some(parser) => extract_ast_with(parser, lang, source, &applicable, min_guard_len),
None => ExtractResult::default(),
},
}
}
fn call_applies(call: &CallSpec, lang: SourceLang) -> bool {
let l = call.lang.to_lowercase();
match lang {
SourceLang::Php => l == "php",
SourceLang::Twig => l == "twig",
SourceLang::Js | SourceLang::Jsx | SourceLang::Ts | SourceLang::Tsx => {
matches!(
l.as_str(),
"js" | "jsx" | "ts" | "tsx" | "javascript" | "typescript"
)
}
}
}
fn extract_ast_with(
parser: &mut Parser,
lang: SourceLang,
source: &str,
calls: &[&CallSpec],
min_guard_len: usize,
) -> ExtractResult {
let Some(tree) = parser.parse(source, None) else {
return ExtractResult::default();
};
let mut res = ExtractResult::default();
let family = lang.family();
for_each_node(tree.root_node(), |node| {
match family {
Family::Php => try_php_call(node, source, calls, min_guard_len, &mut res),
Family::Js => {
try_js_call(node, source, calls, min_guard_len, &mut res);
try_js_index(node, source, calls, min_guard_len, &mut res);
}
Family::Twig => {}
}
collect_ast_literal(node, family, source, &mut res);
});
res
}
fn collect_ast_literal(node: Node, family: Family, src: &str, res: &mut ExtractResult) {
let segments = match family {
Family::Php => match node.kind() {
"string" | "encapsed_string" => php_segments(node, src),
_ => return,
},
Family::Js => match node.kind() {
"string" | "template_string" => js_segments(node, src),
_ => return,
},
Family::Twig => return,
};
res.record_source_literal(&segments);
}
fn for_each_node<F: FnMut(Node)>(root: Node, mut f: F) {
let mut stack = vec![root];
while let Some(node) = stack.pop() {
f(node);
let mut cursor = node.walk();
for child in node.children(&mut cursor) {
stack.push(child);
}
}
}
fn text<'a>(node: Node, src: &'a str) -> &'a str {
node.utf8_text(src.as_bytes()).unwrap_or("")
}
fn receiver_ok(spec: &CallSpec, receiver: &str) -> bool {
match &spec.receiver {
Some(list) => list.iter().any(|r| r == receiver),
None => true,
}
}
fn try_php_call(
node: Node,
src: &str,
calls: &[&CallSpec],
min_guard_len: usize,
res: &mut ExtractResult,
) {
match node.kind() {
"function_call_expression" => {
let Some(func) = node.child_by_field_name("function") else {
return;
};
if func.kind() != "name" {
return;
}
let name = text(func, src);
if let Some(spec) = calls
.iter()
.find(|c| c.kind == CallKind::Function && c.name == name)
{
record_php_key_arg(node, spec, src, min_guard_len, res);
}
}
"member_call_expression" => {
let (Some(name_node), Some(obj)) = (
node.child_by_field_name("name"),
node.child_by_field_name("object"),
) else {
return;
};
let mname = text(name_node, src);
let receiver = normalize_php_receiver(obj, src);
if let Some(spec) = calls.iter().find(|c| {
c.kind == CallKind::Method && c.name == mname && receiver_ok(c, &receiver)
}) {
record_php_key_arg(node, spec, src, min_guard_len, res);
}
}
"scoped_call_expression" => {
let (Some(name_node), Some(scope)) = (
node.child_by_field_name("name"),
node.child_by_field_name("scope"),
) else {
return;
};
let mname = text(name_node, src);
let receiver = text(scope, src).trim_start_matches('$').to_string();
if let Some(spec) = calls.iter().find(|c| {
c.kind == CallKind::Method && c.name == mname && receiver_ok(c, &receiver)
}) {
record_php_key_arg(node, spec, src, min_guard_len, res);
}
}
_ => {}
}
}
fn record_php_key_arg(
call: Node,
spec: &CallSpec,
src: &str,
min_guard_len: usize,
res: &mut ExtractResult,
) {
let Some(args) = call.child_by_field_name("arguments") else {
return;
};
let values = php_arg_values(args);
if let Some(&arg) = values.get(spec.key_arg_index) {
let segments = php_segments(arg, src);
res.record(segments, min_guard_len);
}
}
fn php_arg_values(args: Node) -> Vec<Node> {
let mut out = Vec::new();
let mut cursor = args.walk();
for child in args.named_children(&mut cursor) {
if child.kind() == "argument" {
let count = child.named_child_count();
if count > 0
&& let Some(value) = child.named_child(count as u32 - 1)
{
out.push(value);
}
}
}
out
}
fn normalize_php_receiver(node: Node, src: &str) -> String {
match node.kind() {
"variable_name" => text(node, src).trim_start_matches('$').to_string(),
"name" => text(node, src).to_string(),
"member_access_expression" => {
match (
node.child_by_field_name("object"),
node.child_by_field_name("name"),
) {
(Some(obj), Some(name)) => {
format!("{}.{}", normalize_php_receiver(obj, src), text(name, src))
}
_ => text(node, src).trim_start_matches('$').to_string(),
}
}
"function_call_expression" | "scoped_call_expression" | "member_call_expression" => {
match node.child_by_field_name(if node.kind() == "function_call_expression" {
"function"
} else {
"name"
}) {
Some(callee) => format!("{}()", text(callee, src)),
None => text(node, src).trim_start_matches('$').to_string(),
}
}
_ => text(node, src).trim_start_matches('$').to_string(),
}
}
fn php_segments(node: Node, src: &str) -> Vec<Segment> {
match node.kind() {
"string" => match decode_token(Lang::Php, text(node, src)) {
Decoded::Literal(s) => vec![Segment::Static(s)],
Decoded::Dynamic => vec![Segment::Hole],
},
"encapsed_string" => {
let mut segs = Vec::new();
let mut cursor = node.walk();
for child in node.named_children(&mut cursor) {
match child.kind() {
"string_content" => segs.push(Segment::Static(text(child, src).to_string())),
"escape_sequence" => {
segs.push(Segment::Static(unescape_php_double(text(child, src))))
}
_ => segs.push(Segment::Hole),
}
}
if segs.is_empty() {
vec![Segment::Static(String::new())]
} else {
segs
}
}
"binary_expression" if binary_operator(node, src).as_deref() == Some(".") => {
concat_segments(node, src, php_segments)
}
_ => vec![Segment::Hole],
}
}
fn try_js_call(
node: Node,
src: &str,
calls: &[&CallSpec],
min_guard_len: usize,
res: &mut ExtractResult,
) {
if node.kind() != "call_expression" {
return;
}
let Some(func) = node.child_by_field_name("function") else {
return;
};
match func.kind() {
"identifier" => {
let name = text(func, src);
if let Some(spec) = calls
.iter()
.find(|c| c.kind == CallKind::Function && c.name == name)
{
record_js_key_arg(node, spec, src, min_guard_len, res);
}
}
"member_expression" => {
let (Some(prop), Some(obj)) = (
func.child_by_field_name("property"),
func.child_by_field_name("object"),
) else {
return;
};
let name = text(prop, src);
let receiver = normalize_js_receiver(obj, src);
if let Some(spec) = calls
.iter()
.find(|c| c.kind == CallKind::Method && c.name == name && receiver_ok(c, &receiver))
{
record_js_key_arg(node, spec, src, min_guard_len, res);
}
}
_ => {}
}
}
fn try_js_index(
node: Node,
src: &str,
calls: &[&CallSpec],
min_guard_len: usize,
res: &mut ExtractResult,
) {
if node.kind() != "subscript_expression" {
return;
}
let (Some(obj), Some(index)) = (
node.child_by_field_name("object"),
node.child_by_field_name("index"),
) else {
return;
};
let obj_name = normalize_js_receiver(obj, src);
if calls
.iter()
.any(|c| c.kind == CallKind::Index && c.name == obj_name)
{
let segments = js_segments(index, src);
res.record(segments, min_guard_len);
}
}
fn record_js_key_arg(
call: Node,
spec: &CallSpec,
src: &str,
min_guard_len: usize,
res: &mut ExtractResult,
) {
let Some(args) = call.child_by_field_name("arguments") else {
return;
};
let mut values = Vec::new();
let mut cursor = args.walk();
for child in args.named_children(&mut cursor) {
if child.kind() != "comment" {
values.push(child);
}
}
if let Some(&arg) = values.get(spec.key_arg_index) {
let segments = js_segments(arg, src);
res.record(segments, min_guard_len);
}
}
fn normalize_js_receiver(node: Node, src: &str) -> String {
match node.kind() {
"identifier" | "property_identifier" => text(node, src).to_string(),
"this" => "this".to_string(),
"member_expression" => {
match (
node.child_by_field_name("object"),
node.child_by_field_name("property"),
) {
(Some(obj), Some(prop)) => {
format!("{}.{}", normalize_js_receiver(obj, src), text(prop, src))
}
_ => text(node, src).to_string(),
}
}
_ => text(node, src).to_string(),
}
}
fn js_segments(node: Node, src: &str) -> Vec<Segment> {
match node.kind() {
"string" => match decode_token(Lang::Js, text(node, src)) {
Decoded::Literal(s) => vec![Segment::Static(s)],
Decoded::Dynamic => vec![Segment::Hole],
},
"template_string" => {
let mut segs = Vec::new();
let mut cursor = node.walk();
for child in node.named_children(&mut cursor) {
match child.kind() {
"template_substitution" => segs.push(Segment::Hole),
"escape_sequence" => segs.push(Segment::Static(unescape_js(text(child, src)))),
_ => segs.push(Segment::Static(text(child, src).to_string())),
}
}
if segs.is_empty() {
vec![Segment::Static(String::new())]
} else {
segs
}
}
"binary_expression" if binary_operator(node, src).as_deref() == Some("+") => {
concat_segments(node, src, js_segments)
}
_ => vec![Segment::Hole],
}
}
fn binary_operator(node: Node, src: &str) -> Option<String> {
let mut cursor = node.walk();
for child in node.children(&mut cursor) {
if !child.is_named() {
return Some(text(child, src).to_string());
}
}
None
}
fn concat_segments(node: Node, src: &str, f: fn(Node, &str) -> Vec<Segment>) -> Vec<Segment> {
let mut segs = Vec::new();
if let Some(left) = node.child_by_field_name("left") {
segs.extend(f(left, src));
}
if let Some(right) = node.child_by_field_name("right") {
segs.extend(f(right, src));
}
segs
}
fn extract_twig(source: &str, calls: &[&CallSpec]) -> ExtractResult {
let mut res = ExtractResult::default();
let str_re = Regex::new(r#"'([^'\n]*)'|"([^"\n]*)""#).expect("valid string regex");
for cap in str_re.captures_iter(source) {
if let Some(m) = cap.get(1).or_else(|| cap.get(2)) {
let s = m.as_str();
if !s.is_empty() && !s.contains("#{") {
res.source_literals.insert(s.to_string());
}
}
}
for call in calls.iter().filter(|c| c.kind == CallKind::Filter) {
let name = regex::escape(&call.name);
let total_re = Regex::new(&format!(r"\|\s*{name}\b")).expect("valid total regex");
let lit_re = Regex::new(&format!(r#"['"]([^'"]*)['"]\s*\|\s*{name}\b"#))
.expect("valid literal regex");
let total = total_re.find_iter(source).count();
let mut kept = 0;
for cap in lit_re.captures_iter(source) {
let key = &cap[1];
if key.contains("#{") {
continue;
}
res.literals.insert(key.to_string());
kept += 1;
}
res.blind += total.saturating_sub(kept);
}
res
}
#[cfg(test)]
mod tests {
use super::*;
use crate::config::CallKind;
fn func(lang: &str, name: &str) -> CallSpec {
CallSpec {
lang: lang.to_string(),
kind: CallKind::Function,
name: name.to_string(),
receiver: None,
key_arg_index: 0,
}
}
fn method(lang: &str, name: &str, receiver: &[&str]) -> CallSpec {
CallSpec {
lang: lang.to_string(),
kind: CallKind::Method,
name: name.to_string(),
receiver: Some(receiver.iter().map(|s| s.to_string()).collect()),
key_arg_index: 0,
}
}
fn filter(name: &str) -> CallSpec {
CallSpec {
lang: "twig".to_string(),
kind: CallKind::Filter,
name: name.to_string(),
receiver: None,
key_arg_index: 0,
}
}
fn index(lang: &str, name: &str) -> CallSpec {
CallSpec {
lang: lang.to_string(),
kind: CallKind::Index,
name: name.to_string(),
receiver: None,
key_arg_index: 0,
}
}
fn lit(res: &ExtractResult) -> Vec<String> {
let mut v: Vec<String> = res.literals.iter().cloned().collect();
v.sort();
v
}
#[test]
fn php_function_matcher() {
let src = "<?php i18n('hello'); notI18n('skip'); ?>";
let res = extract(SourceLang::Php, src, &[func("php", "i18n")], 3);
assert_eq!(lit(&res), vec!["hello".to_string()]);
}
#[test]
fn php_method_matcher_with_receiver() {
let src = "<?php $i18n->get('k1'); $this->i18n->get('k2'); $other->get('nope'); ?>";
let res = extract(
SourceLang::Php,
src,
&[method("php", "get", &["i18n", "this.i18n"])],
3,
);
assert_eq!(lit(&res), vec!["k1".to_string(), "k2".to_string()]);
}
#[test]
fn php_factory_call_receiver() {
let src = "<?php Container::get_i18n()->get('k1'); get_i18n()->get('k2'); \
$other->build()->get('nope'); ?>";
let res = extract(
SourceLang::Php,
src,
&[method("php", "get", &["get_i18n()"])],
3,
);
assert_eq!(lit(&res), vec!["k1".to_string(), "k2".to_string()]);
}
#[test]
fn php_concatenation_of_two_literals() {
let src = "<?php i18n('Hello, ' . 'World'); ?>";
let res = extract(SourceLang::Php, src, &[func("php", "i18n")], 3);
assert_eq!(lit(&res), vec!["Hello, World".to_string()]);
}
#[test]
fn php_concatenation_with_variable_yields_prefix_guard() {
let src = "<?php i18n('Hello, ' . $x); ?>";
let res = extract(SourceLang::Php, src, &[func("php", "i18n")], 3);
assert!(res.literals.is_empty());
assert_eq!(res.guards, vec![Guard::Prefix("Hello, ".to_string())]);
assert_eq!(res.blind, 0);
}
#[test]
fn php_double_quote_interpolation_is_dynamic() {
let src = "<?php i18n(\"role_$x\"); ?>";
let res = extract(SourceLang::Php, src, &[func("php", "i18n")], 3);
assert!(res.literals.is_empty());
assert_eq!(res.guards, vec![Guard::Prefix("role_".to_string())]);
}
#[test]
fn js_function_and_template_guard() {
let src = "i18n('k1'); i18n(`cf_${sub}`); i18n($x);";
let res = extract(SourceLang::Js, src, &[func("js", "i18n")], 3);
assert_eq!(lit(&res), vec!["k1".to_string()]);
assert_eq!(res.guards, vec![Guard::Prefix("cf_".to_string())]);
assert_eq!(res.blind, 1, "i18n($x) is a blind site");
}
#[test]
fn js_short_fragment_template_is_blind_not_alive() {
let src = "i18n(`${a}_${b}`);";
let res = extract(SourceLang::Js, src, &[func("js", "i18n")], 3);
assert!(res.guards.is_empty());
assert_eq!(res.blind, 1);
}
#[test]
fn js_method_matcher() {
let src = "i18n.t('k'); other.t('nope');";
let res = extract(SourceLang::Js, src, &[method("js", "t", &["i18n"])], 3);
assert_eq!(lit(&res), vec!["k".to_string()]);
}
#[test]
fn js_index_matcher_literal_dynamic_and_mismatch() {
let src = "locale['Delete email']; locale[varKey]; other['x'];";
let res = extract(SourceLang::Js, src, &[index("js", "locale")], 3);
assert_eq!(lit(&res), vec!["Delete email".to_string()]);
assert_eq!(res.blind, 1, "locale[varKey] is a blind site");
}
#[test]
fn js_index_matcher_guard_and_member_object() {
let src = "this.locale[`cf_subtype_${k}`];";
let res = extract(SourceLang::Js, src, &[index("js", "this.locale")], 3);
assert!(res.literals.is_empty());
assert_eq!(res.guards, vec![Guard::Prefix("cf_subtype_".to_string())]);
}
#[test]
fn twig_filter_literal_and_blind() {
let src = "{{ 'k'|i18n }} and {{ var|i18n }}";
let res = extract(SourceLang::Twig, src, &[filter("i18n")], 3);
assert_eq!(lit(&res), vec!["k".to_string()]);
assert_eq!(res.blind, 1, "{{ var|i18n }} is blind");
}
#[test]
fn php_mixed_html_parses() {
let src = "<html><body><?php i18n('mixed'); ?></body></html>";
let res = extract(SourceLang::Php, src, &[func("php", "i18n")], 3);
assert_eq!(lit(&res), vec!["mixed".to_string()]);
}
#[test]
fn tsx_grammar_extracts() {
let src = "const e = <div>{i18n('tsx_key')}</div>;";
let res = extract(SourceLang::Tsx, src, &[func("ts", "i18n")], 3);
assert_eq!(lit(&res), vec!["tsx_key".to_string()]);
}
#[test]
fn collects_source_literals_outside_calls() {
let src = "<?php const X = 'industry_retail_ecommerce'; \
$a = [RPI::T => 'Count of income calls']; \
$b = \"role_$x\"; ?>";
let res = extract(SourceLang::Php, src, &[], 3);
assert!(res.source_literals.contains("industry_retail_ecommerce"));
assert!(res.source_literals.contains("Count of income calls"));
assert!(!res.source_literals.iter().any(|s| s.contains("role_")));
}
#[test]
fn js_source_literals_include_template_and_plain() {
let src = "i18n('called_key'); const m = {'bare_key': 1};";
let res = extract(SourceLang::Js, src, &[func("js", "i18n")], 3);
assert!(res.source_literals.contains("called_key"));
assert!(res.source_literals.contains("bare_key"));
}
}