#[derive(Debug, Clone, PartialEq)]
pub enum Tok {
Word(String),
Str(String),
Block(String),
Sep,
}
pub fn lex(src: &str, jq_ok: bool) -> Result<Vec<(Tok, usize)>, crate::err::SpecError> {
lex_opts(src, jq_ok, true)
}
pub fn lex_opts(
src: &str,
jq_ok: bool,
hash_comments: bool,
) -> Result<Vec<(Tok, usize)>, crate::err::SpecError> {
use crate::err::SpecError;
let cs: Vec<char> = src.chars().collect();
let n = cs.len();
let mut i = 0;
let mut toks = Vec::new();
let mut at_cmd_start = true;
while i < n {
let c = cs[i];
match c {
' ' | '\t' | '\r' => i += 1,
'\n' | ';' => {
toks.push((Tok::Sep, i));
i += 1;
at_cmd_start = true;
}
'#' if at_cmd_start && hash_comments => {
while i < n && cs[i] != '\n' {
i += 1;
}
}
_ if jq_ok && at_cmd_start && jq_literal_at(&cs, i) => {
let start = i;
let mut depth = 0i32;
let mut def_led = false;
while i < n {
if depth == 0 && !def_led {
def_led = def_keyword_at(&cs, i);
}
match cs[i] {
'"' => i = skip_jq_string(&cs, i).unwrap_or(n - 1),
'#' => {
while i + 1 < n && cs[i + 1] != '\n' {
i += 1;
}
}
'(' | '[' | '{' => depth += 1,
')' | ']' => depth -= 1,
'}' => {
if depth == 0 {
break;
}
depth -= 1;
}
';' if depth == 0 && !def_led => break,
'\n' if depth == 0 => break,
_ => {}
}
i += 1;
}
let w: String = cs[start..i].iter().collect();
toks.push((Tok::Word(w.trim_end().to_string()), start));
at_cmd_start = false;
}
'"' => {
let q = i;
i += 1;
let mut s = String::new();
while i < n && cs[i] != '"' {
if cs[i] == '\\' && i + 1 < n {
i += 1;
s.push(match cs[i] {
'n' => '\n',
't' => '\t',
o => o,
});
} else {
s.push(cs[i]);
}
i += 1;
}
if i >= n {
return Err(SpecError {
msg: "unterminated string".into(),
span: Some((q, n)),
});
}
i += 1; toks.push((Tok::Str(s), q));
at_cmd_start = false;
}
'{' => {
let open = i;
let mut depth = 1;
i += 1;
let start = i;
while i < n {
match cs[i] {
'{' => depth += 1,
'}' => {
depth -= 1;
if depth == 0 {
break;
}
}
'"' => match skip_jq_string(&cs, i) {
Some(close) => i = close,
None => {
i += 1;
while i < n && cs[i] != '"' {
if cs[i] == '\\' && i + 1 < n {
i += 1;
}
i += 1;
}
}
},
'/' if slash_is_division(&cs[start..i]) => {}
'/' => {
let mut j = i + 1;
let mut closed = false;
while j < n {
match cs[j] {
'\\' if j + 1 < n => j += 2,
'/' => {
closed = true;
break;
}
'\n' => break,
_ => j += 1,
}
}
if closed {
i = j; }
}
_ => {}
}
i += 1;
}
if depth != 0 {
return Err(SpecError {
msg: "unterminated block".into(),
span: Some((open, n)),
});
}
let inner: String = cs[start..i].iter().collect();
i += 1; toks.push((Tok::Block(inner), open));
at_cmd_start = false;
}
'/' if at_cmd_start => {
let start = i;
let mut depth = 0i32;
let mut in_str: Option<char> = None;
while i < n {
let c = cs[i];
match in_str {
Some(q) if c == q => in_str = None,
Some(_) => {}
None => match c {
'\'' | '"' if depth > 0 => in_str = Some(c),
'[' => depth += 1,
']' => depth -= 1,
' ' | '\t' | '\r' | '\n' | ';' | '{' | '"' if depth <= 0 => break,
_ => {}
},
}
i += 1;
}
toks.push((Tok::Word(cs[start..i].iter().collect()), start));
at_cmd_start = false;
}
'/' => {
let start = i;
let mut j = i + 1;
let mut closed = false;
while j < n {
match cs[j] {
'\\' if j + 1 < n => j += 2,
'/' => {
j += 1;
closed = true;
break;
}
'\n' => break,
_ => j += 1,
}
}
if closed {
let w: String = cs[start..j].iter().collect();
toks.push((Tok::Word(w), start));
i = j;
} else {
while i < n && !matches!(cs[i], ' ' | '\t' | '\r' | '\n' | ';' | '{' | '"') {
i += 1;
}
let w: String = cs[start..i].iter().collect();
toks.push((Tok::Word(w), start));
}
at_cmd_start = false;
}
_ => {
let start = i;
while i < n && !matches!(cs[i], ' ' | '\t' | '\r' | '\n' | ';' | '{' | '"') {
i += 1;
}
let w: String = cs[start..i].iter().collect();
toks.push((Tok::Word(w), start));
at_cmd_start = false;
}
}
}
Ok(toks)
}
fn skip_jq_string(cs: &[char], open: usize) -> Option<usize> {
let mut i = open + 1;
while i < cs.len() {
match cs[i] {
'\\' if cs.get(i + 1) == Some(&'(') => i = skip_jq_interpolation(cs, i + 2)?,
'\\' => i += 2,
'"' => return Some(i),
_ => i += 1,
}
}
None
}
pub(crate) fn skip_jq_interpolation(cs: &[char], mut i: usize) -> Option<usize> {
let mut depth = 1;
while i < cs.len() {
match cs[i] {
'"' => i = skip_jq_string(cs, i)?,
'(' => depth += 1,
')' => {
depth -= 1;
if depth == 0 {
return Some(i + 1);
}
}
_ => {}
}
i += 1;
}
None
}
const JQ_KEYWORDS: &[&str] = &["reduce", "foreach", "if", "try", "label", "def"];
const JQ_CALLS: &[&str] = &[
"select",
"map",
"map_values",
"has",
"with_entries",
"group_by",
"unique_by",
"sort_by",
"min_by",
"max_by",
"any",
"all",
"range",
"limit",
"first",
"last",
"nth",
"until",
"while",
"repeat",
"recurse",
"paths",
"del",
"path",
"getpath",
"setpath",
"delpaths",
"splits",
"split",
"join",
"ltrimstr",
"rtrimstr",
"startswith",
"endswith",
"test",
"match",
"capture",
"scan",
"sub",
"gsub",
"error",
"add",
"flatten",
"contains",
"inside",
"index",
"rindex",
"indices",
"walk",
"combinations",
"tostream",
"fromstream",
"truncate_stream",
"IN",
"INDEX",
"isempty",
"pick",
"todate",
"strftime",
"strptime",
"strflocaltime",
"ascii",
"implode",
"debug",
"halt_error",
"pow",
"atan2",
"ltrim",
"isvalid",
"env",
"not",
"abs",
"toarray",
"tojson",
"getpath",
"objects",
"arrays",
"values",
"drem",
"fdim",
"fmin",
"fmax",
"fmod",
"hypot",
"ldexp",
"scalb",
"scalbln",
"nextafter",
"nexttoward",
"jn",
"yn",
"fma",
"copysign",
"remainder",
"format",
"bsearch",
"skip",
"JOIN",
"trimstr",
"modulemeta",
"toboolean",
"with",
"ref",
"ireduce",
"with_dtf",
"omit",
"sort_keys",
"sortKeys",
"strenv",
"load",
"load_str",
"load_props",
"load_xml",
"format_datetime",
"tz",
"eval",
"to_json",
"to_yaml",
"to_xml",
];
const JQ_BARE: &[&str] = &[
"null",
"true",
"false",
"env",
"tostring",
"tonumber",
"tojson",
"fromjson",
"type",
"keys_unsorted",
"to_entries",
"from_entries",
"values",
"paths",
"leaf_paths",
"any",
"all",
"unique",
"not",
"empty",
"error",
"input",
"inputs",
"ascii_downcase",
"ascii_upcase",
"explode",
"implode",
"transpose",
"combinations",
"tostream",
"todate",
"fromdate",
"todateiso8601",
"fromdateiso8601",
"now",
"infinite",
"nan",
"isnan",
"isinfinite",
"isnormal",
"halt",
"halt_error",
"arrays",
"objects",
"booleans",
"numbers",
"strings",
"nulls",
"iterables",
"scalars",
"recurse",
"toarray",
"anchor",
"alias",
"tag",
"style",
"kind",
"line",
"column",
"head_comment",
"headComment",
"line_comment",
"lineComment",
"foot_comment",
"footComment",
"comments",
"key",
"is_key",
"parent",
"document_index",
"documentIndex",
"di",
"filename",
"fileIndex",
"splitDoc",
"split_doc",
"from_json",
"from_yaml",
"from_xml",
"to_props",
"from_props",
"to_csv",
"from_csv",
"to_tsv",
"from_tsv",
"envsubst",
"from_unix",
"to_unix",
"shuffle",
"pivot",
"downcase",
"upcase",
"to_string",
"to_number",
"utf8bytelength",
"input_line_number",
"builtins",
"stderr",
"debug",
"mktime",
"gmtime",
"localtime",
"reverse",
"date",
"finites",
"normals",
"ltrim",
"rtrim",
"sqrt",
"log",
"log2",
"log10",
"exp",
"exp2",
"exp10",
"cbrt",
"trunc",
"nearbyint",
"fabs",
"significand",
"logb",
"sin",
"cos",
"tan",
"asin",
"acos",
"atan",
"sinh",
"cosh",
"tanh",
"acosh",
"asinh",
"atanh",
"expm1",
"log1p",
"rint",
"gamma",
"lgamma",
"tgamma",
"erf",
"erfc",
"j0",
"j1",
"y0",
"y1",
"frexp",
"modf",
"lgamma_r",
"isfinite",
"toboolean",
"get_jq_origin",
"get_prog_origin",
"get_search_list",
"modulemeta",
];
use crate::jqlang::FORMAT_NAMES as JQ_FORMATS;
fn slash_is_division(before: &[char]) -> bool {
let mut end = before.len();
while end > 0 && matches!(before[end - 1], ' ' | '\t') {
end -= 1;
}
let Some(&last) = end.checked_sub(1).map(|k| &before[k]) else {
return false;
};
if matches!(last, '.' | ')' | ']') {
return true;
}
let is_tok = |c: char| c.is_alphanumeric() || matches!(c, '_' | '.' | '$');
if !is_tok(last) {
return false;
}
let mut start = end;
while start > 0 && is_tok(before[start - 1]) {
start -= 1;
}
let tok = &before[start..end];
matches!(tok[0], '.' | '$') || tok.iter().all(|c| c.is_ascii_digit() || *c == '.')
}
fn def_keyword_at(cs: &[char], i: usize) -> bool {
let after = cs.get(i + 3).is_some_and(|c| matches!(c, ' ' | '\t'));
let before = i
.checked_sub(1)
.map(|k| cs[k])
.is_none_or(|c| !(c.is_alphanumeric() || matches!(c, '_' | '.' | '$' | '@')));
after && before && cs[i..].starts_with(&['d', 'e', 'f'])
}
fn jq_literal_at(cs: &[char], i: usize) -> bool {
if matches!(cs[i], '.' | '[' | '(' | '$' | '"') {
return true;
}
if cs[i] == '{' {
return jq_object_at(cs, i);
}
if cs[i].is_ascii_digit() {
return true;
}
if cs[i] == '-' {
let mut k = i + 1;
while k < cs.len() && matches!(cs[k], ' ' | '\t') {
k += 1;
}
return cs
.get(k)
.is_some_and(|c| c.is_ascii_digit() || matches!(c, '.' | '(' | '$'));
}
let mut j = i;
if cs[i] == '@' {
j += 1;
let start = j;
while j < cs.len() && (cs[j].is_ascii_alphanumeric() || cs[j] == '_') {
j += 1;
}
let name: String = cs[start..j].iter().collect();
let mut k = j;
while k < cs.len() && matches!(cs[k], ' ' | '\t') {
k += 1;
}
return JQ_FORMATS.contains(&name.as_str()) || (j > start && cs.get(k) == Some(&'"'));
}
while j < cs.len() && (cs[j].is_ascii_alphanumeric() || cs[j] == '_') {
j += 1;
}
let name: String = cs[i..j].iter().collect();
if JQ_KEYWORDS.contains(&name.as_str()) {
return cs
.get(j)
.is_some_and(|c| matches!(c, ' ' | '\t' | '(' | '.' | '$' | '"'));
}
if JQ_BARE.contains(&name.as_str()) {
return !cs.get(j).is_some_and(|c| c.is_alphanumeric() || *c == '_');
}
if cs.get(j) == Some(&'(') && JQ_CALLS.contains(&name.as_str()) {
return true;
}
NATIVE_SHARED.contains(&name.as_str()) && jq_continues_at(cs, j)
}
const NATIVE_SHARED: &[&str] = &[
"abs", "add", "ceil", "first", "flatten", "floor", "last", "length", "max", "min", "round",
"sort", "trim",
];
fn jq_continues_at(cs: &[char], j: usize) -> bool {
let mut k = j;
while k < cs.len() && matches!(cs[k], ' ' | '\t') {
k += 1;
}
if k == j && cs.get(k).is_some_and(|c| c.is_alphanumeric() || *c == '_') {
return false;
}
let word = |w: &str| {
let wc: Vec<char> = w.chars().collect();
cs[k..].starts_with(&wc)
&& cs
.get(k + wc.len())
.is_some_and(|c| matches!(c, ' ' | '\t' | '$' | '[' | '{' | '('))
};
match cs.get(k) {
Some('|' | ',' | '+' | '*' | '/' | '%' | '<' | '>' | '=' | '!' | '?') => true,
Some(_) => word("as") || word("and") || word("or"),
None => false,
}
}
fn jq_object_at(cs: &[char], i: usize) -> bool {
let mut j = i + 1;
while j < cs.len() && matches!(cs[j], ' ' | '\t' | '\r' | '\n') {
j += 1;
}
match cs.get(j) {
None => false,
Some('}') => true,
Some('"' | '$' | '(' | '@') => true,
Some(c) if c.is_alphabetic() || *c == '_' => {
while j < cs.len() && (cs[j].is_alphanumeric() || cs[j] == '_') {
j += 1;
}
while j < cs.len() && matches!(cs[j], ' ' | '\t') {
j += 1;
}
matches!(cs.get(j), Some(':' | ',' | '}'))
}
_ => false,
}
}