use std::collections::{BTreeMap, HashSet};
use base64::{engine::general_purpose::STANDARD, Engine as _};
use percent_encoding::{percent_decode_str, utf8_percent_encode, NON_ALPHANUMERIC};
use regex::Regex;
use scraper::{ElementRef, Html, Selector};
use serde_json::Value;
use crate::expr::Expr;
#[derive(Debug, Clone, PartialEq)]
pub enum FieldSel {
Col(usize),
Key(Vec<String>),
JqKey(Vec<crate::jqval::Seg>),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum OutFmt {
Yaml,
Json,
Props,
Xml,
Csv,
Tsv,
}
#[derive(Debug, Clone)]
pub enum QueryOp {
XPath(Box<crate::xpath::XPath>),
Emit(OutFmt, usize),
Match(Regex),
Reject(Regex),
Field(FieldSel),
Fields(Vec<usize>),
Each,
Where(Expr),
Map(Expr),
Via(std::sync::Arc<crate::actor::ActorDef>, usize),
Count,
Rate,
Tally,
Sum,
Min,
Max,
Avg,
Keys,
JqKeys,
Vals,
NonNull,
JqRawString,
Pick(Vec<String>),
Sort {
numeric: bool,
reverse: bool,
},
Uniq,
Rev,
First,
Last,
Take(usize),
Drop(usize),
Upper,
Lower,
Trim,
Replace(Regex, String),
Join(String),
Nth(usize),
Sel {
css: String,
attr: Option<String>,
},
Find(String),
Attr(String),
Text,
SortBy(String),
UniqueBy(String),
CountBy(String),
GroupBy(String),
MinBy(String),
MaxBy(String),
Has(String),
HasKey(Value),
Entries,
JqEntries,
JqFlatten,
JqMap(Vec<QueryOp>),
JqEach,
JqAdd,
JqLen,
JqProgram(String),
JqSelect(crate::jqval::Expr),
JqCalc(crate::jqval::Expr),
Flatten,
Add,
Over(f64),
Under(f64),
Between(f64, f64),
Enumerate,
Words,
Dedup,
Tailn(usize),
Pad(usize),
Lpad(usize),
Grepf(String, regex::Regex),
Flip,
Basename,
Dirname,
Commafy,
Bytes,
Duration,
Apply(String),
JsonDocs,
Csv,
B64,
B64d,
Hex,
Unhex,
Urlenc,
Urldec,
Extract(Regex),
Split(String),
Substr(usize, usize),
Chars,
Title,
Repeat(usize),
Set(String, String),
Del(String),
Rename(String, String),
Default(String, String),
Merge,
Floor,
Ceil,
Clamp(f64, f64),
Tsv,
Yaml,
Toml,
Calc(Expr),
Contains(String),
Starts(String),
Ends(String),
Nonempty,
Numeric,
Len,
JsonLen,
JsonSlice(Option<i64>, Option<i64>),
Wc,
Abs,
Round,
Delta,
Cumsum,
Sma(usize),
Ewma(f64),
Prepend(String),
Append(String),
Cut(String, usize),
Median,
Percentile(f64),
Stddev,
Range,
Product,
Distinct,
Sample(usize),
Slice(usize, usize),
Index(usize),
Bins(usize),
}
#[derive(Debug, Clone, PartialEq)]
pub enum QueryResult {
Lines(Vec<String>),
Scalar(f64),
Pairs(Vec<(String, u64)>),
Error(String),
}
#[derive(Debug, Default, Clone, PartialEq)]
pub struct JqReport {
pub errors: Vec<String>,
pub last_failed: bool,
pub halt: Option<(i32, String)>,
}
fn uncaught(e: &crate::jqlang::JqErr) -> String {
match e {
crate::jqlang::JqErr::Err(crate::jqlang::JqVal::Str(s)) => format!("jq: {s}"),
crate::jqlang::JqErr::Err(v) => {
format!("jq: (not a string): {}", crate::jqlang::render(v))
}
other => format!("jq: {}", other.to_message()),
}
}
fn halt_text(msg: Option<&crate::jqlang::JqVal>) -> String {
match msg {
None | Some(crate::jqlang::JqVal::Null) => String::new(),
Some(crate::jqlang::JqVal::Str(s)) => s.to_string(),
Some(v) => format!("{}\n", crate::jqlang::render(v)),
}
}
fn record_run(rep: &mut JqReport, r: Result<(), crate::jqlang::JqErr>) -> bool {
match r {
Ok(()) => {
rep.last_failed = false;
false
}
Err(crate::jqlang::JqErr::Halt(code, msg)) => {
rep.halt = Some((code, halt_text(msg.as_ref())));
true
}
Err(e) => {
rep.errors.push(uncaught(&e));
rep.last_failed = true;
false
}
}
}
pub fn eval(ops: &[QueryOp], lines: &[String], elapsed_secs: f64) -> QueryResult {
let mut rep = JqReport::default();
let r = eval_reporting(ops, lines, elapsed_secs, 1, &mut rep);
match (rep.errors.into_iter().next(), rep.halt) {
(Some(e), _) => QueryResult::Error(e),
(None, Some((_, msg))) if !msg.is_empty() => {
QueryResult::Error(format!("jq: {}", msg.trim_end_matches('\n')))
}
_ => r,
}
}
pub fn eval_reporting(
ops: &[QueryOp],
lines: &[String],
elapsed_secs: f64,
first_line: usize,
rep: &mut JqReport,
) -> QueryResult {
let mut cur: Vec<String> = lines.to_vec();
let mut nodes: Option<Vec<crate::jqlang::JqVal>> = None;
for op in ops {
if !matches!(
op,
QueryOp::Yaml | QueryOp::JqProgram(_) | QueryOp::Emit(..)
) {
nodes = None;
}
match op {
QueryOp::Emit(fmt, indent) => {
let docs = match nodes.clone() {
Some(d) => d,
None => cur
.iter()
.map(|l| {
crate::jqlang::parse_json(l)
.unwrap_or_else(|_| crate::jqlang::JqVal::str(l.as_str()))
})
.collect(),
};
cur = emit_stream(&docs, *fmt, *indent);
}
QueryOp::Match(re) => cur.retain(|l| re.is_match(l)),
QueryOp::Reject(re) => cur.retain(|l| !re.is_match(l)),
QueryOp::Field(sel) => {
if let FieldSel::JqKey(segs) = sel {
for l in cur.iter_mut() {
match crate::jqlang::parse_json(l).ok() {
None => *l = "null".to_string(),
Some(v) => match crate::jqlang::get_seg_path(&v, segs) {
Ok(r) => *l = crate::jqlang::render_raw(&r),
Err(e) => return QueryResult::Error(format!("jq: {e}")),
},
}
}
continue;
}
for l in cur.iter_mut() {
*l = extract_field(l, sel);
}
}
QueryOp::Fields(cols) => {
for l in cur.iter_mut() {
*l = cols
.iter()
.map(|&n| nth_col(l, n))
.collect::<Vec<_>>()
.join(" ");
}
}
QueryOp::JqEach => {
let mut out = Vec::with_capacity(cur.len());
for l in &cur {
use crate::jqlang::JqVal;
match crate::jqlang::parse_json(l) {
Ok(v) => match v.bare() {
JqVal::Arr(a) => out.extend(a.iter().map(crate::jqlang::render_raw)),
JqVal::Obj(m) => {
out.extend(m.iter().map(|(_, e)| crate::jqlang::render_raw(e)))
}
other => {
return QueryResult::Error(format!(
"jq: Cannot iterate over {}{}",
other.type_name(),
crate::jqlang::paren_of(other)
))
}
},
Err(_) => out.push(l.clone()),
}
}
cur = out;
}
QueryOp::JqProgram(src) => {
let prog = match jq_program(src) {
Ok(p) => p,
Err(msg) => return QueryResult::Error(format!("jq: {msg}")),
};
let interp = crate::jqlang::Interp::default();
let mut out = Vec::with_capacity(cur.len());
if let Some(docs) = nodes.take().filter(|_| !prog.reads_input_stream()) {
let mut next: Vec<crate::jqlang::JqVal> = Vec::new();
for (i, doc) in docs.iter().enumerate() {
interp.set_line(i + 1);
interp.set_doc(doc);
let r = prog.run_with(&interp, doc, &mut |v| {
out.push(crate::jqlang::render_raw(&v));
next.push(v);
Ok(())
});
if record_run(rep, r) {
break;
}
}
nodes = Some(next);
cur = out;
continue;
}
interp.set_input_lines(std::mem::take(&mut cur));
interp.set_line(first_line.saturating_sub(1));
while let Some(input) = interp.next_input() {
interp.set_doc(&input);
let r = prog.run_with(&interp, &input, &mut |v| {
out.push(crate::jqlang::render_raw(&v));
Ok(())
});
if record_run(rep, r) {
break;
}
}
cur = out;
}
QueryOp::JqSelect(e) => {
let mut out = Vec::with_capacity(cur.len());
for l in &cur {
match crate::jqval::eval(e, &jq_scalar(l)) {
Ok(v) => {
if crate::jqval::truthy(&v) {
out.push(l.clone());
}
}
Err(msg) => return QueryResult::Error(format!("jq: {msg}")),
}
}
cur = out;
}
QueryOp::JqCalc(e) => {
for l in cur.iter_mut() {
match crate::jqval::eval(e, &jq_scalar(l)) {
Ok(v) => *l = crate::jqval::render(&v),
Err(msg) => return QueryResult::Error(format!("jq: {msg}")),
}
}
}
QueryOp::JqAdd => {
if let Some(e) = jq_builtin_lines(&mut cur, "add") {
return e;
}
}
QueryOp::JqLen => {
for l in cur.iter_mut() {
*l = match jq_value(l) {
Some(Value::Array(a)) => a.len().to_string(),
Some(Value::Object(m)) => m.len().to_string(),
Some(Value::String(s)) => s.chars().count().to_string(),
Some(Value::Number(n)) => fmt_num(n.as_f64().unwrap_or(0.0).abs()),
Some(Value::Null) => "0".to_string(),
Some(v @ Value::Bool(_)) => {
return QueryResult::Error(format!(
"jq: {} ({v}) has no length",
crate::jqval::tname(&v)
));
}
None => l.chars().count().to_string(),
};
}
}
QueryOp::Each => {
let mut out = Vec::with_capacity(cur.len());
for l in &cur {
match serde_json::from_str::<Value>(l) {
Ok(Value::Array(arr)) => {
out.extend(arr.iter().map(jq_to_string));
}
Ok(Value::Object(m)) => {
out.extend(m.values().map(jq_to_string));
}
_ => out.push(l.clone()),
}
}
cur = out;
}
QueryOp::Where(e) => {
if crate::expr::expr_has_str(e) {
cur.retain(|l| eval_where(e, l));
} else {
cur.retain(|l| {
let x = l.trim().parse::<f64>().unwrap_or(f64::NAN);
let resolve = |name: &str| field_num(l, name);
crate::expr::eval_pred_ctx(e, x, &resolve).unwrap_or(false)
});
}
}
QueryOp::Map(e) => {
for l in cur.iter_mut() {
let v = {
let x = l.trim().parse::<f64>().unwrap_or(f64::NAN);
let resolve = |name: &str| field_num(l, name);
crate::expr::eval_ctx(e, x, &resolve).unwrap_or(f64::NAN)
};
*l = fmt_num(v);
}
}
QueryOp::Via(def, workers) => {
cur = crate::actor::run_via(def, *workers, &cur);
}
QueryOp::Count => return QueryResult::Scalar(cur.len() as f64),
QueryOp::Rate => {
return QueryResult::Scalar(cur.len() as f64 / elapsed_secs.max(0.001));
}
QueryOp::Tally => {
let mut counts: BTreeMap<String, u64> = BTreeMap::new();
for l in &cur {
*counts.entry(l.clone()).or_insert(0) += 1;
}
let mut pairs: Vec<(String, u64)> = counts.into_iter().collect();
pairs.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
return QueryResult::Pairs(pairs);
}
QueryOp::Calc(e) => {
let x = cur.len() as f64;
return QueryResult::Scalar(crate::expr::eval(e, x).unwrap_or(0.0));
}
QueryOp::Sum => return QueryResult::Scalar(nums(&cur).iter().sum()),
QueryOp::Min => {
let m = nums(&cur).into_iter().fold(f64::INFINITY, f64::min);
return QueryResult::Scalar(if m.is_finite() { m } else { 0.0 });
}
QueryOp::Max => {
let m = nums(&cur).into_iter().fold(f64::NEG_INFINITY, f64::max);
return QueryResult::Scalar(if m.is_finite() { m } else { 0.0 });
}
QueryOp::Avg => {
let ns = nums(&cur);
let a = if ns.is_empty() {
0.0
} else {
ns.iter().sum::<f64>() / ns.len() as f64
};
return QueryResult::Scalar(a);
}
QueryOp::Keys => {
let mut out = Vec::new();
for l in &cur {
match serde_json::from_str::<Value>(l) {
Ok(Value::Object(m)) => out.extend(m.keys().cloned()),
_ => out.push(l.clone()),
}
}
cur = out;
}
QueryOp::JqKeys => {
for l in cur.iter_mut() {
match serde_json::from_str::<Value>(l) {
Ok(Value::Object(m)) => {
let arr: Vec<Value> =
m.keys().map(|k| Value::String(k.clone())).collect();
*l = Value::Array(arr).to_string();
}
Ok(Value::Array(a)) => {
let arr: Vec<Value> = (0..a.len()).map(Value::from).collect();
*l = Value::Array(arr).to_string();
}
Ok(v) => return QueryResult::Error(has_no_keys(&v)),
Err(_) => {}
}
}
}
QueryOp::Vals => {
let mut out = Vec::new();
for l in &cur {
match serde_json::from_str::<Value>(l) {
Ok(Value::Object(m)) => out.extend(m.values().map(json_to_string)),
_ => out.push(l.clone()),
}
}
cur = out;
}
QueryOp::NonNull => {
cur.retain(|l| !matches!(serde_json::from_str::<Value>(l), Ok(Value::Null)));
}
QueryOp::JqRawString => {
for l in cur.iter_mut() {
if let Ok(v) = crate::jqlang::parse_json(l) {
*l = crate::jqlang::render_raw(&v);
}
}
}
QueryOp::Pick(keys) => {
for l in cur.iter_mut() {
if let Ok(Value::Object(m)) = serde_json::from_str::<Value>(l) {
let parts: Vec<String> = keys
.iter()
.filter_map(|k| {
m.get(k)
.map(|v| format!("{}:{}", Value::String(k.clone()), v))
})
.collect();
*l = format!("{{{}}}", parts.join(","));
}
}
}
QueryOp::JsonDocs => cur = crate::jsondocs::documents(&cur),
QueryOp::Csv => cur = to_json_records(&cur, ','),
QueryOp::Tsv => cur = to_json_records(&cur, '\t'),
QueryOp::Yaml => {
let docs = crate::yaml::documents(&cur.join("\n"));
cur = docs.iter().map(crate::jqlang::render).collect();
nodes = Some(docs);
}
QueryOp::Toml => cur = toml_to_json(&cur),
QueryOp::Sort { numeric, reverse } => {
if *numeric {
let key = |s: &str| {
s.split_whitespace()
.next()
.and_then(|t| t.parse::<f64>().ok())
.unwrap_or(f64::NAN)
};
cur.sort_by(|a, b| {
key(a)
.partial_cmp(&key(b))
.unwrap_or(std::cmp::Ordering::Equal)
});
} else {
cur.sort();
}
if *reverse {
cur.reverse();
}
}
QueryOp::Uniq => {
let mut seen = HashSet::new();
cur.retain(|l| seen.insert(l.clone()));
}
QueryOp::Rev => cur.reverse(),
QueryOp::First => cur.truncate(1),
QueryOp::Last => {
if let Some(l) = cur.pop() {
cur = vec![l];
} else {
cur.clear();
}
}
QueryOp::Take(n) => cur.truncate(*n),
QueryOp::Drop(n) => {
cur.drain(0..(*n).min(cur.len()));
}
QueryOp::Upper => {
for l in cur.iter_mut() {
*l = l.to_uppercase();
}
}
QueryOp::Lower => {
for l in cur.iter_mut() {
*l = l.to_lowercase();
}
}
QueryOp::Trim => {
for l in cur.iter_mut() {
*l = l.trim().to_string();
}
}
QueryOp::Replace(re, to) => {
for l in cur.iter_mut() {
*l = re.replace_all(l, to.as_str()).into_owned();
}
}
QueryOp::Join(sep) => {
cur = vec![cur.join(sep)];
}
QueryOp::Nth(n) => {
cur = cur
.get(n.saturating_sub(1))
.cloned()
.map(|l| vec![l])
.unwrap_or_default();
}
QueryOp::Sel { css, attr } => {
let doc = Html::parse_document(&cur.join("\n"));
cur = match Selector::parse(css) {
Ok(sel) => doc
.select(&sel)
.filter_map(|el| match attr {
Some(a) => el.value().attr(a).map(str::to_string),
None => Some(el.text().collect::<String>().trim().to_string()),
})
.collect(),
Err(e) => {
return QueryResult::Error(format!(
"sel: `{css}` is not a valid CSS selector ({e})"
))
}
};
}
QueryOp::XPath(xp) => {
match xp.run(&cur) {
Ok(lines) => cur = lines,
Err(e) => return QueryResult::Error(e),
}
}
QueryOp::Find(css) => {
let doc = Html::parse_document(&cur.join("\n"));
cur = match Selector::parse(css) {
Ok(sel) => doc.select(&sel).map(|el| el.html()).collect(),
Err(e) => {
return QueryResult::Error(format!(
"find: `{css}` is not a valid CSS selector ({e})"
))
}
};
}
QueryOp::Attr(name) => {
let mut out = Vec::with_capacity(cur.len());
for l in &cur {
let frag = Html::parse_fragment(l);
if let Some(v) = first_element(&frag).and_then(|e| e.value().attr(name)) {
out.push(v.to_string());
}
}
cur = out;
}
QueryOp::Text => {
for l in cur.iter_mut() {
let frag = Html::parse_fragment(l);
if let Some(e) = first_element(&frag) {
*l = e
.children()
.filter_map(|c| c.value().as_text().map(|t| t.to_string()))
.collect::<String>()
.trim()
.to_string();
}
}
}
QueryOp::SortBy(field) => {
let mut objs: Vec<(String, String)> = Vec::new();
let mut rest: Vec<String> = Vec::new();
for l in cur.drain(..) {
if let Ok(Value::Object(m)) = serde_json::from_str::<Value>(&l) {
let key = m.get(field).map(json_to_string).unwrap_or_default();
objs.push((key, l));
} else {
rest.push(l);
}
}
let all_numeric =
!objs.is_empty() && objs.iter().all(|(k, _)| k.trim().parse::<f64>().is_ok());
if all_numeric {
objs.sort_by(|a, b| {
let na = a.0.trim().parse::<f64>().unwrap_or(f64::NAN);
let nb = b.0.trim().parse::<f64>().unwrap_or(f64::NAN);
na.partial_cmp(&nb).unwrap_or(std::cmp::Ordering::Equal)
});
} else {
objs.sort_by(|a, b| a.0.cmp(&b.0));
}
cur = objs.into_iter().map(|(_, l)| l).collect();
cur.extend(rest);
}
QueryOp::UniqueBy(field) => {
let mut seen: HashSet<String> = HashSet::new();
cur.retain(|l| {
let key = match serde_json::from_str::<Value>(l) {
Ok(Value::Object(m)) => {
m.get(field).map(json_to_string).unwrap_or_default()
}
_ => l.clone(),
};
seen.insert(key)
});
}
QueryOp::CountBy(field) => {
let mut counts: BTreeMap<String, u64> = BTreeMap::new();
for l in &cur {
let key = match serde_json::from_str::<Value>(l) {
Ok(Value::Object(m)) => {
m.get(field).map(json_to_string).unwrap_or_default()
}
_ => l.clone(),
};
*counts.entry(key).or_insert(0) += 1;
}
let mut pairs: Vec<(String, u64)> = counts.into_iter().collect();
pairs.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
return QueryResult::Pairs(pairs);
}
QueryOp::GroupBy(field) => {
let mut groups: BTreeMap<String, Vec<Value>> = BTreeMap::new();
for l in &cur {
let (key, val) = match serde_json::from_str::<Value>(l) {
Ok(v @ Value::Object(_)) => {
let k = v.get(field).map(json_to_string).unwrap_or_default();
(k, v)
}
Ok(v) => (l.clone(), v),
Err(_) => (l.clone(), Value::String(l.clone())),
};
groups.entry(key).or_default().push(val);
}
cur = groups
.into_values()
.map(|g| Value::Array(g).to_string())
.collect();
}
QueryOp::MinBy(field) => {
let best = cur
.iter()
.filter(|l| !field_num(l, field).is_nan())
.min_by(|a, b| {
field_num(a, field)
.partial_cmp(&field_num(b, field))
.unwrap_or(std::cmp::Ordering::Equal)
});
return QueryResult::Lines(best.into_iter().cloned().collect());
}
QueryOp::MaxBy(field) => {
let best = cur
.iter()
.filter(|l| !field_num(l, field).is_nan())
.max_by(|a, b| {
field_num(a, field)
.partial_cmp(&field_num(b, field))
.unwrap_or(std::cmp::Ordering::Equal)
})
.cloned();
return QueryResult::Lines(best.into_iter().collect());
}
QueryOp::Has(key) => {
cur.retain(|l| {
matches!(
serde_json::from_str::<Value>(l),
Ok(Value::Object(ref m)) if m.contains_key(key)
)
});
}
QueryOp::HasKey(key) => {
for l in cur.iter_mut() {
let present = match (jq_value(l), key) {
(Some(Value::Object(m)), Value::String(k)) => m.contains_key(k),
(Some(Value::Array(a)), Value::Number(n)) => {
let i = n.as_f64().unwrap_or(-1.0);
i >= 0.0 && (i as usize) < a.len()
}
(Some(Value::Null), _) | (None, _) => false,
(Some(v), k) => {
return QueryResult::Error(format!(
"jq: Cannot check whether {} has a {} key",
crate::jqval::tname(&v),
crate::jqval::tname(k)
));
}
};
*l = if present { "true" } else { "false" }.to_string();
}
}
QueryOp::Entries => {
let mut out = Vec::with_capacity(cur.len());
for l in &cur {
match serde_json::from_str::<Value>(l) {
Ok(Value::Object(m)) => {
for (k, v) in &m {
out.push(format!(
"{{\"key\":{},\"value\":{}}}",
Value::String(k.clone()),
v
));
}
}
_ => out.push(l.clone()),
}
}
cur = out;
}
QueryOp::JqEntries => {
if let Some(e) = jq_builtin_lines(&mut cur, "to_entries") {
return e;
}
}
QueryOp::JqMap(inner) => {
let mut out = Vec::with_capacity(cur.len());
for l in &cur {
let elems: Vec<String> = match crate::jqlang::parse_json(l) {
Ok(v) => match v.bare() {
crate::jqlang::JqVal::Arr(a) => {
a.iter().map(crate::jqlang::render_raw).collect()
}
crate::jqlang::JqVal::Obj(m) => m
.iter()
.map(|(_, e)| crate::jqlang::render_raw(e))
.collect(),
other => {
return QueryResult::Error(format!(
"jq: Cannot iterate over {}{}",
other.type_name(),
crate::jqlang::paren_of(other)
))
}
},
Err(_) => {
out.push(l.clone());
continue;
}
};
let mapped = match eval(inner, &elems, elapsed_secs) {
QueryResult::Lines(v) => v,
QueryResult::Scalar(s) => vec![fmt_num(s)],
QueryResult::Pairs(ps) => {
ps.iter().map(|(k, c)| format!("{k}\t{c}")).collect()
}
e @ QueryResult::Error(_) => return e,
};
out.push(jq_array_json(&mapped));
}
cur = out;
}
QueryOp::Flatten => {
fn push_leaves(v: &Value, out: &mut Vec<String>) {
match v {
Value::Array(a) => a.iter().for_each(|e| push_leaves(e, out)),
other => out.push(json_to_string(other)),
}
}
let mut out = Vec::with_capacity(cur.len());
for l in &cur {
match serde_json::from_str::<Value>(l) {
Ok(Value::Array(arr)) => arr.iter().for_each(|e| push_leaves(e, &mut out)),
_ => out.push(l.clone()),
}
}
cur = out;
}
QueryOp::JqFlatten => {
if let Some(e) = jq_builtin_lines(&mut cur, "flatten") {
return e;
}
}
QueryOp::Add => {
for l in cur.iter_mut() {
if let Ok(Value::Array(arr)) = serde_json::from_str::<Value>(l) {
if arr.is_empty() {
*l = String::new();
} else if arr.iter().all(Value::is_number) {
let sum: f64 = arr.iter().filter_map(Value::as_f64).sum();
*l = fmt_num(sum);
} else {
*l = arr.iter().map(json_to_string).collect::<String>();
}
}
}
}
QueryOp::Over(n) => {
cur.retain(|l| l.trim().parse::<f64>().map(|v| v > *n).unwrap_or(false));
}
QueryOp::Under(n) => {
cur.retain(|l| l.trim().parse::<f64>().map(|v| v < *n).unwrap_or(false));
}
QueryOp::Between(lo, hi) => {
cur.retain(|l| {
let x = l.trim().parse::<f64>().unwrap_or(f64::NAN);
x >= *lo && x <= *hi
});
}
QueryOp::Enumerate => {
for (i, l) in cur.iter_mut().enumerate() {
*l = format!("{}\t{}", i + 1, l);
}
}
QueryOp::Words => {
let mut out = Vec::with_capacity(cur.len());
for l in &cur {
out.extend(l.split_whitespace().map(str::to_string));
}
cur = out;
}
QueryOp::Dedup => {
cur.dedup();
}
QueryOp::Tailn(n) => {
let len = cur.len();
if *n < len {
cur.drain(0..len - *n);
}
}
QueryOp::Pad(n) => {
let n = (*n).min(u16::MAX as usize);
for l in cur.iter_mut() {
*l = format!("{:<width$}", l, width = n);
}
}
QueryOp::Lpad(width) => {
let w = (*width).min(u16::MAX as usize);
for l in cur.iter_mut() {
*l = format!("{:>width$}", l, width = w);
}
}
QueryOp::Grepf(field, re) => {
cur.retain(|l| {
let val = if let Ok(Value::Object(m)) = serde_json::from_str::<Value>(l) {
m.get(field).map(json_to_string).unwrap_or_default()
} else if let Ok(idx) = field.parse::<usize>() {
l.split_whitespace()
.nth(idx.saturating_sub(1))
.unwrap_or("")
.to_string()
} else {
String::new()
};
re.is_match(&val)
});
}
QueryOp::Flip => {
for l in cur.iter_mut() {
*l = l.chars().rev().collect();
}
}
QueryOp::Basename => {
for l in cur.iter_mut() {
let t = l.trim_end_matches('/');
*l = t.rsplit('/').next().unwrap_or(t).to_string();
}
}
QueryOp::Dirname => {
for l in cur.iter_mut() {
let t = l.trim_end_matches('/');
*l = match t.rsplit_once('/') {
Some(("", _)) => "/".to_string(),
Some((dir, _)) => dir.to_string(),
None => ".".to_string(),
};
}
}
QueryOp::Commafy => {
for l in cur.iter_mut() {
*l = commafy(l);
}
}
QueryOp::Bytes => {
for l in cur.iter_mut() {
if let Ok(v) = l.trim().parse::<f64>() {
*l = humanize_bytes(v);
}
}
}
QueryOp::Duration => {
for l in cur.iter_mut() {
if let Ok(v) = l.trim().parse::<f64>() {
*l = humanize_duration(v);
}
}
}
QueryOp::Apply(_) => {}
QueryOp::B64 => {
for l in cur.iter_mut() {
*l = STANDARD.encode(l.as_bytes());
}
}
QueryOp::B64d => {
for l in cur.iter_mut() {
if let Some(s) = STANDARD
.decode(l.as_bytes())
.ok()
.and_then(|b| String::from_utf8(b).ok())
{
*l = s;
}
}
}
QueryOp::Hex => {
for l in cur.iter_mut() {
*l = l.bytes().map(|b| format!("{:02x}", b)).collect();
}
}
QueryOp::Unhex => {
for l in cur.iter_mut() {
let chars: Vec<char> = l.chars().collect();
if chars.is_empty() || !chars.len().is_multiple_of(2) {
continue;
}
let mut bytes = Vec::with_capacity(chars.len() / 2);
let mut ok = true;
let mut i = 0;
while i < chars.len() {
let pair: String = chars[i..i + 2].iter().collect();
match u8::from_str_radix(&pair, 16) {
Ok(b) => bytes.push(b),
Err(_) => {
ok = false;
break;
}
}
i += 2;
}
if ok {
if let Ok(decoded) = String::from_utf8(bytes) {
*l = decoded;
}
}
}
}
QueryOp::Urlenc => {
for l in cur.iter_mut() {
*l = utf8_percent_encode(l, NON_ALPHANUMERIC).to_string();
}
}
QueryOp::Urldec => {
for l in cur.iter_mut() {
if let Ok(decoded) = percent_decode_str(l).decode_utf8() {
*l = decoded.into_owned();
}
}
}
QueryOp::Extract(re) => {
cur = cur
.iter()
.filter_map(|l| {
re.captures(l).map(|caps| {
caps.get(1)
.unwrap_or_else(|| caps.get(0).unwrap())
.as_str()
.to_string()
})
})
.collect();
}
QueryOp::Split(delim) => {
let mut out: Vec<String> = Vec::with_capacity(cur.len());
for l in cur.iter() {
for part in l.split(delim.as_str()) {
out.push(part.to_string());
}
}
cur = out;
}
QueryOp::Substr(a, b) => {
for l in cur.iter_mut() {
*l = l.chars().skip(*a).take(b.saturating_sub(*a)).collect();
}
}
QueryOp::Chars => {
let mut out: Vec<String> = Vec::new();
for l in cur.iter() {
for ch in l.chars() {
out.push(ch.to_string());
}
}
cur = out;
}
QueryOp::Title => {
for l in cur.iter_mut() {
*l = l
.split_whitespace()
.map(|w| {
let mut cs = w.chars();
match cs.next() {
Some(f) => {
f.to_uppercase().collect::<String>()
+ &cs.as_str().to_lowercase()
}
None => String::new(),
}
})
.collect::<Vec<_>>()
.join(" ");
}
}
QueryOp::Repeat(n) => {
for l in cur.iter_mut() {
*l = l.repeat(*n);
}
}
QueryOp::Set(key, val) => {
for l in cur.iter_mut() {
if let Ok(Value::Object(mut m)) = serde_json::from_str::<Value>(l) {
m.insert(key.clone(), Value::String(val.clone()));
*l = Value::Object(m).to_string();
}
}
}
QueryOp::Del(key) => {
for l in cur.iter_mut() {
if let Ok(Value::Object(mut m)) = serde_json::from_str::<Value>(l) {
m.remove(key);
*l = Value::Object(m).to_string();
}
}
}
QueryOp::Rename(old, new) => {
for l in cur.iter_mut() {
if let Ok(Value::Object(mut m)) = serde_json::from_str::<Value>(l) {
if let Some(v) = m.remove(old) {
m.insert(new.clone(), v);
*l = Value::Object(m).to_string();
}
}
}
}
QueryOp::Default(key, val) => {
for l in cur.iter_mut() {
if let Ok(Value::Object(mut m)) = serde_json::from_str::<Value>(l) {
m.entry(key.clone()).or_insert(Value::String(val.clone()));
*l = Value::Object(m).to_string();
}
}
}
QueryOp::Merge => {
let mut acc = serde_json::Map::new();
let mut saw_object = false;
for l in cur.iter() {
if let Ok(Value::Object(m)) = serde_json::from_str::<Value>(l) {
saw_object = true;
for (k, v) in m {
acc.insert(k, v);
}
}
}
return if saw_object {
QueryResult::Lines(vec![Value::Object(acc).to_string()])
} else {
QueryResult::Lines(vec![])
};
}
QueryOp::Floor => {
for l in cur.iter_mut() {
if let Ok(x) = l.parse::<f64>() {
*l = fmt_num(x.floor());
}
}
}
QueryOp::Ceil => {
for l in cur.iter_mut() {
if let Ok(x) = l.trim().parse::<f64>() {
*l = fmt_num(x.ceil());
}
}
}
QueryOp::Clamp(lo, hi) => {
for l in cur.iter_mut() {
if let Ok(x) = l.trim().parse::<f64>() {
*l = fmt_num(x.clamp(*lo, *hi));
}
}
}
QueryOp::Contains(s) => cur.retain(|l| l.contains(s.as_str())),
QueryOp::Starts(p) => cur.retain(|l| l.starts_with(p.as_str())),
QueryOp::Ends(s) => cur.retain(|l| l.ends_with(s.as_str())),
QueryOp::Nonempty => cur.retain(|l| !l.trim().is_empty()),
QueryOp::Numeric => cur.retain(|l| l.trim().parse::<f64>().is_ok()),
QueryOp::Len => {
for l in cur.iter_mut() {
*l = l.chars().count().to_string();
}
}
QueryOp::JsonLen => {
for l in cur.iter_mut() {
*l = match serde_json::from_str::<Value>(l) {
Ok(Value::Array(a)) => a.len().to_string(),
Ok(Value::Object(m)) => m.len().to_string(),
Ok(Value::String(s)) => s.chars().count().to_string(),
Ok(Value::Number(n)) => fmt_num(n.as_f64().unwrap_or(0.0).abs()),
Ok(Value::Null) => "0".to_string(),
_ => l.chars().count().to_string(),
};
}
}
QueryOp::JsonSlice(a, b) => {
for l in cur.iter_mut() {
match jq_value(l) {
Some(Value::Array(arr)) => {
let (lo, hi) = slice_bounds(*a, *b, arr.len());
*l = Value::Array(arr[lo..hi].to_vec()).to_string();
}
Some(Value::String(s)) => {
let chars: Vec<char> = s.chars().collect();
let (lo, hi) = slice_bounds(*a, *b, chars.len());
*l = chars[lo..hi].iter().collect();
}
Some(Value::Null) => *l = "null".to_string(),
Some(v) => {
let bound =
|x: &Option<i64>| x.map_or("null".to_string(), |n| n.to_string());
return QueryResult::Error(format!(
"jq: Cannot index {} with object ({{\"start\":{},\"end\":{}}})",
crate::jqval::tname(&v),
bound(a),
bound(b)
));
}
None => {} }
}
}
QueryOp::Wc => {
for l in cur.iter_mut() {
*l = l.split_whitespace().count().to_string();
}
}
QueryOp::Abs => {
for l in cur.iter_mut() {
if let Ok(v) = l.trim().parse::<f64>() {
*l = fmt_num(v.abs());
}
}
}
QueryOp::Round => {
for l in cur.iter_mut() {
if let Ok(v) = l.trim().parse::<f64>() {
*l = fmt_num(v.round());
}
}
}
QueryOp::Delta => {
let ns = nums(&cur);
cur = ns.windows(2).map(|w| fmt_num(w[1] - w[0])).collect();
}
QueryOp::Cumsum => {
let ns = nums(&cur);
let mut acc = 0.0;
cur = ns
.iter()
.map(|&v| {
acc += v;
fmt_num(acc)
})
.collect();
}
QueryOp::Sma(n) => {
let ns = nums(&cur);
let w = (*n).max(1);
cur = (0..ns.len())
.map(|i| {
let lo = i + 1 - (i + 1).min(w);
let win = &ns[lo..=i];
fmt_num(win.iter().sum::<f64>() / win.len() as f64)
})
.collect();
}
QueryOp::Ewma(alpha) => {
let a = alpha.clamp(0.0, 1.0);
let ns = nums(&cur);
let mut s = 0.0;
cur = ns
.iter()
.enumerate()
.map(|(i, &v)| {
s = if i == 0 { v } else { a * v + (1.0 - a) * s };
fmt_num(s)
})
.collect();
}
QueryOp::Prepend(pre) => {
for l in cur.iter_mut() {
*l = format!("{pre}{l}");
}
}
QueryOp::Append(suf) => {
for l in cur.iter_mut() {
l.push_str(suf);
}
}
QueryOp::Cut(delim, n) => {
for l in cur.iter_mut() {
*l = l
.split(delim.as_str())
.nth(n.saturating_sub(1))
.unwrap_or("")
.to_string();
}
}
QueryOp::Median => {
let mut ns = nums(&cur);
ns.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
let m = if ns.is_empty() {
0.0
} else if ns.len() % 2 == 1 {
ns[ns.len() / 2]
} else {
(ns[ns.len() / 2 - 1] + ns[ns.len() / 2]) / 2.0
};
return QueryResult::Scalar(m);
}
QueryOp::Percentile(p) => {
let mut ns = nums(&cur);
ns.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
let v = if ns.is_empty() {
0.0
} else if ns.len() == 1 {
ns[0]
} else {
let frac = p.clamp(0.0, 100.0) / 100.0;
let pos = frac * (ns.len() - 1) as f64;
let lo = pos.floor() as usize;
let hi = pos.ceil() as usize;
ns[lo] + (ns[hi] - ns[lo]) * (pos - lo as f64)
};
return QueryResult::Scalar(v);
}
QueryOp::Stddev => {
let ns = nums(&cur);
let sd = if ns.is_empty() {
0.0
} else {
let mean = ns.iter().sum::<f64>() / ns.len() as f64;
let var = ns.iter().map(|v| (v - mean).powi(2)).sum::<f64>() / ns.len() as f64;
var.sqrt()
};
return QueryResult::Scalar(sd);
}
QueryOp::Range => {
let ns = nums(&cur);
let r = if ns.is_empty() {
0.0
} else {
ns.iter().cloned().fold(f64::NEG_INFINITY, f64::max)
- ns.iter().cloned().fold(f64::INFINITY, f64::min)
};
return QueryResult::Scalar(r);
}
QueryOp::Product => return QueryResult::Scalar(nums(&cur).iter().product()),
QueryOp::Distinct => {
let set: std::collections::HashSet<&String> = cur.iter().collect();
return QueryResult::Scalar(set.len() as f64);
}
QueryOp::Sample(n) => {
if *n >= 1 {
let mut i = 0usize;
cur.retain(|_| {
i += 1;
i.is_multiple_of(*n)
});
}
}
QueryOp::Slice(a, b) => {
let lo = a.saturating_sub(1).min(cur.len());
let hi = (*b).min(cur.len());
cur = if lo < hi {
cur[lo..hi].to_vec()
} else {
Vec::new()
};
}
QueryOp::Index(n) => {
cur = n
.checked_sub(1)
.and_then(|i| cur.get(i).cloned())
.into_iter()
.collect();
}
QueryOp::Bins(n) => {
let vals = nums(&cur);
let n = (*n).clamp(1, 65_536);
if vals.is_empty() {
return QueryResult::Pairs(Vec::new());
}
let min = vals.iter().cloned().fold(f64::INFINITY, f64::min);
let max = vals.iter().cloned().fold(f64::NEG_INFINITY, f64::max);
let width = ((max - min) / n as f64).max(f64::MIN_POSITIVE);
let mut counts = vec![0u64; n];
for v in &vals {
let idx = (((v - min) / width) as usize).min(n - 1);
counts[idx] += 1;
}
let pairs = counts
.iter()
.enumerate()
.map(|(i, &c)| {
let lo = min + i as f64 * width;
(format!("{}-{}", fmt_num(lo), fmt_num(lo + width)), c)
})
.collect();
return QueryResult::Pairs(pairs);
}
}
}
QueryResult::Lines(cur)
}
pub fn table_data(lines: &[String], cols: Option<&str>) -> (Vec<String>, Vec<Vec<String>>) {
let headers: Vec<String> = cols
.map(|c| {
c.split(',')
.map(|s| s.trim().to_string())
.filter(|s| !s.is_empty())
.collect()
})
.unwrap_or_default();
let rows: Vec<Vec<String>> = lines
.iter()
.map(|l| l.split_whitespace().map(str::to_string).collect())
.collect();
(headers, rows)
}
pub fn numeric_series(lines: &[String]) -> Vec<f64> {
lines
.iter()
.filter_map(|l| {
l.split_whitespace()
.next()
.and_then(|t| t.parse::<f64>().ok())
})
.collect()
}
pub fn sparkline(values: &[f64]) -> String {
const TICKS: [char; 8] = ['▁', '▂', '▃', '▄', '▅', '▆', '▇', '█'];
if values.is_empty() {
return String::new();
}
let min = values.iter().cloned().fold(f64::INFINITY, f64::min);
let max = values.iter().cloned().fold(f64::NEG_INFINITY, f64::max);
let range = max - min;
values
.iter()
.map(|&v| {
let idx = if range <= 0.0 {
0
} else {
(((v - min) / range) * 7.0).round() as usize
};
TICKS[idx.min(7)]
})
.collect()
}
pub fn table_ncols(headers: &[String], rows: &[Vec<String>]) -> usize {
rows.iter()
.map(Vec::len)
.max()
.unwrap_or(0)
.max(headers.len())
.max(1)
}
pub fn is_line_streamable(ops: &[QueryOp]) -> bool {
ops.iter().all(|op| {
if let QueryOp::JqMap(inner) = op {
return is_line_streamable(inner);
}
if let QueryOp::JqProgram(src) = op {
return jq_program(src).is_ok_and(|p| !p.reads_input_stream());
}
matches!(
op,
QueryOp::JsonDocs
| QueryOp::Match(_)
| QueryOp::Reject(_)
| QueryOp::Field(_)
| QueryOp::Fields(_)
| QueryOp::Each
| QueryOp::JqEach
| QueryOp::JqSelect(_)
| QueryOp::JqCalc(_)
| QueryOp::JqAdd
| QueryOp::JqLen
| QueryOp::Keys
| QueryOp::JqKeys
| QueryOp::Vals
| QueryOp::NonNull
| QueryOp::JqRawString
| QueryOp::Pick(_)
| QueryOp::Where(_)
| QueryOp::Map(_)
| QueryOp::Contains(_)
| QueryOp::Starts(_)
| QueryOp::Ends(_)
| QueryOp::Nonempty
| QueryOp::Numeric
| QueryOp::Len
| QueryOp::JsonLen
| QueryOp::JsonSlice(_, _)
| QueryOp::Wc
| QueryOp::Abs
| QueryOp::Round
| QueryOp::Prepend(_)
| QueryOp::Append(_)
| QueryOp::Cut(_, _)
| QueryOp::Upper
| QueryOp::Lower
| QueryOp::Trim
| QueryOp::Replace(_, _)
| QueryOp::B64
| QueryOp::B64d
| QueryOp::Hex
| QueryOp::Unhex
| QueryOp::Urlenc
| QueryOp::Urldec
| QueryOp::Extract(_)
| QueryOp::Substr(_, _)
| QueryOp::Title
| QueryOp::Repeat(_)
| QueryOp::Set(_, _)
| QueryOp::Del(_)
| QueryOp::Rename(_, _)
| QueryOp::Default(_, _)
| QueryOp::Floor
| QueryOp::Ceil
| QueryOp::Clamp(_, _)
| QueryOp::Has(_)
| QueryOp::HasKey(_)
| QueryOp::Entries
| QueryOp::JqEntries
| QueryOp::JqFlatten
| QueryOp::Add
| QueryOp::Over(_)
| QueryOp::Under(_)
| QueryOp::Between(_, _)
| QueryOp::Pad(_)
| QueryOp::Lpad(_)
| QueryOp::Grepf(_, _)
| QueryOp::Flip
| QueryOp::Basename
| QueryOp::Dirname
| QueryOp::Commafy
)
})
}
fn extract_field(line: &str, sel: &FieldSel) -> String {
match sel {
FieldSel::Col(n) => nth_col(line, *n).to_string(),
FieldSel::JqKey(segs) => jq_value(line)
.and_then(|v| crate::jqval::get_path(&v, segs).ok())
.map_or_else(|| "null".to_string(), |v| jq_to_string(&v)),
FieldSel::Key(path) => serde_json::from_str::<Value>(line)
.ok()
.and_then(|v| walk(v, path))
.map(|v| json_to_string(&v))
.or_else(|| {
if path.len() == 1 {
logfmt_field(line, &path[0])
} else {
None
}
})
.unwrap_or_default(),
}
}
fn jq_value(line: &str) -> Option<Value> {
serde_json::from_str::<Value>(line).ok()
}
fn jq_scalar(line: &str) -> Value {
jq_value(line).unwrap_or_else(|| Value::String(line.to_string()))
}
fn has_no_keys(v: &Value) -> String {
format!("jq: {} ({v}) has no keys", crate::jqval::tname(v))
}
fn jq_to_string(v: &Value) -> String {
match v {
Value::String(s) => s.clone(),
other => other.to_string(),
}
}
fn jq_array_json(elems: &[String]) -> String {
let mut out = String::from("[");
for (i, e) in elems.iter().enumerate() {
if i > 0 {
out.push(',');
}
match crate::jqlang::parse_json(e) {
Ok(crate::jqlang::JqVal::Num(..)) => out.push_str(e),
Ok(v) => out.push_str(&crate::jqlang::render(&v)),
Err(_) => out.push_str(&Value::String(e.clone()).to_string()),
}
}
out.push(']');
out
}
fn emit_stream(docs: &[crate::jqlang::JqVal], fmt: OutFmt, indent: usize) -> Vec<String> {
let opts = crate::ynode::Emit { indent };
let text = match fmt {
OutFmt::Yaml => crate::ynode::emit_docs(docs, opts),
OutFmt::Json => docs
.iter()
.map(|d| crate::jqlang::render_indented(d, indent))
.collect::<Vec<_>>()
.join("\n"),
OutFmt::Props => docs
.iter()
.map(crate::yqfmt::to_props)
.collect::<Vec<_>>()
.concat(),
OutFmt::Xml => docs
.iter()
.map(|d| crate::yqfmt::to_xml(d, indent))
.collect::<Vec<_>>()
.concat(),
OutFmt::Csv => docs
.iter()
.map(|d| crate::yqfmt::to_delim(d, ','))
.collect::<Vec<_>>()
.concat(),
OutFmt::Tsv => docs
.iter()
.map(|d| crate::yqfmt::to_delim(d, '\t'))
.collect::<Vec<_>>()
.concat(),
};
let mut lines: Vec<String> = text.split('\n').map(str::to_string).collect();
if lines.last().is_some_and(String::is_empty) {
lines.pop();
}
lines
}
fn toml_to_json(lines: &[String]) -> Vec<String> {
match toml::from_str::<toml::Value>(&lines.join("\n")) {
Ok(v) => vec![toml_value_to_json(&v).to_string()],
Err(_) => Vec::new(),
}
}
fn toml_value_to_json(v: &toml::Value) -> Value {
match v {
toml::Value::String(s) => Value::String(s.clone()),
toml::Value::Integer(i) => Value::Number((*i).into()),
toml::Value::Float(f) => serde_json::Number::from_f64(*f)
.map(Value::Number)
.unwrap_or(Value::Null),
toml::Value::Boolean(b) => Value::Bool(*b),
toml::Value::Datetime(dt) => Value::String(dt.to_string()),
toml::Value::Array(a) => Value::Array(a.iter().map(toml_value_to_json).collect()),
toml::Value::Table(t) => Value::Object(
t.iter()
.map(|(k, v)| (k.clone(), toml_value_to_json(v)))
.collect(),
),
}
}
fn join_quoted_records(lines: &[String]) -> Vec<String> {
let mut records = Vec::new();
let mut buf = String::new();
for line in lines {
if !buf.is_empty() {
buf.push('\n');
}
buf.push_str(line);
if buf.matches('"').count().is_multiple_of(2) {
records.push(std::mem::take(&mut buf));
}
}
if !buf.is_empty() {
records.push(buf); }
records
}
fn to_json_records(lines: &[String], delim: char) -> Vec<String> {
if lines.is_empty() {
return Vec::new();
}
let records = join_quoted_records(lines);
let Some(header_row) = records.first() else {
return Vec::new();
};
let header = split_delim(header_row, delim);
records[1..]
.iter()
.map(|row| {
let vals = split_delim(row, delim);
let mut obj = serde_json::Map::new();
for (i, name) in header.iter().enumerate() {
obj.insert(
name.clone(),
Value::String(vals.get(i).cloned().unwrap_or_default()),
);
}
Value::Object(obj).to_string()
})
.collect()
}
fn split_delim(line: &str, delim: char) -> Vec<String> {
let mut fields = Vec::new();
let mut field = String::new();
let mut in_quotes = false;
let mut quoted = false; let mut chars = line.chars().peekable();
let finish = |f: &str, q: bool| -> String {
if q {
f.to_string()
} else {
f.trim().to_string()
}
};
while let Some(c) = chars.next() {
if in_quotes {
if c == '"' {
if chars.peek() == Some(&'"') {
chars.next(); field.push('"');
} else {
in_quotes = false; }
} else {
field.push(c);
}
} else if c == '"' && field.trim().is_empty() {
in_quotes = true; quoted = true;
field.clear(); } else if c == delim {
fields.push(finish(&field, quoted));
field.clear();
quoted = false;
} else {
field.push(c);
}
}
fields.push(finish(&field, quoted));
fields
}
fn logfmt_field(line: &str, key: &str) -> Option<String> {
let cs: Vec<char> = line.chars().collect();
let mut i = 0;
while i < cs.len() {
while i < cs.len() && cs[i].is_whitespace() {
i += 1;
}
let kstart = i;
while i < cs.len() && cs[i] != '=' && !cs[i].is_whitespace() {
i += 1;
}
let k: String = cs[kstart..i].iter().collect();
if i < cs.len() && cs[i] == '=' {
i += 1; let val = if cs.get(i) == Some(&'"') {
i += 1;
let mut v = String::new();
while i < cs.len() && cs[i] != '"' {
if cs[i] == '\\' && i + 1 < cs.len() {
i += 1; }
v.push(cs[i]);
i += 1;
}
i += 1; v
} else {
let vstart = i;
while i < cs.len() && !cs[i].is_whitespace() {
i += 1;
}
cs[vstart..i].iter().collect()
};
if k == key {
return Some(val);
}
}
}
None
}
fn nth_col(line: &str, n: usize) -> &str {
if n == 0 {
return line;
}
line.split_whitespace().nth(n - 1).unwrap_or("")
}
fn walk(mut cur: Value, path: &[String]) -> Option<Value> {
for key in path {
cur = match cur {
Value::Object(mut m) => m.remove(key)?,
Value::Array(mut a) => {
let idx = key.parse::<i64>().ok()?;
let i = if idx < 0 { a.len() as i64 + idx } else { idx };
if i >= 0 && (i as usize) < a.len() {
a.swap_remove(i as usize)
} else {
return None;
}
}
_ => return None,
};
}
Some(cur)
}
fn slice_bounds(a: Option<i64>, b: Option<i64>, len: usize) -> (usize, usize) {
let n = len as i64;
let norm = |x: i64| (if x < 0 { x + n } else { x }).clamp(0, n);
let lo = norm(a.unwrap_or(0));
let hi = norm(b.unwrap_or(n));
(lo as usize, hi.max(lo) as usize)
}
fn nums(lines: &[String]) -> Vec<f64> {
lines
.iter()
.filter_map(|l| l.trim().parse::<f64>().ok())
.collect()
}
pub(crate) fn fmt_num(v: f64) -> String {
if v.is_nan() {
return "null".to_string();
}
let v = v.clamp(-f64::MAX, f64::MAX);
if v == 0.0 {
return "0".to_string();
}
let shortest = format!("{v:e}");
let ndigits = shortest
.split_once('e')
.map_or(1, |(m, _)| m.chars().filter(char::is_ascii_digit).count());
let sci = format!("{:.*e}", ndigits.saturating_sub(1), v);
let (mant, exp) = match sci.split_once('e') {
Some(p) => p,
None => return sci,
};
let exp: i32 = match exp.parse() {
Ok(e) => e,
Err(_) => return sci,
};
let sign = if mant.starts_with('-') { "-" } else { "" };
let mut digits: String = mant.chars().filter(char::is_ascii_digit).collect();
while digits.len() > 1 && digits.ends_with('0') {
digits.pop();
}
let n = digits.len() as i32;
let decpt = exp + 1;
if decpt <= -4 || decpt > n + 15 {
let m = if n == 1 {
digits
} else {
format!("{}.{}", &digits[..1], &digits[1..])
};
format!("{sign}{m}e{:+03}", decpt - 1)
} else if decpt <= 0 {
format!("{sign}0.{}{digits}", "0".repeat(-decpt as usize))
} else if decpt >= n {
format!("{sign}{digits}{}", "0".repeat((decpt - n) as usize))
} else {
let (int, frac) = digits.split_at(decpt as usize);
format!("{sign}{int}.{frac}")
}
}
fn commafy(line: &str) -> String {
let s = line.trim();
if s.parse::<f64>().is_err() {
return line.to_string();
}
let (sign, rest) = match s.strip_prefix('-') {
Some(r) => ("-", r),
None => ("", s),
};
let (int, frac) = match rest.split_once('.') {
Some((i, f)) => (i, Some(f)),
None => (rest, None),
};
let mut grouped = String::new();
for (idx, ch) in int.chars().enumerate() {
if idx > 0 && (int.len() - idx) % 3 == 0 {
grouped.push(',');
}
grouped.push(ch);
}
match frac {
Some(f) => format!("{sign}{grouped}.{f}"),
None => format!("{sign}{grouped}"),
}
}
fn humanize_bytes(v: f64) -> String {
const UNITS: [&str; 6] = ["B", "KB", "MB", "GB", "TB", "PB"];
let sign = if v < 0.0 { "-" } else { "" };
let mut n = v.abs();
let mut u = 0;
while n >= 1024.0 && u < UNITS.len() - 1 {
n /= 1024.0;
u += 1;
}
if u == 0 {
format!("{sign}{} {}", n.round() as i64, UNITS[u])
} else {
let r = (n * 10.0).round() / 10.0;
if (r.fract()).abs() < f64::EPSILON {
format!("{sign}{} {}", r as i64, UNITS[u])
} else {
format!("{sign}{r:.1} {}", UNITS[u])
}
}
}
fn humanize_duration(v: f64) -> String {
let sign = if v < 0.0 { "-" } else { "" };
let total = v.abs().round() as i64;
if total == 0 {
return "0s".to_string();
}
let units = [("d", 86400), ("h", 3600), ("m", 60), ("s", 1)];
let mut rem = total;
let mut parts = Vec::new();
for (label, secs) in units {
let q = rem / secs;
if q > 0 {
parts.push(format!("{q}{label}"));
rem %= secs;
}
}
parts.truncate(2); format!("{sign}{}", parts.join(" "))
}
fn field_num(line: &str, name: &str) -> f64 {
if let Ok(Value::Object(mut m)) = serde_json::from_str::<Value>(line) {
if let Some(v) = m.remove(name) {
return value_to_f64(&v);
}
}
logfmt_field(line, name)
.and_then(|s| s.parse::<f64>().ok())
.unwrap_or(f64::NAN)
}
pub fn field_str_pub(line: &str, name: &str) -> String {
field_str(line, name)
}
pub fn field_num_pub(line: &str, name: &str) -> f64 {
field_num(line, name)
}
pub fn fmt_scalar(v: f64) -> String {
fmt_num(v)
}
fn field_str(line: &str, name: &str) -> String {
if name == "x" {
return line.to_string();
}
if let Ok(Value::Object(m)) = serde_json::from_str::<Value>(line) {
if let Some(v) = m.get(name) {
return json_to_string(v);
}
}
logfmt_field(line, name).unwrap_or_default()
}
fn cmp_str(e: &crate::expr::Expr, line: &str) -> String {
use crate::expr::Expr;
match e {
Expr::Str(s) => s.clone(),
Expr::Field(f) => field_str(line, f),
Expr::Var => line.trim().to_string(),
Expr::Num(n) => fmt_num(*n),
_ => String::new(),
}
}
fn str_of(e: &crate::expr::Expr) -> String {
match e {
crate::expr::Expr::Str(s) => s.clone(),
_ => String::new(),
}
}
fn eval_where(e: &crate::expr::Expr, line: &str) -> bool {
use crate::expr::{BinOp, Expr};
match e {
Expr::Match(inner) => {
let q = str_of(inner);
q.is_empty() || line.to_lowercase().contains(&q.to_lowercase())
}
Expr::InSet(field, inner) => {
let set = str_of(inner);
let items: Vec<&str> = set
.split(',')
.map(str::trim)
.filter(|s| !s.is_empty())
.collect();
if items.is_empty() {
return true; }
let val = field_str(line, field);
items.iter().any(|it| *it == val)
}
Expr::Not(a) => !eval_where(a, line),
Expr::Bin(BinOp::And, a, b) => eval_where(a, line) && eval_where(b, line),
Expr::Bin(BinOp::Or, a, b) => eval_where(a, line) || eval_where(b, line),
Expr::Bin(
op @ (BinOp::Eq | BinOp::Ne | BinOp::Lt | BinOp::Le | BinOp::Gt | BinOp::Ge),
a,
b,
) if matches!(**a, Expr::Str(_)) || matches!(**b, Expr::Str(_)) => {
let sa = cmp_str(a, line);
let sb = cmp_str(b, line);
match op {
BinOp::Eq => sa == sb,
BinOp::Ne => sa != sb,
BinOp::Lt => sa < sb,
BinOp::Le => sa <= sb,
BinOp::Gt => sa > sb,
_ => sa >= sb,
}
}
_ => {
let x = line.trim().parse::<f64>().unwrap_or(f64::NAN);
let resolve = |n: &str| field_num(line, n);
crate::expr::eval_pred_ctx(e, x, &resolve).unwrap_or(false)
}
}
}
fn value_to_f64(v: &Value) -> f64 {
match v {
Value::Number(n) => n.as_f64().unwrap_or(f64::NAN),
Value::String(s) => s.trim().parse().unwrap_or(f64::NAN),
Value::Bool(b) => {
if *b {
1.0
} else {
0.0
}
}
_ => f64::NAN,
}
}
fn first_element(frag: &Html) -> Option<ElementRef<'_>> {
let star = Selector::parse("*").ok()?;
frag.select(&star)
.find(|e| !matches!(e.value().name(), "html" | "head" | "body"))
}
fn json_to_string(v: &Value) -> String {
match v {
Value::String(s) => s.clone(),
Value::Null => String::new(),
other => other.to_string(),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn split_plain() {
assert_eq!(split_delim("a,b,c", ','), vec!["a", "b", "c"]);
}
#[test]
fn split_quoted_comma() {
assert_eq!(split_delim("\"a,b\",c", ','), vec!["a,b", "c"]);
}
#[test]
fn split_doubled_quote() {
assert_eq!(
split_delim("\"she \"\"said\"\"\",x", ','),
vec!["she \"said\"", "x"]
);
}
#[test]
fn split_trailing_empty() {
assert_eq!(split_delim("a,", ','), vec!["a", ""]);
}
#[test]
fn split_tsv() {
assert_eq!(split_delim("a\tb\tc", '\t'), vec!["a", "b", "c"]);
}
#[test]
fn split_unquoted_trims_quoted_keeps_spaces() {
assert_eq!(split_delim(" a , b ", ','), vec!["a", "b"]);
assert_eq!(split_delim("\" a \",b", ','), vec![" a ", "b"]);
}
}
thread_local! {
static JQ_PROGRAMS: std::cell::RefCell<
std::collections::HashMap<String, std::rc::Rc<crate::jqlang::Program>>,
> = std::cell::RefCell::new(std::collections::HashMap::new());
}
fn jq_builtin_lines(cur: &mut Vec<String>, name: &str) -> Option<QueryResult> {
let prog = match jq_program(name) {
Ok(p) => p,
Err(msg) => return Some(QueryResult::Error(format!("jq: {msg}"))),
};
let interp = crate::jqlang::Interp::default();
let mut out = Vec::with_capacity(cur.len());
for l in cur.iter() {
let Ok(v) = crate::jqlang::parse_json(l) else {
out.push(l.clone());
continue;
};
let r = prog.run_with(&interp, &v, &mut |r| {
out.push(crate::jqlang::render(&r));
Ok(())
});
if let Err(e) = r {
return Some(QueryResult::Error(format!("jq: {}", e.to_message())));
}
}
*cur = out;
None
}
pub fn jq_program(src: &str) -> Result<std::rc::Rc<crate::jqlang::Program>, String> {
JQ_PROGRAMS.with(|c| {
if let Some(p) = c.borrow().get(src) {
return Ok(p.clone());
}
let p = std::rc::Rc::new(crate::jqlang::Program::compile(src)?);
c.borrow_mut().insert(src.to_string(), p.clone());
Ok(p)
})
}