Skip to main content

arb/
query.rs

1//! Query pipeline. A `source { … }` body compiles to a sequence of ops applied
2//! to the live stream on each evaluation.
3//!
4//! M2a verbs: `in` (source marker), `match`/`grep RE`, `reject`/`grepv RE`,
5//! `field N` (1-based whitespace column), `count`, `rate`. JSON/CSV field
6//! extraction, `where(pred)`, and aggregation to tables land with later verbs.
7
8use std::collections::{BTreeMap, HashSet};
9
10use regex::Regex;
11use scraper::{ElementRef, Html, Selector};
12use base64::{engine::general_purpose::STANDARD, Engine as _};
13use percent_encoding::{percent_decode_str, utf8_percent_encode, NON_ALPHANUMERIC};
14
15use serde_json::Value;
16
17use crate::expr::Expr;
18
19/// How `field` selects a value: a 1-based whitespace column, or a JSON key path.
20#[derive(Debug, Clone, PartialEq)]
21pub enum FieldSel {
22    Col(usize),
23    Key(Vec<String>),
24}
25
26#[derive(Debug, Clone)]
27pub enum QueryOp {
28    /// Keep lines matching the pattern.
29    Match(Regex),
30    /// Drop lines matching the pattern.
31    Reject(Regex),
32    /// Replace each line with a selected field (whitespace column or JSON key path).
33    Field(FieldSel),
34    /// Flatten JSON-array lines into one line per element (jq `[]`); non-array
35    /// lines pass through unchanged.
36    Each,
37    /// Keep lines whose numeric value (`x` = line parsed as a number) satisfies
38    /// the predicate — compiled to fusevm and evaluated per line.
39    Where(Expr),
40    /// Replace each line with the value of an expression (field-aware; `x` =
41    /// line-as-number), computed on the fusevm VM.
42    Map(Expr),
43    /// Reduce to the current line count.
44    Count,
45    /// Reduce to lines-per-second over the elapsed window.
46    Rate,
47    /// Group identical values and count them, sorted by count desc then key asc.
48    Tally,
49    /// Numeric reductions over lines parsed as numbers (non-numeric ignored).
50    Sum,
51    Min,
52    Max,
53    Avg,
54    /// Flatten a JSON object's keys / values into one line each.
55    Keys,
56    Vals,
57    /// Project a JSON object down to the named keys (jq `{a,b,c}` /
58    /// `pick(.a,.b,.c)`), preserving the listed order. Non-object lines pass
59    /// through unchanged; missing keys are dropped.
60    Pick(Vec<String>),
61    /// Line-list transforms. `Sort` supports numeric (`-n`) and reverse (`-r`).
62    Sort {
63        numeric: bool,
64        reverse: bool,
65    },
66    Uniq,
67    Rev,
68    First,
69    Last,
70    Take(usize),
71    Drop(usize),
72    /// Per-line string transforms.
73    Upper,
74    Lower,
75    Trim,
76    /// Regex replace-all per line (`replace /RE/ TO`; TO may use `$1` captures).
77    Replace(Regex, String),
78    /// Collapse all lines into one, joined by a separator.
79    Join(String),
80    /// Keep only the Nth line (1-based).
81    Nth(usize),
82    /// Parse the accumulated stream as one HTML document and emit, per element
83    /// matching the CSS selector, its text (or a named attribute if `attr` is
84    /// set; elements lacking that attribute are dropped).
85    Sel {
86        css: String,
87        attr: Option<String>,
88    },
89    /// Recursive descent (xpath `//TAG`): parse the accumulated stream as one
90    /// HTML document and emit the outer HTML of every element matching the
91    /// selector, one per line — so `attr`/`text` can then pull from each.
92    Find(String),
93    /// From element fragments (one per line, as emitted by `find`), emit the
94    /// named attribute of each (xpath `@NAME`, css attr); drop lines whose first
95    /// element lacks it.
96    Attr(String),
97    /// From element fragments, emit each element's inner text; non-element lines
98    /// pass through unchanged.
99    Text,
100    /// Stable-sort JSON-record lines by FIELD; numeric if every field value
101    /// parses as a number, else lexicographic on the field's string value.
102    /// Non-object lines sink after the sorted records in input order.
103    SortBy(String),
104    /// Keep the first record for each distinct value of FIELD, preserving input
105    /// order. JSON-object lines dedup by that field's value; other lines dedup by
106    /// the whole line.
107    UniqueBy(String),
108    /// Group JSON records by the value of FIELD and count each group, returning
109    /// value -> count pairs sorted by count desc then value asc; non-object lines
110    /// are counted under their whole-line text. Reducer (early return).
111    CountBy(String),
112    /// Reducer: emit the single JSON record whose numeric FIELD is smallest (records with a missing/non-numeric FIELD are ignored; empty input yields no lines).
113    MinBy(String),
114    /// Reducer: emit the single record whose numeric FIELD is the largest.
115    MaxBy(String),
116    /// Retain only JSON-object lines that contain KEY; all other lines (missing key, non-object, unparseable) are dropped.
117    Has(String),
118    /// jq to_entries: expand each JSON object line into one `{"key":<k>,"value":<v>}` line per key (BTreeMap key order); non-object lines pass through.
119    Entries,
120    /// For each JSON-array line, emit each element (via json_to_string); if an
121    /// element is itself a JSON array, emit ITS elements instead (one level
122    /// deeper than `each`). Non-array lines pass through unchanged.
123    Flatten,
124    /// jq `add`: reduce a JSON array line to a single value — sum numeric
125    /// arrays (fmt_num), concatenate non-numeric arrays via their string
126    /// values, empty array -> "". Non-array lines pass through unchanged.
127    Add,
128    /// Keep only lines that parse as a number strictly greater than `N`.
129    /// Lines that do not parse as `f64` are dropped.
130    Over(f64),
131    /// Keep only numeric lines whose value is strictly less than N; drop non-numeric lines.
132    Under(f64),
133    /// Keep numeric lines x where lo <= x <= hi (inclusive); non-numeric lines are dropped.
134    Between(f64, f64),
135    /// Prefix each line with its 1-based index and a tab: "1\t<line>".
136    Enumerate,
137    /// Split each line on whitespace and emit one word per line (flatten); empty lines produce nothing.
138    Words,
139    /// Collapse runs of adjacent identical lines to a single line (classic uniq), leaving non-adjacent repeats intact.
140    Dedup,
141    /// Keep only the last N lines (complement of `take`, which keeps the first N). N>=len keeps all.
142    Tailn(usize),
143    /// Right-pad each line with spaces to a minimum visible width N (no truncation if the line is already longer).
144    Pad(usize),
145    /// Left-pad each line with spaces to a minimum width of N (lines already >= N are unchanged).
146    Lpad(usize),
147    /// Retain lines whose FIELD (json key or 1-based whitespace column) matches the regex.
148    Grepf(String, regex::Regex),
149    /// Reverse the Unicode scalar characters of each line (chars().rev()).
150    Flip,
151    /// Treat the stream as CSV: the first line is the header; each data row
152    /// becomes a JSON object keyed by the header, so `field NAME` works.
153    Csv,
154    B64,
155    /// Base64-decode each line (STANDARD alphabet) into a UTF-8 string; lines that fail to
156    /// base64-decode or whose bytes aren't valid UTF-8 pass through unchanged.
157    B64d,
158    /// Lowercase hex-encode each line, two hex digits per UTF-8 byte.
159    Hex,
160    /// Decode a hex string to UTF-8 text by parsing byte pairs; on any error (odd length, non-hex digit, invalid UTF-8) the line is left unchanged.
161    Unhex,
162    /// Percent-encode each line, escaping every non-alphanumeric byte (RFC 3986 style).
163    Urlenc,
164    /// Percent-decode each line to UTF-8 (utf8_percent_decode); lines whose decoded bytes are not valid UTF-8 pass through unchanged.
165    Urldec,
166    /// Emit the first regex match per line (capture group 1 if the pattern captures, else the whole match); drop lines with no match.
167    Extract(Regex),
168    /// Explode each line by the literal string DELIM into multiple lines (one part per line).
169    /// One-to-many: unlike `cut` (one field) or `words` (whitespace), every split segment becomes its own line.
170    Split(String),
171    /// Character substring [A,B) 0-based, clamped to the line length (B may exceed len; A>B yields empty).
172    Substr(usize, usize),
173    /// Explode each line into one output line per Unicode scalar (character); one line -> many.
174    Chars,
175    /// Title-case each line: uppercase the first letter of each whitespace-separated word, lowercase the rest, rejoin with single spaces.
176    Title,
177    /// Replace each line with its content repeated N times, concatenated.
178    Repeat(usize),
179    /// Set key K to the string value V (Value::String(V)) in each JSON object line; non-object / unparseable lines pass through unchanged.
180    Set(String, String),
181    /// Remove key K from each JSON object line (jq `del(.K)`); non-object lines pass through.
182    Del(String),
183    /// Rename JSON object key OLD to NEW in each object, preserving the value; no-op if OLD absent. Non-object lines pass through.
184    Rename(String, String),
185    /// Set string key K to V only when K is absent from the JSON object (jq `//=` for a missing key). Present keys keep their value; non-object / unparseable lines pass through unchanged. Key order is normalized on mutation.
186    Default(String, String),
187    /// Reduce all JSON object lines into a single object (later keys overwrite earlier); non-object lines are ignored. Emits one JSON object line, or none if no objects were seen.
188    Merge,
189    /// Floor each numeric line to the nearest lower integer (fmt_num(x.floor())); non-numeric lines pass through unchanged.
190    Floor,
191    /// Round each numeric line up to the nearest integer (ceil); non-numeric lines pass through unchanged.
192    Ceil,
193    /// Clamp each numeric line into the inclusive range [LO, HI]; non-numeric lines pass through unchanged.
194    Clamp(f64, f64),
195    /// Same as `Csv` but tab-separated (TSV).
196    Tsv,
197    /// Parse the accumulated stream as a YAML document (or `---`-separated
198    /// multi-document) and emit each document as a JSON line, so the JSON verbs
199    /// (`field`/`pick`/`keys`/`each`) work over it (the yq leg).
200    Yaml,
201    /// Parse the accumulated stream as one TOML document and emit it as a JSON
202    /// object line.
203    Toml,
204    /// Reduce to a scalar computed by an arithmetic expression over the current
205    /// line count (`x`), evaluated on the fusevm VM.
206    Calc(Expr),
207    /// keep lines containing a literal substring.
208    Contains(String),
209    /// keep lines starting with a literal prefix.
210    Starts(String),
211    /// keep lines ending with a literal suffix.
212    Ends(String),
213    /// drop empty / whitespace-only lines.
214    Nonempty,
215    /// keep only lines that parse as a number.
216    Numeric,
217    /// replace each line with its character count.
218    Len,
219    /// replace each line with its word count.
220    Wc,
221    /// absolute value of each numeric line.
222    Abs,
223    /// round each numeric line to the nearest integer.
224    Round,
225    /// prefix every line with a literal string.
226    Prepend(String),
227    /// suffix every line with a literal string.
228    Append(String),
229    /// split each line by DELIM, keep the Nth (1-based) field.
230    Cut(String, usize),
231    /// median of numeric lines.
232    Median,
233    /// population standard deviation of numeric lines.
234    Stddev,
235    /// 95th percentile (nearest-rank) of numeric lines.
236    P95,
237    /// max minus min of numeric lines.
238    Range,
239    /// product of numeric lines.
240    Product,
241    /// count of distinct lines.
242    Distinct,
243    /// keep every Nth line (1-based).
244    Sample(usize),
245    /// keep lines from index A to B inclusive (1-based).
246    Slice(usize, usize),
247    /// bucket numeric lines into N equal-width ranges -> (range, count) pairs.
248    Bins(usize),
249}
250
251/// The output of evaluating a pipeline: lines, a scalar, or grouped counts.
252#[derive(Debug, Clone, PartialEq)]
253pub enum QueryResult {
254    Lines(Vec<String>),
255    Scalar(f64),
256    Pairs(Vec<(String, u64)>),
257}
258
259/// Evaluate `ops` against `lines`. `elapsed_secs` feeds `rate`.
260pub fn eval(ops: &[QueryOp], lines: &[String], elapsed_secs: f64) -> QueryResult {
261    let mut cur: Vec<String> = lines.to_vec();
262    for op in ops {
263        match op {
264            QueryOp::Match(re) => cur.retain(|l| re.is_match(l)),
265            QueryOp::Reject(re) => cur.retain(|l| !re.is_match(l)),
266            QueryOp::Field(sel) => {
267                for l in cur.iter_mut() {
268                    *l = extract_field(l, sel);
269                }
270            }
271            QueryOp::Each => {
272                let mut out = Vec::with_capacity(cur.len());
273                for l in &cur {
274                    match serde_json::from_str::<Value>(l) {
275                        Ok(Value::Array(arr)) => {
276                            out.extend(arr.iter().map(json_to_string));
277                        }
278                        _ => out.push(l.clone()),
279                    }
280                }
281                cur = out;
282            }
283            QueryOp::Where(e) => {
284                cur.retain(|l| {
285                    let x = l.trim().parse::<f64>().unwrap_or(f64::NAN);
286                    let resolve = |name: &str| field_num(l, name);
287                    crate::expr::eval_pred_ctx(e, x, &resolve).unwrap_or(false)
288                });
289            }
290            QueryOp::Map(e) => {
291                for l in cur.iter_mut() {
292                    let v = {
293                        let x = l.trim().parse::<f64>().unwrap_or(f64::NAN);
294                        let resolve = |name: &str| field_num(l, name);
295                        crate::expr::eval_ctx(e, x, &resolve).unwrap_or(f64::NAN)
296                    };
297                    *l = fmt_num(v);
298                }
299            }
300            QueryOp::Count => return QueryResult::Scalar(cur.len() as f64),
301            QueryOp::Rate => {
302                return QueryResult::Scalar(cur.len() as f64 / elapsed_secs.max(0.001));
303            }
304            QueryOp::Tally => {
305                let mut counts: BTreeMap<String, u64> = BTreeMap::new();
306                for l in &cur {
307                    *counts.entry(l.clone()).or_insert(0) += 1;
308                }
309                let mut pairs: Vec<(String, u64)> = counts.into_iter().collect();
310                pairs.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
311                return QueryResult::Pairs(pairs);
312            }
313            QueryOp::Calc(e) => {
314                let x = cur.len() as f64;
315                return QueryResult::Scalar(crate::expr::eval(e, x).unwrap_or(0.0));
316            }
317            QueryOp::Sum => return QueryResult::Scalar(nums(&cur).iter().sum()),
318            QueryOp::Min => {
319                let m = nums(&cur).into_iter().fold(f64::INFINITY, f64::min);
320                return QueryResult::Scalar(if m.is_finite() { m } else { 0.0 });
321            }
322            QueryOp::Max => {
323                let m = nums(&cur).into_iter().fold(f64::NEG_INFINITY, f64::max);
324                return QueryResult::Scalar(if m.is_finite() { m } else { 0.0 });
325            }
326            QueryOp::Avg => {
327                let ns = nums(&cur);
328                let a = if ns.is_empty() {
329                    0.0
330                } else {
331                    ns.iter().sum::<f64>() / ns.len() as f64
332                };
333                return QueryResult::Scalar(a);
334            }
335            QueryOp::Keys => {
336                let mut out = Vec::new();
337                for l in &cur {
338                    match serde_json::from_str::<Value>(l) {
339                        Ok(Value::Object(m)) => out.extend(m.keys().cloned()),
340                        _ => out.push(l.clone()),
341                    }
342                }
343                cur = out;
344            }
345            QueryOp::Vals => {
346                let mut out = Vec::new();
347                for l in &cur {
348                    match serde_json::from_str::<Value>(l) {
349                        Ok(Value::Object(m)) => out.extend(m.values().map(json_to_string)),
350                        _ => out.push(l.clone()),
351                    }
352                }
353                cur = out;
354            }
355            QueryOp::Pick(keys) => {
356                for l in cur.iter_mut() {
357                    if let Ok(Value::Object(m)) = serde_json::from_str::<Value>(l) {
358                        // Build in pick order — serde_json::Map is a BTreeMap and
359                        // would re-sort the keys, losing the requested order.
360                        let parts: Vec<String> = keys
361                            .iter()
362                            .filter_map(|k| {
363                                m.get(k).map(|v| format!("{}:{}", Value::String(k.clone()), v))
364                            })
365                            .collect();
366                        *l = format!("{{{}}}", parts.join(","));
367                    }
368                }
369            }
370            QueryOp::Csv => cur = to_json_records(&cur, ','),
371            QueryOp::Tsv => cur = to_json_records(&cur, '\t'),
372            QueryOp::Yaml => cur = yaml_to_json(&cur),
373            QueryOp::Toml => cur = toml_to_json(&cur),
374            QueryOp::Sort { numeric, reverse } => {
375                if *numeric {
376                    cur.sort_by(|a, b| {
377                        let na = a.trim().parse::<f64>().unwrap_or(f64::NAN);
378                        let nb = b.trim().parse::<f64>().unwrap_or(f64::NAN);
379                        na.partial_cmp(&nb).unwrap_or(std::cmp::Ordering::Equal)
380                    });
381                } else {
382                    cur.sort();
383                }
384                if *reverse {
385                    cur.reverse();
386                }
387            }
388            QueryOp::Uniq => {
389                let mut seen = HashSet::new();
390                cur.retain(|l| seen.insert(l.clone()));
391            }
392            QueryOp::Rev => cur.reverse(),
393            QueryOp::First => cur.truncate(1),
394            QueryOp::Last => {
395                if let Some(l) = cur.pop() {
396                    cur = vec![l];
397                } else {
398                    cur.clear();
399                }
400            }
401            QueryOp::Take(n) => cur.truncate(*n),
402            QueryOp::Drop(n) => {
403                cur.drain(0..(*n).min(cur.len()));
404            }
405            QueryOp::Upper => {
406                for l in cur.iter_mut() {
407                    *l = l.to_uppercase();
408                }
409            }
410            QueryOp::Lower => {
411                for l in cur.iter_mut() {
412                    *l = l.to_lowercase();
413                }
414            }
415            QueryOp::Trim => {
416                for l in cur.iter_mut() {
417                    *l = l.trim().to_string();
418                }
419            }
420            QueryOp::Replace(re, to) => {
421                for l in cur.iter_mut() {
422                    *l = re.replace_all(l, to.as_str()).into_owned();
423                }
424            }
425            QueryOp::Join(sep) => {
426                cur = vec![cur.join(sep)];
427            }
428            QueryOp::Nth(n) => {
429                cur = cur
430                    .get(n.saturating_sub(1))
431                    .cloned()
432                    .map(|l| vec![l])
433                    .unwrap_or_default();
434            }
435            QueryOp::Sel { css, attr } => {
436                let doc = Html::parse_document(&cur.join("\n"));
437                cur = match Selector::parse(css) {
438                    Ok(sel) => doc
439                        .select(&sel)
440                        .filter_map(|el| match attr {
441                            Some(a) => el.value().attr(a).map(str::to_string),
442                            None => Some(el.text().collect::<String>().trim().to_string()),
443                        })
444                        .collect(),
445                    Err(_) => Vec::new(),
446                };
447            }
448            QueryOp::Find(css) => {
449                let doc = Html::parse_document(&cur.join("\n"));
450                cur = match Selector::parse(css) {
451                    Ok(sel) => doc.select(&sel).map(|el| el.html()).collect(),
452                    Err(_) => Vec::new(),
453                };
454            }
455            QueryOp::Attr(name) => {
456                let mut out = Vec::with_capacity(cur.len());
457                for l in &cur {
458                    let frag = Html::parse_fragment(l);
459                    if let Some(v) = first_element(&frag).and_then(|e| e.value().attr(name)) {
460                        out.push(v.to_string());
461                    }
462                }
463                cur = out;
464            }
465            QueryOp::Text => {
466                for l in cur.iter_mut() {
467                    let frag = Html::parse_fragment(l);
468                    if let Some(e) = first_element(&frag) {
469                        *l = e.text().collect::<String>().trim().to_string();
470                    }
471                }
472            }
473            QueryOp::SortBy(field) => {
474                // Split object lines (carrying their field value) from the rest;
475                // non-objects keep their relative input order and sink to the end.
476                let mut objs: Vec<(String, String)> = Vec::new();
477                let mut rest: Vec<String> = Vec::new();
478                for l in cur.drain(..) {
479                    if let Ok(Value::Object(m)) = serde_json::from_str::<Value>(&l) {
480                        let key = m.get(field).map(json_to_string).unwrap_or_default();
481                        objs.push((key, l));
482                    } else {
483                        rest.push(l);
484                    }
485                }
486                let all_numeric =
487                    !objs.is_empty() && objs.iter().all(|(k, _)| k.trim().parse::<f64>().is_ok());
488                if all_numeric {
489                    // slice::sort_by is stable — equal keys preserve input order.
490                    objs.sort_by(|a, b| {
491                        let na = a.0.trim().parse::<f64>().unwrap_or(f64::NAN);
492                        let nb = b.0.trim().parse::<f64>().unwrap_or(f64::NAN);
493                        na.partial_cmp(&nb).unwrap_or(std::cmp::Ordering::Equal)
494                    });
495                } else {
496                    objs.sort_by(|a, b| a.0.cmp(&b.0));
497                }
498                cur = objs.into_iter().map(|(_, l)| l).collect();
499                cur.extend(rest);
500            }
501            QueryOp::UniqueBy(field) => {
502                let mut seen: HashSet<String> = HashSet::new();
503                cur.retain(|l| {
504                    let key = match serde_json::from_str::<Value>(l) {
505                        Ok(Value::Object(m)) => {
506                            m.get(field).map(json_to_string).unwrap_or_default()
507                        }
508                        _ => l.clone(),
509                    };
510                    seen.insert(key)
511                });
512            }
513            QueryOp::CountBy(field) => {
514                let mut counts: BTreeMap<String, u64> = BTreeMap::new();
515                for l in &cur {
516                    let key = match serde_json::from_str::<Value>(l) {
517                        Ok(Value::Object(m)) => m.get(field).map(json_to_string).unwrap_or_default(),
518                        _ => l.clone(),
519                    };
520                    *counts.entry(key).or_insert(0) += 1;
521                }
522                let mut pairs: Vec<(String, u64)> = counts.into_iter().collect();
523                pairs.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
524                return QueryResult::Pairs(pairs);
525            }
526            QueryOp::MinBy(field) => {
527                let best = cur
528                    .iter()
529                    .filter(|l| !field_num(l, field).is_nan())
530                    .min_by(|a, b| {
531                        field_num(a, field)
532                            .partial_cmp(&field_num(b, field))
533                            .unwrap_or(std::cmp::Ordering::Equal)
534                    });
535                return QueryResult::Lines(best.into_iter().cloned().collect());
536            }
537            QueryOp::MaxBy(field) => {
538                // Ignore lines whose FIELD is absent/non-numeric (field_num -> NaN),
539                // then keep the record with the greatest value. On ties the last
540                // maximal record wins (std max_by semantics). Empty input -> no lines.
541                let best = cur
542                    .iter()
543                    .filter(|l| !field_num(l, field).is_nan())
544                    .max_by(|a, b| {
545                        field_num(a, field)
546                            .partial_cmp(&field_num(b, field))
547                            .unwrap_or(std::cmp::Ordering::Equal)
548                    })
549                    .cloned();
550                return QueryResult::Lines(best.into_iter().collect());
551            }
552            QueryOp::Has(key) => {
553                cur.retain(|l| {
554                    matches!(
555                        serde_json::from_str::<Value>(l),
556                        Ok(Value::Object(ref m)) if m.contains_key(key)
557                    )
558                });
559            }
560            QueryOp::Entries => {
561                let mut out = Vec::with_capacity(cur.len());
562                for l in &cur {
563                    match serde_json::from_str::<Value>(l) {
564                        Ok(Value::Object(m)) => {
565                            for (k, v) in &m {
566                                out.push(format!(
567                                    "{{\"key\":{},\"value\":{}}}",
568                                    Value::String(k.clone()),
569                                    v
570                                ));
571                            }
572                        }
573                        _ => out.push(l.clone()),
574                    }
575                }
576                cur = out;
577            }
578            QueryOp::Flatten => {
579                let mut out = Vec::with_capacity(cur.len());
580                for l in &cur {
581                    match serde_json::from_str::<Value>(l) {
582                        Ok(Value::Array(arr)) => {
583                            for el in &arr {
584                                match el {
585                                    Value::Array(inner) => {
586                                        out.extend(inner.iter().map(json_to_string));
587                                    }
588                                    other => out.push(json_to_string(other)),
589                                }
590                            }
591                        }
592                        _ => out.push(l.clone()),
593                    }
594                }
595                cur = out;
596            }
597            QueryOp::Add => {
598                for l in cur.iter_mut() {
599                    if let Ok(Value::Array(arr)) = serde_json::from_str::<Value>(l) {
600                        if arr.is_empty() {
601                            *l = String::new();
602                        } else if arr.iter().all(Value::is_number) {
603                            let sum: f64 = arr.iter().filter_map(Value::as_f64).sum();
604                            *l = fmt_num(sum);
605                        } else {
606                            *l = arr.iter().map(json_to_string).collect::<String>();
607                        }
608                    }
609                }
610            }
611            QueryOp::Over(n) => {
612                cur.retain(|l| l.trim().parse::<f64>().map(|v| v > *n).unwrap_or(false));
613            }
614            QueryOp::Under(n) => {
615                cur.retain(|l| l.trim().parse::<f64>().map(|v| v < *n).unwrap_or(false));
616            }
617            QueryOp::Between(lo, hi) => {
618                cur.retain(|l| {
619                    let x = l.trim().parse::<f64>().unwrap_or(f64::NAN);
620                    x >= *lo && x <= *hi
621                });
622            }
623            QueryOp::Enumerate => {
624                for (i, l) in cur.iter_mut().enumerate() {
625                    *l = format!("{}\t{}", i + 1, l);
626                }
627            }
628            QueryOp::Words => {
629                let mut out = Vec::with_capacity(cur.len());
630                for l in &cur {
631                    out.extend(l.split_whitespace().map(str::to_string));
632                }
633                cur = out;
634            }
635            QueryOp::Dedup => {
636                cur.dedup();
637            }
638            QueryOp::Tailn(n) => {
639                let len = cur.len();
640                if *n < len {
641                    cur.drain(0..len - *n);
642                }
643            }
644            QueryOp::Pad(n) => {
645                let n = *n;
646                for l in cur.iter_mut() {
647                    *l = format!("{:<width$}", l, width = n);
648                }
649            }
650            QueryOp::Lpad(width) => {
651                let w = *width;
652                for l in cur.iter_mut() {
653                    *l = format!("{:>width$}", l, width = w);
654                }
655            }
656            QueryOp::Grepf(field, re) => {
657                cur.retain(|l| {
658                    let val = if let Ok(Value::Object(m)) = serde_json::from_str::<Value>(l) {
659                        m.get(field).map(json_to_string).unwrap_or_default()
660                    } else if let Ok(idx) = field.parse::<usize>() {
661                        l.split_whitespace()
662                            .nth(idx.saturating_sub(1))
663                            .unwrap_or("")
664                            .to_string()
665                    } else {
666                        String::new()
667                    };
668                    re.is_match(&val)
669                });
670            }
671            QueryOp::Flip => {
672                for l in cur.iter_mut() {
673                    *l = l.chars().rev().collect();
674                }
675            }
676            QueryOp::B64 => {
677                for l in cur.iter_mut() {
678                    *l = STANDARD.encode(l.as_bytes());
679                }
680            }
681            QueryOp::B64d => {
682                for l in cur.iter_mut() {
683                    if let Some(s) = STANDARD.decode(l.as_bytes()).ok().and_then(|b| String::from_utf8(b).ok()) {
684                        *l = s;
685                    }
686                }
687            }
688            QueryOp::Hex => {
689                for l in cur.iter_mut() {
690                    *l = l.bytes().map(|b| format!("{:02x}", b)).collect();
691                }
692            }
693            QueryOp::Unhex => {
694                for l in cur.iter_mut() {
695                    let chars: Vec<char> = l.chars().collect();
696                    if chars.is_empty() || chars.len() % 2 != 0 {
697                        continue;
698                    }
699                    let mut bytes = Vec::with_capacity(chars.len() / 2);
700                    let mut ok = true;
701                    let mut i = 0;
702                    while i < chars.len() {
703                        let pair: String = chars[i..i + 2].iter().collect();
704                        match u8::from_str_radix(&pair, 16) {
705                            Ok(b) => bytes.push(b),
706                            Err(_) => {
707                                ok = false;
708                                break;
709                            }
710                        }
711                        i += 2;
712                    }
713                    if ok {
714                        if let Ok(decoded) = String::from_utf8(bytes) {
715                            *l = decoded;
716                        }
717                    }
718                }
719            }
720            QueryOp::Urlenc => {
721                for l in cur.iter_mut() {
722                    *l = utf8_percent_encode(l, NON_ALPHANUMERIC).to_string();
723                }
724            }
725            QueryOp::Urldec => {
726                for l in cur.iter_mut() {
727                    if let Ok(decoded) = percent_decode_str(l).decode_utf8() {
728                        *l = decoded.into_owned();
729                    }
730                }
731            }
732            QueryOp::Extract(re) => {
733                cur = cur
734                    .iter()
735                    .filter_map(|l| {
736                        re.captures(l).map(|caps| {
737                            caps.get(1)
738                                .unwrap_or_else(|| caps.get(0).unwrap())
739                                .as_str()
740                                .to_string()
741                        })
742                    })
743                    .collect();
744            }
745            QueryOp::Split(delim) => {
746                let mut out: Vec<String> = Vec::with_capacity(cur.len());
747                for l in cur.iter() {
748                    for part in l.split(delim.as_str()) {
749                        out.push(part.to_string());
750                    }
751                }
752                cur = out;
753            }
754            QueryOp::Substr(a, b) => {
755                for l in cur.iter_mut() {
756                    *l = l.chars().skip(*a).take(b.saturating_sub(*a)).collect();
757                }
758            }
759            QueryOp::Chars => {
760                let mut out: Vec<String> = Vec::new();
761                for l in cur.iter() {
762                    for ch in l.chars() {
763                        out.push(ch.to_string());
764                    }
765                }
766                cur = out;
767            }
768            QueryOp::Title => {
769                for l in cur.iter_mut() {
770                    *l = l
771                        .split_whitespace()
772                        .map(|w| {
773                            let mut cs = w.chars();
774                            match cs.next() {
775                                Some(f) => f.to_uppercase().collect::<String>() + &cs.as_str().to_lowercase(),
776                                None => String::new(),
777                            }
778                        })
779                        .collect::<Vec<_>>()
780                        .join(" ");
781                }
782            }
783            QueryOp::Repeat(n) => {
784                for l in cur.iter_mut() {
785                    *l = l.repeat(*n);
786                }
787            }
788            QueryOp::Set(key, val) => {
789                for l in cur.iter_mut() {
790                    if let Ok(Value::Object(mut m)) = serde_json::from_str::<Value>(l) {
791                        m.insert(key.clone(), Value::String(val.clone()));
792                        *l = Value::Object(m).to_string();
793                    }
794                }
795            }
796            QueryOp::Del(key) => {
797                for l in cur.iter_mut() {
798                    if let Ok(Value::Object(mut m)) = serde_json::from_str::<Value>(l) {
799                        m.remove(key);
800                        *l = Value::Object(m).to_string();
801                    }
802                }
803            }
804            QueryOp::Rename(old, new) => {
805                for l in cur.iter_mut() {
806                    if let Ok(Value::Object(mut m)) = serde_json::from_str::<Value>(l) {
807                        if let Some(v) = m.remove(old) {
808                            m.insert(new.clone(), v);
809                            *l = Value::Object(m).to_string();
810                        }
811                    }
812                }
813            }
814            QueryOp::Default(key, val) => {
815                for l in cur.iter_mut() {
816                    if let Ok(Value::Object(mut m)) = serde_json::from_str::<Value>(l) {
817                        m.entry(key.clone()).or_insert(Value::String(val.clone()));
818                        *l = Value::Object(m).to_string();
819                    }
820                }
821            }
822            QueryOp::Merge => {
823                let mut acc = serde_json::Map::new();
824                let mut saw_object = false;
825                for l in cur.iter() {
826                    if let Ok(Value::Object(m)) = serde_json::from_str::<Value>(l) {
827                        saw_object = true;
828                        for (k, v) in m {
829                            acc.insert(k, v);
830                        }
831                    }
832                }
833                return if saw_object {
834                    QueryResult::Lines(vec![Value::Object(acc).to_string()])
835                } else {
836                    QueryResult::Lines(vec![])
837                };
838            }
839            QueryOp::Floor => {
840                for l in cur.iter_mut() {
841                    if let Ok(x) = l.parse::<f64>() {
842                        *l = fmt_num(x.floor());
843                    }
844                }
845            }
846            QueryOp::Ceil => {
847                for l in cur.iter_mut() {
848                    if let Ok(x) = l.trim().parse::<f64>() {
849                        *l = fmt_num(x.ceil());
850                    }
851                }
852            }
853            QueryOp::Clamp(lo, hi) => {
854                for l in cur.iter_mut() {
855                    if let Ok(x) = l.trim().parse::<f64>() {
856                        *l = fmt_num(x.clamp(*lo, *hi));
857                    }
858                }
859            }
860            QueryOp::Contains(s) => cur.retain(|l| l.contains(s.as_str())),
861            QueryOp::Starts(p) => cur.retain(|l| l.starts_with(p.as_str())),
862            QueryOp::Ends(s) => cur.retain(|l| l.ends_with(s.as_str())),
863            QueryOp::Nonempty => cur.retain(|l| !l.trim().is_empty()),
864            QueryOp::Numeric => cur.retain(|l| l.trim().parse::<f64>().is_ok()),
865            QueryOp::Len => {
866                for l in cur.iter_mut() {
867                    *l = l.chars().count().to_string();
868                }
869            }
870            QueryOp::Wc => {
871                for l in cur.iter_mut() {
872                    *l = l.split_whitespace().count().to_string();
873                }
874            }
875            QueryOp::Abs => {
876                for l in cur.iter_mut() {
877                    if let Ok(v) = l.trim().parse::<f64>() {
878                        *l = fmt_num(v.abs());
879                    }
880                }
881            }
882            QueryOp::Round => {
883                for l in cur.iter_mut() {
884                    if let Ok(v) = l.trim().parse::<f64>() {
885                        *l = fmt_num(v.round());
886                    }
887                }
888            }
889            QueryOp::Prepend(pre) => {
890                for l in cur.iter_mut() {
891                    *l = format!("{pre}{l}");
892                }
893            }
894            QueryOp::Append(suf) => {
895                for l in cur.iter_mut() {
896                    l.push_str(suf);
897                }
898            }
899            QueryOp::Cut(delim, n) => {
900                for l in cur.iter_mut() {
901                    *l = l
902                        .split(delim.as_str())
903                        .nth(n.saturating_sub(1))
904                        .unwrap_or("")
905                        .to_string();
906                }
907            }
908            QueryOp::Median => {
909                let mut ns = nums(&cur);
910                ns.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
911                let m = if ns.is_empty() {
912                    0.0
913                } else if ns.len() % 2 == 1 {
914                    ns[ns.len() / 2]
915                } else {
916                    (ns[ns.len() / 2 - 1] + ns[ns.len() / 2]) / 2.0
917                };
918                return QueryResult::Scalar(m);
919            }
920            QueryOp::Stddev => {
921                let ns = nums(&cur);
922                let sd = if ns.is_empty() {
923                    0.0
924                } else {
925                    let mean = ns.iter().sum::<f64>() / ns.len() as f64;
926                    let var =
927                        ns.iter().map(|v| (v - mean).powi(2)).sum::<f64>() / ns.len() as f64;
928                    var.sqrt()
929                };
930                return QueryResult::Scalar(sd);
931            }
932            QueryOp::P95 => {
933                let mut ns = nums(&cur);
934                ns.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
935                let v = if ns.is_empty() {
936                    0.0
937                } else {
938                    let rank = ((0.95 * ns.len() as f64).ceil() as usize).clamp(1, ns.len());
939                    ns[rank - 1]
940                };
941                return QueryResult::Scalar(v);
942            }
943            QueryOp::Range => {
944                let ns = nums(&cur);
945                let r = if ns.is_empty() {
946                    0.0
947                } else {
948                    ns.iter().cloned().fold(f64::NEG_INFINITY, f64::max)
949                        - ns.iter().cloned().fold(f64::INFINITY, f64::min)
950                };
951                return QueryResult::Scalar(r);
952            }
953            QueryOp::Product => return QueryResult::Scalar(nums(&cur).iter().product()),
954            QueryOp::Distinct => {
955                let set: std::collections::HashSet<&String> = cur.iter().collect();
956                return QueryResult::Scalar(set.len() as f64);
957            }
958            QueryOp::Sample(n) => {
959                if *n >= 1 {
960                    let mut i = 0usize;
961                    cur.retain(|_| {
962                        i += 1;
963                        i % *n == 0
964                    });
965                }
966            }
967            QueryOp::Slice(a, b) => {
968                let lo = a.saturating_sub(1).min(cur.len());
969                let hi = (*b).min(cur.len());
970                cur = if lo < hi { cur[lo..hi].to_vec() } else { Vec::new() };
971            }
972            QueryOp::Bins(n) => {
973                let vals = nums(&cur);
974                let n = (*n).max(1);
975                if vals.is_empty() {
976                    return QueryResult::Pairs(Vec::new());
977                }
978                let min = vals.iter().cloned().fold(f64::INFINITY, f64::min);
979                let max = vals.iter().cloned().fold(f64::NEG_INFINITY, f64::max);
980                let width = ((max - min) / n as f64).max(f64::MIN_POSITIVE);
981                let mut counts = vec![0u64; n];
982                for v in &vals {
983                    let idx = (((v - min) / width) as usize).min(n - 1);
984                    counts[idx] += 1;
985                }
986                let pairs = counts
987                    .iter()
988                    .enumerate()
989                    .map(|(i, &c)| {
990                        let lo = min + i as f64 * width;
991                        (format!("{}-{}", fmt_num(lo), fmt_num(lo + width)), c)
992                    })
993                    .collect();
994                return QueryResult::Pairs(pairs);
995            }
996        }
997    }
998    QueryResult::Lines(cur)
999}
1000
1001/// True if every op processes each line independently, so the pipeline can be
1002/// applied per-line and its results emitted incrementally (streaming). Reducers,
1003/// sorts/reorderers, and whole-document ops (`sel`/`csv`/`tsv`) are not.
1004pub fn is_line_streamable(ops: &[QueryOp]) -> bool {
1005    ops.iter().all(|op| {
1006        matches!(
1007            op,
1008            QueryOp::Match(_)
1009                | QueryOp::Reject(_)
1010                | QueryOp::Field(_)
1011                | QueryOp::Each
1012                | QueryOp::Keys
1013                | QueryOp::Vals
1014                | QueryOp::Pick(_)
1015                | QueryOp::Where(_)
1016                | QueryOp::Map(_)
1017                | QueryOp::Contains(_)
1018                | QueryOp::Starts(_)
1019                | QueryOp::Ends(_)
1020                | QueryOp::Nonempty
1021                | QueryOp::Numeric
1022                | QueryOp::Len
1023                | QueryOp::Wc
1024                | QueryOp::Abs
1025                | QueryOp::Round
1026                | QueryOp::Prepend(_)
1027                | QueryOp::Append(_)
1028                | QueryOp::Cut(_, _)
1029                | QueryOp::Upper
1030                | QueryOp::Lower
1031                | QueryOp::Trim
1032                | QueryOp::Replace(_, _)
1033                | QueryOp::B64
1034                | QueryOp::B64d
1035                | QueryOp::Hex
1036                | QueryOp::Unhex
1037                | QueryOp::Urlenc
1038                | QueryOp::Urldec
1039                | QueryOp::Extract(_)
1040                | QueryOp::Substr(_, _)
1041                | QueryOp::Title
1042                | QueryOp::Repeat(_)
1043                | QueryOp::Set(_, _)
1044                | QueryOp::Del(_)
1045                | QueryOp::Rename(_, _)
1046                | QueryOp::Default(_, _)
1047                | QueryOp::Floor
1048                | QueryOp::Ceil
1049                | QueryOp::Clamp(_, _)
1050
1051                | QueryOp::Has(_)
1052                | QueryOp::Entries
1053                | QueryOp::Add
1054                | QueryOp::Over(_)
1055                | QueryOp::Under(_)
1056                | QueryOp::Between(_, _)
1057                | QueryOp::Pad(_)
1058                | QueryOp::Lpad(_)
1059                | QueryOp::Grepf(_, _)
1060                | QueryOp::Flip
1061
1062        )
1063    })
1064}
1065
1066/// Extract a field from a line per the selector.
1067fn extract_field(line: &str, sel: &FieldSel) -> String {
1068    match sel {
1069        FieldSel::Col(n) => nth_col(line, *n).to_string(),
1070        FieldSel::Key(path) => serde_json::from_str::<Value>(line)
1071            .ok()
1072            .and_then(|v| walk(v, path))
1073            .map(|v| json_to_string(&v))
1074            .or_else(|| {
1075                if path.len() == 1 {
1076                    logfmt_field(line, &path[0])
1077                } else {
1078                    None
1079                }
1080            })
1081            .unwrap_or_default(),
1082    }
1083}
1084
1085/// Split a CSV line on commas, trimming each field. (Quoted fields containing
1086/// commas are not yet handled.)
1087/// Parse a header + data rows of a delimited stream into JSON object strings
1088/// keyed by the header, so `field NAME` works over CSV/TSV.
1089/// Parse the stream as a YAML document (or `---`-separated multi-document) and
1090/// emit each document as a compact JSON line. Unparseable documents are dropped.
1091fn yaml_to_json(lines: &[String]) -> Vec<String> {
1092    let doc = lines.join("\n");
1093    doc.split("\n---\n")
1094        .filter(|c| !c.trim().is_empty())
1095        .filter_map(|c| serde_yaml::from_str::<Value>(c).ok())
1096        .map(|v| v.to_string())
1097        .collect()
1098}
1099
1100/// Parse the stream as one TOML document and emit it as a JSON object line
1101/// (empty if it does not parse).
1102fn toml_to_json(lines: &[String]) -> Vec<String> {
1103    match toml::from_str::<Value>(&lines.join("\n")) {
1104        Ok(v) => vec![v.to_string()],
1105        Err(_) => Vec::new(),
1106    }
1107}
1108
1109fn to_json_records(lines: &[String], delim: char) -> Vec<String> {
1110    if lines.is_empty() {
1111        return Vec::new();
1112    }
1113    let header = split_delim(&lines[0], delim);
1114    lines[1..]
1115        .iter()
1116        .map(|row| {
1117            let vals = split_delim(row, delim);
1118            let mut obj = serde_json::Map::new();
1119            for (i, name) in header.iter().enumerate() {
1120                obj.insert(
1121                    name.clone(),
1122                    Value::String(vals.get(i).cloned().unwrap_or_default()),
1123                );
1124            }
1125            Value::Object(obj).to_string()
1126        })
1127        .collect()
1128}
1129
1130fn split_delim(line: &str, delim: char) -> Vec<String> {
1131    line.split(delim).map(|s| s.trim().to_string()).collect()
1132}
1133
1134/// Extract a `key=value` (logfmt) field from a line; strips surrounding quotes.
1135fn logfmt_field(line: &str, key: &str) -> Option<String> {
1136    line.split_whitespace().find_map(|tok| {
1137        tok.split_once('=')
1138            .filter(|(k, _)| *k == key)
1139            .map(|(_, v)| v.trim_matches('"').to_string())
1140    })
1141}
1142
1143/// The 1-based whitespace column `n` of `line` ("" if absent; 0 = whole line).
1144fn nth_col(line: &str, n: usize) -> &str {
1145    if n == 0 {
1146        return line;
1147    }
1148    line.split_whitespace().nth(n - 1).unwrap_or("")
1149}
1150
1151/// Walk a JSON key/array-index path, consuming the value.
1152fn walk(mut cur: Value, path: &[String]) -> Option<Value> {
1153    for key in path {
1154        cur = match cur {
1155            Value::Object(mut m) => m.remove(key)?,
1156            Value::Array(mut a) => {
1157                let i = key.parse::<usize>().ok()?;
1158                if i < a.len() {
1159                    a.swap_remove(i)
1160                } else {
1161                    return None;
1162                }
1163            }
1164            _ => return None,
1165        };
1166    }
1167    Some(cur)
1168}
1169
1170/// Parse the numeric lines of a slice, ignoring non-numeric ones.
1171fn nums(lines: &[String]) -> Vec<f64> {
1172    lines
1173        .iter()
1174        .filter_map(|l| l.trim().parse::<f64>().ok())
1175        .collect()
1176}
1177
1178/// Format a computed number: integers without a decimal point, else default.
1179fn fmt_num(v: f64) -> String {
1180    if v.is_finite() && v.fract() == 0.0 {
1181        format!("{}", v as i64)
1182    } else {
1183        format!("{v}")
1184    }
1185}
1186
1187/// Resolve a JSON field of `line` to a number for expression evaluation
1188/// (missing / non-numeric / non-JSON -> NaN, which fails numeric predicates).
1189fn field_num(line: &str, name: &str) -> f64 {
1190    if let Ok(Value::Object(mut m)) = serde_json::from_str::<Value>(line) {
1191        if let Some(v) = m.remove(name) {
1192            return value_to_f64(&v);
1193        }
1194    }
1195    logfmt_field(line, name)
1196        .and_then(|s| s.parse::<f64>().ok())
1197        .unwrap_or(f64::NAN)
1198}
1199
1200fn value_to_f64(v: &Value) -> f64 {
1201    match v {
1202        Value::Number(n) => n.as_f64().unwrap_or(f64::NAN),
1203        Value::String(s) => s.trim().parse().unwrap_or(f64::NAN),
1204        Value::Bool(b) => {
1205            if *b {
1206                1.0
1207            } else {
1208                0.0
1209            }
1210        }
1211        _ => f64::NAN,
1212    }
1213}
1214
1215/// Render a JSON scalar as a plain string; containers as compact JSON.
1216/// The first real element of a parsed fragment, skipping the synthetic
1217/// `html`/`head`/`body` wrappers that `Html::parse_fragment` inserts.
1218fn first_element(frag: &Html) -> Option<ElementRef<'_>> {
1219    let star = Selector::parse("*").ok()?;
1220    frag.select(&star)
1221        .find(|e| !matches!(e.value().name(), "html" | "head" | "body"))
1222}
1223
1224fn json_to_string(v: &Value) -> String {
1225    match v {
1226        Value::String(s) => s.clone(),
1227        Value::Null => String::new(),
1228        other => other.to_string(),
1229    }
1230}