Skip to main content

arb/
query.rs

1//! Query pipeline. A `source { … }` body compiles to a sequence of ops applied
2//! to the live stream on each evaluation.
3//!
4//! M2a verbs: `in` (source marker), `match`/`grep RE`, `reject`/`grepv RE`,
5//! `field N` (1-based whitespace column), `count`, `rate`. JSON/CSV field
6//! extraction, `where(pred)`, and aggregation to tables land with later verbs.
7
8use std::collections::{BTreeMap, HashSet};
9
10use base64::{engine::general_purpose::STANDARD, Engine as _};
11use percent_encoding::{percent_decode_str, utf8_percent_encode, NON_ALPHANUMERIC};
12use regex::Regex;
13use scraper::{ElementRef, Html, Selector};
14
15use serde_json::Value;
16
17use crate::expr::Expr;
18
19/// How `field` selects a value: a 1-based whitespace column, or a JSON key path.
20#[derive(Debug, Clone, PartialEq)]
21pub enum FieldSel {
22    Col(usize),
23    Key(Vec<String>),
24    /// A JSON key path reached through the jq front-end (`.a.b`, `.foo[0]`).
25    /// Identical to `Key` except for the miss: jq renders an explicit null, an
26    /// absent key and an out-of-range index all as the literal `null`, where
27    /// arb's native `field` yields "" and falls back to logfmt. Keeping the two
28    /// apart lets a jq path answer like jq without changing what `field NAME`
29    /// does to a plain-text or logfmt stream.
30    /// Segments are TYPED, unlike `Key`'s flat strings: `.["0"]` is an object
31    /// key and `.[0]` is an array index, and jq keeps them apart (`[1,2] |
32    /// .["0"]` is an error, not the first element).
33    JqKey(Vec<crate::jqval::Seg>),
34}
35
36/// The serializer an `out.FORMAT` verb selects. `yq`'s `-o=` in arb's spelling.
37#[derive(Debug, Clone, Copy, PartialEq, Eq)]
38pub enum OutFmt {
39    /// Block YAML with every node's comments, anchors, tags and quoting put
40    /// back — the rendering that makes `in.yaml; out.yaml` a round trip.
41    Yaml,
42    /// Indented JSON. `0` is jq's compact line, which is arb's default anyway.
43    Json,
44    /// `dotted.path = value` lines.
45    Props,
46    Xml,
47    Csv,
48    Tsv,
49}
50
51#[derive(Debug, Clone)]
52pub enum QueryOp {
53    /// A compiled XPath 1.0 expression (`crate::xpath`). The expression is
54    /// parsed at spec-build time, so this op always holds a well-formed one and
55    /// a malformed path is refused before any input is read.
56    XPath(Box<crate::xpath::XPath>),
57    /// Re-serialize the whole stream in another format (`out.yaml 4`). The
58    /// number is the indent, `yq`'s `-I`.
59    Emit(OutFmt, usize),
60    /// Keep lines matching the pattern.
61    Match(Regex),
62    /// Drop lines matching the pattern.
63    Reject(Regex),
64    /// Replace each line with a selected field (whitespace column or JSON key path).
65    Field(FieldSel),
66    /// Project multiple whitespace columns (1-based), space-joined, keeping the
67    /// given order — `fields 1 3` on "a b c d" -> "a c". For columnar input.
68    Fields(Vec<usize>),
69    /// Flatten JSON-array lines into one line per element (jq `[]`); non-array
70    /// lines pass through unchanged.
71    Each,
72    /// Keep lines whose numeric value (`x` = line parsed as a number) satisfies
73    /// the predicate — compiled to fusevm and evaluated per line.
74    Where(Expr),
75    /// Replace each line with the value of an expression (field-aware; `x` =
76    /// line-as-number), computed on the fusevm VM.
77    Map(Expr),
78    /// `via NAME [* N]` — fan the stream across a supervised pool of `N` copies
79    /// of actor `NAME` (default: one per hardware thread). Each line's scalar is
80    /// asked as the actor's first-handler message; the reply is the output line,
81    /// order preserved. A parallel actor map (see [`crate::actor::run_via`]).
82    Via(std::sync::Arc<crate::actor::ActorDef>, usize),
83    /// Reduce to the current line count.
84    Count,
85    /// Reduce to lines-per-second over the elapsed window.
86    Rate,
87    /// Group identical values and count them, sorted by count desc then key asc.
88    Tally,
89    /// Numeric reductions over lines parsed as numbers (non-numeric ignored).
90    Sum,
91    Min,
92    Max,
93    Avg,
94    /// Flatten a JSON object's keys / values into one line each.
95    Keys,
96    /// The jq front-end's `keys`. Same key set as `Keys`, but emitted as ONE
97    /// compact array line, which is what jq returns. The native `keys` verb keeps
98    /// its documented line-per-key shape (SPEC §8) — `stdlib/json.arb` runs
99    /// `keys; tally` over it — so the two are separate ops and only the jq
100    /// literal front-end (`. | keys`, `map(…)`) reaches this one.
101    JqKeys,
102    /// Native `vals` verb: expand a JSON object's values, one per line.
103    Vals,
104    /// jq `values` == `select(. != null)`: drop JSON-null lines, pass every other
105    /// line through unchanged. (NOT object-value iteration — that is `vals`/`.[]`.)
106    NonNull,
107    /// Render a PASSTHROUGH line the way `jq -r` prints it, for the one type where
108    /// the raw line and jq's rendering disagree: a top-level JSON STRING.
109    ///
110    /// Identity `.`, `select(…)` and `values` all emit the INPUT LINE verbatim.
111    /// For an object, array, number or boolean that is what jq does too — jq keeps
112    /// the source literal of any number it never computed (`1.50` stays `1.50`,
113    /// `[1,2]` stays `[1,2]`), so passing the line through matches it. A string is
114    /// the exception: `jq -r` strips the quotes and unescapes, so a line reading
115    /// `"hello"` must print `hello`. arb printed `"hello"` — and disagreed with
116    /// ITSELF, since `.[1:3]` on the same input already rendered `el` raw.
117    ///
118    /// Deliberately string-ONLY. Re-rendering the other types would lose more than
119    /// it gained: `serde_json::Map` is a `BTreeMap`, so a rebuilt object comes back
120    /// key-SORTED (`{"b":1,"a":2}` -> `{"a":2,"b":1}`) where jq preserves input
121    /// order, and every number would be reprinted through `fmt_num`, turning jq's
122    /// preserved `1.50` into `1.5`. Both would be new divergences in place of one.
123    JqRawString,
124    /// Project a JSON object down to the named keys (jq `{a,b,c}` /
125    /// `pick(.a,.b,.c)`), preserving the listed order. Non-object lines pass
126    /// through unchanged; missing keys are dropped.
127    Pick(Vec<String>),
128    /// Line-list transforms. `Sort` supports numeric (`-n`) and reverse (`-r`).
129    Sort {
130        numeric: bool,
131        reverse: bool,
132    },
133    Uniq,
134    Rev,
135    First,
136    Last,
137    Take(usize),
138    Drop(usize),
139    /// Per-line string transforms.
140    Upper,
141    Lower,
142    Trim,
143    /// Regex replace-all per line (`replace /RE/ TO`; TO may use `$1` captures).
144    Replace(Regex, String),
145    /// Collapse all lines into one, joined by a separator.
146    Join(String),
147    /// Keep only the Nth line (1-based).
148    Nth(usize),
149    /// Parse the accumulated stream as one HTML document and emit, per element
150    /// matching the CSS selector, its text (or a named attribute if `attr` is
151    /// set; elements lacking that attribute are dropped).
152    Sel {
153        css: String,
154        attr: Option<String>,
155    },
156    /// Recursive descent (xpath `//TAG`): parse the accumulated stream as one
157    /// HTML document and emit the outer HTML of every element matching the
158    /// selector, one per line — so `attr`/`text` can then pull from each.
159    Find(String),
160    /// From element fragments (one per line, as emitted by `find`), emit the
161    /// named attribute of each (xpath `@NAME`, css attr); drop lines whose first
162    /// element lacks it.
163    Attr(String),
164    /// From element fragments, emit each element's inner text; non-element lines
165    /// pass through unchanged.
166    Text,
167    /// Stable-sort JSON-record lines by FIELD; numeric if every field value
168    /// parses as a number, else lexicographic on the field's string value.
169    /// Non-object lines sink after the sorted records in input order.
170    SortBy(String),
171    /// Keep the first record for each distinct value of FIELD, preserving input
172    /// order. JSON-object lines dedup by that field's value; other lines dedup by
173    /// the whole line.
174    UniqueBy(String),
175    /// Group JSON records by the value of FIELD and count each group, returning
176    /// value -> count pairs sorted by count desc then value asc; non-object lines
177    /// are counted under their whole-line text. Reducer (early return).
178    CountBy(String),
179    /// Group lines by the value of FIELD (jq `group_by`): emit one JSON-array
180    /// line per distinct value, each array holding that group's members in input
181    /// order; groups are ordered by key ascending. Object lines group by the
182    /// field's value, other lines by their whole-line text.
183    GroupBy(String),
184    /// Reducer: emit the single JSON record whose numeric FIELD is smallest (records with a missing/non-numeric FIELD are ignored; empty input yields no lines).
185    MinBy(String),
186    /// Reducer: emit the single record whose numeric FIELD is the largest.
187    MaxBy(String),
188    /// Native `has KEY` verb: retain only JSON-object lines that contain KEY; all
189    /// other lines (missing key, non-object, unparseable) are dropped.
190    Has(String),
191    /// jq `has(KEY)`: emit `true`/`false` per input line. The argument is a jq
192    /// VALUE, not a name — jq accepts a string key on an object and a numeric
193    /// index on an array, and refuses every other pairing by name ("Cannot check
194    /// whether array has a string key") rather than answering `false`.
195    HasKey(Value),
196    /// jq to_entries: expand each JSON object line into one `{"key":<k>,"value":<v>}` line per key (BTreeMap key order); non-object lines pass through.
197    Entries,
198    /// The jq front-end's `to_entries`. Same mapping as `Entries`, but emits the
199    /// whole result as ONE compact array line, which is what jq returns. Native
200    /// `entries` keeps its documented line-per-key shape (SPEC §8) so specs built
201    /// on it are unaffected; the two spellings are distinct, so they can differ.
202    JqEntries,
203    /// The jq front-end's `flatten`. Same full-depth leaf walk as `Flatten`, but
204    /// emits them as ONE compact array line, which is what jq returns. Native
205    /// `flatten` keeps its line-per-leaf shape — same context gating as
206    /// `JqKeys`/`JqEntries`.
207    JqFlatten,
208    /// jq `map(f)` == `[.[] | f]`: run the inner ops over the ELEMENTS of each
209    /// input line and re-wrap the results as one compact array line — jq's shape.
210    /// Scoped per input line, so two input lines stay two output arrays. (Iterate
211    /// WITHOUT the re-wrap is jq's own `.[] | f`, which arb also accepts.)
212    JqMap(Vec<QueryOp>),
213    /// The jq front-end's `.[]`. Same expansion as `Each`, but a line that parses
214    /// to a NON-iterable JSON value (a scalar or `null`) is an error rather than a
215    /// pass-through — jq raises "Cannot iterate over null (null)" and SPEC §8
216    /// forbids answering where jq refuses. Native `each` keeps the pass-through,
217    /// because it runs over plain-text streams too.
218    JqEach,
219    /// The jq front-end's `add` — `reduce .[] as $x (null; . + $x)`, so an EMPTY
220    /// array is `null` (native `add` documents `[] -> ""`) and a mixed-type array
221    /// raises the same error `+` would. Iterates an object's values.
222    JqAdd,
223    /// The jq front-end's `length`. Same JSON-aware measure as `JsonLen`, except a
224    /// boolean is an error ("boolean (true) has no length") instead of falling back
225    /// to the raw line's character count.
226    JqLen,
227    /// A COMPLETE jq program, run per line by [`crate::jqlang`].
228    ///
229    /// The ops above are arb's line-stream translations of the jq subset they
230    /// cover, and they keep arb's own promises (identity passes the SOURCE line
231    /// through, so its spacing and number literals survive). They cannot express
232    /// jq's generator semantics — `.a, .b`, `reduce`, `..`, `try`, assignment —
233    /// because a `Vec<QueryOp>` is a stage list, not a language. Anything
234    /// outside that subset compiles to a real jq program instead and runs on
235    /// arb's own jq engine, which is why the constructs SPEC §8 used to list as
236    /// refusals now answer exactly as `jq` does.
237    ///
238    /// The payload is the SOURCE, not a compiled program: `QueryOp` is cloned
239    /// into specs, actors and the web target, so it must stay `Send`-compatible
240    /// plain data. Compilation is memoized per thread by [`jq_program`], so a
241    /// per-line pipeline still parses the program exactly once.
242    JqProgram(String),
243    /// jq `select(f)` over jq VALUE semantics: keep the line when `f` is truthy,
244    /// where only `false` and `null` are falsy. Distinct from native `Where`,
245    /// which evaluates an f64 predicate over a numeric line stream.
246    JqSelect(crate::jqval::Expr),
247    /// A jq value expression stage (`.a + .b`, `. * 2`, `. > 1`). Distinct from
248    /// native `Map`, which computes an f64: this yields a jq VALUE, so a compare
249    /// renders `true`/`false` and a string `+` concatenates.
250    JqCalc(crate::jqval::Expr),
251    /// jq `flatten`: recursively flatten a JSON-array line to its non-array leaves,
252    /// emitting one leaf per line (matching jq's full-depth flatten, unlike `each`
253    /// which descends a single level). Non-array lines pass through unchanged.
254    Flatten,
255    /// jq `add`: reduce a JSON array line to a single value — sum numeric
256    /// arrays (fmt_num), concatenate non-numeric arrays via their string
257    /// values, empty array -> "". Non-array lines pass through unchanged.
258    Add,
259    /// Keep only lines that parse as a number strictly greater than `N`.
260    /// Lines that do not parse as `f64` are dropped.
261    Over(f64),
262    /// Keep only numeric lines whose value is strictly less than N; drop non-numeric lines.
263    Under(f64),
264    /// Keep numeric lines x where lo <= x <= hi (inclusive); non-numeric lines are dropped.
265    Between(f64, f64),
266    /// Prefix each line with its 1-based index and a tab: `"1\t<line>"`.
267    Enumerate,
268    /// Split each line on whitespace and emit one word per line (flatten); empty lines produce nothing.
269    Words,
270    /// Collapse runs of adjacent identical lines to a single line (classic uniq), leaving non-adjacent repeats intact.
271    Dedup,
272    /// Keep only the last N lines (complement of `take`, which keeps the first N). N>=len keeps all.
273    Tailn(usize),
274    /// Right-pad each line with spaces to a minimum visible width N (no truncation if the line is already longer).
275    Pad(usize),
276    /// Left-pad each line with spaces to a minimum width of N (lines already >= N are unchanged).
277    Lpad(usize),
278    /// Retain lines whose FIELD (json key or 1-based whitespace column) matches the regex.
279    Grepf(String, regex::Regex),
280    /// Reverse the Unicode scalar characters of each line (chars().rev()).
281    Flip,
282    /// Path basename: the part after the last `/` (the whole line if none).
283    Basename,
284    /// Path dirname: the part before the last `/` (`.` if none).
285    Dirname,
286    /// Group a numeric line's integer part with thousands separators
287    /// (`1234567` -> `1,234,567`); non-numeric lines pass through.
288    Commafy,
289    /// Humanize a byte count (1024-based): `1536` -> `1.5 KB`; non-numeric passes through.
290    Bytes,
291    /// Humanize a duration in seconds: `3661` -> `1h 1m`; non-numeric passes through.
292    Duration,
293    /// Placeholder for `apply .name`: at render time it is replaced by the query
294    /// pipeline typed into the `input .name` widget (the megafilter/map binding).
295    /// Left in a pipeline unsubstituted it is a no-op.
296    Apply(String),
297    /// `in.json`: regroup the line stream into one item per JSON document — a
298    /// pretty-printed document spanning lines becomes one compact line, and
299    /// several values on one line become one each, as jq reads its input. A
300    /// line that is no part of a document stays the line it was. See
301    /// [`crate::jsondocs`].
302    JsonDocs,
303    /// Treat the stream as CSV: the first line is the header; each data row
304    /// becomes a JSON object keyed by the header, so `field NAME` works.
305    Csv,
306    B64,
307    /// Base64-decode each line (STANDARD alphabet) into a UTF-8 string; lines that fail to
308    /// base64-decode or whose bytes aren't valid UTF-8 pass through unchanged.
309    B64d,
310    /// Lowercase hex-encode each line, two hex digits per UTF-8 byte.
311    Hex,
312    /// Decode a hex string to UTF-8 text by parsing byte pairs; on any error (odd length, non-hex digit, invalid UTF-8) the line is left unchanged.
313    Unhex,
314    /// Percent-encode each line, escaping every non-alphanumeric byte (RFC 3986 style).
315    Urlenc,
316    /// Percent-decode each line to UTF-8 (utf8_percent_decode); lines whose decoded bytes are not valid UTF-8 pass through unchanged.
317    Urldec,
318    /// Emit the first regex match per line (capture group 1 if the pattern captures, else the whole match); drop lines with no match.
319    Extract(Regex),
320    /// Explode each line by the literal string DELIM into multiple lines (one part per line).
321    /// One-to-many: unlike `cut` (one field) or `words` (whitespace), every split segment becomes its own line.
322    Split(String),
323    /// Character substring [A,B) 0-based, clamped to the line length (B may exceed len; A>B yields empty).
324    Substr(usize, usize),
325    /// Explode each line into one output line per Unicode scalar (character); one line -> many.
326    Chars,
327    /// Title-case each line: uppercase the first letter of each whitespace-separated word, lowercase the rest, rejoin with single spaces.
328    Title,
329    /// Replace each line with its content repeated N times, concatenated.
330    Repeat(usize),
331    /// Set key K to the string value V (Value::String(V)) in each JSON object line; non-object / unparseable lines pass through unchanged.
332    Set(String, String),
333    /// Remove key K from each JSON object line (jq `del(.K)`); non-object lines pass through.
334    Del(String),
335    /// Rename JSON object key OLD to NEW in each object, preserving the value; no-op if OLD absent. Non-object lines pass through.
336    Rename(String, String),
337    /// Set string key K to V only when K is absent from the JSON object (jq `//=` for a missing key). Present keys keep their value; non-object / unparseable lines pass through unchanged. Key order is normalized on mutation.
338    Default(String, String),
339    /// Reduce all JSON object lines into a single object (later keys overwrite earlier); non-object lines are ignored. Emits one JSON object line, or none if no objects were seen.
340    Merge,
341    /// Floor each numeric line to the nearest lower integer (fmt_num(x.floor())); non-numeric lines pass through unchanged.
342    Floor,
343    /// Round each numeric line up to the nearest integer (ceil); non-numeric lines pass through unchanged.
344    Ceil,
345    /// Clamp each numeric line into the inclusive range [LO, HI]; non-numeric lines pass through unchanged.
346    Clamp(f64, f64),
347    /// Same as `Csv` but tab-separated (TSV).
348    Tsv,
349    /// Parse the accumulated stream as a YAML document (or `---`-separated
350    /// multi-document) and emit each document as a JSON line, so the JSON verbs
351    /// (`field`/`pick`/`keys`/`each`) work over it (the yq leg).
352    Yaml,
353    /// Parse the accumulated stream as one TOML document and emit it as a JSON
354    /// object line.
355    Toml,
356    /// Reduce to a scalar computed by an arithmetic expression over the current
357    /// line count (`x`), evaluated on the fusevm VM.
358    Calc(Expr),
359    /// keep lines containing a literal substring.
360    Contains(String),
361    /// keep lines starting with a literal prefix.
362    Starts(String),
363    /// keep lines ending with a literal suffix.
364    Ends(String),
365    /// drop empty / whitespace-only lines.
366    Nonempty,
367    /// keep only lines that parse as a number.
368    Numeric,
369    /// replace each line with its character count.
370    Len,
371    /// jq `length` (JSON-aware): array element count / object key count / string
372    /// char count / |number| / null=0; a non-JSON line falls back to char count.
373    JsonLen,
374    /// jq slice `.[a:b]` over a JSON array line (a/b may be negative — counted
375    /// from the end — or omitted); a string line is char-sliced; other lines pass
376    /// through unchanged.
377    JsonSlice(Option<i64>, Option<i64>),
378    /// replace each line with its word count.
379    Wc,
380    /// absolute value of each numeric line.
381    Abs,
382    /// round each numeric line to the nearest integer.
383    Round,
384    /// consecutive differences of the numeric series (n values → n-1 deltas) —
385    /// turns a monotonic counter into a per-step rate-of-change.
386    Delta,
387    /// running (cumulative) total of the numeric series.
388    Cumsum,
389    /// simple moving average over a trailing window of N (length-preserving; the
390    /// first points average a shorter, growing window). Smooths a noisy series.
391    Sma(usize),
392    /// exponentially-weighted moving average, smoothing factor `alpha` in (0,1] —
393    /// higher alpha tracks faster, lower is smoother. `s0 = x0`.
394    Ewma(f64),
395    /// prefix every line with a literal string.
396    Prepend(String),
397    /// suffix every line with a literal string.
398    Append(String),
399    /// split each line by DELIM, keep the Nth (1-based) field.
400    Cut(String, usize),
401    /// median of numeric lines.
402    Median,
403    /// Nth percentile (0–100) of the numeric values, linear interpolation between
404    /// closest ranks (numpy default). `percentile 99` / `p99` for latency tails.
405    Percentile(f64),
406    /// population standard deviation of numeric lines.
407    Stddev,
408    /// max minus min of numeric lines.
409    Range,
410    /// product of numeric lines.
411    Product,
412    /// count of distinct lines.
413    Distinct,
414    /// keep every Nth line (1-based).
415    Sample(usize),
416    /// keep lines from index A to B inclusive (1-based).
417    Slice(usize, usize),
418    /// keep only the Nth line (1-based); out-of-range yields no lines.
419    Index(usize),
420    /// bucket numeric lines into N equal-width ranges -> (range, count) pairs.
421    Bins(usize),
422}
423
424/// The output of evaluating a pipeline: lines, a scalar, grouped counts — or a
425/// runtime REFUSAL.
426///
427/// `Error` is the per-line error channel the jq front-end needs. SPEC §8 promises
428/// that a construct outside the documented subset "is a hard error … never
429/// silently reinterpreted", and a type mismatch (`{"a":1} | . + 3`, `null | .[]`,
430/// `true | length`) is only discoverable once the DATA arrives. Without this
431/// variant every such case had to answer something — `null`, the raw line, the
432/// line's character count — which is the exact failure the SPEC rules out, and
433/// which several comments in this file used to record as "arb has no per-line
434/// error channel".
435///
436/// Only the jq front-end raises it. Native verbs run over plain-text streams and
437/// keep their documented pass-through behaviour.
438#[derive(Debug, Clone, PartialEq)]
439pub enum QueryResult {
440    Lines(Vec<String>),
441    Scalar(f64),
442    Pairs(Vec<(String, u64)>),
443    /// A refused pipeline. The message is already `jq: …`-anchored.
444    Error(String),
445}
446
447/// What a jq program reported besides its output lines — the part of jq's
448/// `process()` (main.c) that is not stdout.
449///
450/// jq does not stop at an uncaught error: the values the input produced BEFORE
451/// it are printed, the error goes to stderr, and the next input runs. The exit
452/// status is the LAST input's (5 when it raised, else 0). `halt`/`halt_error`
453/// end the whole run with their own status and stderr text instead.
454#[derive(Debug, Default, Clone, PartialEq)]
455pub struct JqReport {
456    /// Every uncaught error, in input order, already `jq: …`-anchored.
457    pub errors: Vec<String>,
458    /// The last input a jq program ran on raised, so jq would exit 5.
459    pub last_failed: bool,
460    /// `halt`/`halt_error`: the exit status and the exact stderr text (no
461    /// prefix; jq adds a newline only after a non-string message).
462    pub halt: Option<(i32, String)>,
463}
464
465/// An uncaught jq error as jq's main.c words it, minus the `error (at …)`
466/// position: a string message as-is, anything else as `(not a string): JSON`.
467fn uncaught(e: &crate::jqlang::JqErr) -> String {
468    match e {
469        crate::jqlang::JqErr::Err(crate::jqlang::JqVal::Str(s)) => format!("jq: {s}"),
470        crate::jqlang::JqErr::Err(v) => {
471            format!("jq: (not a string): {}", crate::jqlang::render(v))
472        }
473        other => format!("jq: {}", other.to_message()),
474    }
475}
476
477/// The stderr text of `halt_error`, transcribed from main.c: a string is written
478/// raw with no newline, `null` writes nothing, anything else is its compact JSON
479/// plus a newline.
480fn halt_text(msg: Option<&crate::jqlang::JqVal>) -> String {
481    match msg {
482        None | Some(crate::jqlang::JqVal::Null) => String::new(),
483        Some(crate::jqlang::JqVal::Str(s)) => s.to_string(),
484        Some(v) => format!("{}\n", crate::jqlang::render(v)),
485    }
486}
487
488/// Fold one input's run into `rep` the way jq's main loop does. `true` means
489/// the program halted, which ends the whole run, not just this input.
490fn record_run(rep: &mut JqReport, r: Result<(), crate::jqlang::JqErr>) -> bool {
491    match r {
492        Ok(()) => {
493            rep.last_failed = false;
494            false
495        }
496        Err(crate::jqlang::JqErr::Halt(code, msg)) => {
497            rep.halt = Some((code, halt_text(msg.as_ref())));
498            true
499        }
500        Err(e) => {
501            rep.errors.push(uncaught(&e));
502            rep.last_failed = true;
503            false
504        }
505    }
506}
507
508/// Evaluate `ops` against `lines`. `elapsed_secs` feeds `rate`.
509///
510/// A jq program's uncaught error REFUSES the pipeline here: the TUI, the REPL,
511/// `serve` and `--test` show a refusal rather than a partial answer. The `out`
512/// filter path wants jq's stream semantics instead and calls [`eval_reporting`].
513pub fn eval(ops: &[QueryOp], lines: &[String], elapsed_secs: f64) -> QueryResult {
514    let mut rep = JqReport::default();
515    let r = eval_reporting(ops, lines, elapsed_secs, 1, &mut rep);
516    match (rep.errors.into_iter().next(), rep.halt) {
517        (Some(e), _) => QueryResult::Error(e),
518        (None, Some((_, msg))) if !msg.is_empty() => {
519            QueryResult::Error(format!("jq: {}", msg.trim_end_matches('\n')))
520        }
521        _ => r,
522    }
523}
524
525/// [`eval`] with jq's uncaught-error model: a jq program's errors and `halt`
526/// land in `rep` and the lines produced before them are kept. `first_line` is
527/// the 1-based number of `lines[0]` in the whole input, so a caller that feeds
528/// one line at a time still gets jq's `input_line_number`.
529pub fn eval_reporting(
530    ops: &[QueryOp],
531    lines: &[String],
532    elapsed_secs: f64,
533    first_line: usize,
534    rep: &mut JqReport,
535) -> QueryResult {
536    let mut cur: Vec<String> = lines.to_vec();
537    // The YAML NODE channel. `cur` is text, so a YAML node's comments, anchors
538    // and quoting cannot survive it — this carries the composed documents beside
539    // it so `in.yaml`, a jq program and `out.yaml` can hand nodes to each other
540    // without a JSON round trip in between.
541    //
542    // It is invalidated by any op that is not one of those three, in ONE place
543    // rather than at every arm, so an op added later drops the metadata (which
544    // is merely a lost comment) instead of answering from a stale one (which
545    // would be a wrong answer).
546    let mut nodes: Option<Vec<crate::jqlang::JqVal>> = None;
547    for op in ops {
548        if !matches!(
549            op,
550            QueryOp::Yaml | QueryOp::JqProgram(_) | QueryOp::Emit(..)
551        ) {
552            nodes = None;
553        }
554        match op {
555            QueryOp::Emit(fmt, indent) => {
556                let docs = match nodes.clone() {
557                    Some(d) => d,
558                    // Not YAML in: every line is its own document, read through
559                    // the jq model so a non-JSON line stays a string.
560                    None => cur
561                        .iter()
562                        .map(|l| {
563                            crate::jqlang::parse_json(l)
564                                .unwrap_or_else(|_| crate::jqlang::JqVal::str(l.as_str()))
565                        })
566                        .collect(),
567                };
568                cur = emit_stream(&docs, *fmt, *indent);
569            }
570            QueryOp::Match(re) => cur.retain(|l| re.is_match(l)),
571            QueryOp::Reject(re) => cur.retain(|l| !re.is_match(l)),
572            QueryOp::Field(sel) => {
573                if let FieldSel::JqKey(segs) = sel {
574                    for l in cur.iter_mut() {
575                        // Through the jq value model, so an object this returns
576                        // keeps the document's KEY ORDER — `serde_json::Map` is a
577                        // `BTreeMap` and gave `.nested` back alphabetised, which
578                        // `yq -o=json` on the same YAML does not.
579                        match crate::jqlang::parse_json(l).ok() {
580                            // A line that is not JSON at all has no jq TYPE to
581                            // check, so it keeps the documented miss rendering.
582                            None => *l = "null".to_string(),
583                            Some(v) => match crate::jqlang::get_seg_path(&v, segs) {
584                                Ok(r) => *l = crate::jqlang::render_raw(&r),
585                                Err(e) => return QueryResult::Error(format!("jq: {e}")),
586                            },
587                        }
588                    }
589                    continue;
590                }
591                for l in cur.iter_mut() {
592                    *l = extract_field(l, sel);
593                }
594            }
595            QueryOp::Fields(cols) => {
596                for l in cur.iter_mut() {
597                    *l = cols
598                        .iter()
599                        .map(|&n| nth_col(l, n))
600                        .collect::<Vec<_>>()
601                        .join(" ");
602                }
603            }
604            QueryOp::JqEach => {
605                let mut out = Vec::with_capacity(cur.len());
606                // Through the jq value model, so each element keeps its source
607                // literal (`[1.50,1E1000]` iterates as `1.50`, `1E+1000`, as jq
608                // prints them) and an object yields its values in KEY ORDER.
609                for l in &cur {
610                    use crate::jqlang::JqVal;
611                    match crate::jqlang::parse_json(l) {
612                        Ok(v) => match v.bare() {
613                            JqVal::Arr(a) => out.extend(a.iter().map(crate::jqlang::render_raw)),
614                            JqVal::Obj(m) => {
615                                out.extend(m.iter().map(|(_, e)| crate::jqlang::render_raw(e)))
616                            }
617                            other => {
618                                return QueryResult::Error(format!(
619                                    "jq: Cannot iterate over {}{}",
620                                    other.type_name(),
621                                    crate::jqlang::paren_of(other)
622                                ))
623                            }
624                        },
625                        Err(_) => out.push(l.clone()),
626                    }
627                }
628                cur = out;
629            }
630            QueryOp::JqProgram(src) => {
631                let prog = match jq_program(src) {
632                    Ok(p) => p,
633                    Err(msg) => return QueryResult::Error(format!("jq: {msg}")),
634                };
635                let interp = crate::jqlang::Interp::default();
636                let mut out = Vec::with_capacity(cur.len());
637                // ONE cursor over the stream, which is jq's model: the next
638                // document is `.`, and `input` takes the one after it — so a
639                // document `input` consumed is not replayed by this loop.
640                //
641                // The first shape of this was per-line: it handed every
642                // iteration a freshly parsed copy of the whole REMAINING stream
643                // so `inputs` would see it. That is O(n^2) parses, and it showed
644                // — `{n: .name, v: .v}` over 2,000 records took 9.7s against
645                // jq's 0.01s. Lines are moved in raw and parsed once, on the way
646                // out.
647                // The YAML path: run over the composed NODES so a program can
648                // read and write comments, anchors, tags and styles. Only taken
649                // when the program does not pull from the input STREAM —
650                // `input`/`inputs` share one cursor over raw text lines, and
651                // splitting that cursor across two representations would let a
652                // document be seen twice.
653                if let Some(docs) = nodes.take().filter(|_| !prog.reads_input_stream()) {
654                    let mut next: Vec<crate::jqlang::JqVal> = Vec::new();
655                    for (i, doc) in docs.iter().enumerate() {
656                        interp.set_line(i + 1);
657                        interp.set_doc(doc);
658                        let r = prog.run_with(&interp, doc, &mut |v| {
659                            out.push(crate::jqlang::render_raw(&v));
660                            next.push(v);
661                            Ok(())
662                        });
663                        if record_run(rep, r) {
664                            break;
665                        }
666                    }
667                    nodes = Some(next);
668                    cur = out;
669                    continue;
670                }
671                interp.set_input_lines(std::mem::take(&mut cur));
672                interp.set_line(first_line.saturating_sub(1));
673                while let Some(input) = interp.next_input() {
674                    interp.set_doc(&input);
675                    let r = prog.run_with(&interp, &input, &mut |v| {
676                        out.push(crate::jqlang::render_raw(&v));
677                        Ok(())
678                    });
679                    if record_run(rep, r) {
680                        break;
681                    }
682                }
683                cur = out;
684            }
685            QueryOp::JqSelect(e) => {
686                let mut out = Vec::with_capacity(cur.len());
687                for l in &cur {
688                    match crate::jqval::eval(e, &jq_scalar(l)) {
689                        Ok(v) => {
690                            if crate::jqval::truthy(&v) {
691                                out.push(l.clone());
692                            }
693                        }
694                        Err(msg) => return QueryResult::Error(format!("jq: {msg}")),
695                    }
696                }
697                cur = out;
698            }
699            QueryOp::JqCalc(e) => {
700                for l in cur.iter_mut() {
701                    match crate::jqval::eval(e, &jq_scalar(l)) {
702                        Ok(v) => *l = crate::jqval::render(&v),
703                        Err(msg) => return QueryResult::Error(format!("jq: {msg}")),
704                    }
705                }
706            }
707            QueryOp::JqAdd => {
708                // jq `add` is `reduce .[] as $x (null; . + $x)`: it starts from
709                // null (so `[]` is `null`, not ""), folds with jq's `+` (so
710                // `[1,"a"]` raises), iterates an object's values, and keeps a
711                // lone element's literal (`[1.50] | add` is `1.50`).
712                if let Some(e) = jq_builtin_lines(&mut cur, "add") {
713                    return e;
714                }
715            }
716            QueryOp::JqLen => {
717                for l in cur.iter_mut() {
718                    *l = match jq_value(l) {
719                        Some(Value::Array(a)) => a.len().to_string(),
720                        Some(Value::Object(m)) => m.len().to_string(),
721                        Some(Value::String(s)) => s.chars().count().to_string(),
722                        // jq's `length` on a number is its ABSOLUTE value.
723                        Some(Value::Number(n)) => fmt_num(n.as_f64().unwrap_or(0.0).abs()),
724                        Some(Value::Null) => "0".to_string(),
725                        Some(v @ Value::Bool(_)) => {
726                            return QueryResult::Error(format!(
727                                "jq: {} ({v}) has no length",
728                                crate::jqval::tname(&v)
729                            ));
730                        }
731                        None => l.chars().count().to_string(),
732                    };
733                }
734            }
735            QueryOp::Each => {
736                let mut out = Vec::with_capacity(cur.len());
737                for l in &cur {
738                    match serde_json::from_str::<Value>(l) {
739                        // `each` IS jq's `.[]` (SPEC §8), so a null ELEMENT keeps
740                        // jq's rendering — dropping it to "" loses the element's
741                        // existence, which is what `[1,null,2]` must not do.
742                        Ok(Value::Array(arr)) => {
743                            out.extend(arr.iter().map(jq_to_string));
744                        }
745                        // jq `.[]` over an object iterates its VALUES.
746                        Ok(Value::Object(m)) => {
747                            out.extend(m.values().map(jq_to_string));
748                        }
749                        _ => out.push(l.clone()),
750                    }
751                }
752                cur = out;
753            }
754            QueryOp::Where(e) => {
755                // A string predicate (`match(.q)` / `field in .lv`) can't run on
756                // the numeric VM — route it to the Rust evaluator; purely numeric
757                // predicates stay on fusevm.
758                if crate::expr::expr_has_str(e) {
759                    cur.retain(|l| eval_where(e, l));
760                } else {
761                    cur.retain(|l| {
762                        let x = l.trim().parse::<f64>().unwrap_or(f64::NAN);
763                        let resolve = |name: &str| field_num(l, name);
764                        crate::expr::eval_pred_ctx(e, x, &resolve).unwrap_or(false)
765                    });
766                }
767            }
768            QueryOp::Map(e) => {
769                for l in cur.iter_mut() {
770                    let v = {
771                        let x = l.trim().parse::<f64>().unwrap_or(f64::NAN);
772                        let resolve = |name: &str| field_num(l, name);
773                        crate::expr::eval_ctx(e, x, &resolve).unwrap_or(f64::NAN)
774                    };
775                    *l = fmt_num(v);
776                }
777            }
778            QueryOp::Via(def, workers) => {
779                cur = crate::actor::run_via(def, *workers, &cur);
780            }
781            QueryOp::Count => return QueryResult::Scalar(cur.len() as f64),
782            QueryOp::Rate => {
783                return QueryResult::Scalar(cur.len() as f64 / elapsed_secs.max(0.001));
784            }
785            QueryOp::Tally => {
786                let mut counts: BTreeMap<String, u64> = BTreeMap::new();
787                for l in &cur {
788                    *counts.entry(l.clone()).or_insert(0) += 1;
789                }
790                let mut pairs: Vec<(String, u64)> = counts.into_iter().collect();
791                pairs.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
792                return QueryResult::Pairs(pairs);
793            }
794            QueryOp::Calc(e) => {
795                let x = cur.len() as f64;
796                return QueryResult::Scalar(crate::expr::eval(e, x).unwrap_or(0.0));
797            }
798            QueryOp::Sum => return QueryResult::Scalar(nums(&cur).iter().sum()),
799            QueryOp::Min => {
800                let m = nums(&cur).into_iter().fold(f64::INFINITY, f64::min);
801                return QueryResult::Scalar(if m.is_finite() { m } else { 0.0 });
802            }
803            QueryOp::Max => {
804                let m = nums(&cur).into_iter().fold(f64::NEG_INFINITY, f64::max);
805                return QueryResult::Scalar(if m.is_finite() { m } else { 0.0 });
806            }
807            QueryOp::Avg => {
808                let ns = nums(&cur);
809                let a = if ns.is_empty() {
810                    0.0
811                } else {
812                    ns.iter().sum::<f64>() / ns.len() as f64
813                };
814                return QueryResult::Scalar(a);
815            }
816            QueryOp::Keys => {
817                let mut out = Vec::new();
818                for l in &cur {
819                    match serde_json::from_str::<Value>(l) {
820                        Ok(Value::Object(m)) => out.extend(m.keys().cloned()),
821                        _ => out.push(l.clone()),
822                    }
823                }
824                cur = out;
825            }
826            QueryOp::JqKeys => {
827                for l in cur.iter_mut() {
828                    // jq `keys` sorts; serde_json's Map is a BTreeMap here, so
829                    // iteration is already sorted. An ARRAY's keys are its indices.
830                    match serde_json::from_str::<Value>(l) {
831                        Ok(Value::Object(m)) => {
832                            let arr: Vec<Value> =
833                                m.keys().map(|k| Value::String(k.clone())).collect();
834                            *l = Value::Array(arr).to_string();
835                        }
836                        Ok(Value::Array(a)) => {
837                            let arr: Vec<Value> = (0..a.len()).map(Value::from).collect();
838                            *l = Value::Array(arr).to_string();
839                        }
840                        // A scalar (or null) has no keys and jq says so by name.
841                        Ok(v) => return QueryResult::Error(has_no_keys(&v)),
842                        Err(_) => {}
843                    }
844                }
845            }
846            QueryOp::Vals => {
847                let mut out = Vec::new();
848                for l in &cur {
849                    match serde_json::from_str::<Value>(l) {
850                        Ok(Value::Object(m)) => out.extend(m.values().map(json_to_string)),
851                        _ => out.push(l.clone()),
852                    }
853                }
854                cur = out;
855            }
856            QueryOp::NonNull => {
857                // jq `values` == `select(. != null)`: keep every non-null input
858                // unchanged, drop only the lines that parse to JSON `null`.
859                cur.retain(|l| !matches!(serde_json::from_str::<Value>(l), Ok(Value::Null)));
860            }
861            QueryOp::JqRawString => {
862                for l in cur.iter_mut() {
863                    // Re-render through the jq value model, which is exactly what
864                    // `jq -rc` prints: a top-level string goes out RAW, and every
865                    // other value comes back COMPACT. A line that is not JSON at
866                    // all has no jq rendering and passes through as written.
867                    if let Ok(v) = crate::jqlang::parse_json(l) {
868                        *l = crate::jqlang::render_raw(&v);
869                    }
870                }
871            }
872            QueryOp::Pick(keys) => {
873                for l in cur.iter_mut() {
874                    if let Ok(Value::Object(m)) = serde_json::from_str::<Value>(l) {
875                        // Build in pick order — serde_json::Map is a BTreeMap and
876                        // would re-sort the keys, losing the requested order.
877                        let parts: Vec<String> = keys
878                            .iter()
879                            .filter_map(|k| {
880                                m.get(k)
881                                    .map(|v| format!("{}:{}", Value::String(k.clone()), v))
882                            })
883                            .collect();
884                        *l = format!("{{{}}}", parts.join(","));
885                    }
886                }
887            }
888            QueryOp::JsonDocs => cur = crate::jsondocs::documents(&cur),
889            QueryOp::Csv => cur = to_json_records(&cur, ','),
890            QueryOp::Tsv => cur = to_json_records(&cur, '\t'),
891            QueryOp::Yaml => {
892                let docs = crate::yaml::documents(&cur.join("\n"));
893                cur = docs.iter().map(crate::jqlang::render).collect();
894                nodes = Some(docs);
895            }
896            QueryOp::Toml => cur = toml_to_json(&cur),
897            QueryOp::Sort { numeric, reverse } => {
898                if *numeric {
899                    // Like Unix `sort -n`: order by each line's LEADING numeric
900                    // token (the first whitespace-delimited field), so mixed rows
901                    // such as `2.1 claude` sort by 2.1 — not the whole line, which
902                    // would parse as NaN and leave the order untouched.
903                    let key = |s: &str| {
904                        s.split_whitespace()
905                            .next()
906                            .and_then(|t| t.parse::<f64>().ok())
907                            .unwrap_or(f64::NAN)
908                    };
909                    cur.sort_by(|a, b| {
910                        key(a)
911                            .partial_cmp(&key(b))
912                            .unwrap_or(std::cmp::Ordering::Equal)
913                    });
914                } else {
915                    cur.sort();
916                }
917                if *reverse {
918                    cur.reverse();
919                }
920            }
921            QueryOp::Uniq => {
922                let mut seen = HashSet::new();
923                cur.retain(|l| seen.insert(l.clone()));
924            }
925            QueryOp::Rev => cur.reverse(),
926            QueryOp::First => cur.truncate(1),
927            QueryOp::Last => {
928                if let Some(l) = cur.pop() {
929                    cur = vec![l];
930                } else {
931                    cur.clear();
932                }
933            }
934            QueryOp::Take(n) => cur.truncate(*n),
935            QueryOp::Drop(n) => {
936                cur.drain(0..(*n).min(cur.len()));
937            }
938            QueryOp::Upper => {
939                for l in cur.iter_mut() {
940                    *l = l.to_uppercase();
941                }
942            }
943            QueryOp::Lower => {
944                for l in cur.iter_mut() {
945                    *l = l.to_lowercase();
946                }
947            }
948            QueryOp::Trim => {
949                for l in cur.iter_mut() {
950                    *l = l.trim().to_string();
951                }
952            }
953            QueryOp::Replace(re, to) => {
954                for l in cur.iter_mut() {
955                    *l = re.replace_all(l, to.as_str()).into_owned();
956                }
957            }
958            QueryOp::Join(sep) => {
959                cur = vec![cur.join(sep)];
960            }
961            QueryOp::Nth(n) => {
962                cur = cur
963                    .get(n.saturating_sub(1))
964                    .cloned()
965                    .map(|l| vec![l])
966                    .unwrap_or_default();
967            }
968            QueryOp::Sel { css, attr } => {
969                let doc = Html::parse_document(&cur.join("\n"));
970                cur = match Selector::parse(css) {
971                    Ok(sel) => doc
972                        .select(&sel)
973                        .filter_map(|el| match attr {
974                            Some(a) => el.value().attr(a).map(str::to_string),
975                            None => Some(el.text().collect::<String>().trim().to_string()),
976                        })
977                        .collect(),
978                    // A selector no CSS engine accepts is an ERROR, not "no
979                    // matches" — `spec::check_css` refuses it at build time, and
980                    // this is the second line of defence for a selector that
981                    // reached here another way (an `input` value substituted by
982                    // `resolve_pipeline`).
983                    Err(e) => {
984                        return QueryResult::Error(format!(
985                            "sel: `{css}` is not a valid CSS selector ({e})"
986                        ))
987                    }
988                };
989            }
990            QueryOp::XPath(xp) => {
991                // An expression that is a RELATIVE location path is evaluated
992                // per line, so it composes with the step that produced those
993                // lines; anything else is one question about the DOCUMENT and is
994                // answered once. See `crate::xpath`'s module docs.
995                match xp.run(&cur) {
996                    Ok(lines) => cur = lines,
997                    // SPEC §8: a construct outside the subset is a hard error,
998                    // never a silent empty node-set.
999                    Err(e) => return QueryResult::Error(e),
1000                }
1001            }
1002            QueryOp::Find(css) => {
1003                let doc = Html::parse_document(&cur.join("\n"));
1004                cur = match Selector::parse(css) {
1005                    Ok(sel) => doc.select(&sel).map(|el| el.html()).collect(),
1006                    Err(e) => {
1007                        return QueryResult::Error(format!(
1008                            "find: `{css}` is not a valid CSS selector ({e})"
1009                        ))
1010                    }
1011                };
1012            }
1013            QueryOp::Attr(name) => {
1014                let mut out = Vec::with_capacity(cur.len());
1015                for l in &cur {
1016                    let frag = Html::parse_fragment(l);
1017                    if let Some(v) = first_element(&frag).and_then(|e| e.value().attr(name)) {
1018                        out.push(v.to_string());
1019                    }
1020                }
1021                cur = out;
1022            }
1023            QueryOp::Text => {
1024                for l in cur.iter_mut() {
1025                    let frag = Html::parse_fragment(l);
1026                    if let Some(e) = first_element(&frag) {
1027                        // xpath `text()` is the element's DIRECT child text nodes,
1028                        // not all descendant text — `e.text()` would also fold in a
1029                        // child element's text (e.g. `<a>1<b>X</b>2</a>` -> `1X2`).
1030                        *l = e
1031                            .children()
1032                            .filter_map(|c| c.value().as_text().map(|t| t.to_string()))
1033                            .collect::<String>()
1034                            .trim()
1035                            .to_string();
1036                    }
1037                }
1038            }
1039            QueryOp::SortBy(field) => {
1040                // Split object lines (carrying their field value) from the rest;
1041                // non-objects keep their relative input order and sink to the end.
1042                let mut objs: Vec<(String, String)> = Vec::new();
1043                let mut rest: Vec<String> = Vec::new();
1044                for l in cur.drain(..) {
1045                    if let Ok(Value::Object(m)) = serde_json::from_str::<Value>(&l) {
1046                        let key = m.get(field).map(json_to_string).unwrap_or_default();
1047                        objs.push((key, l));
1048                    } else {
1049                        rest.push(l);
1050                    }
1051                }
1052                let all_numeric =
1053                    !objs.is_empty() && objs.iter().all(|(k, _)| k.trim().parse::<f64>().is_ok());
1054                if all_numeric {
1055                    // slice::sort_by is stable — equal keys preserve input order.
1056                    objs.sort_by(|a, b| {
1057                        let na = a.0.trim().parse::<f64>().unwrap_or(f64::NAN);
1058                        let nb = b.0.trim().parse::<f64>().unwrap_or(f64::NAN);
1059                        na.partial_cmp(&nb).unwrap_or(std::cmp::Ordering::Equal)
1060                    });
1061                } else {
1062                    objs.sort_by(|a, b| a.0.cmp(&b.0));
1063                }
1064                cur = objs.into_iter().map(|(_, l)| l).collect();
1065                cur.extend(rest);
1066            }
1067            QueryOp::UniqueBy(field) => {
1068                let mut seen: HashSet<String> = HashSet::new();
1069                cur.retain(|l| {
1070                    let key = match serde_json::from_str::<Value>(l) {
1071                        Ok(Value::Object(m)) => {
1072                            m.get(field).map(json_to_string).unwrap_or_default()
1073                        }
1074                        _ => l.clone(),
1075                    };
1076                    seen.insert(key)
1077                });
1078            }
1079            QueryOp::CountBy(field) => {
1080                let mut counts: BTreeMap<String, u64> = BTreeMap::new();
1081                for l in &cur {
1082                    let key = match serde_json::from_str::<Value>(l) {
1083                        Ok(Value::Object(m)) => {
1084                            m.get(field).map(json_to_string).unwrap_or_default()
1085                        }
1086                        _ => l.clone(),
1087                    };
1088                    *counts.entry(key).or_insert(0) += 1;
1089                }
1090                let mut pairs: Vec<(String, u64)> = counts.into_iter().collect();
1091                pairs.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
1092                return QueryResult::Pairs(pairs);
1093            }
1094            QueryOp::GroupBy(field) => {
1095                // jq group_by: one array per distinct key, groups sorted by key.
1096                let mut groups: BTreeMap<String, Vec<Value>> = BTreeMap::new();
1097                for l in &cur {
1098                    let (key, val) = match serde_json::from_str::<Value>(l) {
1099                        Ok(v @ Value::Object(_)) => {
1100                            let k = v.get(field).map(json_to_string).unwrap_or_default();
1101                            (k, v)
1102                        }
1103                        Ok(v) => (l.clone(), v),
1104                        Err(_) => (l.clone(), Value::String(l.clone())),
1105                    };
1106                    groups.entry(key).or_default().push(val);
1107                }
1108                cur = groups
1109                    .into_values()
1110                    .map(|g| Value::Array(g).to_string())
1111                    .collect();
1112            }
1113            QueryOp::MinBy(field) => {
1114                let best = cur
1115                    .iter()
1116                    .filter(|l| !field_num(l, field).is_nan())
1117                    .min_by(|a, b| {
1118                        field_num(a, field)
1119                            .partial_cmp(&field_num(b, field))
1120                            .unwrap_or(std::cmp::Ordering::Equal)
1121                    });
1122                return QueryResult::Lines(best.into_iter().cloned().collect());
1123            }
1124            QueryOp::MaxBy(field) => {
1125                // Ignore lines whose FIELD is absent/non-numeric (field_num -> NaN),
1126                // then keep the record with the greatest value. On ties the last
1127                // maximal record wins (std max_by semantics). Empty input -> no lines.
1128                let best = cur
1129                    .iter()
1130                    .filter(|l| !field_num(l, field).is_nan())
1131                    .max_by(|a, b| {
1132                        field_num(a, field)
1133                            .partial_cmp(&field_num(b, field))
1134                            .unwrap_or(std::cmp::Ordering::Equal)
1135                    })
1136                    .cloned();
1137                return QueryResult::Lines(best.into_iter().collect());
1138            }
1139            QueryOp::Has(key) => {
1140                cur.retain(|l| {
1141                    matches!(
1142                        serde_json::from_str::<Value>(l),
1143                        Ok(Value::Object(ref m)) if m.contains_key(key)
1144                    )
1145                });
1146            }
1147            QueryOp::HasKey(key) => {
1148                // jq `has`: a per-input boolean test, NOT a filter — every input
1149                // yields exactly one `true`/`false` line. The key must MATCH the
1150                // container: a string key on an object, a numeric index on an
1151                // array. Every other pairing is an error in jq, where arb used to
1152                // answer `false` — including `[1,2] | has(0)`, which is `true`.
1153                for l in cur.iter_mut() {
1154                    let present = match (jq_value(l), key) {
1155                        (Some(Value::Object(m)), Value::String(k)) => m.contains_key(k),
1156                        (Some(Value::Array(a)), Value::Number(n)) => {
1157                            let i = n.as_f64().unwrap_or(-1.0);
1158                            i >= 0.0 && (i as usize) < a.len()
1159                        }
1160                        // jq answers `false` for null rather than raising.
1161                        (Some(Value::Null), _) | (None, _) => false,
1162                        (Some(v), k) => {
1163                            return QueryResult::Error(format!(
1164                                "jq: Cannot check whether {} has a {} key",
1165                                crate::jqval::tname(&v),
1166                                crate::jqval::tname(k)
1167                            ));
1168                        }
1169                    };
1170                    *l = if present { "true" } else { "false" }.to_string();
1171                }
1172            }
1173            QueryOp::Entries => {
1174                let mut out = Vec::with_capacity(cur.len());
1175                for l in &cur {
1176                    match serde_json::from_str::<Value>(l) {
1177                        Ok(Value::Object(m)) => {
1178                            for (k, v) in &m {
1179                                out.push(format!(
1180                                    "{{\"key\":{},\"value\":{}}}",
1181                                    Value::String(k.clone()),
1182                                    v
1183                                ));
1184                            }
1185                        }
1186                        _ => out.push(l.clone()),
1187                    }
1188                }
1189                cur = out;
1190            }
1191            QueryOp::JqEntries => {
1192                if let Some(e) = jq_builtin_lines(&mut cur, "to_entries") {
1193                    return e;
1194                }
1195            }
1196            QueryOp::JqMap(inner) => {
1197                // jq `map(f)` is `[.[] | f]` — a per-input SCOPED iterate, so two
1198                // input lines stay two output arrays. Running the inner ops over
1199                // the whole `cur` instead (which dropping the re-wrap amounts to)
1200                // both loses the array and merges the inputs together.
1201                let mut out = Vec::with_capacity(cur.len());
1202                for l in &cur {
1203                    // Through the jq value model: elements keep their source
1204                    // literals and an object's values come in its KEY ORDER.
1205                    let elems: Vec<String> = match crate::jqlang::parse_json(l) {
1206                        Ok(v) => match v.bare() {
1207                            crate::jqlang::JqVal::Arr(a) => {
1208                                a.iter().map(crate::jqlang::render_raw).collect()
1209                            }
1210                            // jq `map` over an object maps its VALUES (and still
1211                            // returns an array).
1212                            crate::jqlang::JqVal::Obj(m) => m
1213                                .iter()
1214                                .map(|(_, e)| crate::jqlang::render_raw(e))
1215                                .collect(),
1216                            // Not iterable: jq raises, and so does arb now.
1217                            other => {
1218                                return QueryResult::Error(format!(
1219                                    "jq: Cannot iterate over {}{}",
1220                                    other.type_name(),
1221                                    crate::jqlang::paren_of(other)
1222                                ))
1223                            }
1224                        },
1225                        // Not JSON at all: no jq type to check, so it passes.
1226                        Err(_) => {
1227                            out.push(l.clone());
1228                            continue;
1229                        }
1230                    };
1231                    let mapped = match eval(inner, &elems, elapsed_secs) {
1232                        QueryResult::Lines(v) => v,
1233                        QueryResult::Scalar(s) => vec![fmt_num(s)],
1234                        QueryResult::Pairs(ps) => {
1235                            ps.iter().map(|(k, c)| format!("{k}\t{c}")).collect()
1236                        }
1237                        // An inner refusal is the whole `map`'s refusal.
1238                        e @ QueryResult::Error(_) => return e,
1239                    };
1240                    out.push(jq_array_json(&mapped));
1241                }
1242                cur = out;
1243            }
1244            QueryOp::Flatten => {
1245                // jq `flatten` fully flattens all nesting levels; recurse into
1246                // every sub-array and emit only the non-array leaves.
1247                fn push_leaves(v: &Value, out: &mut Vec<String>) {
1248                    match v {
1249                        Value::Array(a) => a.iter().for_each(|e| push_leaves(e, out)),
1250                        other => out.push(json_to_string(other)),
1251                    }
1252                }
1253                let mut out = Vec::with_capacity(cur.len());
1254                for l in &cur {
1255                    match serde_json::from_str::<Value>(l) {
1256                        Ok(Value::Array(arr)) => arr.iter().for_each(|e| push_leaves(e, &mut out)),
1257                        _ => out.push(l.clone()),
1258                    }
1259                }
1260                cur = out;
1261            }
1262            QueryOp::JqFlatten => {
1263                // jq's `flatten` is defined over `reduce .[] as $x`, so it
1264                // iterates an OBJECT's values too and refuses a scalar.
1265                if let Some(e) = jq_builtin_lines(&mut cur, "flatten") {
1266                    return e;
1267                }
1268            }
1269            QueryOp::Add => {
1270                for l in cur.iter_mut() {
1271                    if let Ok(Value::Array(arr)) = serde_json::from_str::<Value>(l) {
1272                        if arr.is_empty() {
1273                            *l = String::new();
1274                        } else if arr.iter().all(Value::is_number) {
1275                            let sum: f64 = arr.iter().filter_map(Value::as_f64).sum();
1276                            *l = fmt_num(sum);
1277                        } else {
1278                            *l = arr.iter().map(json_to_string).collect::<String>();
1279                        }
1280                    }
1281                }
1282            }
1283            QueryOp::Over(n) => {
1284                cur.retain(|l| l.trim().parse::<f64>().map(|v| v > *n).unwrap_or(false));
1285            }
1286            QueryOp::Under(n) => {
1287                cur.retain(|l| l.trim().parse::<f64>().map(|v| v < *n).unwrap_or(false));
1288            }
1289            QueryOp::Between(lo, hi) => {
1290                cur.retain(|l| {
1291                    let x = l.trim().parse::<f64>().unwrap_or(f64::NAN);
1292                    x >= *lo && x <= *hi
1293                });
1294            }
1295            QueryOp::Enumerate => {
1296                for (i, l) in cur.iter_mut().enumerate() {
1297                    *l = format!("{}\t{}", i + 1, l);
1298                }
1299            }
1300            QueryOp::Words => {
1301                let mut out = Vec::with_capacity(cur.len());
1302                for l in &cur {
1303                    out.extend(l.split_whitespace().map(str::to_string));
1304                }
1305                cur = out;
1306            }
1307            QueryOp::Dedup => {
1308                cur.dedup();
1309            }
1310            QueryOp::Tailn(n) => {
1311                let len = cur.len();
1312                if *n < len {
1313                    cur.drain(0..len - *n);
1314                }
1315            }
1316            QueryOp::Pad(n) => {
1317                // `format!` width is a u16 internally: a width >= 65536 panics
1318                // ("Formatting argument out of range"). Cap it — no real pane is
1319                // that wide and the widget layer clips lines to the pane anyway.
1320                let n = (*n).min(u16::MAX as usize);
1321                for l in cur.iter_mut() {
1322                    *l = format!("{:<width$}", l, width = n);
1323                }
1324            }
1325            QueryOp::Lpad(width) => {
1326                let w = (*width).min(u16::MAX as usize);
1327                for l in cur.iter_mut() {
1328                    *l = format!("{:>width$}", l, width = w);
1329                }
1330            }
1331            QueryOp::Grepf(field, re) => {
1332                cur.retain(|l| {
1333                    let val = if let Ok(Value::Object(m)) = serde_json::from_str::<Value>(l) {
1334                        m.get(field).map(json_to_string).unwrap_or_default()
1335                    } else if let Ok(idx) = field.parse::<usize>() {
1336                        l.split_whitespace()
1337                            .nth(idx.saturating_sub(1))
1338                            .unwrap_or("")
1339                            .to_string()
1340                    } else {
1341                        String::new()
1342                    };
1343                    re.is_match(&val)
1344                });
1345            }
1346            QueryOp::Flip => {
1347                for l in cur.iter_mut() {
1348                    *l = l.chars().rev().collect();
1349                }
1350            }
1351            QueryOp::Basename => {
1352                for l in cur.iter_mut() {
1353                    let t = l.trim_end_matches('/');
1354                    *l = t.rsplit('/').next().unwrap_or(t).to_string();
1355                }
1356            }
1357            QueryOp::Dirname => {
1358                for l in cur.iter_mut() {
1359                    let t = l.trim_end_matches('/');
1360                    *l = match t.rsplit_once('/') {
1361                        Some(("", _)) => "/".to_string(),
1362                        Some((dir, _)) => dir.to_string(),
1363                        None => ".".to_string(),
1364                    };
1365                }
1366            }
1367            QueryOp::Commafy => {
1368                for l in cur.iter_mut() {
1369                    *l = commafy(l);
1370                }
1371            }
1372            QueryOp::Bytes => {
1373                for l in cur.iter_mut() {
1374                    if let Ok(v) = l.trim().parse::<f64>() {
1375                        *l = humanize_bytes(v);
1376                    }
1377                }
1378            }
1379            QueryOp::Duration => {
1380                for l in cur.iter_mut() {
1381                    if let Ok(v) = l.trim().parse::<f64>() {
1382                        *l = humanize_duration(v);
1383                    }
1384                }
1385            }
1386            // Resolved to the input widget's pipeline before eval; a no-op if it
1387            // survives (empty input, or eval reached without substitution).
1388            QueryOp::Apply(_) => {}
1389            QueryOp::B64 => {
1390                for l in cur.iter_mut() {
1391                    *l = STANDARD.encode(l.as_bytes());
1392                }
1393            }
1394            QueryOp::B64d => {
1395                for l in cur.iter_mut() {
1396                    if let Some(s) = STANDARD
1397                        .decode(l.as_bytes())
1398                        .ok()
1399                        .and_then(|b| String::from_utf8(b).ok())
1400                    {
1401                        *l = s;
1402                    }
1403                }
1404            }
1405            QueryOp::Hex => {
1406                for l in cur.iter_mut() {
1407                    *l = l.bytes().map(|b| format!("{:02x}", b)).collect();
1408                }
1409            }
1410            QueryOp::Unhex => {
1411                for l in cur.iter_mut() {
1412                    let chars: Vec<char> = l.chars().collect();
1413                    if chars.is_empty() || !chars.len().is_multiple_of(2) {
1414                        continue;
1415                    }
1416                    let mut bytes = Vec::with_capacity(chars.len() / 2);
1417                    let mut ok = true;
1418                    let mut i = 0;
1419                    while i < chars.len() {
1420                        let pair: String = chars[i..i + 2].iter().collect();
1421                        match u8::from_str_radix(&pair, 16) {
1422                            Ok(b) => bytes.push(b),
1423                            Err(_) => {
1424                                ok = false;
1425                                break;
1426                            }
1427                        }
1428                        i += 2;
1429                    }
1430                    if ok {
1431                        if let Ok(decoded) = String::from_utf8(bytes) {
1432                            *l = decoded;
1433                        }
1434                    }
1435                }
1436            }
1437            QueryOp::Urlenc => {
1438                for l in cur.iter_mut() {
1439                    *l = utf8_percent_encode(l, NON_ALPHANUMERIC).to_string();
1440                }
1441            }
1442            QueryOp::Urldec => {
1443                for l in cur.iter_mut() {
1444                    if let Ok(decoded) = percent_decode_str(l).decode_utf8() {
1445                        *l = decoded.into_owned();
1446                    }
1447                }
1448            }
1449            QueryOp::Extract(re) => {
1450                cur = cur
1451                    .iter()
1452                    .filter_map(|l| {
1453                        re.captures(l).map(|caps| {
1454                            caps.get(1)
1455                                .unwrap_or_else(|| caps.get(0).unwrap())
1456                                .as_str()
1457                                .to_string()
1458                        })
1459                    })
1460                    .collect();
1461            }
1462            QueryOp::Split(delim) => {
1463                let mut out: Vec<String> = Vec::with_capacity(cur.len());
1464                for l in cur.iter() {
1465                    for part in l.split(delim.as_str()) {
1466                        out.push(part.to_string());
1467                    }
1468                }
1469                cur = out;
1470            }
1471            QueryOp::Substr(a, b) => {
1472                for l in cur.iter_mut() {
1473                    *l = l.chars().skip(*a).take(b.saturating_sub(*a)).collect();
1474                }
1475            }
1476            QueryOp::Chars => {
1477                let mut out: Vec<String> = Vec::new();
1478                for l in cur.iter() {
1479                    for ch in l.chars() {
1480                        out.push(ch.to_string());
1481                    }
1482                }
1483                cur = out;
1484            }
1485            QueryOp::Title => {
1486                for l in cur.iter_mut() {
1487                    *l = l
1488                        .split_whitespace()
1489                        .map(|w| {
1490                            let mut cs = w.chars();
1491                            match cs.next() {
1492                                Some(f) => {
1493                                    f.to_uppercase().collect::<String>()
1494                                        + &cs.as_str().to_lowercase()
1495                                }
1496                                None => String::new(),
1497                            }
1498                        })
1499                        .collect::<Vec<_>>()
1500                        .join(" ");
1501                }
1502            }
1503            QueryOp::Repeat(n) => {
1504                for l in cur.iter_mut() {
1505                    *l = l.repeat(*n);
1506                }
1507            }
1508            QueryOp::Set(key, val) => {
1509                for l in cur.iter_mut() {
1510                    if let Ok(Value::Object(mut m)) = serde_json::from_str::<Value>(l) {
1511                        m.insert(key.clone(), Value::String(val.clone()));
1512                        *l = Value::Object(m).to_string();
1513                    }
1514                }
1515            }
1516            QueryOp::Del(key) => {
1517                for l in cur.iter_mut() {
1518                    if let Ok(Value::Object(mut m)) = serde_json::from_str::<Value>(l) {
1519                        m.remove(key);
1520                        *l = Value::Object(m).to_string();
1521                    }
1522                }
1523            }
1524            QueryOp::Rename(old, new) => {
1525                for l in cur.iter_mut() {
1526                    if let Ok(Value::Object(mut m)) = serde_json::from_str::<Value>(l) {
1527                        if let Some(v) = m.remove(old) {
1528                            m.insert(new.clone(), v);
1529                            *l = Value::Object(m).to_string();
1530                        }
1531                    }
1532                }
1533            }
1534            QueryOp::Default(key, val) => {
1535                for l in cur.iter_mut() {
1536                    if let Ok(Value::Object(mut m)) = serde_json::from_str::<Value>(l) {
1537                        m.entry(key.clone()).or_insert(Value::String(val.clone()));
1538                        *l = Value::Object(m).to_string();
1539                    }
1540                }
1541            }
1542            QueryOp::Merge => {
1543                let mut acc = serde_json::Map::new();
1544                let mut saw_object = false;
1545                for l in cur.iter() {
1546                    if let Ok(Value::Object(m)) = serde_json::from_str::<Value>(l) {
1547                        saw_object = true;
1548                        for (k, v) in m {
1549                            acc.insert(k, v);
1550                        }
1551                    }
1552                }
1553                return if saw_object {
1554                    QueryResult::Lines(vec![Value::Object(acc).to_string()])
1555                } else {
1556                    QueryResult::Lines(vec![])
1557                };
1558            }
1559            QueryOp::Floor => {
1560                for l in cur.iter_mut() {
1561                    if let Ok(x) = l.parse::<f64>() {
1562                        *l = fmt_num(x.floor());
1563                    }
1564                }
1565            }
1566            QueryOp::Ceil => {
1567                for l in cur.iter_mut() {
1568                    if let Ok(x) = l.trim().parse::<f64>() {
1569                        *l = fmt_num(x.ceil());
1570                    }
1571                }
1572            }
1573            QueryOp::Clamp(lo, hi) => {
1574                for l in cur.iter_mut() {
1575                    if let Ok(x) = l.trim().parse::<f64>() {
1576                        *l = fmt_num(x.clamp(*lo, *hi));
1577                    }
1578                }
1579            }
1580            QueryOp::Contains(s) => cur.retain(|l| l.contains(s.as_str())),
1581            QueryOp::Starts(p) => cur.retain(|l| l.starts_with(p.as_str())),
1582            QueryOp::Ends(s) => cur.retain(|l| l.ends_with(s.as_str())),
1583            QueryOp::Nonempty => cur.retain(|l| !l.trim().is_empty()),
1584            QueryOp::Numeric => cur.retain(|l| l.trim().parse::<f64>().is_ok()),
1585            QueryOp::Len => {
1586                for l in cur.iter_mut() {
1587                    *l = l.chars().count().to_string();
1588                }
1589            }
1590            QueryOp::JsonLen => {
1591                for l in cur.iter_mut() {
1592                    *l = match serde_json::from_str::<Value>(l) {
1593                        Ok(Value::Array(a)) => a.len().to_string(),
1594                        Ok(Value::Object(m)) => m.len().to_string(),
1595                        Ok(Value::String(s)) => s.chars().count().to_string(),
1596                        Ok(Value::Number(n)) => fmt_num(n.as_f64().unwrap_or(0.0).abs()),
1597                        Ok(Value::Null) => "0".to_string(),
1598                        // jq errors on a bool length; arb has no per-line error
1599                        // channel, so fall back to the raw line's char count.
1600                        _ => l.chars().count().to_string(),
1601                    };
1602                }
1603            }
1604            QueryOp::JsonSlice(a, b) => {
1605                for l in cur.iter_mut() {
1606                    match jq_value(l) {
1607                        Some(Value::Array(arr)) => {
1608                            let (lo, hi) = slice_bounds(*a, *b, arr.len());
1609                            *l = Value::Array(arr[lo..hi].to_vec()).to_string();
1610                        }
1611                        Some(Value::String(s)) => {
1612                            let chars: Vec<char> = s.chars().collect();
1613                            let (lo, hi) = slice_bounds(*a, *b, chars.len());
1614                            *l = chars[lo..hi].iter().collect();
1615                        }
1616                        // jq slices null to null, and refuses every other type —
1617                        // it reports the slice as the OBJECT it is internally.
1618                        Some(Value::Null) => *l = "null".to_string(),
1619                        Some(v) => {
1620                            let bound =
1621                                |x: &Option<i64>| x.map_or("null".to_string(), |n| n.to_string());
1622                            return QueryResult::Error(format!(
1623                                "jq: Cannot index {} with object ({{\"start\":{},\"end\":{}}})",
1624                                crate::jqval::tname(&v),
1625                                bound(a),
1626                                bound(b)
1627                            ));
1628                        }
1629                        None => {} // not JSON at all: passes through
1630                    }
1631                }
1632            }
1633            QueryOp::Wc => {
1634                for l in cur.iter_mut() {
1635                    *l = l.split_whitespace().count().to_string();
1636                }
1637            }
1638            QueryOp::Abs => {
1639                for l in cur.iter_mut() {
1640                    if let Ok(v) = l.trim().parse::<f64>() {
1641                        *l = fmt_num(v.abs());
1642                    }
1643                }
1644            }
1645            QueryOp::Round => {
1646                for l in cur.iter_mut() {
1647                    if let Ok(v) = l.trim().parse::<f64>() {
1648                        *l = fmt_num(v.round());
1649                    }
1650                }
1651            }
1652            QueryOp::Delta => {
1653                let ns = nums(&cur);
1654                cur = ns.windows(2).map(|w| fmt_num(w[1] - w[0])).collect();
1655            }
1656            QueryOp::Cumsum => {
1657                let ns = nums(&cur);
1658                let mut acc = 0.0;
1659                cur = ns
1660                    .iter()
1661                    .map(|&v| {
1662                        acc += v;
1663                        fmt_num(acc)
1664                    })
1665                    .collect();
1666            }
1667            QueryOp::Sma(n) => {
1668                let ns = nums(&cur);
1669                let w = (*n).max(1);
1670                cur = (0..ns.len())
1671                    .map(|i| {
1672                        let lo = i + 1 - (i + 1).min(w);
1673                        let win = &ns[lo..=i];
1674                        fmt_num(win.iter().sum::<f64>() / win.len() as f64)
1675                    })
1676                    .collect();
1677            }
1678            QueryOp::Ewma(alpha) => {
1679                let a = alpha.clamp(0.0, 1.0);
1680                let ns = nums(&cur);
1681                let mut s = 0.0;
1682                cur = ns
1683                    .iter()
1684                    .enumerate()
1685                    .map(|(i, &v)| {
1686                        s = if i == 0 { v } else { a * v + (1.0 - a) * s };
1687                        fmt_num(s)
1688                    })
1689                    .collect();
1690            }
1691            QueryOp::Prepend(pre) => {
1692                for l in cur.iter_mut() {
1693                    *l = format!("{pre}{l}");
1694                }
1695            }
1696            QueryOp::Append(suf) => {
1697                for l in cur.iter_mut() {
1698                    l.push_str(suf);
1699                }
1700            }
1701            QueryOp::Cut(delim, n) => {
1702                for l in cur.iter_mut() {
1703                    *l = l
1704                        .split(delim.as_str())
1705                        .nth(n.saturating_sub(1))
1706                        .unwrap_or("")
1707                        .to_string();
1708                }
1709            }
1710            QueryOp::Median => {
1711                let mut ns = nums(&cur);
1712                ns.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
1713                let m = if ns.is_empty() {
1714                    0.0
1715                } else if ns.len() % 2 == 1 {
1716                    ns[ns.len() / 2]
1717                } else {
1718                    (ns[ns.len() / 2 - 1] + ns[ns.len() / 2]) / 2.0
1719                };
1720                return QueryResult::Scalar(m);
1721            }
1722            QueryOp::Percentile(p) => {
1723                let mut ns = nums(&cur);
1724                ns.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
1725                let v = if ns.is_empty() {
1726                    0.0
1727                } else if ns.len() == 1 {
1728                    ns[0]
1729                } else {
1730                    // Linear interpolation between ranks (numpy default), so `p50`
1731                    // equals `median` and `p90` of 1..10 is 9.1, matching the docs.
1732                    let frac = p.clamp(0.0, 100.0) / 100.0;
1733                    let pos = frac * (ns.len() - 1) as f64;
1734                    let lo = pos.floor() as usize;
1735                    let hi = pos.ceil() as usize;
1736                    ns[lo] + (ns[hi] - ns[lo]) * (pos - lo as f64)
1737                };
1738                return QueryResult::Scalar(v);
1739            }
1740            QueryOp::Stddev => {
1741                let ns = nums(&cur);
1742                let sd = if ns.is_empty() {
1743                    0.0
1744                } else {
1745                    let mean = ns.iter().sum::<f64>() / ns.len() as f64;
1746                    let var = ns.iter().map(|v| (v - mean).powi(2)).sum::<f64>() / ns.len() as f64;
1747                    var.sqrt()
1748                };
1749                return QueryResult::Scalar(sd);
1750            }
1751            QueryOp::Range => {
1752                let ns = nums(&cur);
1753                let r = if ns.is_empty() {
1754                    0.0
1755                } else {
1756                    ns.iter().cloned().fold(f64::NEG_INFINITY, f64::max)
1757                        - ns.iter().cloned().fold(f64::INFINITY, f64::min)
1758                };
1759                return QueryResult::Scalar(r);
1760            }
1761            QueryOp::Product => return QueryResult::Scalar(nums(&cur).iter().product()),
1762            QueryOp::Distinct => {
1763                let set: std::collections::HashSet<&String> = cur.iter().collect();
1764                return QueryResult::Scalar(set.len() as f64);
1765            }
1766            QueryOp::Sample(n) => {
1767                if *n >= 1 {
1768                    let mut i = 0usize;
1769                    cur.retain(|_| {
1770                        i += 1;
1771                        i.is_multiple_of(*n)
1772                    });
1773                }
1774            }
1775            QueryOp::Slice(a, b) => {
1776                let lo = a.saturating_sub(1).min(cur.len());
1777                let hi = (*b).min(cur.len());
1778                cur = if lo < hi {
1779                    cur[lo..hi].to_vec()
1780                } else {
1781                    Vec::new()
1782                };
1783            }
1784            QueryOp::Index(n) => {
1785                cur = n
1786                    .checked_sub(1)
1787                    .and_then(|i| cur.get(i).cloned())
1788                    .into_iter()
1789                    .collect();
1790            }
1791            QueryOp::Bins(n) => {
1792                let vals = nums(&cur);
1793                // Bound the bucket count: it allocates `vec![0u64; n]` and builds
1794                // n formatted pairs, so an over-large N (e.g. 1e8) balloons memory
1795                // and hangs. No histogram display needs more than this many bars.
1796                let n = (*n).clamp(1, 65_536);
1797                if vals.is_empty() {
1798                    return QueryResult::Pairs(Vec::new());
1799                }
1800                let min = vals.iter().cloned().fold(f64::INFINITY, f64::min);
1801                let max = vals.iter().cloned().fold(f64::NEG_INFINITY, f64::max);
1802                let width = ((max - min) / n as f64).max(f64::MIN_POSITIVE);
1803                let mut counts = vec![0u64; n];
1804                for v in &vals {
1805                    let idx = (((v - min) / width) as usize).min(n - 1);
1806                    counts[idx] += 1;
1807                }
1808                let pairs = counts
1809                    .iter()
1810                    .enumerate()
1811                    .map(|(i, &c)| {
1812                        let lo = min + i as f64 * width;
1813                        (format!("{}-{}", fmt_num(lo), fmt_num(lo + width)), c)
1814                    })
1815                    .collect();
1816                return QueryResult::Pairs(pairs);
1817            }
1818        }
1819    }
1820    QueryResult::Lines(cur)
1821}
1822
1823/// True if every op processes each line independently, so the pipeline can be
1824/// applied per-line and its results emitted incrementally (streaming). Reducers,
1825/// sorts/reorderers, and whole-document ops (`sel`/`csv`/`tsv`) are not.
1826/// Split lines into table `(headers, rows)` for the `table` widget. Each line is
1827/// split on whitespace into cells; `cols` (a comma-separated list, from `-cols`)
1828/// names the header row when present. Renderers pad short rows to the column
1829/// count. Shared by the ratatui TUI and the web dashboard so both agree.
1830pub fn table_data(lines: &[String], cols: Option<&str>) -> (Vec<String>, Vec<Vec<String>>) {
1831    let headers: Vec<String> = cols
1832        .map(|c| {
1833            c.split(',')
1834                .map(|s| s.trim().to_string())
1835                .filter(|s| !s.is_empty())
1836                .collect()
1837        })
1838        .unwrap_or_default();
1839    let rows: Vec<Vec<String>> = lines
1840        .iter()
1841        .map(|l| l.split_whitespace().map(str::to_string).collect())
1842        .collect();
1843    (headers, rows)
1844}
1845
1846/// Parse lines as a numeric series for the `spark` widget — each line's first
1847/// whitespace token that parses as a number; non-numeric lines are skipped.
1848pub fn numeric_series(lines: &[String]) -> Vec<f64> {
1849    lines
1850        .iter()
1851        .filter_map(|l| {
1852            l.split_whitespace()
1853                .next()
1854                .and_then(|t| t.parse::<f64>().ok())
1855        })
1856        .collect()
1857}
1858
1859/// Render a numeric series as a unicode sparkline (`▁▂▃▄▅▆▇█`), scaled between
1860/// the series min and max. Shared by the TUI and web so both draw the same shape.
1861/// A flat series renders as the lowest tick; an empty series is the empty string.
1862pub fn sparkline(values: &[f64]) -> String {
1863    const TICKS: [char; 8] = ['▁', '▂', '▃', '▄', '▅', '▆', '▇', '█'];
1864    if values.is_empty() {
1865        return String::new();
1866    }
1867    let min = values.iter().cloned().fold(f64::INFINITY, f64::min);
1868    let max = values.iter().cloned().fold(f64::NEG_INFINITY, f64::max);
1869    let range = max - min;
1870    values
1871        .iter()
1872        .map(|&v| {
1873            let idx = if range <= 0.0 {
1874                0
1875            } else {
1876                (((v - min) / range) * 7.0).round() as usize
1877            };
1878            TICKS[idx.min(7)]
1879        })
1880        .collect()
1881}
1882
1883/// The number of columns a table needs to hold `headers` and `rows` (at least 1).
1884pub fn table_ncols(headers: &[String], rows: &[Vec<String>]) -> usize {
1885    rows.iter()
1886        .map(Vec::len)
1887        .max()
1888        .unwrap_or(0)
1889        .max(headers.len())
1890        .max(1)
1891}
1892
1893pub fn is_line_streamable(ops: &[QueryOp]) -> bool {
1894    ops.iter().all(|op| {
1895        // `map(f)` is scoped to one input line, so it streams exactly when its
1896        // inner pipeline does — a reducer inside it (`map(add)`) does not.
1897        if let QueryOp::JqMap(inner) = op {
1898            return is_line_streamable(inner);
1899        }
1900        // A jq PROGRAM is per-line unless it reads the stream itself: `input`
1901        // and `inputs` pull the following documents, so they need the whole
1902        // buffer in hand. A program that does not touch them streams, which is
1903        // what keeps `arb -e '… | .name'` emitting as lines arrive instead of
1904        // buffering to EOF.
1905        if let QueryOp::JqProgram(src) = op {
1906            return jq_program(src).is_ok_and(|p| !p.reads_input_stream());
1907        }
1908        matches!(
1909            op,
1910            QueryOp::JsonDocs
1911                | QueryOp::Match(_)
1912                | QueryOp::Reject(_)
1913                | QueryOp::Field(_)
1914                | QueryOp::Fields(_)
1915                | QueryOp::Each
1916                | QueryOp::JqEach
1917                | QueryOp::JqSelect(_)
1918                | QueryOp::JqCalc(_)
1919                | QueryOp::JqAdd
1920                | QueryOp::JqLen
1921                | QueryOp::Keys
1922                | QueryOp::JqKeys
1923                | QueryOp::Vals
1924                | QueryOp::NonNull
1925                | QueryOp::JqRawString
1926                | QueryOp::Pick(_)
1927                | QueryOp::Where(_)
1928                | QueryOp::Map(_)
1929                | QueryOp::Contains(_)
1930                | QueryOp::Starts(_)
1931                | QueryOp::Ends(_)
1932                | QueryOp::Nonempty
1933                | QueryOp::Numeric
1934                | QueryOp::Len
1935                | QueryOp::JsonLen
1936                | QueryOp::JsonSlice(_, _)
1937                | QueryOp::Wc
1938                | QueryOp::Abs
1939                | QueryOp::Round
1940                | QueryOp::Prepend(_)
1941                | QueryOp::Append(_)
1942                | QueryOp::Cut(_, _)
1943                | QueryOp::Upper
1944                | QueryOp::Lower
1945                | QueryOp::Trim
1946                | QueryOp::Replace(_, _)
1947                | QueryOp::B64
1948                | QueryOp::B64d
1949                | QueryOp::Hex
1950                | QueryOp::Unhex
1951                | QueryOp::Urlenc
1952                | QueryOp::Urldec
1953                | QueryOp::Extract(_)
1954                | QueryOp::Substr(_, _)
1955                | QueryOp::Title
1956                | QueryOp::Repeat(_)
1957                | QueryOp::Set(_, _)
1958                | QueryOp::Del(_)
1959                | QueryOp::Rename(_, _)
1960                | QueryOp::Default(_, _)
1961                | QueryOp::Floor
1962                | QueryOp::Ceil
1963                | QueryOp::Clamp(_, _)
1964                | QueryOp::Has(_)
1965                | QueryOp::HasKey(_)
1966                | QueryOp::Entries
1967                | QueryOp::JqEntries
1968                | QueryOp::JqFlatten
1969                | QueryOp::Add
1970                | QueryOp::Over(_)
1971                | QueryOp::Under(_)
1972                | QueryOp::Between(_, _)
1973                | QueryOp::Pad(_)
1974                | QueryOp::Lpad(_)
1975                | QueryOp::Grepf(_, _)
1976                | QueryOp::Flip
1977                | QueryOp::Basename
1978                | QueryOp::Dirname
1979                | QueryOp::Commafy
1980        )
1981    })
1982}
1983
1984/// Extract a field from a line per the selector.
1985///
1986/// `JqKey` is NOT handled here: a jq path can refuse (`3 | .a`), and a `String`
1987/// return has nowhere to put the refusal, so `eval`'s `Field` arm resolves it
1988/// through `jqval::get_path` and raises `QueryResult::Error` instead.
1989fn extract_field(line: &str, sel: &FieldSel) -> String {
1990    match sel {
1991        FieldSel::Col(n) => nth_col(line, *n).to_string(),
1992        FieldSel::JqKey(segs) => jq_value(line)
1993            .and_then(|v| crate::jqval::get_path(&v, segs).ok())
1994            .map_or_else(|| "null".to_string(), |v| jq_to_string(&v)),
1995        FieldSel::Key(path) => serde_json::from_str::<Value>(line)
1996            .ok()
1997            .and_then(|v| walk(v, path))
1998            .map(|v| json_to_string(&v))
1999            .or_else(|| {
2000                if path.len() == 1 {
2001                    logfmt_field(line, &path[0])
2002                } else {
2003                    None
2004                }
2005            })
2006            .unwrap_or_default(),
2007    }
2008}
2009
2010/// Parse a line as JSON for a jq-context op. `None` means the line is not JSON
2011/// at all — arb's stream is TEXT, so such a line has no jq type to check and
2012/// every caller keeps its documented pass-through instead of raising.
2013fn jq_value(line: &str) -> Option<Value> {
2014    serde_json::from_str::<Value>(line).ok()
2015}
2016
2017/// The current value for a jq EXPRESSION stage. Unlike [`jq_value`] there is no
2018/// "not JSON" case: a bare text line is jq's string, so `"ab" | . * 2` gives
2019/// `abab` the way jq does.
2020fn jq_scalar(line: &str) -> Value {
2021    jq_value(line).unwrap_or_else(|| Value::String(line.to_string()))
2022}
2023
2024/// jq's message for asking a non-container for its keys.
2025fn has_no_keys(v: &Value) -> String {
2026    format!("jq: {} ({v}) has no keys", crate::jqval::tname(v))
2027}
2028
2029/// Render a JSON value the way jq's `-r`/`-c` pair does: a string raw, a null as
2030/// the literal `null`, everything else compact. `json_to_string` renders a null
2031/// as "" instead, which is right for arb's native text verbs and wrong for jq.
2032fn jq_to_string(v: &Value) -> String {
2033    match v {
2034        Value::String(s) => s.clone(),
2035        other => other.to_string(),
2036    }
2037}
2038
2039/// Rebuild a jq array from element strings that have ALREADY been rendered.
2040///
2041/// `fmt_num` is the one place an arb number becomes text, but parsing its
2042/// output back into a `serde_json::Value` and letting serde_json print it again
2043/// runs the value through a SECOND, different float formatter (ryu) and undoes
2044/// it: `1e-06` came back out as `1e-6`, `1e-05` as `0.00001`, and
2045/// `123456789012345000000` as `1.23456789012345e+20`. Every one of those
2046/// disagreed with `jq -rc` on the same input while the scalar path next to it
2047/// agreed — one invariant, applied at one site and reverted at the other.
2048///
2049/// So an element that is already a JSON NUMBER is emitted verbatim (`1E+1000`
2050/// included, which serde refuses as out of range). Everything else is rendered
2051/// by the jq value model, which gives an object, a nested array and a string
2052/// their escaping while keeping KEY ORDER and nested number literals (serde's
2053/// `BTreeMap` re-sorted `[{"b":1,"a":2}] | map(.)`), and a line that is not
2054/// JSON at all becomes a JSON string. That last rule is lossy in exactly one place — a string whose
2055/// text is itself valid JSON (`"123"`) comes back as the number — which is
2056/// inherent to a raw line stream and is probed, not hidden, by
2057/// `scripts/jq_parity.sh`.
2058fn jq_array_json(elems: &[String]) -> String {
2059    let mut out = String::from("[");
2060    for (i, e) in elems.iter().enumerate() {
2061        if i > 0 {
2062            out.push(',');
2063        }
2064        match crate::jqlang::parse_json(e) {
2065            Ok(crate::jqlang::JqVal::Num(..)) => out.push_str(e),
2066            Ok(v) => out.push_str(&crate::jqlang::render(&v)),
2067            Err(_) => out.push_str(&Value::String(e.clone()).to_string()),
2068        }
2069    }
2070    out.push(']');
2071    out
2072}
2073
2074/// Re-serialize a stream of documents in `fmt`, as the lines the pipeline
2075/// carries. Splitting on `\n` and dropping the trailing empty element is what
2076/// makes the printed output byte-identical to the rendered text: the printer
2077/// writes one `\n` per line.
2078fn emit_stream(docs: &[crate::jqlang::JqVal], fmt: OutFmt, indent: usize) -> Vec<String> {
2079    let opts = crate::ynode::Emit { indent };
2080    let text = match fmt {
2081        OutFmt::Yaml => crate::ynode::emit_docs(docs, opts),
2082        OutFmt::Json => docs
2083            .iter()
2084            .map(|d| crate::jqlang::render_indented(d, indent))
2085            .collect::<Vec<_>>()
2086            .join("\n"),
2087        OutFmt::Props => docs
2088            .iter()
2089            .map(crate::yqfmt::to_props)
2090            .collect::<Vec<_>>()
2091            .concat(),
2092        OutFmt::Xml => docs
2093            .iter()
2094            .map(|d| crate::yqfmt::to_xml(d, indent))
2095            .collect::<Vec<_>>()
2096            .concat(),
2097        OutFmt::Csv => docs
2098            .iter()
2099            .map(|d| crate::yqfmt::to_delim(d, ','))
2100            .collect::<Vec<_>>()
2101            .concat(),
2102        OutFmt::Tsv => docs
2103            .iter()
2104            .map(|d| crate::yqfmt::to_delim(d, '\t'))
2105            .collect::<Vec<_>>()
2106            .concat(),
2107    };
2108    let mut lines: Vec<String> = text.split('\n').map(str::to_string).collect();
2109    if lines.last().is_some_and(String::is_empty) {
2110        lines.pop();
2111    }
2112    lines
2113}
2114
2115/// Parse the stream as one TOML document and emit it as a JSON object line
2116/// (empty if it does not parse). Goes through `toml::Value` and converts
2117/// explicitly so a TOML datetime becomes a clean scalar string — deserializing
2118/// straight into `serde_json::Value` instead leaks the toml crate's internal
2119/// `{"$__toml_private_datetime": …}` tagging map into the output.
2120fn toml_to_json(lines: &[String]) -> Vec<String> {
2121    match toml::from_str::<toml::Value>(&lines.join("\n")) {
2122        Ok(v) => vec![toml_value_to_json(&v).to_string()],
2123        Err(_) => Vec::new(),
2124    }
2125}
2126
2127/// Convert a `toml::Value` to a `serde_json::Value`, rendering datetimes as their
2128/// string form (RFC-3339 / date / time) rather than a tagged object.
2129fn toml_value_to_json(v: &toml::Value) -> Value {
2130    match v {
2131        toml::Value::String(s) => Value::String(s.clone()),
2132        toml::Value::Integer(i) => Value::Number((*i).into()),
2133        toml::Value::Float(f) => serde_json::Number::from_f64(*f)
2134            .map(Value::Number)
2135            .unwrap_or(Value::Null),
2136        toml::Value::Boolean(b) => Value::Bool(*b),
2137        toml::Value::Datetime(dt) => Value::String(dt.to_string()),
2138        toml::Value::Array(a) => Value::Array(a.iter().map(toml_value_to_json).collect()),
2139        toml::Value::Table(t) => Value::Object(
2140            t.iter()
2141                .map(|(k, v)| (k.clone(), toml_value_to_json(v)))
2142                .collect(),
2143        ),
2144    }
2145}
2146
2147/// Reassemble physical lines into logical CSV/TSV records. A quoted field may
2148/// contain newlines (RFC 4180), so a line whose double-quotes are unbalanced is
2149/// still inside a quoted field and continues on the next line until they balance
2150/// — otherwise a multi-line field is torn into corrupt phantom rows. A doubled
2151/// `""` (an escaped quote) contributes two quotes, so parity tracks correctly.
2152fn join_quoted_records(lines: &[String]) -> Vec<String> {
2153    let mut records = Vec::new();
2154    let mut buf = String::new();
2155    for line in lines {
2156        if !buf.is_empty() {
2157            buf.push('\n');
2158        }
2159        buf.push_str(line);
2160        if buf.matches('"').count().is_multiple_of(2) {
2161            records.push(std::mem::take(&mut buf));
2162        }
2163    }
2164    if !buf.is_empty() {
2165        records.push(buf); // trailing unbalanced record: emit what we have
2166    }
2167    records
2168}
2169
2170/// Parse a header + data rows of a delimited stream into JSON object strings
2171/// keyed by the header, so `field NAME` works over CSV/TSV.
2172fn to_json_records(lines: &[String], delim: char) -> Vec<String> {
2173    if lines.is_empty() {
2174        return Vec::new();
2175    }
2176    let records = join_quoted_records(lines);
2177    let Some(header_row) = records.first() else {
2178        return Vec::new();
2179    };
2180    let header = split_delim(header_row, delim);
2181    records[1..]
2182        .iter()
2183        .map(|row| {
2184            let vals = split_delim(row, delim);
2185            let mut obj = serde_json::Map::new();
2186            for (i, name) in header.iter().enumerate() {
2187                obj.insert(
2188                    name.clone(),
2189                    Value::String(vals.get(i).cloned().unwrap_or_default()),
2190                );
2191            }
2192            Value::Object(obj).to_string()
2193        })
2194        .collect()
2195}
2196
2197/// Split one delimited line into fields per RFC 4180 (line-oriented: no embedded
2198/// newlines). A field may be double-quoted; a quoted field may contain the
2199/// delimiter, and a doubled `""` inside it is one literal `"`. Unquoted fields
2200/// are trimmed (preserving prior behavior); quoted fields are returned verbatim.
2201fn split_delim(line: &str, delim: char) -> Vec<String> {
2202    let mut fields = Vec::new();
2203    let mut field = String::new();
2204    let mut in_quotes = false;
2205    let mut quoted = false; // this field opened with a quote
2206    let mut chars = line.chars().peekable();
2207
2208    // Unquoted fields are trimmed; a quoted field is returned exactly.
2209    let finish = |f: &str, q: bool| -> String {
2210        if q {
2211            f.to_string()
2212        } else {
2213            f.trim().to_string()
2214        }
2215    };
2216
2217    while let Some(c) = chars.next() {
2218        if in_quotes {
2219            if c == '"' {
2220                if chars.peek() == Some(&'"') {
2221                    chars.next(); // "" -> one literal quote
2222                    field.push('"');
2223                } else {
2224                    in_quotes = false; // closing quote
2225                }
2226            } else {
2227                field.push(c);
2228            }
2229        } else if c == '"' && field.trim().is_empty() {
2230            in_quotes = true; // opening quote at field start
2231            quoted = true;
2232            field.clear(); // drop any leading whitespace
2233        } else if c == delim {
2234            fields.push(finish(&field, quoted));
2235            field.clear();
2236            quoted = false;
2237        } else {
2238            field.push(c);
2239        }
2240    }
2241    fields.push(finish(&field, quoted));
2242    fields
2243}
2244
2245/// Extract a `key=value` (logfmt) field. A value may be double-quoted and then
2246/// contain spaces (`msg="hello world"`), so this scans key=value pairs honoring
2247/// quotes rather than splitting on whitespace first (which would truncate a
2248/// quoted value at its first space).
2249fn logfmt_field(line: &str, key: &str) -> Option<String> {
2250    let cs: Vec<char> = line.chars().collect();
2251    let mut i = 0;
2252    while i < cs.len() {
2253        while i < cs.len() && cs[i].is_whitespace() {
2254            i += 1;
2255        }
2256        let kstart = i;
2257        while i < cs.len() && cs[i] != '=' && !cs[i].is_whitespace() {
2258            i += 1;
2259        }
2260        let k: String = cs[kstart..i].iter().collect();
2261        if i < cs.len() && cs[i] == '=' {
2262            i += 1; // consume '='
2263            let val = if cs.get(i) == Some(&'"') {
2264                i += 1;
2265                let mut v = String::new();
2266                while i < cs.len() && cs[i] != '"' {
2267                    if cs[i] == '\\' && i + 1 < cs.len() {
2268                        i += 1; // keep the escaped char verbatim
2269                    }
2270                    v.push(cs[i]);
2271                    i += 1;
2272                }
2273                i += 1; // closing quote (or past end)
2274                v
2275            } else {
2276                let vstart = i;
2277                while i < cs.len() && !cs[i].is_whitespace() {
2278                    i += 1;
2279                }
2280                cs[vstart..i].iter().collect()
2281            };
2282            if k == key {
2283                return Some(val);
2284            }
2285        }
2286    }
2287    None
2288}
2289
2290/// The 1-based whitespace column `n` of `line` ("" if absent; 0 = whole line).
2291fn nth_col(line: &str, n: usize) -> &str {
2292    if n == 0 {
2293        return line;
2294    }
2295    line.split_whitespace().nth(n - 1).unwrap_or("")
2296}
2297
2298/// Walk a JSON key/array-index path, consuming the value.
2299fn walk(mut cur: Value, path: &[String]) -> Option<Value> {
2300    for key in path {
2301        cur = match cur {
2302            Value::Object(mut m) => m.remove(key)?,
2303            Value::Array(mut a) => {
2304                // jq array index: a negative index counts from the end (`.[-1]`).
2305                let idx = key.parse::<i64>().ok()?;
2306                let i = if idx < 0 { a.len() as i64 + idx } else { idx };
2307                if i >= 0 && (i as usize) < a.len() {
2308                    a.swap_remove(i as usize)
2309                } else {
2310                    return None;
2311                }
2312            }
2313            _ => return None,
2314        };
2315    }
2316    Some(cur)
2317}
2318
2319/// Resolve jq slice bounds `[a:b]` to a clamped `lo..hi` over `len`. A negative
2320/// bound counts from the end; `None` is the start/end. Clamped into `0..=len`,
2321/// and `hi < lo` collapses to an empty slice.
2322fn slice_bounds(a: Option<i64>, b: Option<i64>, len: usize) -> (usize, usize) {
2323    let n = len as i64;
2324    let norm = |x: i64| (if x < 0 { x + n } else { x }).clamp(0, n);
2325    let lo = norm(a.unwrap_or(0));
2326    let hi = norm(b.unwrap_or(n));
2327    (lo as usize, hi.max(lo) as usize)
2328}
2329
2330/// Parse the numeric lines of a slice, ignoring non-numeric ones.
2331fn nums(lines: &[String]) -> Vec<f64> {
2332    lines
2333        .iter()
2334        .filter_map(|l| l.trim().parse::<f64>().ok())
2335        .collect()
2336}
2337
2338/// Format a computed number the way `jq` prints a computed double.
2339///
2340/// Every arb value is an f64 (SPEC §6), so this is the one place a number
2341/// becomes text — `map`, `sum`, `avg`, `floor`, `diff`, the scalar result and
2342/// every control readout all arrive here. The reference is `jq -r`: arb claims
2343/// jq parity for its numeric core, and `scripts/expr_paths.sh` checks it.
2344///
2345/// jq renders a computed double with netlib `g_fmt`: take the SHORTEST
2346/// round-trip decimal digits `d1…dn` and the decimal-point position `decpt`
2347/// (the value is `0.d1…dn × 10^decpt`), then pick exponential notation when
2348/// `decpt <= -4 || decpt > n + 15` and plain positional notation otherwise.
2349/// The exponent carries a sign and at least two digits (`1e+17`, `1e-05`).
2350///
2351/// Both cutoffs are measured, not guessed: rendering 33,618 doubles this way —
2352/// every decade from 1e-330 to 1e308 at seven mantissas, 20,000 random bit
2353/// patterns and 5,000 random decimals — and diffing against `jq 1.8.2 -rc '.*1'`
2354/// gives zero mismatches. `.*1` forces jq to COMPUTE, so the comparison is
2355/// against its double formatter and not against its decNumber literal
2356/// preservation, which arb has no equivalent for (see the module docs).
2357///
2358/// The two behaviours this replaces were both wrong against that reference:
2359///   * Rust's `{}` never uses exponential notation at all, so `1e308` printed
2360///     as 309 digits and `5e-324` as 324, instead of `1e+308` / `5e-324`.
2361///   * The `v as i64` fast path printed the EXACT binary value of a whole
2362///     number, inventing precision the double does not carry: `1e18 / 3`
2363///     printed `333333333333333312` where jq prints `333333333333333300`, the
2364///     16 round-trip digits zero-padded. (That cast also saturated outside
2365///     i64's range — `tests/query.rs` still pins that it never reappears.)
2366pub(crate) fn fmt_num(v: f64) -> String {
2367    // A non-finite double has no decimal expansion, and neither JSON nor jq has
2368    // a spelling for one, so jq maps both onto values that do: an infinity
2369    // CLAMPS to ±DBL_MAX and a NaN renders as `null`. This function's contract is
2370    // jq's rendering, so it follows jq here too. Measured against jq 1.8.2:
2371    // `1e308 * 2` -> `1.7976931348623157e+308`, `-1e308 * 2` -> the negation of
2372    // that, and `(1e308 * 2) * 0` -> `null`.
2373    //
2374    // Rust's own spelling (`inf` / `NaN`) was returned here before. That was a
2375    // deviation from the contract, justified in this comment by the claim that
2376    // clamping "would hide the divergence the harness reports for a zero
2377    // divisor". It does not, and that is checked rather than argued: the split
2378    // `scripts/expr_paths.sh` reports for `x / 0` is `Value::Undef` on the
2379    // interpreter against an IEEE infinity in compiled code, and `Value::Undef`
2380    // renders `0` — not through this branch at all. Clamping moves only the
2381    // native side, from `inf` to `1.7976931348623157e+308`, so the two tiers
2382    // still disagree and the probe stays red. Both `x / 0` probes are still
2383    // listed as divergences by that harness after this change.
2384    if v.is_nan() {
2385        return "null".to_string();
2386    }
2387    // Clamp the VALUE and fall through to the normal path rather than returning a
2388    // literal string, so the digits an infinity prints as are by construction the
2389    // ones this formatter gives DBL_MAX — they can never drift apart.
2390    let v = v.clamp(-f64::MAX, f64::MAX);
2391    // `-0.0 == 0.0`, so this is also what keeps a negated zero printing as `0`.
2392    if v == 0.0 {
2393        return "0".to_string();
2394    }
2395    // `{:e}` is Rust's shortest round-trip form: `d[.ddd]e<exp>`, value =
2396    // d.ddd × 10^exp. It gives the DIGIT COUNT to work from, but not always the
2397    // right digits — shortest and shortest-AND-CLOSEST are different rules.
2398    //
2399    // Where a double's neighbours are further apart than one unit in the last
2400    // decimal place, SEVERAL decimals of that shortest length parse back to the
2401    // same double, and the two rules may pick different ones. jq's dtoa (David
2402    // Gay's) emits the closest; Rust emits one that round-trips, which need not
2403    // be. `191510495617760.12` came out of Rust as `…13` and `611630169981189.25`
2404    // as `…89.3` against jq's `…89.2`. Both of Rust's answers round-trip; neither
2405    // is the nearest decimal of that length.
2406    //
2407    // So: take the LENGTH from Rust's shortest form, then re-render at that many
2408    // significant digits with `{:.*e}`, which rounds correctly. That is Gay's
2409    // rule, and it cannot cost round-tripping — if some n-digit decimal parses
2410    // back to `v` then the CLOSEST n-digit decimal does too, being at least as
2411    // near. Measured over 200,000 doubles (150,000 random bit patterns plus
2412    // 50,000 in the 1e14..1e17 band where the ties concentrate) against
2413    // `jq 1.8.2 -rc '.*1'`: the shortest-only form missed 1,286 of them, this one
2414    // misses none. The band matters — a pool spread evenly over the exponent
2415    // range hits it so rarely that 33,618 doubles scored clean while 1 in 160 of
2416    // the numbers a person actually writes at that magnitude was wrong.
2417    let shortest = format!("{v:e}");
2418    let ndigits = shortest
2419        .split_once('e')
2420        .map_or(1, |(m, _)| m.chars().filter(char::is_ascii_digit).count());
2421    let sci = format!("{:.*e}", ndigits.saturating_sub(1), v);
2422    let (mant, exp) = match sci.split_once('e') {
2423        Some(p) => p,
2424        None => return sci,
2425    };
2426    let exp: i32 = match exp.parse() {
2427        Ok(e) => e,
2428        Err(_) => return sci,
2429    };
2430    let sign = if mant.starts_with('-') { "-" } else { "" };
2431    let mut digits: String = mant.chars().filter(char::is_ascii_digit).collect();
2432    // Correct rounding can carry into a trailing zero (`…95` at 4 digits becomes
2433    // `…0`), and a trailing zero would both lengthen the output and move the
2434    // fixed-vs-exponential cutoff below, which counts SIGNIFICANT digits.
2435    while digits.len() > 1 && digits.ends_with('0') {
2436        digits.pop();
2437    }
2438    let n = digits.len() as i32;
2439    let decpt = exp + 1;
2440
2441    if decpt <= -4 || decpt > n + 15 {
2442        let m = if n == 1 {
2443            digits
2444        } else {
2445            format!("{}.{}", &digits[..1], &digits[1..])
2446        };
2447        // `{:+03}` is sign + at least two digits: `+17`, `-05`, `-300`.
2448        format!("{sign}{m}e{:+03}", decpt - 1)
2449    } else if decpt <= 0 {
2450        format!("{sign}0.{}{digits}", "0".repeat(-decpt as usize))
2451    } else if decpt >= n {
2452        format!("{sign}{digits}{}", "0".repeat((decpt - n) as usize))
2453    } else {
2454        let (int, frac) = digits.split_at(decpt as usize);
2455        format!("{sign}{int}.{frac}")
2456    }
2457}
2458
2459/// Group the integer part of a numeric line with thousands separators; leaves a
2460/// non-numeric line (and any fractional/sign parts) intact.
2461fn commafy(line: &str) -> String {
2462    let s = line.trim();
2463    if s.parse::<f64>().is_err() {
2464        return line.to_string();
2465    }
2466    let (sign, rest) = match s.strip_prefix('-') {
2467        Some(r) => ("-", r),
2468        None => ("", s),
2469    };
2470    let (int, frac) = match rest.split_once('.') {
2471        Some((i, f)) => (i, Some(f)),
2472        None => (rest, None),
2473    };
2474    let mut grouped = String::new();
2475    for (idx, ch) in int.chars().enumerate() {
2476        if idx > 0 && (int.len() - idx) % 3 == 0 {
2477            grouped.push(',');
2478        }
2479        grouped.push(ch);
2480    }
2481    match frac {
2482        Some(f) => format!("{sign}{grouped}.{f}"),
2483        None => format!("{sign}{grouped}"),
2484    }
2485}
2486
2487/// Humanize a byte count (1024-based): `1536` -> `1.5 KB`, `1024` -> `1 KB`,
2488/// `500` -> `500 B`. One decimal, trailing `.0` trimmed. Negatives keep the sign.
2489fn humanize_bytes(v: f64) -> String {
2490    const UNITS: [&str; 6] = ["B", "KB", "MB", "GB", "TB", "PB"];
2491    let sign = if v < 0.0 { "-" } else { "" };
2492    let mut n = v.abs();
2493    let mut u = 0;
2494    while n >= 1024.0 && u < UNITS.len() - 1 {
2495        n /= 1024.0;
2496        u += 1;
2497    }
2498    // Bytes are whole; scaled values show one decimal (unless it rounds to .0).
2499    if u == 0 {
2500        format!("{sign}{} {}", n.round() as i64, UNITS[u])
2501    } else {
2502        let r = (n * 10.0).round() / 10.0;
2503        if (r.fract()).abs() < f64::EPSILON {
2504            format!("{sign}{} {}", r as i64, UNITS[u])
2505        } else {
2506            format!("{sign}{r:.1} {}", UNITS[u])
2507        }
2508    }
2509}
2510
2511/// Humanize a duration in seconds as the two largest non-zero units: `3661` ->
2512/// `1h 1m`, `45` -> `45s`, `90061` -> `1d 1h`, `0` -> `0s`. Negatives keep the sign.
2513fn humanize_duration(v: f64) -> String {
2514    let sign = if v < 0.0 { "-" } else { "" };
2515    let total = v.abs().round() as i64;
2516    if total == 0 {
2517        return "0s".to_string();
2518    }
2519    let units = [("d", 86400), ("h", 3600), ("m", 60), ("s", 1)];
2520    let mut rem = total;
2521    let mut parts = Vec::new();
2522    for (label, secs) in units {
2523        let q = rem / secs;
2524        if q > 0 {
2525            parts.push(format!("{q}{label}"));
2526            rem %= secs;
2527        }
2528    }
2529    parts.truncate(2); // two largest non-zero units
2530    format!("{sign}{}", parts.join(" "))
2531}
2532
2533/// Resolve a JSON field of `line` to a number for expression evaluation
2534/// (missing / non-numeric / non-JSON -> NaN, which fails numeric predicates).
2535fn field_num(line: &str, name: &str) -> f64 {
2536    if let Ok(Value::Object(mut m)) = serde_json::from_str::<Value>(line) {
2537        if let Some(v) = m.remove(name) {
2538            return value_to_f64(&v);
2539        }
2540    }
2541    logfmt_field(line, name)
2542        .and_then(|s| s.parse::<f64>().ok())
2543        .unwrap_or(f64::NAN)
2544}
2545
2546/// Public wrapper for `field_str` — used by the TUI facet control to derive
2547/// candidate values from a stream field.
2548pub fn field_str_pub(line: &str, name: &str) -> String {
2549    field_str(line, name)
2550}
2551
2552/// Public wrapper for `field_num` — used by the DAP `evaluate` request to
2553/// resolve `.field` references against the paused stream line (same resolver the
2554/// per-line `where`/`map` evaluation uses).
2555pub fn field_num_pub(line: &str, name: &str) -> f64 {
2556    field_num(line, name)
2557}
2558
2559/// Format a control scalar: integers without a decimal, else the shortest repr.
2560pub fn fmt_scalar(v: f64) -> String {
2561    fmt_num(v)
2562}
2563
2564/// A field's value as a string: a JSON object key, else a logfmt `key=value`,
2565/// else "". `x` (or `.` the whole line via Var) resolves to the whole line.
2566fn field_str(line: &str, name: &str) -> String {
2567    if name == "x" {
2568        return line.to_string();
2569    }
2570    if let Ok(Value::Object(m)) = serde_json::from_str::<Value>(line) {
2571        if let Some(v) = m.get(name) {
2572            return json_to_string(v);
2573        }
2574    }
2575    logfmt_field(line, name).unwrap_or_default()
2576}
2577
2578/// One side of a string comparison, resolved against the current line: a literal
2579/// as itself, a field as its value, and `x`/`.` as the whole line.
2580fn cmp_str(e: &crate::expr::Expr, line: &str) -> String {
2581    use crate::expr::Expr;
2582    match e {
2583        Expr::Str(s) => s.clone(),
2584        Expr::Field(f) => field_str(line, f),
2585        Expr::Var => line.trim().to_string(),
2586        Expr::Num(n) => fmt_num(*n),
2587        _ => String::new(),
2588    }
2589}
2590
2591/// The string value of a substituted string node (`Str`), else "".
2592fn str_of(e: &crate::expr::Expr) -> String {
2593    match e {
2594        crate::expr::Expr::Str(s) => s.clone(),
2595        _ => String::new(),
2596    }
2597}
2598
2599/// Evaluate a string-bearing `where` predicate against one line (Rust, not the
2600/// numeric VM). `match`/`in .set` test strings; and/or/not compose; any purely
2601/// numeric subtree falls back to the fusevm predicate path.
2602fn eval_where(e: &crate::expr::Expr, line: &str) -> bool {
2603    use crate::expr::{BinOp, Expr};
2604    match e {
2605        Expr::Match(inner) => {
2606            let q = str_of(inner);
2607            q.is_empty() || line.to_lowercase().contains(&q.to_lowercase())
2608        }
2609        Expr::InSet(field, inner) => {
2610            let set = str_of(inner);
2611            let items: Vec<&str> = set
2612                .split(',')
2613                .map(str::trim)
2614                .filter(|s| !s.is_empty())
2615                .collect();
2616            if items.is_empty() {
2617                return true; // empty selection -> no filter
2618            }
2619            let val = field_str(line, field);
2620            items.iter().any(|it| *it == val)
2621        }
2622        Expr::Not(a) => !eval_where(a, line),
2623        Expr::Bin(BinOp::And, a, b) => eval_where(a, line) && eval_where(b, line),
2624        Expr::Bin(BinOp::Or, a, b) => eval_where(a, line) || eval_where(b, line),
2625        // A compare where one side is a string literal — jq's `select(.status ==
2626        // "ok")`. Compared as TEXT: the numeric fallback below reads a non-numeric
2627        // field as NaN, and every NaN comparison is false, so without this arm such
2628        // a predicate silently matched NOTHING (a filter that drops every row
2629        // rather than erroring). The ORDERED forms belong here for the same reason:
2630        // SPEC §8 says "a compare may test strings as well as numbers", and jq
2631        // orders strings by codepoint — which is Rust's `str` ordering, since UTF-8
2632        // byte order and codepoint order agree.
2633        Expr::Bin(
2634            op @ (BinOp::Eq | BinOp::Ne | BinOp::Lt | BinOp::Le | BinOp::Gt | BinOp::Ge),
2635            a,
2636            b,
2637        ) if matches!(**a, Expr::Str(_)) || matches!(**b, Expr::Str(_)) => {
2638            let sa = cmp_str(a, line);
2639            let sb = cmp_str(b, line);
2640            match op {
2641                BinOp::Eq => sa == sb,
2642                BinOp::Ne => sa != sb,
2643                BinOp::Lt => sa < sb,
2644                BinOp::Le => sa <= sb,
2645                BinOp::Gt => sa > sb,
2646                _ => sa >= sb,
2647            }
2648        }
2649        // A numeric subtree: evaluate it on fusevm as usual.
2650        _ => {
2651            let x = line.trim().parse::<f64>().unwrap_or(f64::NAN);
2652            let resolve = |n: &str| field_num(line, n);
2653            crate::expr::eval_pred_ctx(e, x, &resolve).unwrap_or(false)
2654        }
2655    }
2656}
2657
2658fn value_to_f64(v: &Value) -> f64 {
2659    match v {
2660        Value::Number(n) => n.as_f64().unwrap_or(f64::NAN),
2661        Value::String(s) => s.trim().parse().unwrap_or(f64::NAN),
2662        Value::Bool(b) => {
2663            if *b {
2664                1.0
2665            } else {
2666                0.0
2667            }
2668        }
2669        _ => f64::NAN,
2670    }
2671}
2672
2673/// Render a JSON scalar as a plain string; containers as compact JSON.
2674/// The first real element of a parsed fragment, skipping the synthetic
2675/// `html`/`head`/`body` wrappers that `Html::parse_fragment` inserts.
2676fn first_element(frag: &Html) -> Option<ElementRef<'_>> {
2677    let star = Selector::parse("*").ok()?;
2678    frag.select(&star)
2679        .find(|e| !matches!(e.value().name(), "html" | "head" | "body"))
2680}
2681
2682fn json_to_string(v: &Value) -> String {
2683    match v {
2684        Value::String(s) => s.clone(),
2685        Value::Null => String::new(),
2686        other => other.to_string(),
2687    }
2688}
2689
2690#[cfg(test)]
2691mod tests {
2692    use super::*;
2693
2694    #[test]
2695    fn split_plain() {
2696        assert_eq!(split_delim("a,b,c", ','), vec!["a", "b", "c"]);
2697    }
2698
2699    #[test]
2700    fn split_quoted_comma() {
2701        // A delimiter inside quotes is not a separator.
2702        assert_eq!(split_delim("\"a,b\",c", ','), vec!["a,b", "c"]);
2703    }
2704
2705    #[test]
2706    fn split_doubled_quote() {
2707        // "" inside a quoted field is one literal quote.
2708        assert_eq!(
2709            split_delim("\"she \"\"said\"\"\",x", ','),
2710            vec!["she \"said\"", "x"]
2711        );
2712    }
2713
2714    #[test]
2715    fn split_trailing_empty() {
2716        assert_eq!(split_delim("a,", ','), vec!["a", ""]);
2717    }
2718
2719    #[test]
2720    fn split_tsv() {
2721        assert_eq!(split_delim("a\tb\tc", '\t'), vec!["a", "b", "c"]);
2722    }
2723
2724    #[test]
2725    fn split_unquoted_trims_quoted_keeps_spaces() {
2726        // Unquoted fields are trimmed; a quoted field keeps its inner spaces.
2727        assert_eq!(split_delim(" a , b ", ','), vec!["a", "b"]);
2728        assert_eq!(split_delim("\" a \",b", ','), vec![" a ", "b"]);
2729    }
2730}
2731
2732thread_local! {
2733    /// Compiled jq programs, keyed by source. A `QueryOp::JqProgram` is
2734    /// re-evaluated on every stream tick (the TUI re-runs the pipeline as lines
2735    /// arrive), so without this the same program would be lexed and parsed once
2736    /// per frame. Thread-local because a compiled program holds `Rc`s.
2737    static JQ_PROGRAMS: std::cell::RefCell<
2738        std::collections::HashMap<String, std::rc::Rc<crate::jqlang::Program>>,
2739    > = std::cell::RefCell::new(std::collections::HashMap::new());
2740}
2741
2742/// Replace every JSON line of `cur` with the one value jq's builtin `name`
2743/// answers for it, through [`crate::jqlang`] so key order and number literals
2744/// are jq's. A line that is not JSON has no jq type and PASSES THROUGH, which
2745/// is the documented carve-out these stages always had. `Some` is the
2746/// refusal that ends the pipeline.
2747///
2748/// The answer is written as compact JSON, never `jq -r`'s raw string: these
2749/// stages run MID-pipeline, where `add`'s raw `ab` is not a JSON line, so the
2750/// next translated stage (`.[1:]`, `keys`, `.[0]`) took it for plain text and
2751/// passed it through instead of slicing or refusing it. `jq::translate`
2752/// appends the one final `JqRawString` that `jq -r` owes.
2753fn jq_builtin_lines(cur: &mut Vec<String>, name: &str) -> Option<QueryResult> {
2754    let prog = match jq_program(name) {
2755        Ok(p) => p,
2756        Err(msg) => return Some(QueryResult::Error(format!("jq: {msg}"))),
2757    };
2758    let interp = crate::jqlang::Interp::default();
2759    let mut out = Vec::with_capacity(cur.len());
2760    for l in cur.iter() {
2761        let Ok(v) = crate::jqlang::parse_json(l) else {
2762            out.push(l.clone());
2763            continue;
2764        };
2765        let r = prog.run_with(&interp, &v, &mut |r| {
2766            out.push(crate::jqlang::render(&r));
2767            Ok(())
2768        });
2769        if let Err(e) = r {
2770            return Some(QueryResult::Error(format!("jq: {}", e.to_message())));
2771        }
2772    }
2773    *cur = out;
2774    None
2775}
2776
2777/// Compile (or fetch from this thread's cache) the jq program `src`.
2778pub fn jq_program(src: &str) -> Result<std::rc::Rc<crate::jqlang::Program>, String> {
2779    JQ_PROGRAMS.with(|c| {
2780        if let Some(p) = c.borrow().get(src) {
2781            return Ok(p.clone());
2782        }
2783        let p = std::rc::Rc::new(crate::jqlang::Program::compile(src)?);
2784        c.borrow_mut().insert(src.to_string(), p.clone());
2785        Ok(p)
2786    })
2787}