arb/query.rs
1//! Query pipeline. A `source { … }` body compiles to a sequence of ops applied
2//! to the live stream on each evaluation.
3//!
4//! M2a verbs: `in` (source marker), `match`/`grep RE`, `reject`/`grepv RE`,
5//! `field N` (1-based whitespace column), `count`, `rate`. JSON/CSV field
6//! extraction, `where(pred)`, and aggregation to tables land with later verbs.
7
8use std::collections::{BTreeMap, HashSet};
9
10use base64::{engine::general_purpose::STANDARD, Engine as _};
11use percent_encoding::{percent_decode_str, utf8_percent_encode, NON_ALPHANUMERIC};
12use regex::Regex;
13use scraper::{ElementRef, Html, Selector};
14
15use serde_json::Value;
16
17use crate::expr::Expr;
18
19/// How `field` selects a value: a 1-based whitespace column, or a JSON key path.
20#[derive(Debug, Clone, PartialEq)]
21pub enum FieldSel {
22 Col(usize),
23 Key(Vec<String>),
24 /// A JSON key path reached through the jq front-end (`.a.b`, `.foo[0]`).
25 /// Identical to `Key` except for the miss: jq renders an explicit null, an
26 /// absent key and an out-of-range index all as the literal `null`, where
27 /// arb's native `field` yields "" and falls back to logfmt. Keeping the two
28 /// apart lets a jq path answer like jq without changing what `field NAME`
29 /// does to a plain-text or logfmt stream.
30 /// Segments are TYPED, unlike `Key`'s flat strings: `.["0"]` is an object
31 /// key and `.[0]` is an array index, and jq keeps them apart (`[1,2] |
32 /// .["0"]` is an error, not the first element).
33 JqKey(Vec<crate::jqval::Seg>),
34}
35
36/// The serializer an `out.FORMAT` verb selects. `yq`'s `-o=` in arb's spelling.
37#[derive(Debug, Clone, Copy, PartialEq, Eq)]
38pub enum OutFmt {
39 /// Block YAML with every node's comments, anchors, tags and quoting put
40 /// back — the rendering that makes `in.yaml; out.yaml` a round trip.
41 Yaml,
42 /// Indented JSON. `0` is jq's compact line, which is arb's default anyway.
43 Json,
44 /// `dotted.path = value` lines.
45 Props,
46 Xml,
47 Csv,
48 Tsv,
49}
50
51#[derive(Debug, Clone)]
52pub enum QueryOp {
53 /// A compiled XPath 1.0 expression (`crate::xpath`). The expression is
54 /// parsed at spec-build time, so this op always holds a well-formed one and
55 /// a malformed path is refused before any input is read.
56 XPath(Box<crate::xpath::XPath>),
57 /// Re-serialize the whole stream in another format (`out.yaml 4`). The
58 /// number is the indent, `yq`'s `-I`.
59 Emit(OutFmt, usize),
60 /// Keep lines matching the pattern.
61 Match(Regex),
62 /// Drop lines matching the pattern.
63 Reject(Regex),
64 /// Replace each line with a selected field (whitespace column or JSON key path).
65 Field(FieldSel),
66 /// Project multiple whitespace columns (1-based), space-joined, keeping the
67 /// given order — `fields 1 3` on "a b c d" -> "a c". For columnar input.
68 Fields(Vec<usize>),
69 /// Flatten JSON-array lines into one line per element (jq `[]`); non-array
70 /// lines pass through unchanged.
71 Each,
72 /// Keep lines whose numeric value (`x` = line parsed as a number) satisfies
73 /// the predicate — compiled to fusevm and evaluated per line.
74 Where(Expr),
75 /// Replace each line with the value of an expression (field-aware; `x` =
76 /// line-as-number), computed on the fusevm VM.
77 Map(Expr),
78 /// `via NAME [* N]` — fan the stream across a supervised pool of `N` copies
79 /// of actor `NAME` (default: one per hardware thread). Each line's scalar is
80 /// asked as the actor's first-handler message; the reply is the output line,
81 /// order preserved. A parallel actor map (see [`crate::actor::run_via`]).
82 Via(std::sync::Arc<crate::actor::ActorDef>, usize),
83 /// Reduce to the current line count.
84 Count,
85 /// Reduce to lines-per-second over the elapsed window.
86 Rate,
87 /// Group identical values and count them, sorted by count desc then key asc.
88 Tally,
89 /// Numeric reductions over lines parsed as numbers (non-numeric ignored).
90 Sum,
91 Min,
92 Max,
93 Avg,
94 /// Flatten a JSON object's keys / values into one line each.
95 Keys,
96 /// The jq front-end's `keys`. Same key set as `Keys`, but emitted as ONE
97 /// compact array line, which is what jq returns. The native `keys` verb keeps
98 /// its documented line-per-key shape (SPEC §8) — `stdlib/json.arb` runs
99 /// `keys; tally` over it — so the two are separate ops and only the jq
100 /// literal front-end (`. | keys`, `map(…)`) reaches this one.
101 JqKeys,
102 /// Native `vals` verb: expand a JSON object's values, one per line.
103 Vals,
104 /// jq `values` == `select(. != null)`: drop JSON-null lines, pass every other
105 /// line through unchanged. (NOT object-value iteration — that is `vals`/`.[]`.)
106 NonNull,
107 /// Render a PASSTHROUGH line the way `jq -r` prints it, for the one type where
108 /// the raw line and jq's rendering disagree: a top-level JSON STRING.
109 ///
110 /// Identity `.`, `select(…)` and `values` all emit the INPUT LINE verbatim.
111 /// For an object, array, number or boolean that is what jq does too — jq keeps
112 /// the source literal of any number it never computed (`1.50` stays `1.50`,
113 /// `[1,2]` stays `[1,2]`), so passing the line through matches it. A string is
114 /// the exception: `jq -r` strips the quotes and unescapes, so a line reading
115 /// `"hello"` must print `hello`. arb printed `"hello"` — and disagreed with
116 /// ITSELF, since `.[1:3]` on the same input already rendered `el` raw.
117 ///
118 /// Deliberately string-ONLY. Re-rendering the other types would lose more than
119 /// it gained: `serde_json::Map` is a `BTreeMap`, so a rebuilt object comes back
120 /// key-SORTED (`{"b":1,"a":2}` -> `{"a":2,"b":1}`) where jq preserves input
121 /// order, and every number would be reprinted through `fmt_num`, turning jq's
122 /// preserved `1.50` into `1.5`. Both would be new divergences in place of one.
123 JqRawString,
124 /// Project a JSON object down to the named keys (jq `{a,b,c}` /
125 /// `pick(.a,.b,.c)`), preserving the listed order. Non-object lines pass
126 /// through unchanged; missing keys are dropped.
127 Pick(Vec<String>),
128 /// Line-list transforms. `Sort` supports numeric (`-n`) and reverse (`-r`).
129 Sort {
130 numeric: bool,
131 reverse: bool,
132 },
133 Uniq,
134 Rev,
135 First,
136 Last,
137 Take(usize),
138 Drop(usize),
139 /// Per-line string transforms.
140 Upper,
141 Lower,
142 Trim,
143 /// Regex replace-all per line (`replace /RE/ TO`; TO may use `$1` captures).
144 Replace(Regex, String),
145 /// Collapse all lines into one, joined by a separator.
146 Join(String),
147 /// Keep only the Nth line (1-based).
148 Nth(usize),
149 /// Parse the accumulated stream as one HTML document and emit, per element
150 /// matching the CSS selector, its text (or a named attribute if `attr` is
151 /// set; elements lacking that attribute are dropped).
152 Sel {
153 css: String,
154 attr: Option<String>,
155 },
156 /// Recursive descent (xpath `//TAG`): parse the accumulated stream as one
157 /// HTML document and emit the outer HTML of every element matching the
158 /// selector, one per line — so `attr`/`text` can then pull from each.
159 Find(String),
160 /// From element fragments (one per line, as emitted by `find`), emit the
161 /// named attribute of each (xpath `@NAME`, css attr); drop lines whose first
162 /// element lacks it.
163 Attr(String),
164 /// From element fragments, emit each element's inner text; non-element lines
165 /// pass through unchanged.
166 Text,
167 /// Stable-sort JSON-record lines by FIELD; numeric if every field value
168 /// parses as a number, else lexicographic on the field's string value.
169 /// Non-object lines sink after the sorted records in input order.
170 SortBy(String),
171 /// Keep the first record for each distinct value of FIELD, preserving input
172 /// order. JSON-object lines dedup by that field's value; other lines dedup by
173 /// the whole line.
174 UniqueBy(String),
175 /// Group JSON records by the value of FIELD and count each group, returning
176 /// value -> count pairs sorted by count desc then value asc; non-object lines
177 /// are counted under their whole-line text. Reducer (early return).
178 CountBy(String),
179 /// Group lines by the value of FIELD (jq `group_by`): emit one JSON-array
180 /// line per distinct value, each array holding that group's members in input
181 /// order; groups are ordered by key ascending. Object lines group by the
182 /// field's value, other lines by their whole-line text.
183 GroupBy(String),
184 /// Reducer: emit the single JSON record whose numeric FIELD is smallest (records with a missing/non-numeric FIELD are ignored; empty input yields no lines).
185 MinBy(String),
186 /// Reducer: emit the single record whose numeric FIELD is the largest.
187 MaxBy(String),
188 /// Native `has KEY` verb: retain only JSON-object lines that contain KEY; all
189 /// other lines (missing key, non-object, unparseable) are dropped.
190 Has(String),
191 /// jq `has(KEY)`: emit `true`/`false` per input line. The argument is a jq
192 /// VALUE, not a name — jq accepts a string key on an object and a numeric
193 /// index on an array, and refuses every other pairing by name ("Cannot check
194 /// whether array has a string key") rather than answering `false`.
195 HasKey(Value),
196 /// jq to_entries: expand each JSON object line into one `{"key":<k>,"value":<v>}` line per key (BTreeMap key order); non-object lines pass through.
197 Entries,
198 /// The jq front-end's `to_entries`. Same mapping as `Entries`, but emits the
199 /// whole result as ONE compact array line, which is what jq returns. Native
200 /// `entries` keeps its documented line-per-key shape (SPEC §8) so specs built
201 /// on it are unaffected; the two spellings are distinct, so they can differ.
202 JqEntries,
203 /// The jq front-end's `flatten`. Same full-depth leaf walk as `Flatten`, but
204 /// emits them as ONE compact array line, which is what jq returns. Native
205 /// `flatten` keeps its line-per-leaf shape — same context gating as
206 /// `JqKeys`/`JqEntries`.
207 JqFlatten,
208 /// jq `map(f)` == `[.[] | f]`: run the inner ops over the ELEMENTS of each
209 /// input line and re-wrap the results as one compact array line — jq's shape.
210 /// Scoped per input line, so two input lines stay two output arrays. (Iterate
211 /// WITHOUT the re-wrap is jq's own `.[] | f`, which arb also accepts.)
212 JqMap(Vec<QueryOp>),
213 /// The jq front-end's `.[]`. Same expansion as `Each`, but a line that parses
214 /// to a NON-iterable JSON value (a scalar or `null`) is an error rather than a
215 /// pass-through — jq raises "Cannot iterate over null (null)" and SPEC §8
216 /// forbids answering where jq refuses. Native `each` keeps the pass-through,
217 /// because it runs over plain-text streams too.
218 JqEach,
219 /// The jq front-end's `add` — `reduce .[] as $x (null; . + $x)`, so an EMPTY
220 /// array is `null` (native `add` documents `[] -> ""`) and a mixed-type array
221 /// raises the same error `+` would. Iterates an object's values.
222 JqAdd,
223 /// The jq front-end's `length`. Same JSON-aware measure as `JsonLen`, except a
224 /// boolean is an error ("boolean (true) has no length") instead of falling back
225 /// to the raw line's character count.
226 JqLen,
227 /// A COMPLETE jq program, run per line by [`crate::jqlang`].
228 ///
229 /// The ops above are arb's line-stream translations of the jq subset they
230 /// cover, and they keep arb's own promises (identity passes the SOURCE line
231 /// through, so its spacing and number literals survive). They cannot express
232 /// jq's generator semantics — `.a, .b`, `reduce`, `..`, `try`, assignment —
233 /// because a `Vec<QueryOp>` is a stage list, not a language. Anything
234 /// outside that subset compiles to a real jq program instead and runs on
235 /// arb's own jq engine, which is why the constructs SPEC §8 used to list as
236 /// refusals now answer exactly as `jq` does.
237 ///
238 /// The payload is the SOURCE, not a compiled program: `QueryOp` is cloned
239 /// into specs, actors and the web target, so it must stay `Send`-compatible
240 /// plain data. Compilation is memoized per thread by [`jq_program`], so a
241 /// per-line pipeline still parses the program exactly once.
242 JqProgram(String),
243 /// jq `select(f)` over jq VALUE semantics: keep the line when `f` is truthy,
244 /// where only `false` and `null` are falsy. Distinct from native `Where`,
245 /// which evaluates an f64 predicate over a numeric line stream.
246 JqSelect(crate::jqval::Expr),
247 /// A jq value expression stage (`.a + .b`, `. * 2`, `. > 1`). Distinct from
248 /// native `Map`, which computes an f64: this yields a jq VALUE, so a compare
249 /// renders `true`/`false` and a string `+` concatenates.
250 JqCalc(crate::jqval::Expr),
251 /// jq `flatten`: recursively flatten a JSON-array line to its non-array leaves,
252 /// emitting one leaf per line (matching jq's full-depth flatten, unlike `each`
253 /// which descends a single level). Non-array lines pass through unchanged.
254 Flatten,
255 /// jq `add`: reduce a JSON array line to a single value — sum numeric
256 /// arrays (fmt_num), concatenate non-numeric arrays via their string
257 /// values, empty array -> "". Non-array lines pass through unchanged.
258 Add,
259 /// Keep only lines that parse as a number strictly greater than `N`.
260 /// Lines that do not parse as `f64` are dropped.
261 Over(f64),
262 /// Keep only numeric lines whose value is strictly less than N; drop non-numeric lines.
263 Under(f64),
264 /// Keep numeric lines x where lo <= x <= hi (inclusive); non-numeric lines are dropped.
265 Between(f64, f64),
266 /// Prefix each line with its 1-based index and a tab: `"1\t<line>"`.
267 Enumerate,
268 /// Split each line on whitespace and emit one word per line (flatten); empty lines produce nothing.
269 Words,
270 /// Collapse runs of adjacent identical lines to a single line (classic uniq), leaving non-adjacent repeats intact.
271 Dedup,
272 /// Keep only the last N lines (complement of `take`, which keeps the first N). N>=len keeps all.
273 Tailn(usize),
274 /// Right-pad each line with spaces to a minimum visible width N (no truncation if the line is already longer).
275 Pad(usize),
276 /// Left-pad each line with spaces to a minimum width of N (lines already >= N are unchanged).
277 Lpad(usize),
278 /// Retain lines whose FIELD (json key or 1-based whitespace column) matches the regex.
279 Grepf(String, regex::Regex),
280 /// Reverse the Unicode scalar characters of each line (chars().rev()).
281 Flip,
282 /// Path basename: the part after the last `/` (the whole line if none).
283 Basename,
284 /// Path dirname: the part before the last `/` (`.` if none).
285 Dirname,
286 /// Group a numeric line's integer part with thousands separators
287 /// (`1234567` -> `1,234,567`); non-numeric lines pass through.
288 Commafy,
289 /// Humanize a byte count (1024-based): `1536` -> `1.5 KB`; non-numeric passes through.
290 Bytes,
291 /// Humanize a duration in seconds: `3661` -> `1h 1m`; non-numeric passes through.
292 Duration,
293 /// Placeholder for `apply .name`: at render time it is replaced by the query
294 /// pipeline typed into the `input .name` widget (the megafilter/map binding).
295 /// Left in a pipeline unsubstituted it is a no-op.
296 Apply(String),
297 /// `in.json`: regroup the line stream into one item per JSON document — a
298 /// pretty-printed document spanning lines becomes one compact line, and
299 /// several values on one line become one each, as jq reads its input. A
300 /// line that is no part of a document stays the line it was. See
301 /// [`crate::jsondocs`].
302 JsonDocs,
303 /// Treat the stream as CSV: the first line is the header; each data row
304 /// becomes a JSON object keyed by the header, so `field NAME` works.
305 Csv,
306 B64,
307 /// Base64-decode each line (STANDARD alphabet) into a UTF-8 string; lines that fail to
308 /// base64-decode or whose bytes aren't valid UTF-8 pass through unchanged.
309 B64d,
310 /// Lowercase hex-encode each line, two hex digits per UTF-8 byte.
311 Hex,
312 /// Decode a hex string to UTF-8 text by parsing byte pairs; on any error (odd length, non-hex digit, invalid UTF-8) the line is left unchanged.
313 Unhex,
314 /// Percent-encode each line, escaping every non-alphanumeric byte (RFC 3986 style).
315 Urlenc,
316 /// Percent-decode each line to UTF-8 (utf8_percent_decode); lines whose decoded bytes are not valid UTF-8 pass through unchanged.
317 Urldec,
318 /// Emit the first regex match per line (capture group 1 if the pattern captures, else the whole match); drop lines with no match.
319 Extract(Regex),
320 /// Explode each line by the literal string DELIM into multiple lines (one part per line).
321 /// One-to-many: unlike `cut` (one field) or `words` (whitespace), every split segment becomes its own line.
322 Split(String),
323 /// Character substring [A,B) 0-based, clamped to the line length (B may exceed len; A>B yields empty).
324 Substr(usize, usize),
325 /// Explode each line into one output line per Unicode scalar (character); one line -> many.
326 Chars,
327 /// Title-case each line: uppercase the first letter of each whitespace-separated word, lowercase the rest, rejoin with single spaces.
328 Title,
329 /// Replace each line with its content repeated N times, concatenated.
330 Repeat(usize),
331 /// Set key K to the string value V (Value::String(V)) in each JSON object line; non-object / unparseable lines pass through unchanged.
332 Set(String, String),
333 /// Remove key K from each JSON object line (jq `del(.K)`); non-object lines pass through.
334 Del(String),
335 /// Rename JSON object key OLD to NEW in each object, preserving the value; no-op if OLD absent. Non-object lines pass through.
336 Rename(String, String),
337 /// Set string key K to V only when K is absent from the JSON object (jq `//=` for a missing key). Present keys keep their value; non-object / unparseable lines pass through unchanged. Key order is normalized on mutation.
338 Default(String, String),
339 /// Reduce all JSON object lines into a single object (later keys overwrite earlier); non-object lines are ignored. Emits one JSON object line, or none if no objects were seen.
340 Merge,
341 /// Floor each numeric line to the nearest lower integer (fmt_num(x.floor())); non-numeric lines pass through unchanged.
342 Floor,
343 /// Round each numeric line up to the nearest integer (ceil); non-numeric lines pass through unchanged.
344 Ceil,
345 /// Clamp each numeric line into the inclusive range [LO, HI]; non-numeric lines pass through unchanged.
346 Clamp(f64, f64),
347 /// Same as `Csv` but tab-separated (TSV).
348 Tsv,
349 /// Parse the accumulated stream as a YAML document (or `---`-separated
350 /// multi-document) and emit each document as a JSON line, so the JSON verbs
351 /// (`field`/`pick`/`keys`/`each`) work over it (the yq leg).
352 Yaml,
353 /// Parse the accumulated stream as one TOML document and emit it as a JSON
354 /// object line.
355 Toml,
356 /// Reduce to a scalar computed by an arithmetic expression over the current
357 /// line count (`x`), evaluated on the fusevm VM.
358 Calc(Expr),
359 /// keep lines containing a literal substring.
360 Contains(String),
361 /// keep lines starting with a literal prefix.
362 Starts(String),
363 /// keep lines ending with a literal suffix.
364 Ends(String),
365 /// drop empty / whitespace-only lines.
366 Nonempty,
367 /// keep only lines that parse as a number.
368 Numeric,
369 /// replace each line with its character count.
370 Len,
371 /// jq `length` (JSON-aware): array element count / object key count / string
372 /// char count / |number| / null=0; a non-JSON line falls back to char count.
373 JsonLen,
374 /// jq slice `.[a:b]` over a JSON array line (a/b may be negative — counted
375 /// from the end — or omitted); a string line is char-sliced; other lines pass
376 /// through unchanged.
377 JsonSlice(Option<i64>, Option<i64>),
378 /// replace each line with its word count.
379 Wc,
380 /// absolute value of each numeric line.
381 Abs,
382 /// round each numeric line to the nearest integer.
383 Round,
384 /// consecutive differences of the numeric series (n values → n-1 deltas) —
385 /// turns a monotonic counter into a per-step rate-of-change.
386 Delta,
387 /// running (cumulative) total of the numeric series.
388 Cumsum,
389 /// simple moving average over a trailing window of N (length-preserving; the
390 /// first points average a shorter, growing window). Smooths a noisy series.
391 Sma(usize),
392 /// exponentially-weighted moving average, smoothing factor `alpha` in (0,1] —
393 /// higher alpha tracks faster, lower is smoother. `s0 = x0`.
394 Ewma(f64),
395 /// prefix every line with a literal string.
396 Prepend(String),
397 /// suffix every line with a literal string.
398 Append(String),
399 /// split each line by DELIM, keep the Nth (1-based) field.
400 Cut(String, usize),
401 /// median of numeric lines.
402 Median,
403 /// Nth percentile (0–100) of the numeric values, linear interpolation between
404 /// closest ranks (numpy default). `percentile 99` / `p99` for latency tails.
405 Percentile(f64),
406 /// population standard deviation of numeric lines.
407 Stddev,
408 /// max minus min of numeric lines.
409 Range,
410 /// product of numeric lines.
411 Product,
412 /// count of distinct lines.
413 Distinct,
414 /// keep every Nth line (1-based).
415 Sample(usize),
416 /// keep lines from index A to B inclusive (1-based).
417 Slice(usize, usize),
418 /// keep only the Nth line (1-based); out-of-range yields no lines.
419 Index(usize),
420 /// bucket numeric lines into N equal-width ranges -> (range, count) pairs.
421 Bins(usize),
422}
423
424/// The output of evaluating a pipeline: lines, a scalar, grouped counts — or a
425/// runtime REFUSAL.
426///
427/// `Error` is the per-line error channel the jq front-end needs. SPEC §8 promises
428/// that a construct outside the documented subset "is a hard error … never
429/// silently reinterpreted", and a type mismatch (`{"a":1} | . + 3`, `null | .[]`,
430/// `true | length`) is only discoverable once the DATA arrives. Without this
431/// variant every such case had to answer something — `null`, the raw line, the
432/// line's character count — which is the exact failure the SPEC rules out, and
433/// which several comments in this file used to record as "arb has no per-line
434/// error channel".
435///
436/// Only the jq front-end raises it. Native verbs run over plain-text streams and
437/// keep their documented pass-through behaviour.
438#[derive(Debug, Clone, PartialEq)]
439pub enum QueryResult {
440 Lines(Vec<String>),
441 Scalar(f64),
442 Pairs(Vec<(String, u64)>),
443 /// A refused pipeline. The message is already `jq: …`-anchored.
444 Error(String),
445}
446
447/// What a jq program reported besides its output lines — the part of jq's
448/// `process()` (main.c) that is not stdout.
449///
450/// jq does not stop at an uncaught error: the values the input produced BEFORE
451/// it are printed, the error goes to stderr, and the next input runs. The exit
452/// status is the LAST input's (5 when it raised, else 0). `halt`/`halt_error`
453/// end the whole run with their own status and stderr text instead.
454#[derive(Debug, Default, Clone, PartialEq)]
455pub struct JqReport {
456 /// Every uncaught error, in input order, already `jq: …`-anchored.
457 pub errors: Vec<String>,
458 /// The last input a jq program ran on raised, so jq would exit 5.
459 pub last_failed: bool,
460 /// `halt`/`halt_error`: the exit status and the exact stderr text (no
461 /// prefix; jq adds a newline only after a non-string message).
462 pub halt: Option<(i32, String)>,
463}
464
465/// An uncaught jq error as jq's main.c words it, minus the `error (at …)`
466/// position: a string message as-is, anything else as `(not a string): JSON`.
467fn uncaught(e: &crate::jqlang::JqErr) -> String {
468 match e {
469 crate::jqlang::JqErr::Err(crate::jqlang::JqVal::Str(s)) => format!("jq: {s}"),
470 crate::jqlang::JqErr::Err(v) => {
471 format!("jq: (not a string): {}", crate::jqlang::render(v))
472 }
473 other => format!("jq: {}", other.to_message()),
474 }
475}
476
477/// The stderr text of `halt_error`, transcribed from main.c: a string is written
478/// raw with no newline, `null` writes nothing, anything else is its compact JSON
479/// plus a newline.
480fn halt_text(msg: Option<&crate::jqlang::JqVal>) -> String {
481 match msg {
482 None | Some(crate::jqlang::JqVal::Null) => String::new(),
483 Some(crate::jqlang::JqVal::Str(s)) => s.to_string(),
484 Some(v) => format!("{}\n", crate::jqlang::render(v)),
485 }
486}
487
488/// Fold one input's run into `rep` the way jq's main loop does. `true` means
489/// the program halted, which ends the whole run, not just this input.
490fn record_run(rep: &mut JqReport, r: Result<(), crate::jqlang::JqErr>) -> bool {
491 match r {
492 Ok(()) => {
493 rep.last_failed = false;
494 false
495 }
496 Err(crate::jqlang::JqErr::Halt(code, msg)) => {
497 rep.halt = Some((code, halt_text(msg.as_ref())));
498 true
499 }
500 Err(e) => {
501 rep.errors.push(uncaught(&e));
502 rep.last_failed = true;
503 false
504 }
505 }
506}
507
508/// Evaluate `ops` against `lines`. `elapsed_secs` feeds `rate`.
509///
510/// A jq program's uncaught error REFUSES the pipeline here: the TUI, the REPL,
511/// `serve` and `--test` show a refusal rather than a partial answer. The `out`
512/// filter path wants jq's stream semantics instead and calls [`eval_reporting`].
513pub fn eval(ops: &[QueryOp], lines: &[String], elapsed_secs: f64) -> QueryResult {
514 let mut rep = JqReport::default();
515 let r = eval_reporting(ops, lines, elapsed_secs, 1, &mut rep);
516 match (rep.errors.into_iter().next(), rep.halt) {
517 (Some(e), _) => QueryResult::Error(e),
518 (None, Some((_, msg))) if !msg.is_empty() => {
519 QueryResult::Error(format!("jq: {}", msg.trim_end_matches('\n')))
520 }
521 _ => r,
522 }
523}
524
525/// [`eval`] with jq's uncaught-error model: a jq program's errors and `halt`
526/// land in `rep` and the lines produced before them are kept. `first_line` is
527/// the 1-based number of `lines[0]` in the whole input, so a caller that feeds
528/// one line at a time still gets jq's `input_line_number`.
529pub fn eval_reporting(
530 ops: &[QueryOp],
531 lines: &[String],
532 elapsed_secs: f64,
533 first_line: usize,
534 rep: &mut JqReport,
535) -> QueryResult {
536 let mut cur: Vec<String> = lines.to_vec();
537 // The YAML NODE channel. `cur` is text, so a YAML node's comments, anchors
538 // and quoting cannot survive it — this carries the composed documents beside
539 // it so `in.yaml`, a jq program and `out.yaml` can hand nodes to each other
540 // without a JSON round trip in between.
541 //
542 // It is invalidated by any op that is not one of those three, in ONE place
543 // rather than at every arm, so an op added later drops the metadata (which
544 // is merely a lost comment) instead of answering from a stale one (which
545 // would be a wrong answer).
546 let mut nodes: Option<Vec<crate::jqlang::JqVal>> = None;
547 for op in ops {
548 if !matches!(
549 op,
550 QueryOp::Yaml | QueryOp::JqProgram(_) | QueryOp::Emit(..)
551 ) {
552 nodes = None;
553 }
554 match op {
555 QueryOp::Emit(fmt, indent) => {
556 let docs = match nodes.clone() {
557 Some(d) => d,
558 // Not YAML in: every line is its own document, read through
559 // the jq model so a non-JSON line stays a string.
560 None => cur
561 .iter()
562 .map(|l| {
563 crate::jqlang::parse_json(l)
564 .unwrap_or_else(|_| crate::jqlang::JqVal::str(l.as_str()))
565 })
566 .collect(),
567 };
568 cur = emit_stream(&docs, *fmt, *indent);
569 }
570 QueryOp::Match(re) => cur.retain(|l| re.is_match(l)),
571 QueryOp::Reject(re) => cur.retain(|l| !re.is_match(l)),
572 QueryOp::Field(sel) => {
573 if let FieldSel::JqKey(segs) = sel {
574 for l in cur.iter_mut() {
575 // Through the jq value model, so an object this returns
576 // keeps the document's KEY ORDER — `serde_json::Map` is a
577 // `BTreeMap` and gave `.nested` back alphabetised, which
578 // `yq -o=json` on the same YAML does not.
579 match crate::jqlang::parse_json(l).ok() {
580 // A line that is not JSON at all has no jq TYPE to
581 // check, so it keeps the documented miss rendering.
582 None => *l = "null".to_string(),
583 Some(v) => match crate::jqlang::get_seg_path(&v, segs) {
584 Ok(r) => *l = crate::jqlang::render_raw(&r),
585 Err(e) => return QueryResult::Error(format!("jq: {e}")),
586 },
587 }
588 }
589 continue;
590 }
591 for l in cur.iter_mut() {
592 *l = extract_field(l, sel);
593 }
594 }
595 QueryOp::Fields(cols) => {
596 for l in cur.iter_mut() {
597 *l = cols
598 .iter()
599 .map(|&n| nth_col(l, n))
600 .collect::<Vec<_>>()
601 .join(" ");
602 }
603 }
604 QueryOp::JqEach => {
605 let mut out = Vec::with_capacity(cur.len());
606 // Through the jq value model, so each element keeps its source
607 // literal (`[1.50,1E1000]` iterates as `1.50`, `1E+1000`, as jq
608 // prints them) and an object yields its values in KEY ORDER.
609 for l in &cur {
610 use crate::jqlang::JqVal;
611 match crate::jqlang::parse_json(l) {
612 Ok(v) => match v.bare() {
613 JqVal::Arr(a) => out.extend(a.iter().map(crate::jqlang::render_raw)),
614 JqVal::Obj(m) => {
615 out.extend(m.iter().map(|(_, e)| crate::jqlang::render_raw(e)))
616 }
617 other => {
618 return QueryResult::Error(format!(
619 "jq: Cannot iterate over {}{}",
620 other.type_name(),
621 crate::jqlang::paren_of(other)
622 ))
623 }
624 },
625 Err(_) => out.push(l.clone()),
626 }
627 }
628 cur = out;
629 }
630 QueryOp::JqProgram(src) => {
631 let prog = match jq_program(src) {
632 Ok(p) => p,
633 Err(msg) => return QueryResult::Error(format!("jq: {msg}")),
634 };
635 let interp = crate::jqlang::Interp::default();
636 let mut out = Vec::with_capacity(cur.len());
637 // ONE cursor over the stream, which is jq's model: the next
638 // document is `.`, and `input` takes the one after it — so a
639 // document `input` consumed is not replayed by this loop.
640 //
641 // The first shape of this was per-line: it handed every
642 // iteration a freshly parsed copy of the whole REMAINING stream
643 // so `inputs` would see it. That is O(n^2) parses, and it showed
644 // — `{n: .name, v: .v}` over 2,000 records took 9.7s against
645 // jq's 0.01s. Lines are moved in raw and parsed once, on the way
646 // out.
647 // The YAML path: run over the composed NODES so a program can
648 // read and write comments, anchors, tags and styles. Only taken
649 // when the program does not pull from the input STREAM —
650 // `input`/`inputs` share one cursor over raw text lines, and
651 // splitting that cursor across two representations would let a
652 // document be seen twice.
653 if let Some(docs) = nodes.take().filter(|_| !prog.reads_input_stream()) {
654 let mut next: Vec<crate::jqlang::JqVal> = Vec::new();
655 for (i, doc) in docs.iter().enumerate() {
656 interp.set_line(i + 1);
657 interp.set_doc(doc);
658 let r = prog.run_with(&interp, doc, &mut |v| {
659 out.push(crate::jqlang::render_raw(&v));
660 next.push(v);
661 Ok(())
662 });
663 if record_run(rep, r) {
664 break;
665 }
666 }
667 nodes = Some(next);
668 cur = out;
669 continue;
670 }
671 interp.set_input_lines(std::mem::take(&mut cur));
672 interp.set_line(first_line.saturating_sub(1));
673 while let Some(input) = interp.next_input() {
674 interp.set_doc(&input);
675 let r = prog.run_with(&interp, &input, &mut |v| {
676 out.push(crate::jqlang::render_raw(&v));
677 Ok(())
678 });
679 if record_run(rep, r) {
680 break;
681 }
682 }
683 cur = out;
684 }
685 QueryOp::JqSelect(e) => {
686 let mut out = Vec::with_capacity(cur.len());
687 for l in &cur {
688 match crate::jqval::eval(e, &jq_scalar(l)) {
689 Ok(v) => {
690 if crate::jqval::truthy(&v) {
691 out.push(l.clone());
692 }
693 }
694 Err(msg) => return QueryResult::Error(format!("jq: {msg}")),
695 }
696 }
697 cur = out;
698 }
699 QueryOp::JqCalc(e) => {
700 for l in cur.iter_mut() {
701 match crate::jqval::eval(e, &jq_scalar(l)) {
702 Ok(v) => *l = crate::jqval::render(&v),
703 Err(msg) => return QueryResult::Error(format!("jq: {msg}")),
704 }
705 }
706 }
707 QueryOp::JqAdd => {
708 // jq `add` is `reduce .[] as $x (null; . + $x)`: it starts from
709 // null (so `[]` is `null`, not ""), folds with jq's `+` (so
710 // `[1,"a"]` raises), iterates an object's values, and keeps a
711 // lone element's literal (`[1.50] | add` is `1.50`).
712 if let Some(e) = jq_builtin_lines(&mut cur, "add") {
713 return e;
714 }
715 }
716 QueryOp::JqLen => {
717 for l in cur.iter_mut() {
718 *l = match jq_value(l) {
719 Some(Value::Array(a)) => a.len().to_string(),
720 Some(Value::Object(m)) => m.len().to_string(),
721 Some(Value::String(s)) => s.chars().count().to_string(),
722 // jq's `length` on a number is its ABSOLUTE value.
723 Some(Value::Number(n)) => fmt_num(n.as_f64().unwrap_or(0.0).abs()),
724 Some(Value::Null) => "0".to_string(),
725 Some(v @ Value::Bool(_)) => {
726 return QueryResult::Error(format!(
727 "jq: {} ({v}) has no length",
728 crate::jqval::tname(&v)
729 ));
730 }
731 None => l.chars().count().to_string(),
732 };
733 }
734 }
735 QueryOp::Each => {
736 let mut out = Vec::with_capacity(cur.len());
737 for l in &cur {
738 match serde_json::from_str::<Value>(l) {
739 // `each` IS jq's `.[]` (SPEC §8), so a null ELEMENT keeps
740 // jq's rendering — dropping it to "" loses the element's
741 // existence, which is what `[1,null,2]` must not do.
742 Ok(Value::Array(arr)) => {
743 out.extend(arr.iter().map(jq_to_string));
744 }
745 // jq `.[]` over an object iterates its VALUES.
746 Ok(Value::Object(m)) => {
747 out.extend(m.values().map(jq_to_string));
748 }
749 _ => out.push(l.clone()),
750 }
751 }
752 cur = out;
753 }
754 QueryOp::Where(e) => {
755 // A string predicate (`match(.q)` / `field in .lv`) can't run on
756 // the numeric VM — route it to the Rust evaluator; purely numeric
757 // predicates stay on fusevm.
758 if crate::expr::expr_has_str(e) {
759 cur.retain(|l| eval_where(e, l));
760 } else {
761 cur.retain(|l| {
762 let x = l.trim().parse::<f64>().unwrap_or(f64::NAN);
763 let resolve = |name: &str| field_num(l, name);
764 crate::expr::eval_pred_ctx(e, x, &resolve).unwrap_or(false)
765 });
766 }
767 }
768 QueryOp::Map(e) => {
769 for l in cur.iter_mut() {
770 let v = {
771 let x = l.trim().parse::<f64>().unwrap_or(f64::NAN);
772 let resolve = |name: &str| field_num(l, name);
773 crate::expr::eval_ctx(e, x, &resolve).unwrap_or(f64::NAN)
774 };
775 *l = fmt_num(v);
776 }
777 }
778 QueryOp::Via(def, workers) => {
779 cur = crate::actor::run_via(def, *workers, &cur);
780 }
781 QueryOp::Count => return QueryResult::Scalar(cur.len() as f64),
782 QueryOp::Rate => {
783 return QueryResult::Scalar(cur.len() as f64 / elapsed_secs.max(0.001));
784 }
785 QueryOp::Tally => {
786 let mut counts: BTreeMap<String, u64> = BTreeMap::new();
787 for l in &cur {
788 *counts.entry(l.clone()).or_insert(0) += 1;
789 }
790 let mut pairs: Vec<(String, u64)> = counts.into_iter().collect();
791 pairs.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
792 return QueryResult::Pairs(pairs);
793 }
794 QueryOp::Calc(e) => {
795 let x = cur.len() as f64;
796 return QueryResult::Scalar(crate::expr::eval(e, x).unwrap_or(0.0));
797 }
798 QueryOp::Sum => return QueryResult::Scalar(nums(&cur).iter().sum()),
799 QueryOp::Min => {
800 let m = nums(&cur).into_iter().fold(f64::INFINITY, f64::min);
801 return QueryResult::Scalar(if m.is_finite() { m } else { 0.0 });
802 }
803 QueryOp::Max => {
804 let m = nums(&cur).into_iter().fold(f64::NEG_INFINITY, f64::max);
805 return QueryResult::Scalar(if m.is_finite() { m } else { 0.0 });
806 }
807 QueryOp::Avg => {
808 let ns = nums(&cur);
809 let a = if ns.is_empty() {
810 0.0
811 } else {
812 ns.iter().sum::<f64>() / ns.len() as f64
813 };
814 return QueryResult::Scalar(a);
815 }
816 QueryOp::Keys => {
817 let mut out = Vec::new();
818 for l in &cur {
819 match serde_json::from_str::<Value>(l) {
820 Ok(Value::Object(m)) => out.extend(m.keys().cloned()),
821 _ => out.push(l.clone()),
822 }
823 }
824 cur = out;
825 }
826 QueryOp::JqKeys => {
827 for l in cur.iter_mut() {
828 // jq `keys` sorts; serde_json's Map is a BTreeMap here, so
829 // iteration is already sorted. An ARRAY's keys are its indices.
830 match serde_json::from_str::<Value>(l) {
831 Ok(Value::Object(m)) => {
832 let arr: Vec<Value> =
833 m.keys().map(|k| Value::String(k.clone())).collect();
834 *l = Value::Array(arr).to_string();
835 }
836 Ok(Value::Array(a)) => {
837 let arr: Vec<Value> = (0..a.len()).map(Value::from).collect();
838 *l = Value::Array(arr).to_string();
839 }
840 // A scalar (or null) has no keys and jq says so by name.
841 Ok(v) => return QueryResult::Error(has_no_keys(&v)),
842 Err(_) => {}
843 }
844 }
845 }
846 QueryOp::Vals => {
847 let mut out = Vec::new();
848 for l in &cur {
849 match serde_json::from_str::<Value>(l) {
850 Ok(Value::Object(m)) => out.extend(m.values().map(json_to_string)),
851 _ => out.push(l.clone()),
852 }
853 }
854 cur = out;
855 }
856 QueryOp::NonNull => {
857 // jq `values` == `select(. != null)`: keep every non-null input
858 // unchanged, drop only the lines that parse to JSON `null`.
859 cur.retain(|l| !matches!(serde_json::from_str::<Value>(l), Ok(Value::Null)));
860 }
861 QueryOp::JqRawString => {
862 for l in cur.iter_mut() {
863 // Re-render through the jq value model, which is exactly what
864 // `jq -rc` prints: a top-level string goes out RAW, and every
865 // other value comes back COMPACT. A line that is not JSON at
866 // all has no jq rendering and passes through as written.
867 if let Ok(v) = crate::jqlang::parse_json(l) {
868 *l = crate::jqlang::render_raw(&v);
869 }
870 }
871 }
872 QueryOp::Pick(keys) => {
873 for l in cur.iter_mut() {
874 if let Ok(Value::Object(m)) = serde_json::from_str::<Value>(l) {
875 // Build in pick order — serde_json::Map is a BTreeMap and
876 // would re-sort the keys, losing the requested order.
877 let parts: Vec<String> = keys
878 .iter()
879 .filter_map(|k| {
880 m.get(k)
881 .map(|v| format!("{}:{}", Value::String(k.clone()), v))
882 })
883 .collect();
884 *l = format!("{{{}}}", parts.join(","));
885 }
886 }
887 }
888 QueryOp::JsonDocs => cur = crate::jsondocs::documents(&cur),
889 QueryOp::Csv => cur = to_json_records(&cur, ','),
890 QueryOp::Tsv => cur = to_json_records(&cur, '\t'),
891 QueryOp::Yaml => {
892 let docs = crate::yaml::documents(&cur.join("\n"));
893 cur = docs.iter().map(crate::jqlang::render).collect();
894 nodes = Some(docs);
895 }
896 QueryOp::Toml => cur = toml_to_json(&cur),
897 QueryOp::Sort { numeric, reverse } => {
898 if *numeric {
899 // Like Unix `sort -n`: order by each line's LEADING numeric
900 // token (the first whitespace-delimited field), so mixed rows
901 // such as `2.1 claude` sort by 2.1 — not the whole line, which
902 // would parse as NaN and leave the order untouched.
903 let key = |s: &str| {
904 s.split_whitespace()
905 .next()
906 .and_then(|t| t.parse::<f64>().ok())
907 .unwrap_or(f64::NAN)
908 };
909 cur.sort_by(|a, b| {
910 key(a)
911 .partial_cmp(&key(b))
912 .unwrap_or(std::cmp::Ordering::Equal)
913 });
914 } else {
915 cur.sort();
916 }
917 if *reverse {
918 cur.reverse();
919 }
920 }
921 QueryOp::Uniq => {
922 let mut seen = HashSet::new();
923 cur.retain(|l| seen.insert(l.clone()));
924 }
925 QueryOp::Rev => cur.reverse(),
926 QueryOp::First => cur.truncate(1),
927 QueryOp::Last => {
928 if let Some(l) = cur.pop() {
929 cur = vec![l];
930 } else {
931 cur.clear();
932 }
933 }
934 QueryOp::Take(n) => cur.truncate(*n),
935 QueryOp::Drop(n) => {
936 cur.drain(0..(*n).min(cur.len()));
937 }
938 QueryOp::Upper => {
939 for l in cur.iter_mut() {
940 *l = l.to_uppercase();
941 }
942 }
943 QueryOp::Lower => {
944 for l in cur.iter_mut() {
945 *l = l.to_lowercase();
946 }
947 }
948 QueryOp::Trim => {
949 for l in cur.iter_mut() {
950 *l = l.trim().to_string();
951 }
952 }
953 QueryOp::Replace(re, to) => {
954 for l in cur.iter_mut() {
955 *l = re.replace_all(l, to.as_str()).into_owned();
956 }
957 }
958 QueryOp::Join(sep) => {
959 cur = vec![cur.join(sep)];
960 }
961 QueryOp::Nth(n) => {
962 cur = cur
963 .get(n.saturating_sub(1))
964 .cloned()
965 .map(|l| vec![l])
966 .unwrap_or_default();
967 }
968 QueryOp::Sel { css, attr } => {
969 let doc = Html::parse_document(&cur.join("\n"));
970 cur = match Selector::parse(css) {
971 Ok(sel) => doc
972 .select(&sel)
973 .filter_map(|el| match attr {
974 Some(a) => el.value().attr(a).map(str::to_string),
975 None => Some(el.text().collect::<String>().trim().to_string()),
976 })
977 .collect(),
978 // A selector no CSS engine accepts is an ERROR, not "no
979 // matches" — `spec::check_css` refuses it at build time, and
980 // this is the second line of defence for a selector that
981 // reached here another way (an `input` value substituted by
982 // `resolve_pipeline`).
983 Err(e) => {
984 return QueryResult::Error(format!(
985 "sel: `{css}` is not a valid CSS selector ({e})"
986 ))
987 }
988 };
989 }
990 QueryOp::XPath(xp) => {
991 // An expression that is a RELATIVE location path is evaluated
992 // per line, so it composes with the step that produced those
993 // lines; anything else is one question about the DOCUMENT and is
994 // answered once. See `crate::xpath`'s module docs.
995 match xp.run(&cur) {
996 Ok(lines) => cur = lines,
997 // SPEC §8: a construct outside the subset is a hard error,
998 // never a silent empty node-set.
999 Err(e) => return QueryResult::Error(e),
1000 }
1001 }
1002 QueryOp::Find(css) => {
1003 let doc = Html::parse_document(&cur.join("\n"));
1004 cur = match Selector::parse(css) {
1005 Ok(sel) => doc.select(&sel).map(|el| el.html()).collect(),
1006 Err(e) => {
1007 return QueryResult::Error(format!(
1008 "find: `{css}` is not a valid CSS selector ({e})"
1009 ))
1010 }
1011 };
1012 }
1013 QueryOp::Attr(name) => {
1014 let mut out = Vec::with_capacity(cur.len());
1015 for l in &cur {
1016 let frag = Html::parse_fragment(l);
1017 if let Some(v) = first_element(&frag).and_then(|e| e.value().attr(name)) {
1018 out.push(v.to_string());
1019 }
1020 }
1021 cur = out;
1022 }
1023 QueryOp::Text => {
1024 for l in cur.iter_mut() {
1025 let frag = Html::parse_fragment(l);
1026 if let Some(e) = first_element(&frag) {
1027 // xpath `text()` is the element's DIRECT child text nodes,
1028 // not all descendant text — `e.text()` would also fold in a
1029 // child element's text (e.g. `<a>1<b>X</b>2</a>` -> `1X2`).
1030 *l = e
1031 .children()
1032 .filter_map(|c| c.value().as_text().map(|t| t.to_string()))
1033 .collect::<String>()
1034 .trim()
1035 .to_string();
1036 }
1037 }
1038 }
1039 QueryOp::SortBy(field) => {
1040 // Split object lines (carrying their field value) from the rest;
1041 // non-objects keep their relative input order and sink to the end.
1042 let mut objs: Vec<(String, String)> = Vec::new();
1043 let mut rest: Vec<String> = Vec::new();
1044 for l in cur.drain(..) {
1045 if let Ok(Value::Object(m)) = serde_json::from_str::<Value>(&l) {
1046 let key = m.get(field).map(json_to_string).unwrap_or_default();
1047 objs.push((key, l));
1048 } else {
1049 rest.push(l);
1050 }
1051 }
1052 let all_numeric =
1053 !objs.is_empty() && objs.iter().all(|(k, _)| k.trim().parse::<f64>().is_ok());
1054 if all_numeric {
1055 // slice::sort_by is stable — equal keys preserve input order.
1056 objs.sort_by(|a, b| {
1057 let na = a.0.trim().parse::<f64>().unwrap_or(f64::NAN);
1058 let nb = b.0.trim().parse::<f64>().unwrap_or(f64::NAN);
1059 na.partial_cmp(&nb).unwrap_or(std::cmp::Ordering::Equal)
1060 });
1061 } else {
1062 objs.sort_by(|a, b| a.0.cmp(&b.0));
1063 }
1064 cur = objs.into_iter().map(|(_, l)| l).collect();
1065 cur.extend(rest);
1066 }
1067 QueryOp::UniqueBy(field) => {
1068 let mut seen: HashSet<String> = HashSet::new();
1069 cur.retain(|l| {
1070 let key = match serde_json::from_str::<Value>(l) {
1071 Ok(Value::Object(m)) => {
1072 m.get(field).map(json_to_string).unwrap_or_default()
1073 }
1074 _ => l.clone(),
1075 };
1076 seen.insert(key)
1077 });
1078 }
1079 QueryOp::CountBy(field) => {
1080 let mut counts: BTreeMap<String, u64> = BTreeMap::new();
1081 for l in &cur {
1082 let key = match serde_json::from_str::<Value>(l) {
1083 Ok(Value::Object(m)) => {
1084 m.get(field).map(json_to_string).unwrap_or_default()
1085 }
1086 _ => l.clone(),
1087 };
1088 *counts.entry(key).or_insert(0) += 1;
1089 }
1090 let mut pairs: Vec<(String, u64)> = counts.into_iter().collect();
1091 pairs.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
1092 return QueryResult::Pairs(pairs);
1093 }
1094 QueryOp::GroupBy(field) => {
1095 // jq group_by: one array per distinct key, groups sorted by key.
1096 let mut groups: BTreeMap<String, Vec<Value>> = BTreeMap::new();
1097 for l in &cur {
1098 let (key, val) = match serde_json::from_str::<Value>(l) {
1099 Ok(v @ Value::Object(_)) => {
1100 let k = v.get(field).map(json_to_string).unwrap_or_default();
1101 (k, v)
1102 }
1103 Ok(v) => (l.clone(), v),
1104 Err(_) => (l.clone(), Value::String(l.clone())),
1105 };
1106 groups.entry(key).or_default().push(val);
1107 }
1108 cur = groups
1109 .into_values()
1110 .map(|g| Value::Array(g).to_string())
1111 .collect();
1112 }
1113 QueryOp::MinBy(field) => {
1114 let best = cur
1115 .iter()
1116 .filter(|l| !field_num(l, field).is_nan())
1117 .min_by(|a, b| {
1118 field_num(a, field)
1119 .partial_cmp(&field_num(b, field))
1120 .unwrap_or(std::cmp::Ordering::Equal)
1121 });
1122 return QueryResult::Lines(best.into_iter().cloned().collect());
1123 }
1124 QueryOp::MaxBy(field) => {
1125 // Ignore lines whose FIELD is absent/non-numeric (field_num -> NaN),
1126 // then keep the record with the greatest value. On ties the last
1127 // maximal record wins (std max_by semantics). Empty input -> no lines.
1128 let best = cur
1129 .iter()
1130 .filter(|l| !field_num(l, field).is_nan())
1131 .max_by(|a, b| {
1132 field_num(a, field)
1133 .partial_cmp(&field_num(b, field))
1134 .unwrap_or(std::cmp::Ordering::Equal)
1135 })
1136 .cloned();
1137 return QueryResult::Lines(best.into_iter().collect());
1138 }
1139 QueryOp::Has(key) => {
1140 cur.retain(|l| {
1141 matches!(
1142 serde_json::from_str::<Value>(l),
1143 Ok(Value::Object(ref m)) if m.contains_key(key)
1144 )
1145 });
1146 }
1147 QueryOp::HasKey(key) => {
1148 // jq `has`: a per-input boolean test, NOT a filter — every input
1149 // yields exactly one `true`/`false` line. The key must MATCH the
1150 // container: a string key on an object, a numeric index on an
1151 // array. Every other pairing is an error in jq, where arb used to
1152 // answer `false` — including `[1,2] | has(0)`, which is `true`.
1153 for l in cur.iter_mut() {
1154 let present = match (jq_value(l), key) {
1155 (Some(Value::Object(m)), Value::String(k)) => m.contains_key(k),
1156 (Some(Value::Array(a)), Value::Number(n)) => {
1157 let i = n.as_f64().unwrap_or(-1.0);
1158 i >= 0.0 && (i as usize) < a.len()
1159 }
1160 // jq answers `false` for null rather than raising.
1161 (Some(Value::Null), _) | (None, _) => false,
1162 (Some(v), k) => {
1163 return QueryResult::Error(format!(
1164 "jq: Cannot check whether {} has a {} key",
1165 crate::jqval::tname(&v),
1166 crate::jqval::tname(k)
1167 ));
1168 }
1169 };
1170 *l = if present { "true" } else { "false" }.to_string();
1171 }
1172 }
1173 QueryOp::Entries => {
1174 let mut out = Vec::with_capacity(cur.len());
1175 for l in &cur {
1176 match serde_json::from_str::<Value>(l) {
1177 Ok(Value::Object(m)) => {
1178 for (k, v) in &m {
1179 out.push(format!(
1180 "{{\"key\":{},\"value\":{}}}",
1181 Value::String(k.clone()),
1182 v
1183 ));
1184 }
1185 }
1186 _ => out.push(l.clone()),
1187 }
1188 }
1189 cur = out;
1190 }
1191 QueryOp::JqEntries => {
1192 if let Some(e) = jq_builtin_lines(&mut cur, "to_entries") {
1193 return e;
1194 }
1195 }
1196 QueryOp::JqMap(inner) => {
1197 // jq `map(f)` is `[.[] | f]` — a per-input SCOPED iterate, so two
1198 // input lines stay two output arrays. Running the inner ops over
1199 // the whole `cur` instead (which dropping the re-wrap amounts to)
1200 // both loses the array and merges the inputs together.
1201 let mut out = Vec::with_capacity(cur.len());
1202 for l in &cur {
1203 // Through the jq value model: elements keep their source
1204 // literals and an object's values come in its KEY ORDER.
1205 let elems: Vec<String> = match crate::jqlang::parse_json(l) {
1206 Ok(v) => match v.bare() {
1207 crate::jqlang::JqVal::Arr(a) => {
1208 a.iter().map(crate::jqlang::render_raw).collect()
1209 }
1210 // jq `map` over an object maps its VALUES (and still
1211 // returns an array).
1212 crate::jqlang::JqVal::Obj(m) => m
1213 .iter()
1214 .map(|(_, e)| crate::jqlang::render_raw(e))
1215 .collect(),
1216 // Not iterable: jq raises, and so does arb now.
1217 other => {
1218 return QueryResult::Error(format!(
1219 "jq: Cannot iterate over {}{}",
1220 other.type_name(),
1221 crate::jqlang::paren_of(other)
1222 ))
1223 }
1224 },
1225 // Not JSON at all: no jq type to check, so it passes.
1226 Err(_) => {
1227 out.push(l.clone());
1228 continue;
1229 }
1230 };
1231 let mapped = match eval(inner, &elems, elapsed_secs) {
1232 QueryResult::Lines(v) => v,
1233 QueryResult::Scalar(s) => vec![fmt_num(s)],
1234 QueryResult::Pairs(ps) => {
1235 ps.iter().map(|(k, c)| format!("{k}\t{c}")).collect()
1236 }
1237 // An inner refusal is the whole `map`'s refusal.
1238 e @ QueryResult::Error(_) => return e,
1239 };
1240 out.push(jq_array_json(&mapped));
1241 }
1242 cur = out;
1243 }
1244 QueryOp::Flatten => {
1245 // jq `flatten` fully flattens all nesting levels; recurse into
1246 // every sub-array and emit only the non-array leaves.
1247 fn push_leaves(v: &Value, out: &mut Vec<String>) {
1248 match v {
1249 Value::Array(a) => a.iter().for_each(|e| push_leaves(e, out)),
1250 other => out.push(json_to_string(other)),
1251 }
1252 }
1253 let mut out = Vec::with_capacity(cur.len());
1254 for l in &cur {
1255 match serde_json::from_str::<Value>(l) {
1256 Ok(Value::Array(arr)) => arr.iter().for_each(|e| push_leaves(e, &mut out)),
1257 _ => out.push(l.clone()),
1258 }
1259 }
1260 cur = out;
1261 }
1262 QueryOp::JqFlatten => {
1263 // jq's `flatten` is defined over `reduce .[] as $x`, so it
1264 // iterates an OBJECT's values too and refuses a scalar.
1265 if let Some(e) = jq_builtin_lines(&mut cur, "flatten") {
1266 return e;
1267 }
1268 }
1269 QueryOp::Add => {
1270 for l in cur.iter_mut() {
1271 if let Ok(Value::Array(arr)) = serde_json::from_str::<Value>(l) {
1272 if arr.is_empty() {
1273 *l = String::new();
1274 } else if arr.iter().all(Value::is_number) {
1275 let sum: f64 = arr.iter().filter_map(Value::as_f64).sum();
1276 *l = fmt_num(sum);
1277 } else {
1278 *l = arr.iter().map(json_to_string).collect::<String>();
1279 }
1280 }
1281 }
1282 }
1283 QueryOp::Over(n) => {
1284 cur.retain(|l| l.trim().parse::<f64>().map(|v| v > *n).unwrap_or(false));
1285 }
1286 QueryOp::Under(n) => {
1287 cur.retain(|l| l.trim().parse::<f64>().map(|v| v < *n).unwrap_or(false));
1288 }
1289 QueryOp::Between(lo, hi) => {
1290 cur.retain(|l| {
1291 let x = l.trim().parse::<f64>().unwrap_or(f64::NAN);
1292 x >= *lo && x <= *hi
1293 });
1294 }
1295 QueryOp::Enumerate => {
1296 for (i, l) in cur.iter_mut().enumerate() {
1297 *l = format!("{}\t{}", i + 1, l);
1298 }
1299 }
1300 QueryOp::Words => {
1301 let mut out = Vec::with_capacity(cur.len());
1302 for l in &cur {
1303 out.extend(l.split_whitespace().map(str::to_string));
1304 }
1305 cur = out;
1306 }
1307 QueryOp::Dedup => {
1308 cur.dedup();
1309 }
1310 QueryOp::Tailn(n) => {
1311 let len = cur.len();
1312 if *n < len {
1313 cur.drain(0..len - *n);
1314 }
1315 }
1316 QueryOp::Pad(n) => {
1317 // `format!` width is a u16 internally: a width >= 65536 panics
1318 // ("Formatting argument out of range"). Cap it — no real pane is
1319 // that wide and the widget layer clips lines to the pane anyway.
1320 let n = (*n).min(u16::MAX as usize);
1321 for l in cur.iter_mut() {
1322 *l = format!("{:<width$}", l, width = n);
1323 }
1324 }
1325 QueryOp::Lpad(width) => {
1326 let w = (*width).min(u16::MAX as usize);
1327 for l in cur.iter_mut() {
1328 *l = format!("{:>width$}", l, width = w);
1329 }
1330 }
1331 QueryOp::Grepf(field, re) => {
1332 cur.retain(|l| {
1333 let val = if let Ok(Value::Object(m)) = serde_json::from_str::<Value>(l) {
1334 m.get(field).map(json_to_string).unwrap_or_default()
1335 } else if let Ok(idx) = field.parse::<usize>() {
1336 l.split_whitespace()
1337 .nth(idx.saturating_sub(1))
1338 .unwrap_or("")
1339 .to_string()
1340 } else {
1341 String::new()
1342 };
1343 re.is_match(&val)
1344 });
1345 }
1346 QueryOp::Flip => {
1347 for l in cur.iter_mut() {
1348 *l = l.chars().rev().collect();
1349 }
1350 }
1351 QueryOp::Basename => {
1352 for l in cur.iter_mut() {
1353 let t = l.trim_end_matches('/');
1354 *l = t.rsplit('/').next().unwrap_or(t).to_string();
1355 }
1356 }
1357 QueryOp::Dirname => {
1358 for l in cur.iter_mut() {
1359 let t = l.trim_end_matches('/');
1360 *l = match t.rsplit_once('/') {
1361 Some(("", _)) => "/".to_string(),
1362 Some((dir, _)) => dir.to_string(),
1363 None => ".".to_string(),
1364 };
1365 }
1366 }
1367 QueryOp::Commafy => {
1368 for l in cur.iter_mut() {
1369 *l = commafy(l);
1370 }
1371 }
1372 QueryOp::Bytes => {
1373 for l in cur.iter_mut() {
1374 if let Ok(v) = l.trim().parse::<f64>() {
1375 *l = humanize_bytes(v);
1376 }
1377 }
1378 }
1379 QueryOp::Duration => {
1380 for l in cur.iter_mut() {
1381 if let Ok(v) = l.trim().parse::<f64>() {
1382 *l = humanize_duration(v);
1383 }
1384 }
1385 }
1386 // Resolved to the input widget's pipeline before eval; a no-op if it
1387 // survives (empty input, or eval reached without substitution).
1388 QueryOp::Apply(_) => {}
1389 QueryOp::B64 => {
1390 for l in cur.iter_mut() {
1391 *l = STANDARD.encode(l.as_bytes());
1392 }
1393 }
1394 QueryOp::B64d => {
1395 for l in cur.iter_mut() {
1396 if let Some(s) = STANDARD
1397 .decode(l.as_bytes())
1398 .ok()
1399 .and_then(|b| String::from_utf8(b).ok())
1400 {
1401 *l = s;
1402 }
1403 }
1404 }
1405 QueryOp::Hex => {
1406 for l in cur.iter_mut() {
1407 *l = l.bytes().map(|b| format!("{:02x}", b)).collect();
1408 }
1409 }
1410 QueryOp::Unhex => {
1411 for l in cur.iter_mut() {
1412 let chars: Vec<char> = l.chars().collect();
1413 if chars.is_empty() || !chars.len().is_multiple_of(2) {
1414 continue;
1415 }
1416 let mut bytes = Vec::with_capacity(chars.len() / 2);
1417 let mut ok = true;
1418 let mut i = 0;
1419 while i < chars.len() {
1420 let pair: String = chars[i..i + 2].iter().collect();
1421 match u8::from_str_radix(&pair, 16) {
1422 Ok(b) => bytes.push(b),
1423 Err(_) => {
1424 ok = false;
1425 break;
1426 }
1427 }
1428 i += 2;
1429 }
1430 if ok {
1431 if let Ok(decoded) = String::from_utf8(bytes) {
1432 *l = decoded;
1433 }
1434 }
1435 }
1436 }
1437 QueryOp::Urlenc => {
1438 for l in cur.iter_mut() {
1439 *l = utf8_percent_encode(l, NON_ALPHANUMERIC).to_string();
1440 }
1441 }
1442 QueryOp::Urldec => {
1443 for l in cur.iter_mut() {
1444 if let Ok(decoded) = percent_decode_str(l).decode_utf8() {
1445 *l = decoded.into_owned();
1446 }
1447 }
1448 }
1449 QueryOp::Extract(re) => {
1450 cur = cur
1451 .iter()
1452 .filter_map(|l| {
1453 re.captures(l).map(|caps| {
1454 caps.get(1)
1455 .unwrap_or_else(|| caps.get(0).unwrap())
1456 .as_str()
1457 .to_string()
1458 })
1459 })
1460 .collect();
1461 }
1462 QueryOp::Split(delim) => {
1463 let mut out: Vec<String> = Vec::with_capacity(cur.len());
1464 for l in cur.iter() {
1465 for part in l.split(delim.as_str()) {
1466 out.push(part.to_string());
1467 }
1468 }
1469 cur = out;
1470 }
1471 QueryOp::Substr(a, b) => {
1472 for l in cur.iter_mut() {
1473 *l = l.chars().skip(*a).take(b.saturating_sub(*a)).collect();
1474 }
1475 }
1476 QueryOp::Chars => {
1477 let mut out: Vec<String> = Vec::new();
1478 for l in cur.iter() {
1479 for ch in l.chars() {
1480 out.push(ch.to_string());
1481 }
1482 }
1483 cur = out;
1484 }
1485 QueryOp::Title => {
1486 for l in cur.iter_mut() {
1487 *l = l
1488 .split_whitespace()
1489 .map(|w| {
1490 let mut cs = w.chars();
1491 match cs.next() {
1492 Some(f) => {
1493 f.to_uppercase().collect::<String>()
1494 + &cs.as_str().to_lowercase()
1495 }
1496 None => String::new(),
1497 }
1498 })
1499 .collect::<Vec<_>>()
1500 .join(" ");
1501 }
1502 }
1503 QueryOp::Repeat(n) => {
1504 for l in cur.iter_mut() {
1505 *l = l.repeat(*n);
1506 }
1507 }
1508 QueryOp::Set(key, val) => {
1509 for l in cur.iter_mut() {
1510 if let Ok(Value::Object(mut m)) = serde_json::from_str::<Value>(l) {
1511 m.insert(key.clone(), Value::String(val.clone()));
1512 *l = Value::Object(m).to_string();
1513 }
1514 }
1515 }
1516 QueryOp::Del(key) => {
1517 for l in cur.iter_mut() {
1518 if let Ok(Value::Object(mut m)) = serde_json::from_str::<Value>(l) {
1519 m.remove(key);
1520 *l = Value::Object(m).to_string();
1521 }
1522 }
1523 }
1524 QueryOp::Rename(old, new) => {
1525 for l in cur.iter_mut() {
1526 if let Ok(Value::Object(mut m)) = serde_json::from_str::<Value>(l) {
1527 if let Some(v) = m.remove(old) {
1528 m.insert(new.clone(), v);
1529 *l = Value::Object(m).to_string();
1530 }
1531 }
1532 }
1533 }
1534 QueryOp::Default(key, val) => {
1535 for l in cur.iter_mut() {
1536 if let Ok(Value::Object(mut m)) = serde_json::from_str::<Value>(l) {
1537 m.entry(key.clone()).or_insert(Value::String(val.clone()));
1538 *l = Value::Object(m).to_string();
1539 }
1540 }
1541 }
1542 QueryOp::Merge => {
1543 let mut acc = serde_json::Map::new();
1544 let mut saw_object = false;
1545 for l in cur.iter() {
1546 if let Ok(Value::Object(m)) = serde_json::from_str::<Value>(l) {
1547 saw_object = true;
1548 for (k, v) in m {
1549 acc.insert(k, v);
1550 }
1551 }
1552 }
1553 return if saw_object {
1554 QueryResult::Lines(vec![Value::Object(acc).to_string()])
1555 } else {
1556 QueryResult::Lines(vec![])
1557 };
1558 }
1559 QueryOp::Floor => {
1560 for l in cur.iter_mut() {
1561 if let Ok(x) = l.parse::<f64>() {
1562 *l = fmt_num(x.floor());
1563 }
1564 }
1565 }
1566 QueryOp::Ceil => {
1567 for l in cur.iter_mut() {
1568 if let Ok(x) = l.trim().parse::<f64>() {
1569 *l = fmt_num(x.ceil());
1570 }
1571 }
1572 }
1573 QueryOp::Clamp(lo, hi) => {
1574 for l in cur.iter_mut() {
1575 if let Ok(x) = l.trim().parse::<f64>() {
1576 *l = fmt_num(x.clamp(*lo, *hi));
1577 }
1578 }
1579 }
1580 QueryOp::Contains(s) => cur.retain(|l| l.contains(s.as_str())),
1581 QueryOp::Starts(p) => cur.retain(|l| l.starts_with(p.as_str())),
1582 QueryOp::Ends(s) => cur.retain(|l| l.ends_with(s.as_str())),
1583 QueryOp::Nonempty => cur.retain(|l| !l.trim().is_empty()),
1584 QueryOp::Numeric => cur.retain(|l| l.trim().parse::<f64>().is_ok()),
1585 QueryOp::Len => {
1586 for l in cur.iter_mut() {
1587 *l = l.chars().count().to_string();
1588 }
1589 }
1590 QueryOp::JsonLen => {
1591 for l in cur.iter_mut() {
1592 *l = match serde_json::from_str::<Value>(l) {
1593 Ok(Value::Array(a)) => a.len().to_string(),
1594 Ok(Value::Object(m)) => m.len().to_string(),
1595 Ok(Value::String(s)) => s.chars().count().to_string(),
1596 Ok(Value::Number(n)) => fmt_num(n.as_f64().unwrap_or(0.0).abs()),
1597 Ok(Value::Null) => "0".to_string(),
1598 // jq errors on a bool length; arb has no per-line error
1599 // channel, so fall back to the raw line's char count.
1600 _ => l.chars().count().to_string(),
1601 };
1602 }
1603 }
1604 QueryOp::JsonSlice(a, b) => {
1605 for l in cur.iter_mut() {
1606 match jq_value(l) {
1607 Some(Value::Array(arr)) => {
1608 let (lo, hi) = slice_bounds(*a, *b, arr.len());
1609 *l = Value::Array(arr[lo..hi].to_vec()).to_string();
1610 }
1611 Some(Value::String(s)) => {
1612 let chars: Vec<char> = s.chars().collect();
1613 let (lo, hi) = slice_bounds(*a, *b, chars.len());
1614 *l = chars[lo..hi].iter().collect();
1615 }
1616 // jq slices null to null, and refuses every other type —
1617 // it reports the slice as the OBJECT it is internally.
1618 Some(Value::Null) => *l = "null".to_string(),
1619 Some(v) => {
1620 let bound =
1621 |x: &Option<i64>| x.map_or("null".to_string(), |n| n.to_string());
1622 return QueryResult::Error(format!(
1623 "jq: Cannot index {} with object ({{\"start\":{},\"end\":{}}})",
1624 crate::jqval::tname(&v),
1625 bound(a),
1626 bound(b)
1627 ));
1628 }
1629 None => {} // not JSON at all: passes through
1630 }
1631 }
1632 }
1633 QueryOp::Wc => {
1634 for l in cur.iter_mut() {
1635 *l = l.split_whitespace().count().to_string();
1636 }
1637 }
1638 QueryOp::Abs => {
1639 for l in cur.iter_mut() {
1640 if let Ok(v) = l.trim().parse::<f64>() {
1641 *l = fmt_num(v.abs());
1642 }
1643 }
1644 }
1645 QueryOp::Round => {
1646 for l in cur.iter_mut() {
1647 if let Ok(v) = l.trim().parse::<f64>() {
1648 *l = fmt_num(v.round());
1649 }
1650 }
1651 }
1652 QueryOp::Delta => {
1653 let ns = nums(&cur);
1654 cur = ns.windows(2).map(|w| fmt_num(w[1] - w[0])).collect();
1655 }
1656 QueryOp::Cumsum => {
1657 let ns = nums(&cur);
1658 let mut acc = 0.0;
1659 cur = ns
1660 .iter()
1661 .map(|&v| {
1662 acc += v;
1663 fmt_num(acc)
1664 })
1665 .collect();
1666 }
1667 QueryOp::Sma(n) => {
1668 let ns = nums(&cur);
1669 let w = (*n).max(1);
1670 cur = (0..ns.len())
1671 .map(|i| {
1672 let lo = i + 1 - (i + 1).min(w);
1673 let win = &ns[lo..=i];
1674 fmt_num(win.iter().sum::<f64>() / win.len() as f64)
1675 })
1676 .collect();
1677 }
1678 QueryOp::Ewma(alpha) => {
1679 let a = alpha.clamp(0.0, 1.0);
1680 let ns = nums(&cur);
1681 let mut s = 0.0;
1682 cur = ns
1683 .iter()
1684 .enumerate()
1685 .map(|(i, &v)| {
1686 s = if i == 0 { v } else { a * v + (1.0 - a) * s };
1687 fmt_num(s)
1688 })
1689 .collect();
1690 }
1691 QueryOp::Prepend(pre) => {
1692 for l in cur.iter_mut() {
1693 *l = format!("{pre}{l}");
1694 }
1695 }
1696 QueryOp::Append(suf) => {
1697 for l in cur.iter_mut() {
1698 l.push_str(suf);
1699 }
1700 }
1701 QueryOp::Cut(delim, n) => {
1702 for l in cur.iter_mut() {
1703 *l = l
1704 .split(delim.as_str())
1705 .nth(n.saturating_sub(1))
1706 .unwrap_or("")
1707 .to_string();
1708 }
1709 }
1710 QueryOp::Median => {
1711 let mut ns = nums(&cur);
1712 ns.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
1713 let m = if ns.is_empty() {
1714 0.0
1715 } else if ns.len() % 2 == 1 {
1716 ns[ns.len() / 2]
1717 } else {
1718 (ns[ns.len() / 2 - 1] + ns[ns.len() / 2]) / 2.0
1719 };
1720 return QueryResult::Scalar(m);
1721 }
1722 QueryOp::Percentile(p) => {
1723 let mut ns = nums(&cur);
1724 ns.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
1725 let v = if ns.is_empty() {
1726 0.0
1727 } else if ns.len() == 1 {
1728 ns[0]
1729 } else {
1730 // Linear interpolation between ranks (numpy default), so `p50`
1731 // equals `median` and `p90` of 1..10 is 9.1, matching the docs.
1732 let frac = p.clamp(0.0, 100.0) / 100.0;
1733 let pos = frac * (ns.len() - 1) as f64;
1734 let lo = pos.floor() as usize;
1735 let hi = pos.ceil() as usize;
1736 ns[lo] + (ns[hi] - ns[lo]) * (pos - lo as f64)
1737 };
1738 return QueryResult::Scalar(v);
1739 }
1740 QueryOp::Stddev => {
1741 let ns = nums(&cur);
1742 let sd = if ns.is_empty() {
1743 0.0
1744 } else {
1745 let mean = ns.iter().sum::<f64>() / ns.len() as f64;
1746 let var = ns.iter().map(|v| (v - mean).powi(2)).sum::<f64>() / ns.len() as f64;
1747 var.sqrt()
1748 };
1749 return QueryResult::Scalar(sd);
1750 }
1751 QueryOp::Range => {
1752 let ns = nums(&cur);
1753 let r = if ns.is_empty() {
1754 0.0
1755 } else {
1756 ns.iter().cloned().fold(f64::NEG_INFINITY, f64::max)
1757 - ns.iter().cloned().fold(f64::INFINITY, f64::min)
1758 };
1759 return QueryResult::Scalar(r);
1760 }
1761 QueryOp::Product => return QueryResult::Scalar(nums(&cur).iter().product()),
1762 QueryOp::Distinct => {
1763 let set: std::collections::HashSet<&String> = cur.iter().collect();
1764 return QueryResult::Scalar(set.len() as f64);
1765 }
1766 QueryOp::Sample(n) => {
1767 if *n >= 1 {
1768 let mut i = 0usize;
1769 cur.retain(|_| {
1770 i += 1;
1771 i.is_multiple_of(*n)
1772 });
1773 }
1774 }
1775 QueryOp::Slice(a, b) => {
1776 let lo = a.saturating_sub(1).min(cur.len());
1777 let hi = (*b).min(cur.len());
1778 cur = if lo < hi {
1779 cur[lo..hi].to_vec()
1780 } else {
1781 Vec::new()
1782 };
1783 }
1784 QueryOp::Index(n) => {
1785 cur = n
1786 .checked_sub(1)
1787 .and_then(|i| cur.get(i).cloned())
1788 .into_iter()
1789 .collect();
1790 }
1791 QueryOp::Bins(n) => {
1792 let vals = nums(&cur);
1793 // Bound the bucket count: it allocates `vec![0u64; n]` and builds
1794 // n formatted pairs, so an over-large N (e.g. 1e8) balloons memory
1795 // and hangs. No histogram display needs more than this many bars.
1796 let n = (*n).clamp(1, 65_536);
1797 if vals.is_empty() {
1798 return QueryResult::Pairs(Vec::new());
1799 }
1800 let min = vals.iter().cloned().fold(f64::INFINITY, f64::min);
1801 let max = vals.iter().cloned().fold(f64::NEG_INFINITY, f64::max);
1802 let width = ((max - min) / n as f64).max(f64::MIN_POSITIVE);
1803 let mut counts = vec![0u64; n];
1804 for v in &vals {
1805 let idx = (((v - min) / width) as usize).min(n - 1);
1806 counts[idx] += 1;
1807 }
1808 let pairs = counts
1809 .iter()
1810 .enumerate()
1811 .map(|(i, &c)| {
1812 let lo = min + i as f64 * width;
1813 (format!("{}-{}", fmt_num(lo), fmt_num(lo + width)), c)
1814 })
1815 .collect();
1816 return QueryResult::Pairs(pairs);
1817 }
1818 }
1819 }
1820 QueryResult::Lines(cur)
1821}
1822
1823/// True if every op processes each line independently, so the pipeline can be
1824/// applied per-line and its results emitted incrementally (streaming). Reducers,
1825/// sorts/reorderers, and whole-document ops (`sel`/`csv`/`tsv`) are not.
1826/// Split lines into table `(headers, rows)` for the `table` widget. Each line is
1827/// split on whitespace into cells; `cols` (a comma-separated list, from `-cols`)
1828/// names the header row when present. Renderers pad short rows to the column
1829/// count. Shared by the ratatui TUI and the web dashboard so both agree.
1830pub fn table_data(lines: &[String], cols: Option<&str>) -> (Vec<String>, Vec<Vec<String>>) {
1831 let headers: Vec<String> = cols
1832 .map(|c| {
1833 c.split(',')
1834 .map(|s| s.trim().to_string())
1835 .filter(|s| !s.is_empty())
1836 .collect()
1837 })
1838 .unwrap_or_default();
1839 let rows: Vec<Vec<String>> = lines
1840 .iter()
1841 .map(|l| l.split_whitespace().map(str::to_string).collect())
1842 .collect();
1843 (headers, rows)
1844}
1845
1846/// Parse lines as a numeric series for the `spark` widget — each line's first
1847/// whitespace token that parses as a number; non-numeric lines are skipped.
1848pub fn numeric_series(lines: &[String]) -> Vec<f64> {
1849 lines
1850 .iter()
1851 .filter_map(|l| {
1852 l.split_whitespace()
1853 .next()
1854 .and_then(|t| t.parse::<f64>().ok())
1855 })
1856 .collect()
1857}
1858
1859/// Render a numeric series as a unicode sparkline (`▁▂▃▄▅▆▇█`), scaled between
1860/// the series min and max. Shared by the TUI and web so both draw the same shape.
1861/// A flat series renders as the lowest tick; an empty series is the empty string.
1862pub fn sparkline(values: &[f64]) -> String {
1863 const TICKS: [char; 8] = ['▁', '▂', '▃', '▄', '▅', '▆', '▇', '█'];
1864 if values.is_empty() {
1865 return String::new();
1866 }
1867 let min = values.iter().cloned().fold(f64::INFINITY, f64::min);
1868 let max = values.iter().cloned().fold(f64::NEG_INFINITY, f64::max);
1869 let range = max - min;
1870 values
1871 .iter()
1872 .map(|&v| {
1873 let idx = if range <= 0.0 {
1874 0
1875 } else {
1876 (((v - min) / range) * 7.0).round() as usize
1877 };
1878 TICKS[idx.min(7)]
1879 })
1880 .collect()
1881}
1882
1883/// The number of columns a table needs to hold `headers` and `rows` (at least 1).
1884pub fn table_ncols(headers: &[String], rows: &[Vec<String>]) -> usize {
1885 rows.iter()
1886 .map(Vec::len)
1887 .max()
1888 .unwrap_or(0)
1889 .max(headers.len())
1890 .max(1)
1891}
1892
1893pub fn is_line_streamable(ops: &[QueryOp]) -> bool {
1894 ops.iter().all(|op| {
1895 // `map(f)` is scoped to one input line, so it streams exactly when its
1896 // inner pipeline does — a reducer inside it (`map(add)`) does not.
1897 if let QueryOp::JqMap(inner) = op {
1898 return is_line_streamable(inner);
1899 }
1900 // A jq PROGRAM is per-line unless it reads the stream itself: `input`
1901 // and `inputs` pull the following documents, so they need the whole
1902 // buffer in hand. A program that does not touch them streams, which is
1903 // what keeps `arb -e '… | .name'` emitting as lines arrive instead of
1904 // buffering to EOF.
1905 if let QueryOp::JqProgram(src) = op {
1906 return jq_program(src).is_ok_and(|p| !p.reads_input_stream());
1907 }
1908 matches!(
1909 op,
1910 QueryOp::JsonDocs
1911 | QueryOp::Match(_)
1912 | QueryOp::Reject(_)
1913 | QueryOp::Field(_)
1914 | QueryOp::Fields(_)
1915 | QueryOp::Each
1916 | QueryOp::JqEach
1917 | QueryOp::JqSelect(_)
1918 | QueryOp::JqCalc(_)
1919 | QueryOp::JqAdd
1920 | QueryOp::JqLen
1921 | QueryOp::Keys
1922 | QueryOp::JqKeys
1923 | QueryOp::Vals
1924 | QueryOp::NonNull
1925 | QueryOp::JqRawString
1926 | QueryOp::Pick(_)
1927 | QueryOp::Where(_)
1928 | QueryOp::Map(_)
1929 | QueryOp::Contains(_)
1930 | QueryOp::Starts(_)
1931 | QueryOp::Ends(_)
1932 | QueryOp::Nonempty
1933 | QueryOp::Numeric
1934 | QueryOp::Len
1935 | QueryOp::JsonLen
1936 | QueryOp::JsonSlice(_, _)
1937 | QueryOp::Wc
1938 | QueryOp::Abs
1939 | QueryOp::Round
1940 | QueryOp::Prepend(_)
1941 | QueryOp::Append(_)
1942 | QueryOp::Cut(_, _)
1943 | QueryOp::Upper
1944 | QueryOp::Lower
1945 | QueryOp::Trim
1946 | QueryOp::Replace(_, _)
1947 | QueryOp::B64
1948 | QueryOp::B64d
1949 | QueryOp::Hex
1950 | QueryOp::Unhex
1951 | QueryOp::Urlenc
1952 | QueryOp::Urldec
1953 | QueryOp::Extract(_)
1954 | QueryOp::Substr(_, _)
1955 | QueryOp::Title
1956 | QueryOp::Repeat(_)
1957 | QueryOp::Set(_, _)
1958 | QueryOp::Del(_)
1959 | QueryOp::Rename(_, _)
1960 | QueryOp::Default(_, _)
1961 | QueryOp::Floor
1962 | QueryOp::Ceil
1963 | QueryOp::Clamp(_, _)
1964 | QueryOp::Has(_)
1965 | QueryOp::HasKey(_)
1966 | QueryOp::Entries
1967 | QueryOp::JqEntries
1968 | QueryOp::JqFlatten
1969 | QueryOp::Add
1970 | QueryOp::Over(_)
1971 | QueryOp::Under(_)
1972 | QueryOp::Between(_, _)
1973 | QueryOp::Pad(_)
1974 | QueryOp::Lpad(_)
1975 | QueryOp::Grepf(_, _)
1976 | QueryOp::Flip
1977 | QueryOp::Basename
1978 | QueryOp::Dirname
1979 | QueryOp::Commafy
1980 )
1981 })
1982}
1983
1984/// Extract a field from a line per the selector.
1985///
1986/// `JqKey` is NOT handled here: a jq path can refuse (`3 | .a`), and a `String`
1987/// return has nowhere to put the refusal, so `eval`'s `Field` arm resolves it
1988/// through `jqval::get_path` and raises `QueryResult::Error` instead.
1989fn extract_field(line: &str, sel: &FieldSel) -> String {
1990 match sel {
1991 FieldSel::Col(n) => nth_col(line, *n).to_string(),
1992 FieldSel::JqKey(segs) => jq_value(line)
1993 .and_then(|v| crate::jqval::get_path(&v, segs).ok())
1994 .map_or_else(|| "null".to_string(), |v| jq_to_string(&v)),
1995 FieldSel::Key(path) => serde_json::from_str::<Value>(line)
1996 .ok()
1997 .and_then(|v| walk(v, path))
1998 .map(|v| json_to_string(&v))
1999 .or_else(|| {
2000 if path.len() == 1 {
2001 logfmt_field(line, &path[0])
2002 } else {
2003 None
2004 }
2005 })
2006 .unwrap_or_default(),
2007 }
2008}
2009
2010/// Parse a line as JSON for a jq-context op. `None` means the line is not JSON
2011/// at all — arb's stream is TEXT, so such a line has no jq type to check and
2012/// every caller keeps its documented pass-through instead of raising.
2013fn jq_value(line: &str) -> Option<Value> {
2014 serde_json::from_str::<Value>(line).ok()
2015}
2016
2017/// The current value for a jq EXPRESSION stage. Unlike [`jq_value`] there is no
2018/// "not JSON" case: a bare text line is jq's string, so `"ab" | . * 2` gives
2019/// `abab` the way jq does.
2020fn jq_scalar(line: &str) -> Value {
2021 jq_value(line).unwrap_or_else(|| Value::String(line.to_string()))
2022}
2023
2024/// jq's message for asking a non-container for its keys.
2025fn has_no_keys(v: &Value) -> String {
2026 format!("jq: {} ({v}) has no keys", crate::jqval::tname(v))
2027}
2028
2029/// Render a JSON value the way jq's `-r`/`-c` pair does: a string raw, a null as
2030/// the literal `null`, everything else compact. `json_to_string` renders a null
2031/// as "" instead, which is right for arb's native text verbs and wrong for jq.
2032fn jq_to_string(v: &Value) -> String {
2033 match v {
2034 Value::String(s) => s.clone(),
2035 other => other.to_string(),
2036 }
2037}
2038
2039/// Rebuild a jq array from element strings that have ALREADY been rendered.
2040///
2041/// `fmt_num` is the one place an arb number becomes text, but parsing its
2042/// output back into a `serde_json::Value` and letting serde_json print it again
2043/// runs the value through a SECOND, different float formatter (ryu) and undoes
2044/// it: `1e-06` came back out as `1e-6`, `1e-05` as `0.00001`, and
2045/// `123456789012345000000` as `1.23456789012345e+20`. Every one of those
2046/// disagreed with `jq -rc` on the same input while the scalar path next to it
2047/// agreed — one invariant, applied at one site and reverted at the other.
2048///
2049/// So an element that is already a JSON NUMBER is emitted verbatim (`1E+1000`
2050/// included, which serde refuses as out of range). Everything else is rendered
2051/// by the jq value model, which gives an object, a nested array and a string
2052/// their escaping while keeping KEY ORDER and nested number literals (serde's
2053/// `BTreeMap` re-sorted `[{"b":1,"a":2}] | map(.)`), and a line that is not
2054/// JSON at all becomes a JSON string. That last rule is lossy in exactly one place — a string whose
2055/// text is itself valid JSON (`"123"`) comes back as the number — which is
2056/// inherent to a raw line stream and is probed, not hidden, by
2057/// `scripts/jq_parity.sh`.
2058fn jq_array_json(elems: &[String]) -> String {
2059 let mut out = String::from("[");
2060 for (i, e) in elems.iter().enumerate() {
2061 if i > 0 {
2062 out.push(',');
2063 }
2064 match crate::jqlang::parse_json(e) {
2065 Ok(crate::jqlang::JqVal::Num(..)) => out.push_str(e),
2066 Ok(v) => out.push_str(&crate::jqlang::render(&v)),
2067 Err(_) => out.push_str(&Value::String(e.clone()).to_string()),
2068 }
2069 }
2070 out.push(']');
2071 out
2072}
2073
2074/// Re-serialize a stream of documents in `fmt`, as the lines the pipeline
2075/// carries. Splitting on `\n` and dropping the trailing empty element is what
2076/// makes the printed output byte-identical to the rendered text: the printer
2077/// writes one `\n` per line.
2078fn emit_stream(docs: &[crate::jqlang::JqVal], fmt: OutFmt, indent: usize) -> Vec<String> {
2079 let opts = crate::ynode::Emit { indent };
2080 let text = match fmt {
2081 OutFmt::Yaml => crate::ynode::emit_docs(docs, opts),
2082 OutFmt::Json => docs
2083 .iter()
2084 .map(|d| crate::jqlang::render_indented(d, indent))
2085 .collect::<Vec<_>>()
2086 .join("\n"),
2087 OutFmt::Props => docs
2088 .iter()
2089 .map(crate::yqfmt::to_props)
2090 .collect::<Vec<_>>()
2091 .concat(),
2092 OutFmt::Xml => docs
2093 .iter()
2094 .map(|d| crate::yqfmt::to_xml(d, indent))
2095 .collect::<Vec<_>>()
2096 .concat(),
2097 OutFmt::Csv => docs
2098 .iter()
2099 .map(|d| crate::yqfmt::to_delim(d, ','))
2100 .collect::<Vec<_>>()
2101 .concat(),
2102 OutFmt::Tsv => docs
2103 .iter()
2104 .map(|d| crate::yqfmt::to_delim(d, '\t'))
2105 .collect::<Vec<_>>()
2106 .concat(),
2107 };
2108 let mut lines: Vec<String> = text.split('\n').map(str::to_string).collect();
2109 if lines.last().is_some_and(String::is_empty) {
2110 lines.pop();
2111 }
2112 lines
2113}
2114
2115/// Parse the stream as one TOML document and emit it as a JSON object line
2116/// (empty if it does not parse). Goes through `toml::Value` and converts
2117/// explicitly so a TOML datetime becomes a clean scalar string — deserializing
2118/// straight into `serde_json::Value` instead leaks the toml crate's internal
2119/// `{"$__toml_private_datetime": …}` tagging map into the output.
2120fn toml_to_json(lines: &[String]) -> Vec<String> {
2121 match toml::from_str::<toml::Value>(&lines.join("\n")) {
2122 Ok(v) => vec![toml_value_to_json(&v).to_string()],
2123 Err(_) => Vec::new(),
2124 }
2125}
2126
2127/// Convert a `toml::Value` to a `serde_json::Value`, rendering datetimes as their
2128/// string form (RFC-3339 / date / time) rather than a tagged object.
2129fn toml_value_to_json(v: &toml::Value) -> Value {
2130 match v {
2131 toml::Value::String(s) => Value::String(s.clone()),
2132 toml::Value::Integer(i) => Value::Number((*i).into()),
2133 toml::Value::Float(f) => serde_json::Number::from_f64(*f)
2134 .map(Value::Number)
2135 .unwrap_or(Value::Null),
2136 toml::Value::Boolean(b) => Value::Bool(*b),
2137 toml::Value::Datetime(dt) => Value::String(dt.to_string()),
2138 toml::Value::Array(a) => Value::Array(a.iter().map(toml_value_to_json).collect()),
2139 toml::Value::Table(t) => Value::Object(
2140 t.iter()
2141 .map(|(k, v)| (k.clone(), toml_value_to_json(v)))
2142 .collect(),
2143 ),
2144 }
2145}
2146
2147/// Reassemble physical lines into logical CSV/TSV records. A quoted field may
2148/// contain newlines (RFC 4180), so a line whose double-quotes are unbalanced is
2149/// still inside a quoted field and continues on the next line until they balance
2150/// — otherwise a multi-line field is torn into corrupt phantom rows. A doubled
2151/// `""` (an escaped quote) contributes two quotes, so parity tracks correctly.
2152fn join_quoted_records(lines: &[String]) -> Vec<String> {
2153 let mut records = Vec::new();
2154 let mut buf = String::new();
2155 for line in lines {
2156 if !buf.is_empty() {
2157 buf.push('\n');
2158 }
2159 buf.push_str(line);
2160 if buf.matches('"').count().is_multiple_of(2) {
2161 records.push(std::mem::take(&mut buf));
2162 }
2163 }
2164 if !buf.is_empty() {
2165 records.push(buf); // trailing unbalanced record: emit what we have
2166 }
2167 records
2168}
2169
2170/// Parse a header + data rows of a delimited stream into JSON object strings
2171/// keyed by the header, so `field NAME` works over CSV/TSV.
2172fn to_json_records(lines: &[String], delim: char) -> Vec<String> {
2173 if lines.is_empty() {
2174 return Vec::new();
2175 }
2176 let records = join_quoted_records(lines);
2177 let Some(header_row) = records.first() else {
2178 return Vec::new();
2179 };
2180 let header = split_delim(header_row, delim);
2181 records[1..]
2182 .iter()
2183 .map(|row| {
2184 let vals = split_delim(row, delim);
2185 let mut obj = serde_json::Map::new();
2186 for (i, name) in header.iter().enumerate() {
2187 obj.insert(
2188 name.clone(),
2189 Value::String(vals.get(i).cloned().unwrap_or_default()),
2190 );
2191 }
2192 Value::Object(obj).to_string()
2193 })
2194 .collect()
2195}
2196
2197/// Split one delimited line into fields per RFC 4180 (line-oriented: no embedded
2198/// newlines). A field may be double-quoted; a quoted field may contain the
2199/// delimiter, and a doubled `""` inside it is one literal `"`. Unquoted fields
2200/// are trimmed (preserving prior behavior); quoted fields are returned verbatim.
2201fn split_delim(line: &str, delim: char) -> Vec<String> {
2202 let mut fields = Vec::new();
2203 let mut field = String::new();
2204 let mut in_quotes = false;
2205 let mut quoted = false; // this field opened with a quote
2206 let mut chars = line.chars().peekable();
2207
2208 // Unquoted fields are trimmed; a quoted field is returned exactly.
2209 let finish = |f: &str, q: bool| -> String {
2210 if q {
2211 f.to_string()
2212 } else {
2213 f.trim().to_string()
2214 }
2215 };
2216
2217 while let Some(c) = chars.next() {
2218 if in_quotes {
2219 if c == '"' {
2220 if chars.peek() == Some(&'"') {
2221 chars.next(); // "" -> one literal quote
2222 field.push('"');
2223 } else {
2224 in_quotes = false; // closing quote
2225 }
2226 } else {
2227 field.push(c);
2228 }
2229 } else if c == '"' && field.trim().is_empty() {
2230 in_quotes = true; // opening quote at field start
2231 quoted = true;
2232 field.clear(); // drop any leading whitespace
2233 } else if c == delim {
2234 fields.push(finish(&field, quoted));
2235 field.clear();
2236 quoted = false;
2237 } else {
2238 field.push(c);
2239 }
2240 }
2241 fields.push(finish(&field, quoted));
2242 fields
2243}
2244
2245/// Extract a `key=value` (logfmt) field. A value may be double-quoted and then
2246/// contain spaces (`msg="hello world"`), so this scans key=value pairs honoring
2247/// quotes rather than splitting on whitespace first (which would truncate a
2248/// quoted value at its first space).
2249fn logfmt_field(line: &str, key: &str) -> Option<String> {
2250 let cs: Vec<char> = line.chars().collect();
2251 let mut i = 0;
2252 while i < cs.len() {
2253 while i < cs.len() && cs[i].is_whitespace() {
2254 i += 1;
2255 }
2256 let kstart = i;
2257 while i < cs.len() && cs[i] != '=' && !cs[i].is_whitespace() {
2258 i += 1;
2259 }
2260 let k: String = cs[kstart..i].iter().collect();
2261 if i < cs.len() && cs[i] == '=' {
2262 i += 1; // consume '='
2263 let val = if cs.get(i) == Some(&'"') {
2264 i += 1;
2265 let mut v = String::new();
2266 while i < cs.len() && cs[i] != '"' {
2267 if cs[i] == '\\' && i + 1 < cs.len() {
2268 i += 1; // keep the escaped char verbatim
2269 }
2270 v.push(cs[i]);
2271 i += 1;
2272 }
2273 i += 1; // closing quote (or past end)
2274 v
2275 } else {
2276 let vstart = i;
2277 while i < cs.len() && !cs[i].is_whitespace() {
2278 i += 1;
2279 }
2280 cs[vstart..i].iter().collect()
2281 };
2282 if k == key {
2283 return Some(val);
2284 }
2285 }
2286 }
2287 None
2288}
2289
2290/// The 1-based whitespace column `n` of `line` ("" if absent; 0 = whole line).
2291fn nth_col(line: &str, n: usize) -> &str {
2292 if n == 0 {
2293 return line;
2294 }
2295 line.split_whitespace().nth(n - 1).unwrap_or("")
2296}
2297
2298/// Walk a JSON key/array-index path, consuming the value.
2299fn walk(mut cur: Value, path: &[String]) -> Option<Value> {
2300 for key in path {
2301 cur = match cur {
2302 Value::Object(mut m) => m.remove(key)?,
2303 Value::Array(mut a) => {
2304 // jq array index: a negative index counts from the end (`.[-1]`).
2305 let idx = key.parse::<i64>().ok()?;
2306 let i = if idx < 0 { a.len() as i64 + idx } else { idx };
2307 if i >= 0 && (i as usize) < a.len() {
2308 a.swap_remove(i as usize)
2309 } else {
2310 return None;
2311 }
2312 }
2313 _ => return None,
2314 };
2315 }
2316 Some(cur)
2317}
2318
2319/// Resolve jq slice bounds `[a:b]` to a clamped `lo..hi` over `len`. A negative
2320/// bound counts from the end; `None` is the start/end. Clamped into `0..=len`,
2321/// and `hi < lo` collapses to an empty slice.
2322fn slice_bounds(a: Option<i64>, b: Option<i64>, len: usize) -> (usize, usize) {
2323 let n = len as i64;
2324 let norm = |x: i64| (if x < 0 { x + n } else { x }).clamp(0, n);
2325 let lo = norm(a.unwrap_or(0));
2326 let hi = norm(b.unwrap_or(n));
2327 (lo as usize, hi.max(lo) as usize)
2328}
2329
2330/// Parse the numeric lines of a slice, ignoring non-numeric ones.
2331fn nums(lines: &[String]) -> Vec<f64> {
2332 lines
2333 .iter()
2334 .filter_map(|l| l.trim().parse::<f64>().ok())
2335 .collect()
2336}
2337
2338/// Format a computed number the way `jq` prints a computed double.
2339///
2340/// Every arb value is an f64 (SPEC §6), so this is the one place a number
2341/// becomes text — `map`, `sum`, `avg`, `floor`, `diff`, the scalar result and
2342/// every control readout all arrive here. The reference is `jq -r`: arb claims
2343/// jq parity for its numeric core, and `scripts/expr_paths.sh` checks it.
2344///
2345/// jq renders a computed double with netlib `g_fmt`: take the SHORTEST
2346/// round-trip decimal digits `d1…dn` and the decimal-point position `decpt`
2347/// (the value is `0.d1…dn × 10^decpt`), then pick exponential notation when
2348/// `decpt <= -4 || decpt > n + 15` and plain positional notation otherwise.
2349/// The exponent carries a sign and at least two digits (`1e+17`, `1e-05`).
2350///
2351/// Both cutoffs are measured, not guessed: rendering 33,618 doubles this way —
2352/// every decade from 1e-330 to 1e308 at seven mantissas, 20,000 random bit
2353/// patterns and 5,000 random decimals — and diffing against `jq 1.8.2 -rc '.*1'`
2354/// gives zero mismatches. `.*1` forces jq to COMPUTE, so the comparison is
2355/// against its double formatter and not against its decNumber literal
2356/// preservation, which arb has no equivalent for (see the module docs).
2357///
2358/// The two behaviours this replaces were both wrong against that reference:
2359/// * Rust's `{}` never uses exponential notation at all, so `1e308` printed
2360/// as 309 digits and `5e-324` as 324, instead of `1e+308` / `5e-324`.
2361/// * The `v as i64` fast path printed the EXACT binary value of a whole
2362/// number, inventing precision the double does not carry: `1e18 / 3`
2363/// printed `333333333333333312` where jq prints `333333333333333300`, the
2364/// 16 round-trip digits zero-padded. (That cast also saturated outside
2365/// i64's range — `tests/query.rs` still pins that it never reappears.)
2366pub(crate) fn fmt_num(v: f64) -> String {
2367 // A non-finite double has no decimal expansion, and neither JSON nor jq has
2368 // a spelling for one, so jq maps both onto values that do: an infinity
2369 // CLAMPS to ±DBL_MAX and a NaN renders as `null`. This function's contract is
2370 // jq's rendering, so it follows jq here too. Measured against jq 1.8.2:
2371 // `1e308 * 2` -> `1.7976931348623157e+308`, `-1e308 * 2` -> the negation of
2372 // that, and `(1e308 * 2) * 0` -> `null`.
2373 //
2374 // Rust's own spelling (`inf` / `NaN`) was returned here before. That was a
2375 // deviation from the contract, justified in this comment by the claim that
2376 // clamping "would hide the divergence the harness reports for a zero
2377 // divisor". It does not, and that is checked rather than argued: the split
2378 // `scripts/expr_paths.sh` reports for `x / 0` is `Value::Undef` on the
2379 // interpreter against an IEEE infinity in compiled code, and `Value::Undef`
2380 // renders `0` — not through this branch at all. Clamping moves only the
2381 // native side, from `inf` to `1.7976931348623157e+308`, so the two tiers
2382 // still disagree and the probe stays red. Both `x / 0` probes are still
2383 // listed as divergences by that harness after this change.
2384 if v.is_nan() {
2385 return "null".to_string();
2386 }
2387 // Clamp the VALUE and fall through to the normal path rather than returning a
2388 // literal string, so the digits an infinity prints as are by construction the
2389 // ones this formatter gives DBL_MAX — they can never drift apart.
2390 let v = v.clamp(-f64::MAX, f64::MAX);
2391 // `-0.0 == 0.0`, so this is also what keeps a negated zero printing as `0`.
2392 if v == 0.0 {
2393 return "0".to_string();
2394 }
2395 // `{:e}` is Rust's shortest round-trip form: `d[.ddd]e<exp>`, value =
2396 // d.ddd × 10^exp. It gives the DIGIT COUNT to work from, but not always the
2397 // right digits — shortest and shortest-AND-CLOSEST are different rules.
2398 //
2399 // Where a double's neighbours are further apart than one unit in the last
2400 // decimal place, SEVERAL decimals of that shortest length parse back to the
2401 // same double, and the two rules may pick different ones. jq's dtoa (David
2402 // Gay's) emits the closest; Rust emits one that round-trips, which need not
2403 // be. `191510495617760.12` came out of Rust as `…13` and `611630169981189.25`
2404 // as `…89.3` against jq's `…89.2`. Both of Rust's answers round-trip; neither
2405 // is the nearest decimal of that length.
2406 //
2407 // So: take the LENGTH from Rust's shortest form, then re-render at that many
2408 // significant digits with `{:.*e}`, which rounds correctly. That is Gay's
2409 // rule, and it cannot cost round-tripping — if some n-digit decimal parses
2410 // back to `v` then the CLOSEST n-digit decimal does too, being at least as
2411 // near. Measured over 200,000 doubles (150,000 random bit patterns plus
2412 // 50,000 in the 1e14..1e17 band where the ties concentrate) against
2413 // `jq 1.8.2 -rc '.*1'`: the shortest-only form missed 1,286 of them, this one
2414 // misses none. The band matters — a pool spread evenly over the exponent
2415 // range hits it so rarely that 33,618 doubles scored clean while 1 in 160 of
2416 // the numbers a person actually writes at that magnitude was wrong.
2417 let shortest = format!("{v:e}");
2418 let ndigits = shortest
2419 .split_once('e')
2420 .map_or(1, |(m, _)| m.chars().filter(char::is_ascii_digit).count());
2421 let sci = format!("{:.*e}", ndigits.saturating_sub(1), v);
2422 let (mant, exp) = match sci.split_once('e') {
2423 Some(p) => p,
2424 None => return sci,
2425 };
2426 let exp: i32 = match exp.parse() {
2427 Ok(e) => e,
2428 Err(_) => return sci,
2429 };
2430 let sign = if mant.starts_with('-') { "-" } else { "" };
2431 let mut digits: String = mant.chars().filter(char::is_ascii_digit).collect();
2432 // Correct rounding can carry into a trailing zero (`…95` at 4 digits becomes
2433 // `…0`), and a trailing zero would both lengthen the output and move the
2434 // fixed-vs-exponential cutoff below, which counts SIGNIFICANT digits.
2435 while digits.len() > 1 && digits.ends_with('0') {
2436 digits.pop();
2437 }
2438 let n = digits.len() as i32;
2439 let decpt = exp + 1;
2440
2441 if decpt <= -4 || decpt > n + 15 {
2442 let m = if n == 1 {
2443 digits
2444 } else {
2445 format!("{}.{}", &digits[..1], &digits[1..])
2446 };
2447 // `{:+03}` is sign + at least two digits: `+17`, `-05`, `-300`.
2448 format!("{sign}{m}e{:+03}", decpt - 1)
2449 } else if decpt <= 0 {
2450 format!("{sign}0.{}{digits}", "0".repeat(-decpt as usize))
2451 } else if decpt >= n {
2452 format!("{sign}{digits}{}", "0".repeat((decpt - n) as usize))
2453 } else {
2454 let (int, frac) = digits.split_at(decpt as usize);
2455 format!("{sign}{int}.{frac}")
2456 }
2457}
2458
2459/// Group the integer part of a numeric line with thousands separators; leaves a
2460/// non-numeric line (and any fractional/sign parts) intact.
2461fn commafy(line: &str) -> String {
2462 let s = line.trim();
2463 if s.parse::<f64>().is_err() {
2464 return line.to_string();
2465 }
2466 let (sign, rest) = match s.strip_prefix('-') {
2467 Some(r) => ("-", r),
2468 None => ("", s),
2469 };
2470 let (int, frac) = match rest.split_once('.') {
2471 Some((i, f)) => (i, Some(f)),
2472 None => (rest, None),
2473 };
2474 let mut grouped = String::new();
2475 for (idx, ch) in int.chars().enumerate() {
2476 if idx > 0 && (int.len() - idx) % 3 == 0 {
2477 grouped.push(',');
2478 }
2479 grouped.push(ch);
2480 }
2481 match frac {
2482 Some(f) => format!("{sign}{grouped}.{f}"),
2483 None => format!("{sign}{grouped}"),
2484 }
2485}
2486
2487/// Humanize a byte count (1024-based): `1536` -> `1.5 KB`, `1024` -> `1 KB`,
2488/// `500` -> `500 B`. One decimal, trailing `.0` trimmed. Negatives keep the sign.
2489fn humanize_bytes(v: f64) -> String {
2490 const UNITS: [&str; 6] = ["B", "KB", "MB", "GB", "TB", "PB"];
2491 let sign = if v < 0.0 { "-" } else { "" };
2492 let mut n = v.abs();
2493 let mut u = 0;
2494 while n >= 1024.0 && u < UNITS.len() - 1 {
2495 n /= 1024.0;
2496 u += 1;
2497 }
2498 // Bytes are whole; scaled values show one decimal (unless it rounds to .0).
2499 if u == 0 {
2500 format!("{sign}{} {}", n.round() as i64, UNITS[u])
2501 } else {
2502 let r = (n * 10.0).round() / 10.0;
2503 if (r.fract()).abs() < f64::EPSILON {
2504 format!("{sign}{} {}", r as i64, UNITS[u])
2505 } else {
2506 format!("{sign}{r:.1} {}", UNITS[u])
2507 }
2508 }
2509}
2510
2511/// Humanize a duration in seconds as the two largest non-zero units: `3661` ->
2512/// `1h 1m`, `45` -> `45s`, `90061` -> `1d 1h`, `0` -> `0s`. Negatives keep the sign.
2513fn humanize_duration(v: f64) -> String {
2514 let sign = if v < 0.0 { "-" } else { "" };
2515 let total = v.abs().round() as i64;
2516 if total == 0 {
2517 return "0s".to_string();
2518 }
2519 let units = [("d", 86400), ("h", 3600), ("m", 60), ("s", 1)];
2520 let mut rem = total;
2521 let mut parts = Vec::new();
2522 for (label, secs) in units {
2523 let q = rem / secs;
2524 if q > 0 {
2525 parts.push(format!("{q}{label}"));
2526 rem %= secs;
2527 }
2528 }
2529 parts.truncate(2); // two largest non-zero units
2530 format!("{sign}{}", parts.join(" "))
2531}
2532
2533/// Resolve a JSON field of `line` to a number for expression evaluation
2534/// (missing / non-numeric / non-JSON -> NaN, which fails numeric predicates).
2535fn field_num(line: &str, name: &str) -> f64 {
2536 if let Ok(Value::Object(mut m)) = serde_json::from_str::<Value>(line) {
2537 if let Some(v) = m.remove(name) {
2538 return value_to_f64(&v);
2539 }
2540 }
2541 logfmt_field(line, name)
2542 .and_then(|s| s.parse::<f64>().ok())
2543 .unwrap_or(f64::NAN)
2544}
2545
2546/// Public wrapper for `field_str` — used by the TUI facet control to derive
2547/// candidate values from a stream field.
2548pub fn field_str_pub(line: &str, name: &str) -> String {
2549 field_str(line, name)
2550}
2551
2552/// Public wrapper for `field_num` — used by the DAP `evaluate` request to
2553/// resolve `.field` references against the paused stream line (same resolver the
2554/// per-line `where`/`map` evaluation uses).
2555pub fn field_num_pub(line: &str, name: &str) -> f64 {
2556 field_num(line, name)
2557}
2558
2559/// Format a control scalar: integers without a decimal, else the shortest repr.
2560pub fn fmt_scalar(v: f64) -> String {
2561 fmt_num(v)
2562}
2563
2564/// A field's value as a string: a JSON object key, else a logfmt `key=value`,
2565/// else "". `x` (or `.` the whole line via Var) resolves to the whole line.
2566fn field_str(line: &str, name: &str) -> String {
2567 if name == "x" {
2568 return line.to_string();
2569 }
2570 if let Ok(Value::Object(m)) = serde_json::from_str::<Value>(line) {
2571 if let Some(v) = m.get(name) {
2572 return json_to_string(v);
2573 }
2574 }
2575 logfmt_field(line, name).unwrap_or_default()
2576}
2577
2578/// One side of a string comparison, resolved against the current line: a literal
2579/// as itself, a field as its value, and `x`/`.` as the whole line.
2580fn cmp_str(e: &crate::expr::Expr, line: &str) -> String {
2581 use crate::expr::Expr;
2582 match e {
2583 Expr::Str(s) => s.clone(),
2584 Expr::Field(f) => field_str(line, f),
2585 Expr::Var => line.trim().to_string(),
2586 Expr::Num(n) => fmt_num(*n),
2587 _ => String::new(),
2588 }
2589}
2590
2591/// The string value of a substituted string node (`Str`), else "".
2592fn str_of(e: &crate::expr::Expr) -> String {
2593 match e {
2594 crate::expr::Expr::Str(s) => s.clone(),
2595 _ => String::new(),
2596 }
2597}
2598
2599/// Evaluate a string-bearing `where` predicate against one line (Rust, not the
2600/// numeric VM). `match`/`in .set` test strings; and/or/not compose; any purely
2601/// numeric subtree falls back to the fusevm predicate path.
2602fn eval_where(e: &crate::expr::Expr, line: &str) -> bool {
2603 use crate::expr::{BinOp, Expr};
2604 match e {
2605 Expr::Match(inner) => {
2606 let q = str_of(inner);
2607 q.is_empty() || line.to_lowercase().contains(&q.to_lowercase())
2608 }
2609 Expr::InSet(field, inner) => {
2610 let set = str_of(inner);
2611 let items: Vec<&str> = set
2612 .split(',')
2613 .map(str::trim)
2614 .filter(|s| !s.is_empty())
2615 .collect();
2616 if items.is_empty() {
2617 return true; // empty selection -> no filter
2618 }
2619 let val = field_str(line, field);
2620 items.iter().any(|it| *it == val)
2621 }
2622 Expr::Not(a) => !eval_where(a, line),
2623 Expr::Bin(BinOp::And, a, b) => eval_where(a, line) && eval_where(b, line),
2624 Expr::Bin(BinOp::Or, a, b) => eval_where(a, line) || eval_where(b, line),
2625 // A compare where one side is a string literal — jq's `select(.status ==
2626 // "ok")`. Compared as TEXT: the numeric fallback below reads a non-numeric
2627 // field as NaN, and every NaN comparison is false, so without this arm such
2628 // a predicate silently matched NOTHING (a filter that drops every row
2629 // rather than erroring). The ORDERED forms belong here for the same reason:
2630 // SPEC §8 says "a compare may test strings as well as numbers", and jq
2631 // orders strings by codepoint — which is Rust's `str` ordering, since UTF-8
2632 // byte order and codepoint order agree.
2633 Expr::Bin(
2634 op @ (BinOp::Eq | BinOp::Ne | BinOp::Lt | BinOp::Le | BinOp::Gt | BinOp::Ge),
2635 a,
2636 b,
2637 ) if matches!(**a, Expr::Str(_)) || matches!(**b, Expr::Str(_)) => {
2638 let sa = cmp_str(a, line);
2639 let sb = cmp_str(b, line);
2640 match op {
2641 BinOp::Eq => sa == sb,
2642 BinOp::Ne => sa != sb,
2643 BinOp::Lt => sa < sb,
2644 BinOp::Le => sa <= sb,
2645 BinOp::Gt => sa > sb,
2646 _ => sa >= sb,
2647 }
2648 }
2649 // A numeric subtree: evaluate it on fusevm as usual.
2650 _ => {
2651 let x = line.trim().parse::<f64>().unwrap_or(f64::NAN);
2652 let resolve = |n: &str| field_num(line, n);
2653 crate::expr::eval_pred_ctx(e, x, &resolve).unwrap_or(false)
2654 }
2655 }
2656}
2657
2658fn value_to_f64(v: &Value) -> f64 {
2659 match v {
2660 Value::Number(n) => n.as_f64().unwrap_or(f64::NAN),
2661 Value::String(s) => s.trim().parse().unwrap_or(f64::NAN),
2662 Value::Bool(b) => {
2663 if *b {
2664 1.0
2665 } else {
2666 0.0
2667 }
2668 }
2669 _ => f64::NAN,
2670 }
2671}
2672
2673/// Render a JSON scalar as a plain string; containers as compact JSON.
2674/// The first real element of a parsed fragment, skipping the synthetic
2675/// `html`/`head`/`body` wrappers that `Html::parse_fragment` inserts.
2676fn first_element(frag: &Html) -> Option<ElementRef<'_>> {
2677 let star = Selector::parse("*").ok()?;
2678 frag.select(&star)
2679 .find(|e| !matches!(e.value().name(), "html" | "head" | "body"))
2680}
2681
2682fn json_to_string(v: &Value) -> String {
2683 match v {
2684 Value::String(s) => s.clone(),
2685 Value::Null => String::new(),
2686 other => other.to_string(),
2687 }
2688}
2689
2690#[cfg(test)]
2691mod tests {
2692 use super::*;
2693
2694 #[test]
2695 fn split_plain() {
2696 assert_eq!(split_delim("a,b,c", ','), vec!["a", "b", "c"]);
2697 }
2698
2699 #[test]
2700 fn split_quoted_comma() {
2701 // A delimiter inside quotes is not a separator.
2702 assert_eq!(split_delim("\"a,b\",c", ','), vec!["a,b", "c"]);
2703 }
2704
2705 #[test]
2706 fn split_doubled_quote() {
2707 // "" inside a quoted field is one literal quote.
2708 assert_eq!(
2709 split_delim("\"she \"\"said\"\"\",x", ','),
2710 vec!["she \"said\"", "x"]
2711 );
2712 }
2713
2714 #[test]
2715 fn split_trailing_empty() {
2716 assert_eq!(split_delim("a,", ','), vec!["a", ""]);
2717 }
2718
2719 #[test]
2720 fn split_tsv() {
2721 assert_eq!(split_delim("a\tb\tc", '\t'), vec!["a", "b", "c"]);
2722 }
2723
2724 #[test]
2725 fn split_unquoted_trims_quoted_keeps_spaces() {
2726 // Unquoted fields are trimmed; a quoted field keeps its inner spaces.
2727 assert_eq!(split_delim(" a , b ", ','), vec!["a", "b"]);
2728 assert_eq!(split_delim("\" a \",b", ','), vec![" a ", "b"]);
2729 }
2730}
2731
2732thread_local! {
2733 /// Compiled jq programs, keyed by source. A `QueryOp::JqProgram` is
2734 /// re-evaluated on every stream tick (the TUI re-runs the pipeline as lines
2735 /// arrive), so without this the same program would be lexed and parsed once
2736 /// per frame. Thread-local because a compiled program holds `Rc`s.
2737 static JQ_PROGRAMS: std::cell::RefCell<
2738 std::collections::HashMap<String, std::rc::Rc<crate::jqlang::Program>>,
2739 > = std::cell::RefCell::new(std::collections::HashMap::new());
2740}
2741
2742/// Replace every JSON line of `cur` with the one value jq's builtin `name`
2743/// answers for it, through [`crate::jqlang`] so key order and number literals
2744/// are jq's. A line that is not JSON has no jq type and PASSES THROUGH, which
2745/// is the documented carve-out these stages always had. `Some` is the
2746/// refusal that ends the pipeline.
2747///
2748/// The answer is written as compact JSON, never `jq -r`'s raw string: these
2749/// stages run MID-pipeline, where `add`'s raw `ab` is not a JSON line, so the
2750/// next translated stage (`.[1:]`, `keys`, `.[0]`) took it for plain text and
2751/// passed it through instead of slicing or refusing it. `jq::translate`
2752/// appends the one final `JqRawString` that `jq -r` owes.
2753fn jq_builtin_lines(cur: &mut Vec<String>, name: &str) -> Option<QueryResult> {
2754 let prog = match jq_program(name) {
2755 Ok(p) => p,
2756 Err(msg) => return Some(QueryResult::Error(format!("jq: {msg}"))),
2757 };
2758 let interp = crate::jqlang::Interp::default();
2759 let mut out = Vec::with_capacity(cur.len());
2760 for l in cur.iter() {
2761 let Ok(v) = crate::jqlang::parse_json(l) else {
2762 out.push(l.clone());
2763 continue;
2764 };
2765 let r = prog.run_with(&interp, &v, &mut |r| {
2766 out.push(crate::jqlang::render(&r));
2767 Ok(())
2768 });
2769 if let Err(e) = r {
2770 return Some(QueryResult::Error(format!("jq: {}", e.to_message())));
2771 }
2772 }
2773 *cur = out;
2774 None
2775}
2776
2777/// Compile (or fetch from this thread's cache) the jq program `src`.
2778pub fn jq_program(src: &str) -> Result<std::rc::Rc<crate::jqlang::Program>, String> {
2779 JQ_PROGRAMS.with(|c| {
2780 if let Some(p) = c.borrow().get(src) {
2781 return Ok(p.clone());
2782 }
2783 let p = std::rc::Rc::new(crate::jqlang::Program::compile(src)?);
2784 c.borrow_mut().insert(src.to_string(), p.clone());
2785 Ok(p)
2786 })
2787}