arb/query.rs
1//! Query pipeline. A `source { … }` body compiles to a sequence of ops applied
2//! to the live stream on each evaluation.
3//!
4//! M2a verbs: `in` (source marker), `match`/`grep RE`, `reject`/`grepv RE`,
5//! `field N` (1-based whitespace column), `count`, `rate`. JSON/CSV field
6//! extraction, `where(pred)`, and aggregation to tables land with later verbs.
7
8use std::collections::{BTreeMap, HashSet};
9
10use base64::{engine::general_purpose::STANDARD, Engine as _};
11use percent_encoding::{percent_decode_str, utf8_percent_encode, NON_ALPHANUMERIC};
12use regex::Regex;
13use scraper::{ElementRef, Html, Selector};
14
15use serde_json::Value;
16
17use crate::expr::Expr;
18
19/// How `field` selects a value: a 1-based whitespace column, or a JSON key path.
20#[derive(Debug, Clone, PartialEq)]
21pub enum FieldSel {
22 Col(usize),
23 Key(Vec<String>),
24}
25
26#[derive(Debug, Clone)]
27pub enum QueryOp {
28 /// Keep lines matching the pattern.
29 Match(Regex),
30 /// Drop lines matching the pattern.
31 Reject(Regex),
32 /// Replace each line with a selected field (whitespace column or JSON key path).
33 Field(FieldSel),
34 /// Project multiple whitespace columns (1-based), space-joined, keeping the
35 /// given order — `fields 1 3` on "a b c d" -> "a c". For columnar input.
36 Fields(Vec<usize>),
37 /// Flatten JSON-array lines into one line per element (jq `[]`); non-array
38 /// lines pass through unchanged.
39 Each,
40 /// Keep lines whose numeric value (`x` = line parsed as a number) satisfies
41 /// the predicate — compiled to fusevm and evaluated per line.
42 Where(Expr),
43 /// Replace each line with the value of an expression (field-aware; `x` =
44 /// line-as-number), computed on the fusevm VM.
45 Map(Expr),
46 /// `via NAME [* N]` — fan the stream across a supervised pool of `N` copies
47 /// of actor `NAME` (default: one per hardware thread). Each line's scalar is
48 /// asked as the actor's first-handler message; the reply is the output line,
49 /// order preserved. A parallel actor map (see [`crate::actor::run_via`]).
50 Via(std::sync::Arc<crate::actor::ActorDef>, usize),
51 /// Reduce to the current line count.
52 Count,
53 /// Reduce to lines-per-second over the elapsed window.
54 Rate,
55 /// Group identical values and count them, sorted by count desc then key asc.
56 Tally,
57 /// Numeric reductions over lines parsed as numbers (non-numeric ignored).
58 Sum,
59 Min,
60 Max,
61 Avg,
62 /// Flatten a JSON object's keys / values into one line each.
63 Keys,
64 /// Native `vals` verb: expand a JSON object's values, one per line.
65 Vals,
66 /// jq `values` == `select(. != null)`: drop JSON-null lines, pass every other
67 /// line through unchanged. (NOT object-value iteration — that is `vals`/`.[]`.)
68 NonNull,
69 /// Project a JSON object down to the named keys (jq `{a,b,c}` /
70 /// `pick(.a,.b,.c)`), preserving the listed order. Non-object lines pass
71 /// through unchanged; missing keys are dropped.
72 Pick(Vec<String>),
73 /// Line-list transforms. `Sort` supports numeric (`-n`) and reverse (`-r`).
74 Sort {
75 numeric: bool,
76 reverse: bool,
77 },
78 Uniq,
79 Rev,
80 First,
81 Last,
82 Take(usize),
83 Drop(usize),
84 /// Per-line string transforms.
85 Upper,
86 Lower,
87 Trim,
88 /// Regex replace-all per line (`replace /RE/ TO`; TO may use `$1` captures).
89 Replace(Regex, String),
90 /// Collapse all lines into one, joined by a separator.
91 Join(String),
92 /// Keep only the Nth line (1-based).
93 Nth(usize),
94 /// Parse the accumulated stream as one HTML document and emit, per element
95 /// matching the CSS selector, its text (or a named attribute if `attr` is
96 /// set; elements lacking that attribute are dropped).
97 Sel {
98 css: String,
99 attr: Option<String>,
100 },
101 /// Recursive descent (xpath `//TAG`): parse the accumulated stream as one
102 /// HTML document and emit the outer HTML of every element matching the
103 /// selector, one per line — so `attr`/`text` can then pull from each.
104 Find(String),
105 /// From element fragments (one per line, as emitted by `find`), emit the
106 /// named attribute of each (xpath `@NAME`, css attr); drop lines whose first
107 /// element lacks it.
108 Attr(String),
109 /// From element fragments, emit each element's inner text; non-element lines
110 /// pass through unchanged.
111 Text,
112 /// Stable-sort JSON-record lines by FIELD; numeric if every field value
113 /// parses as a number, else lexicographic on the field's string value.
114 /// Non-object lines sink after the sorted records in input order.
115 SortBy(String),
116 /// Keep the first record for each distinct value of FIELD, preserving input
117 /// order. JSON-object lines dedup by that field's value; other lines dedup by
118 /// the whole line.
119 UniqueBy(String),
120 /// Group JSON records by the value of FIELD and count each group, returning
121 /// value -> count pairs sorted by count desc then value asc; non-object lines
122 /// are counted under their whole-line text. Reducer (early return).
123 CountBy(String),
124 /// Group lines by the value of FIELD (jq `group_by`): emit one JSON-array
125 /// line per distinct value, each array holding that group's members in input
126 /// order; groups are ordered by key ascending. Object lines group by the
127 /// field's value, other lines by their whole-line text.
128 GroupBy(String),
129 /// Reducer: emit the single JSON record whose numeric FIELD is smallest (records with a missing/non-numeric FIELD are ignored; empty input yields no lines).
130 MinBy(String),
131 /// Reducer: emit the single record whose numeric FIELD is the largest.
132 MaxBy(String),
133 /// Native `has KEY` verb: retain only JSON-object lines that contain KEY; all
134 /// other lines (missing key, non-object, unparseable) are dropped.
135 Has(String),
136 /// jq `has(KEY)`: emit `true`/`false` per input line — `true` iff the line is a
137 /// JSON object containing KEY, else `false`. A per-input boolean, not a filter.
138 HasKey(String),
139 /// jq to_entries: expand each JSON object line into one `{"key":<k>,"value":<v>}` line per key (BTreeMap key order); non-object lines pass through.
140 Entries,
141 /// jq `flatten`: recursively flatten a JSON-array line to its non-array leaves,
142 /// emitting one leaf per line (matching jq's full-depth flatten, unlike `each`
143 /// which descends a single level). Non-array lines pass through unchanged.
144 Flatten,
145 /// jq `add`: reduce a JSON array line to a single value — sum numeric
146 /// arrays (fmt_num), concatenate non-numeric arrays via their string
147 /// values, empty array -> "". Non-array lines pass through unchanged.
148 Add,
149 /// Keep only lines that parse as a number strictly greater than `N`.
150 /// Lines that do not parse as `f64` are dropped.
151 Over(f64),
152 /// Keep only numeric lines whose value is strictly less than N; drop non-numeric lines.
153 Under(f64),
154 /// Keep numeric lines x where lo <= x <= hi (inclusive); non-numeric lines are dropped.
155 Between(f64, f64),
156 /// Prefix each line with its 1-based index and a tab: `"1\t<line>"`.
157 Enumerate,
158 /// Split each line on whitespace and emit one word per line (flatten); empty lines produce nothing.
159 Words,
160 /// Collapse runs of adjacent identical lines to a single line (classic uniq), leaving non-adjacent repeats intact.
161 Dedup,
162 /// Keep only the last N lines (complement of `take`, which keeps the first N). N>=len keeps all.
163 Tailn(usize),
164 /// Right-pad each line with spaces to a minimum visible width N (no truncation if the line is already longer).
165 Pad(usize),
166 /// Left-pad each line with spaces to a minimum width of N (lines already >= N are unchanged).
167 Lpad(usize),
168 /// Retain lines whose FIELD (json key or 1-based whitespace column) matches the regex.
169 Grepf(String, regex::Regex),
170 /// Reverse the Unicode scalar characters of each line (chars().rev()).
171 Flip,
172 /// Path basename: the part after the last `/` (the whole line if none).
173 Basename,
174 /// Path dirname: the part before the last `/` (`.` if none).
175 Dirname,
176 /// Group a numeric line's integer part with thousands separators
177 /// (`1234567` -> `1,234,567`); non-numeric lines pass through.
178 Commafy,
179 /// Humanize a byte count (1024-based): `1536` -> `1.5 KB`; non-numeric passes through.
180 Bytes,
181 /// Humanize a duration in seconds: `3661` -> `1h 1m`; non-numeric passes through.
182 Duration,
183 /// Placeholder for `apply .name`: at render time it is replaced by the query
184 /// pipeline typed into the `input .name` widget (the megafilter/map binding).
185 /// Left in a pipeline unsubstituted it is a no-op.
186 Apply(String),
187 /// Treat the stream as CSV: the first line is the header; each data row
188 /// becomes a JSON object keyed by the header, so `field NAME` works.
189 Csv,
190 B64,
191 /// Base64-decode each line (STANDARD alphabet) into a UTF-8 string; lines that fail to
192 /// base64-decode or whose bytes aren't valid UTF-8 pass through unchanged.
193 B64d,
194 /// Lowercase hex-encode each line, two hex digits per UTF-8 byte.
195 Hex,
196 /// Decode a hex string to UTF-8 text by parsing byte pairs; on any error (odd length, non-hex digit, invalid UTF-8) the line is left unchanged.
197 Unhex,
198 /// Percent-encode each line, escaping every non-alphanumeric byte (RFC 3986 style).
199 Urlenc,
200 /// Percent-decode each line to UTF-8 (utf8_percent_decode); lines whose decoded bytes are not valid UTF-8 pass through unchanged.
201 Urldec,
202 /// Emit the first regex match per line (capture group 1 if the pattern captures, else the whole match); drop lines with no match.
203 Extract(Regex),
204 /// Explode each line by the literal string DELIM into multiple lines (one part per line).
205 /// One-to-many: unlike `cut` (one field) or `words` (whitespace), every split segment becomes its own line.
206 Split(String),
207 /// Character substring [A,B) 0-based, clamped to the line length (B may exceed len; A>B yields empty).
208 Substr(usize, usize),
209 /// Explode each line into one output line per Unicode scalar (character); one line -> many.
210 Chars,
211 /// Title-case each line: uppercase the first letter of each whitespace-separated word, lowercase the rest, rejoin with single spaces.
212 Title,
213 /// Replace each line with its content repeated N times, concatenated.
214 Repeat(usize),
215 /// Set key K to the string value V (Value::String(V)) in each JSON object line; non-object / unparseable lines pass through unchanged.
216 Set(String, String),
217 /// Remove key K from each JSON object line (jq `del(.K)`); non-object lines pass through.
218 Del(String),
219 /// Rename JSON object key OLD to NEW in each object, preserving the value; no-op if OLD absent. Non-object lines pass through.
220 Rename(String, String),
221 /// Set string key K to V only when K is absent from the JSON object (jq `//=` for a missing key). Present keys keep their value; non-object / unparseable lines pass through unchanged. Key order is normalized on mutation.
222 Default(String, String),
223 /// Reduce all JSON object lines into a single object (later keys overwrite earlier); non-object lines are ignored. Emits one JSON object line, or none if no objects were seen.
224 Merge,
225 /// Floor each numeric line to the nearest lower integer (fmt_num(x.floor())); non-numeric lines pass through unchanged.
226 Floor,
227 /// Round each numeric line up to the nearest integer (ceil); non-numeric lines pass through unchanged.
228 Ceil,
229 /// Clamp each numeric line into the inclusive range [LO, HI]; non-numeric lines pass through unchanged.
230 Clamp(f64, f64),
231 /// Same as `Csv` but tab-separated (TSV).
232 Tsv,
233 /// Parse the accumulated stream as a YAML document (or `---`-separated
234 /// multi-document) and emit each document as a JSON line, so the JSON verbs
235 /// (`field`/`pick`/`keys`/`each`) work over it (the yq leg).
236 Yaml,
237 /// Parse the accumulated stream as one TOML document and emit it as a JSON
238 /// object line.
239 Toml,
240 /// Reduce to a scalar computed by an arithmetic expression over the current
241 /// line count (`x`), evaluated on the fusevm VM.
242 Calc(Expr),
243 /// keep lines containing a literal substring.
244 Contains(String),
245 /// keep lines starting with a literal prefix.
246 Starts(String),
247 /// keep lines ending with a literal suffix.
248 Ends(String),
249 /// drop empty / whitespace-only lines.
250 Nonempty,
251 /// keep only lines that parse as a number.
252 Numeric,
253 /// replace each line with its character count.
254 Len,
255 /// jq `length` (JSON-aware): array element count / object key count / string
256 /// char count / |number| / null=0; a non-JSON line falls back to char count.
257 JsonLen,
258 /// jq slice `.[a:b]` over a JSON array line (a/b may be negative — counted
259 /// from the end — or omitted); a string line is char-sliced; other lines pass
260 /// through unchanged.
261 JsonSlice(Option<i64>, Option<i64>),
262 /// replace each line with its word count.
263 Wc,
264 /// absolute value of each numeric line.
265 Abs,
266 /// round each numeric line to the nearest integer.
267 Round,
268 /// consecutive differences of the numeric series (n values → n-1 deltas) —
269 /// turns a monotonic counter into a per-step rate-of-change.
270 Delta,
271 /// running (cumulative) total of the numeric series.
272 Cumsum,
273 /// simple moving average over a trailing window of N (length-preserving; the
274 /// first points average a shorter, growing window). Smooths a noisy series.
275 Sma(usize),
276 /// exponentially-weighted moving average, smoothing factor `alpha` in (0,1] —
277 /// higher alpha tracks faster, lower is smoother. `s0 = x0`.
278 Ewma(f64),
279 /// prefix every line with a literal string.
280 Prepend(String),
281 /// suffix every line with a literal string.
282 Append(String),
283 /// split each line by DELIM, keep the Nth (1-based) field.
284 Cut(String, usize),
285 /// median of numeric lines.
286 Median,
287 /// Nth percentile (0–100) of the numeric values, linear interpolation between
288 /// closest ranks (numpy default). `percentile 99` / `p99` for latency tails.
289 Percentile(f64),
290 /// population standard deviation of numeric lines.
291 Stddev,
292 /// max minus min of numeric lines.
293 Range,
294 /// product of numeric lines.
295 Product,
296 /// count of distinct lines.
297 Distinct,
298 /// keep every Nth line (1-based).
299 Sample(usize),
300 /// keep lines from index A to B inclusive (1-based).
301 Slice(usize, usize),
302 /// keep only the Nth line (1-based); out-of-range yields no lines.
303 Index(usize),
304 /// bucket numeric lines into N equal-width ranges -> (range, count) pairs.
305 Bins(usize),
306}
307
308/// The output of evaluating a pipeline: lines, a scalar, or grouped counts.
309#[derive(Debug, Clone, PartialEq)]
310pub enum QueryResult {
311 Lines(Vec<String>),
312 Scalar(f64),
313 Pairs(Vec<(String, u64)>),
314}
315
316/// Evaluate `ops` against `lines`. `elapsed_secs` feeds `rate`.
317pub fn eval(ops: &[QueryOp], lines: &[String], elapsed_secs: f64) -> QueryResult {
318 let mut cur: Vec<String> = lines.to_vec();
319 for op in ops {
320 match op {
321 QueryOp::Match(re) => cur.retain(|l| re.is_match(l)),
322 QueryOp::Reject(re) => cur.retain(|l| !re.is_match(l)),
323 QueryOp::Field(sel) => {
324 for l in cur.iter_mut() {
325 *l = extract_field(l, sel);
326 }
327 }
328 QueryOp::Fields(cols) => {
329 for l in cur.iter_mut() {
330 *l = cols
331 .iter()
332 .map(|&n| nth_col(l, n))
333 .collect::<Vec<_>>()
334 .join(" ");
335 }
336 }
337 QueryOp::Each => {
338 let mut out = Vec::with_capacity(cur.len());
339 for l in &cur {
340 match serde_json::from_str::<Value>(l) {
341 Ok(Value::Array(arr)) => {
342 out.extend(arr.iter().map(json_to_string));
343 }
344 // jq `.[]` over an object iterates its VALUES.
345 Ok(Value::Object(m)) => {
346 out.extend(m.values().map(json_to_string));
347 }
348 _ => out.push(l.clone()),
349 }
350 }
351 cur = out;
352 }
353 QueryOp::Where(e) => {
354 // A string predicate (`match(.q)` / `field in .lv`) can't run on
355 // the numeric VM — route it to the Rust evaluator; purely numeric
356 // predicates stay on fusevm.
357 if crate::expr::expr_has_str(e) {
358 cur.retain(|l| eval_where(e, l));
359 } else {
360 cur.retain(|l| {
361 let x = l.trim().parse::<f64>().unwrap_or(f64::NAN);
362 let resolve = |name: &str| field_num(l, name);
363 crate::expr::eval_pred_ctx(e, x, &resolve).unwrap_or(false)
364 });
365 }
366 }
367 QueryOp::Map(e) => {
368 for l in cur.iter_mut() {
369 let v = {
370 let x = l.trim().parse::<f64>().unwrap_or(f64::NAN);
371 let resolve = |name: &str| field_num(l, name);
372 crate::expr::eval_ctx(e, x, &resolve).unwrap_or(f64::NAN)
373 };
374 *l = fmt_num(v);
375 }
376 }
377 QueryOp::Via(def, workers) => {
378 cur = crate::actor::run_via(def, *workers, &cur);
379 }
380 QueryOp::Count => return QueryResult::Scalar(cur.len() as f64),
381 QueryOp::Rate => {
382 return QueryResult::Scalar(cur.len() as f64 / elapsed_secs.max(0.001));
383 }
384 QueryOp::Tally => {
385 let mut counts: BTreeMap<String, u64> = BTreeMap::new();
386 for l in &cur {
387 *counts.entry(l.clone()).or_insert(0) += 1;
388 }
389 let mut pairs: Vec<(String, u64)> = counts.into_iter().collect();
390 pairs.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
391 return QueryResult::Pairs(pairs);
392 }
393 QueryOp::Calc(e) => {
394 let x = cur.len() as f64;
395 return QueryResult::Scalar(crate::expr::eval(e, x).unwrap_or(0.0));
396 }
397 QueryOp::Sum => return QueryResult::Scalar(nums(&cur).iter().sum()),
398 QueryOp::Min => {
399 let m = nums(&cur).into_iter().fold(f64::INFINITY, f64::min);
400 return QueryResult::Scalar(if m.is_finite() { m } else { 0.0 });
401 }
402 QueryOp::Max => {
403 let m = nums(&cur).into_iter().fold(f64::NEG_INFINITY, f64::max);
404 return QueryResult::Scalar(if m.is_finite() { m } else { 0.0 });
405 }
406 QueryOp::Avg => {
407 let ns = nums(&cur);
408 let a = if ns.is_empty() {
409 0.0
410 } else {
411 ns.iter().sum::<f64>() / ns.len() as f64
412 };
413 return QueryResult::Scalar(a);
414 }
415 QueryOp::Keys => {
416 let mut out = Vec::new();
417 for l in &cur {
418 match serde_json::from_str::<Value>(l) {
419 Ok(Value::Object(m)) => out.extend(m.keys().cloned()),
420 _ => out.push(l.clone()),
421 }
422 }
423 cur = out;
424 }
425 QueryOp::Vals => {
426 let mut out = Vec::new();
427 for l in &cur {
428 match serde_json::from_str::<Value>(l) {
429 Ok(Value::Object(m)) => out.extend(m.values().map(json_to_string)),
430 _ => out.push(l.clone()),
431 }
432 }
433 cur = out;
434 }
435 QueryOp::NonNull => {
436 // jq `values` == `select(. != null)`: keep every non-null input
437 // unchanged, drop only the lines that parse to JSON `null`.
438 cur.retain(|l| !matches!(serde_json::from_str::<Value>(l), Ok(Value::Null)));
439 }
440 QueryOp::Pick(keys) => {
441 for l in cur.iter_mut() {
442 if let Ok(Value::Object(m)) = serde_json::from_str::<Value>(l) {
443 // Build in pick order — serde_json::Map is a BTreeMap and
444 // would re-sort the keys, losing the requested order.
445 let parts: Vec<String> = keys
446 .iter()
447 .filter_map(|k| {
448 m.get(k)
449 .map(|v| format!("{}:{}", Value::String(k.clone()), v))
450 })
451 .collect();
452 *l = format!("{{{}}}", parts.join(","));
453 }
454 }
455 }
456 QueryOp::Csv => cur = to_json_records(&cur, ','),
457 QueryOp::Tsv => cur = to_json_records(&cur, '\t'),
458 QueryOp::Yaml => cur = yaml_to_json(&cur),
459 QueryOp::Toml => cur = toml_to_json(&cur),
460 QueryOp::Sort { numeric, reverse } => {
461 if *numeric {
462 // Like Unix `sort -n`: order by each line's LEADING numeric
463 // token (the first whitespace-delimited field), so mixed rows
464 // such as `2.1 claude` sort by 2.1 — not the whole line, which
465 // would parse as NaN and leave the order untouched.
466 let key = |s: &str| {
467 s.split_whitespace()
468 .next()
469 .and_then(|t| t.parse::<f64>().ok())
470 .unwrap_or(f64::NAN)
471 };
472 cur.sort_by(|a, b| {
473 key(a)
474 .partial_cmp(&key(b))
475 .unwrap_or(std::cmp::Ordering::Equal)
476 });
477 } else {
478 cur.sort();
479 }
480 if *reverse {
481 cur.reverse();
482 }
483 }
484 QueryOp::Uniq => {
485 let mut seen = HashSet::new();
486 cur.retain(|l| seen.insert(l.clone()));
487 }
488 QueryOp::Rev => cur.reverse(),
489 QueryOp::First => cur.truncate(1),
490 QueryOp::Last => {
491 if let Some(l) = cur.pop() {
492 cur = vec![l];
493 } else {
494 cur.clear();
495 }
496 }
497 QueryOp::Take(n) => cur.truncate(*n),
498 QueryOp::Drop(n) => {
499 cur.drain(0..(*n).min(cur.len()));
500 }
501 QueryOp::Upper => {
502 for l in cur.iter_mut() {
503 *l = l.to_uppercase();
504 }
505 }
506 QueryOp::Lower => {
507 for l in cur.iter_mut() {
508 *l = l.to_lowercase();
509 }
510 }
511 QueryOp::Trim => {
512 for l in cur.iter_mut() {
513 *l = l.trim().to_string();
514 }
515 }
516 QueryOp::Replace(re, to) => {
517 for l in cur.iter_mut() {
518 *l = re.replace_all(l, to.as_str()).into_owned();
519 }
520 }
521 QueryOp::Join(sep) => {
522 cur = vec![cur.join(sep)];
523 }
524 QueryOp::Nth(n) => {
525 cur = cur
526 .get(n.saturating_sub(1))
527 .cloned()
528 .map(|l| vec![l])
529 .unwrap_or_default();
530 }
531 QueryOp::Sel { css, attr } => {
532 let doc = Html::parse_document(&cur.join("\n"));
533 cur = match Selector::parse(css) {
534 Ok(sel) => doc
535 .select(&sel)
536 .filter_map(|el| match attr {
537 Some(a) => el.value().attr(a).map(str::to_string),
538 None => Some(el.text().collect::<String>().trim().to_string()),
539 })
540 .collect(),
541 Err(_) => Vec::new(),
542 };
543 }
544 QueryOp::Find(css) => {
545 let doc = Html::parse_document(&cur.join("\n"));
546 cur = match Selector::parse(css) {
547 Ok(sel) => doc.select(&sel).map(|el| el.html()).collect(),
548 Err(_) => Vec::new(),
549 };
550 }
551 QueryOp::Attr(name) => {
552 let mut out = Vec::with_capacity(cur.len());
553 for l in &cur {
554 let frag = Html::parse_fragment(l);
555 if let Some(v) = first_element(&frag).and_then(|e| e.value().attr(name)) {
556 out.push(v.to_string());
557 }
558 }
559 cur = out;
560 }
561 QueryOp::Text => {
562 for l in cur.iter_mut() {
563 let frag = Html::parse_fragment(l);
564 if let Some(e) = first_element(&frag) {
565 // xpath `text()` is the element's DIRECT child text nodes,
566 // not all descendant text — `e.text()` would also fold in a
567 // child element's text (e.g. `<a>1<b>X</b>2</a>` -> `1X2`).
568 *l = e
569 .children()
570 .filter_map(|c| c.value().as_text().map(|t| t.to_string()))
571 .collect::<String>()
572 .trim()
573 .to_string();
574 }
575 }
576 }
577 QueryOp::SortBy(field) => {
578 // Split object lines (carrying their field value) from the rest;
579 // non-objects keep their relative input order and sink to the end.
580 let mut objs: Vec<(String, String)> = Vec::new();
581 let mut rest: Vec<String> = Vec::new();
582 for l in cur.drain(..) {
583 if let Ok(Value::Object(m)) = serde_json::from_str::<Value>(&l) {
584 let key = m.get(field).map(json_to_string).unwrap_or_default();
585 objs.push((key, l));
586 } else {
587 rest.push(l);
588 }
589 }
590 let all_numeric =
591 !objs.is_empty() && objs.iter().all(|(k, _)| k.trim().parse::<f64>().is_ok());
592 if all_numeric {
593 // slice::sort_by is stable — equal keys preserve input order.
594 objs.sort_by(|a, b| {
595 let na = a.0.trim().parse::<f64>().unwrap_or(f64::NAN);
596 let nb = b.0.trim().parse::<f64>().unwrap_or(f64::NAN);
597 na.partial_cmp(&nb).unwrap_or(std::cmp::Ordering::Equal)
598 });
599 } else {
600 objs.sort_by(|a, b| a.0.cmp(&b.0));
601 }
602 cur = objs.into_iter().map(|(_, l)| l).collect();
603 cur.extend(rest);
604 }
605 QueryOp::UniqueBy(field) => {
606 let mut seen: HashSet<String> = HashSet::new();
607 cur.retain(|l| {
608 let key = match serde_json::from_str::<Value>(l) {
609 Ok(Value::Object(m)) => {
610 m.get(field).map(json_to_string).unwrap_or_default()
611 }
612 _ => l.clone(),
613 };
614 seen.insert(key)
615 });
616 }
617 QueryOp::CountBy(field) => {
618 let mut counts: BTreeMap<String, u64> = BTreeMap::new();
619 for l in &cur {
620 let key = match serde_json::from_str::<Value>(l) {
621 Ok(Value::Object(m)) => {
622 m.get(field).map(json_to_string).unwrap_or_default()
623 }
624 _ => l.clone(),
625 };
626 *counts.entry(key).or_insert(0) += 1;
627 }
628 let mut pairs: Vec<(String, u64)> = counts.into_iter().collect();
629 pairs.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
630 return QueryResult::Pairs(pairs);
631 }
632 QueryOp::GroupBy(field) => {
633 // jq group_by: one array per distinct key, groups sorted by key.
634 let mut groups: BTreeMap<String, Vec<Value>> = BTreeMap::new();
635 for l in &cur {
636 let (key, val) = match serde_json::from_str::<Value>(l) {
637 Ok(v @ Value::Object(_)) => {
638 let k = v.get(field).map(json_to_string).unwrap_or_default();
639 (k, v)
640 }
641 Ok(v) => (l.clone(), v),
642 Err(_) => (l.clone(), Value::String(l.clone())),
643 };
644 groups.entry(key).or_default().push(val);
645 }
646 cur = groups
647 .into_values()
648 .map(|g| Value::Array(g).to_string())
649 .collect();
650 }
651 QueryOp::MinBy(field) => {
652 let best = cur
653 .iter()
654 .filter(|l| !field_num(l, field).is_nan())
655 .min_by(|a, b| {
656 field_num(a, field)
657 .partial_cmp(&field_num(b, field))
658 .unwrap_or(std::cmp::Ordering::Equal)
659 });
660 return QueryResult::Lines(best.into_iter().cloned().collect());
661 }
662 QueryOp::MaxBy(field) => {
663 // Ignore lines whose FIELD is absent/non-numeric (field_num -> NaN),
664 // then keep the record with the greatest value. On ties the last
665 // maximal record wins (std max_by semantics). Empty input -> no lines.
666 let best = cur
667 .iter()
668 .filter(|l| !field_num(l, field).is_nan())
669 .max_by(|a, b| {
670 field_num(a, field)
671 .partial_cmp(&field_num(b, field))
672 .unwrap_or(std::cmp::Ordering::Equal)
673 })
674 .cloned();
675 return QueryResult::Lines(best.into_iter().collect());
676 }
677 QueryOp::Has(key) => {
678 cur.retain(|l| {
679 matches!(
680 serde_json::from_str::<Value>(l),
681 Ok(Value::Object(ref m)) if m.contains_key(key)
682 )
683 });
684 }
685 QueryOp::HasKey(key) => {
686 // jq `has`: a per-input boolean test, NOT a filter — every input
687 // yields exactly one `true`/`false` line.
688 for l in cur.iter_mut() {
689 let present = matches!(
690 serde_json::from_str::<Value>(l),
691 Ok(Value::Object(ref m)) if m.contains_key(key)
692 );
693 *l = if present { "true" } else { "false" }.to_string();
694 }
695 }
696 QueryOp::Entries => {
697 let mut out = Vec::with_capacity(cur.len());
698 for l in &cur {
699 match serde_json::from_str::<Value>(l) {
700 Ok(Value::Object(m)) => {
701 for (k, v) in &m {
702 out.push(format!(
703 "{{\"key\":{},\"value\":{}}}",
704 Value::String(k.clone()),
705 v
706 ));
707 }
708 }
709 _ => out.push(l.clone()),
710 }
711 }
712 cur = out;
713 }
714 QueryOp::Flatten => {
715 // jq `flatten` fully flattens all nesting levels; recurse into
716 // every sub-array and emit only the non-array leaves.
717 fn push_leaves(v: &Value, out: &mut Vec<String>) {
718 match v {
719 Value::Array(a) => a.iter().for_each(|e| push_leaves(e, out)),
720 other => out.push(json_to_string(other)),
721 }
722 }
723 let mut out = Vec::with_capacity(cur.len());
724 for l in &cur {
725 match serde_json::from_str::<Value>(l) {
726 Ok(Value::Array(arr)) => arr.iter().for_each(|e| push_leaves(e, &mut out)),
727 _ => out.push(l.clone()),
728 }
729 }
730 cur = out;
731 }
732 QueryOp::Add => {
733 for l in cur.iter_mut() {
734 if let Ok(Value::Array(arr)) = serde_json::from_str::<Value>(l) {
735 if arr.is_empty() {
736 *l = String::new();
737 } else if arr.iter().all(Value::is_number) {
738 let sum: f64 = arr.iter().filter_map(Value::as_f64).sum();
739 *l = fmt_num(sum);
740 } else {
741 *l = arr.iter().map(json_to_string).collect::<String>();
742 }
743 }
744 }
745 }
746 QueryOp::Over(n) => {
747 cur.retain(|l| l.trim().parse::<f64>().map(|v| v > *n).unwrap_or(false));
748 }
749 QueryOp::Under(n) => {
750 cur.retain(|l| l.trim().parse::<f64>().map(|v| v < *n).unwrap_or(false));
751 }
752 QueryOp::Between(lo, hi) => {
753 cur.retain(|l| {
754 let x = l.trim().parse::<f64>().unwrap_or(f64::NAN);
755 x >= *lo && x <= *hi
756 });
757 }
758 QueryOp::Enumerate => {
759 for (i, l) in cur.iter_mut().enumerate() {
760 *l = format!("{}\t{}", i + 1, l);
761 }
762 }
763 QueryOp::Words => {
764 let mut out = Vec::with_capacity(cur.len());
765 for l in &cur {
766 out.extend(l.split_whitespace().map(str::to_string));
767 }
768 cur = out;
769 }
770 QueryOp::Dedup => {
771 cur.dedup();
772 }
773 QueryOp::Tailn(n) => {
774 let len = cur.len();
775 if *n < len {
776 cur.drain(0..len - *n);
777 }
778 }
779 QueryOp::Pad(n) => {
780 // `format!` width is a u16 internally: a width >= 65536 panics
781 // ("Formatting argument out of range"). Cap it — no real pane is
782 // that wide and the widget layer clips lines to the pane anyway.
783 let n = (*n).min(u16::MAX as usize);
784 for l in cur.iter_mut() {
785 *l = format!("{:<width$}", l, width = n);
786 }
787 }
788 QueryOp::Lpad(width) => {
789 let w = (*width).min(u16::MAX as usize);
790 for l in cur.iter_mut() {
791 *l = format!("{:>width$}", l, width = w);
792 }
793 }
794 QueryOp::Grepf(field, re) => {
795 cur.retain(|l| {
796 let val = if let Ok(Value::Object(m)) = serde_json::from_str::<Value>(l) {
797 m.get(field).map(json_to_string).unwrap_or_default()
798 } else if let Ok(idx) = field.parse::<usize>() {
799 l.split_whitespace()
800 .nth(idx.saturating_sub(1))
801 .unwrap_or("")
802 .to_string()
803 } else {
804 String::new()
805 };
806 re.is_match(&val)
807 });
808 }
809 QueryOp::Flip => {
810 for l in cur.iter_mut() {
811 *l = l.chars().rev().collect();
812 }
813 }
814 QueryOp::Basename => {
815 for l in cur.iter_mut() {
816 let t = l.trim_end_matches('/');
817 *l = t.rsplit('/').next().unwrap_or(t).to_string();
818 }
819 }
820 QueryOp::Dirname => {
821 for l in cur.iter_mut() {
822 let t = l.trim_end_matches('/');
823 *l = match t.rsplit_once('/') {
824 Some(("", _)) => "/".to_string(),
825 Some((dir, _)) => dir.to_string(),
826 None => ".".to_string(),
827 };
828 }
829 }
830 QueryOp::Commafy => {
831 for l in cur.iter_mut() {
832 *l = commafy(l);
833 }
834 }
835 QueryOp::Bytes => {
836 for l in cur.iter_mut() {
837 if let Ok(v) = l.trim().parse::<f64>() {
838 *l = humanize_bytes(v);
839 }
840 }
841 }
842 QueryOp::Duration => {
843 for l in cur.iter_mut() {
844 if let Ok(v) = l.trim().parse::<f64>() {
845 *l = humanize_duration(v);
846 }
847 }
848 }
849 // Resolved to the input widget's pipeline before eval; a no-op if it
850 // survives (empty input, or eval reached without substitution).
851 QueryOp::Apply(_) => {}
852 QueryOp::B64 => {
853 for l in cur.iter_mut() {
854 *l = STANDARD.encode(l.as_bytes());
855 }
856 }
857 QueryOp::B64d => {
858 for l in cur.iter_mut() {
859 if let Some(s) = STANDARD
860 .decode(l.as_bytes())
861 .ok()
862 .and_then(|b| String::from_utf8(b).ok())
863 {
864 *l = s;
865 }
866 }
867 }
868 QueryOp::Hex => {
869 for l in cur.iter_mut() {
870 *l = l.bytes().map(|b| format!("{:02x}", b)).collect();
871 }
872 }
873 QueryOp::Unhex => {
874 for l in cur.iter_mut() {
875 let chars: Vec<char> = l.chars().collect();
876 if chars.is_empty() || !chars.len().is_multiple_of(2) {
877 continue;
878 }
879 let mut bytes = Vec::with_capacity(chars.len() / 2);
880 let mut ok = true;
881 let mut i = 0;
882 while i < chars.len() {
883 let pair: String = chars[i..i + 2].iter().collect();
884 match u8::from_str_radix(&pair, 16) {
885 Ok(b) => bytes.push(b),
886 Err(_) => {
887 ok = false;
888 break;
889 }
890 }
891 i += 2;
892 }
893 if ok {
894 if let Ok(decoded) = String::from_utf8(bytes) {
895 *l = decoded;
896 }
897 }
898 }
899 }
900 QueryOp::Urlenc => {
901 for l in cur.iter_mut() {
902 *l = utf8_percent_encode(l, NON_ALPHANUMERIC).to_string();
903 }
904 }
905 QueryOp::Urldec => {
906 for l in cur.iter_mut() {
907 if let Ok(decoded) = percent_decode_str(l).decode_utf8() {
908 *l = decoded.into_owned();
909 }
910 }
911 }
912 QueryOp::Extract(re) => {
913 cur = cur
914 .iter()
915 .filter_map(|l| {
916 re.captures(l).map(|caps| {
917 caps.get(1)
918 .unwrap_or_else(|| caps.get(0).unwrap())
919 .as_str()
920 .to_string()
921 })
922 })
923 .collect();
924 }
925 QueryOp::Split(delim) => {
926 let mut out: Vec<String> = Vec::with_capacity(cur.len());
927 for l in cur.iter() {
928 for part in l.split(delim.as_str()) {
929 out.push(part.to_string());
930 }
931 }
932 cur = out;
933 }
934 QueryOp::Substr(a, b) => {
935 for l in cur.iter_mut() {
936 *l = l.chars().skip(*a).take(b.saturating_sub(*a)).collect();
937 }
938 }
939 QueryOp::Chars => {
940 let mut out: Vec<String> = Vec::new();
941 for l in cur.iter() {
942 for ch in l.chars() {
943 out.push(ch.to_string());
944 }
945 }
946 cur = out;
947 }
948 QueryOp::Title => {
949 for l in cur.iter_mut() {
950 *l = l
951 .split_whitespace()
952 .map(|w| {
953 let mut cs = w.chars();
954 match cs.next() {
955 Some(f) => {
956 f.to_uppercase().collect::<String>()
957 + &cs.as_str().to_lowercase()
958 }
959 None => String::new(),
960 }
961 })
962 .collect::<Vec<_>>()
963 .join(" ");
964 }
965 }
966 QueryOp::Repeat(n) => {
967 for l in cur.iter_mut() {
968 *l = l.repeat(*n);
969 }
970 }
971 QueryOp::Set(key, val) => {
972 for l in cur.iter_mut() {
973 if let Ok(Value::Object(mut m)) = serde_json::from_str::<Value>(l) {
974 m.insert(key.clone(), Value::String(val.clone()));
975 *l = Value::Object(m).to_string();
976 }
977 }
978 }
979 QueryOp::Del(key) => {
980 for l in cur.iter_mut() {
981 if let Ok(Value::Object(mut m)) = serde_json::from_str::<Value>(l) {
982 m.remove(key);
983 *l = Value::Object(m).to_string();
984 }
985 }
986 }
987 QueryOp::Rename(old, new) => {
988 for l in cur.iter_mut() {
989 if let Ok(Value::Object(mut m)) = serde_json::from_str::<Value>(l) {
990 if let Some(v) = m.remove(old) {
991 m.insert(new.clone(), v);
992 *l = Value::Object(m).to_string();
993 }
994 }
995 }
996 }
997 QueryOp::Default(key, val) => {
998 for l in cur.iter_mut() {
999 if let Ok(Value::Object(mut m)) = serde_json::from_str::<Value>(l) {
1000 m.entry(key.clone()).or_insert(Value::String(val.clone()));
1001 *l = Value::Object(m).to_string();
1002 }
1003 }
1004 }
1005 QueryOp::Merge => {
1006 let mut acc = serde_json::Map::new();
1007 let mut saw_object = false;
1008 for l in cur.iter() {
1009 if let Ok(Value::Object(m)) = serde_json::from_str::<Value>(l) {
1010 saw_object = true;
1011 for (k, v) in m {
1012 acc.insert(k, v);
1013 }
1014 }
1015 }
1016 return if saw_object {
1017 QueryResult::Lines(vec![Value::Object(acc).to_string()])
1018 } else {
1019 QueryResult::Lines(vec![])
1020 };
1021 }
1022 QueryOp::Floor => {
1023 for l in cur.iter_mut() {
1024 if let Ok(x) = l.parse::<f64>() {
1025 *l = fmt_num(x.floor());
1026 }
1027 }
1028 }
1029 QueryOp::Ceil => {
1030 for l in cur.iter_mut() {
1031 if let Ok(x) = l.trim().parse::<f64>() {
1032 *l = fmt_num(x.ceil());
1033 }
1034 }
1035 }
1036 QueryOp::Clamp(lo, hi) => {
1037 for l in cur.iter_mut() {
1038 if let Ok(x) = l.trim().parse::<f64>() {
1039 *l = fmt_num(x.clamp(*lo, *hi));
1040 }
1041 }
1042 }
1043 QueryOp::Contains(s) => cur.retain(|l| l.contains(s.as_str())),
1044 QueryOp::Starts(p) => cur.retain(|l| l.starts_with(p.as_str())),
1045 QueryOp::Ends(s) => cur.retain(|l| l.ends_with(s.as_str())),
1046 QueryOp::Nonempty => cur.retain(|l| !l.trim().is_empty()),
1047 QueryOp::Numeric => cur.retain(|l| l.trim().parse::<f64>().is_ok()),
1048 QueryOp::Len => {
1049 for l in cur.iter_mut() {
1050 *l = l.chars().count().to_string();
1051 }
1052 }
1053 QueryOp::JsonLen => {
1054 for l in cur.iter_mut() {
1055 *l = match serde_json::from_str::<Value>(l) {
1056 Ok(Value::Array(a)) => a.len().to_string(),
1057 Ok(Value::Object(m)) => m.len().to_string(),
1058 Ok(Value::String(s)) => s.chars().count().to_string(),
1059 Ok(Value::Number(n)) => fmt_num(n.as_f64().unwrap_or(0.0).abs()),
1060 Ok(Value::Null) => "0".to_string(),
1061 // jq errors on a bool length; arb has no per-line error
1062 // channel, so fall back to the raw line's char count.
1063 _ => l.chars().count().to_string(),
1064 };
1065 }
1066 }
1067 QueryOp::JsonSlice(a, b) => {
1068 for l in cur.iter_mut() {
1069 match serde_json::from_str::<Value>(l) {
1070 Ok(Value::Array(arr)) => {
1071 let (lo, hi) = slice_bounds(*a, *b, arr.len());
1072 *l = Value::Array(arr[lo..hi].to_vec()).to_string();
1073 }
1074 Ok(Value::String(s)) => {
1075 let chars: Vec<char> = s.chars().collect();
1076 let (lo, hi) = slice_bounds(*a, *b, chars.len());
1077 *l = chars[lo..hi].iter().collect();
1078 }
1079 _ => {} // non-array/string lines pass through
1080 }
1081 }
1082 }
1083 QueryOp::Wc => {
1084 for l in cur.iter_mut() {
1085 *l = l.split_whitespace().count().to_string();
1086 }
1087 }
1088 QueryOp::Abs => {
1089 for l in cur.iter_mut() {
1090 if let Ok(v) = l.trim().parse::<f64>() {
1091 *l = fmt_num(v.abs());
1092 }
1093 }
1094 }
1095 QueryOp::Round => {
1096 for l in cur.iter_mut() {
1097 if let Ok(v) = l.trim().parse::<f64>() {
1098 *l = fmt_num(v.round());
1099 }
1100 }
1101 }
1102 QueryOp::Delta => {
1103 let ns = nums(&cur);
1104 cur = ns.windows(2).map(|w| fmt_num(w[1] - w[0])).collect();
1105 }
1106 QueryOp::Cumsum => {
1107 let ns = nums(&cur);
1108 let mut acc = 0.0;
1109 cur = ns
1110 .iter()
1111 .map(|&v| {
1112 acc += v;
1113 fmt_num(acc)
1114 })
1115 .collect();
1116 }
1117 QueryOp::Sma(n) => {
1118 let ns = nums(&cur);
1119 let w = (*n).max(1);
1120 cur = (0..ns.len())
1121 .map(|i| {
1122 let lo = i + 1 - (i + 1).min(w);
1123 let win = &ns[lo..=i];
1124 fmt_num(win.iter().sum::<f64>() / win.len() as f64)
1125 })
1126 .collect();
1127 }
1128 QueryOp::Ewma(alpha) => {
1129 let a = alpha.clamp(0.0, 1.0);
1130 let ns = nums(&cur);
1131 let mut s = 0.0;
1132 cur = ns
1133 .iter()
1134 .enumerate()
1135 .map(|(i, &v)| {
1136 s = if i == 0 { v } else { a * v + (1.0 - a) * s };
1137 fmt_num(s)
1138 })
1139 .collect();
1140 }
1141 QueryOp::Prepend(pre) => {
1142 for l in cur.iter_mut() {
1143 *l = format!("{pre}{l}");
1144 }
1145 }
1146 QueryOp::Append(suf) => {
1147 for l in cur.iter_mut() {
1148 l.push_str(suf);
1149 }
1150 }
1151 QueryOp::Cut(delim, n) => {
1152 for l in cur.iter_mut() {
1153 *l = l
1154 .split(delim.as_str())
1155 .nth(n.saturating_sub(1))
1156 .unwrap_or("")
1157 .to_string();
1158 }
1159 }
1160 QueryOp::Median => {
1161 let mut ns = nums(&cur);
1162 ns.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
1163 let m = if ns.is_empty() {
1164 0.0
1165 } else if ns.len() % 2 == 1 {
1166 ns[ns.len() / 2]
1167 } else {
1168 (ns[ns.len() / 2 - 1] + ns[ns.len() / 2]) / 2.0
1169 };
1170 return QueryResult::Scalar(m);
1171 }
1172 QueryOp::Percentile(p) => {
1173 let mut ns = nums(&cur);
1174 ns.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
1175 let v = if ns.is_empty() {
1176 0.0
1177 } else if ns.len() == 1 {
1178 ns[0]
1179 } else {
1180 // Linear interpolation between ranks (numpy default), so `p50`
1181 // equals `median` and `p90` of 1..10 is 9.1, matching the docs.
1182 let frac = p.clamp(0.0, 100.0) / 100.0;
1183 let pos = frac * (ns.len() - 1) as f64;
1184 let lo = pos.floor() as usize;
1185 let hi = pos.ceil() as usize;
1186 ns[lo] + (ns[hi] - ns[lo]) * (pos - lo as f64)
1187 };
1188 return QueryResult::Scalar(v);
1189 }
1190 QueryOp::Stddev => {
1191 let ns = nums(&cur);
1192 let sd = if ns.is_empty() {
1193 0.0
1194 } else {
1195 let mean = ns.iter().sum::<f64>() / ns.len() as f64;
1196 let var = ns.iter().map(|v| (v - mean).powi(2)).sum::<f64>() / ns.len() as f64;
1197 var.sqrt()
1198 };
1199 return QueryResult::Scalar(sd);
1200 }
1201 QueryOp::Range => {
1202 let ns = nums(&cur);
1203 let r = if ns.is_empty() {
1204 0.0
1205 } else {
1206 ns.iter().cloned().fold(f64::NEG_INFINITY, f64::max)
1207 - ns.iter().cloned().fold(f64::INFINITY, f64::min)
1208 };
1209 return QueryResult::Scalar(r);
1210 }
1211 QueryOp::Product => return QueryResult::Scalar(nums(&cur).iter().product()),
1212 QueryOp::Distinct => {
1213 let set: std::collections::HashSet<&String> = cur.iter().collect();
1214 return QueryResult::Scalar(set.len() as f64);
1215 }
1216 QueryOp::Sample(n) => {
1217 if *n >= 1 {
1218 let mut i = 0usize;
1219 cur.retain(|_| {
1220 i += 1;
1221 i.is_multiple_of(*n)
1222 });
1223 }
1224 }
1225 QueryOp::Slice(a, b) => {
1226 let lo = a.saturating_sub(1).min(cur.len());
1227 let hi = (*b).min(cur.len());
1228 cur = if lo < hi {
1229 cur[lo..hi].to_vec()
1230 } else {
1231 Vec::new()
1232 };
1233 }
1234 QueryOp::Index(n) => {
1235 cur = n
1236 .checked_sub(1)
1237 .and_then(|i| cur.get(i).cloned())
1238 .into_iter()
1239 .collect();
1240 }
1241 QueryOp::Bins(n) => {
1242 let vals = nums(&cur);
1243 // Bound the bucket count: it allocates `vec![0u64; n]` and builds
1244 // n formatted pairs, so an over-large N (e.g. 1e8) balloons memory
1245 // and hangs. No histogram display needs more than this many bars.
1246 let n = (*n).clamp(1, 65_536);
1247 if vals.is_empty() {
1248 return QueryResult::Pairs(Vec::new());
1249 }
1250 let min = vals.iter().cloned().fold(f64::INFINITY, f64::min);
1251 let max = vals.iter().cloned().fold(f64::NEG_INFINITY, f64::max);
1252 let width = ((max - min) / n as f64).max(f64::MIN_POSITIVE);
1253 let mut counts = vec![0u64; n];
1254 for v in &vals {
1255 let idx = (((v - min) / width) as usize).min(n - 1);
1256 counts[idx] += 1;
1257 }
1258 let pairs = counts
1259 .iter()
1260 .enumerate()
1261 .map(|(i, &c)| {
1262 let lo = min + i as f64 * width;
1263 (format!("{}-{}", fmt_num(lo), fmt_num(lo + width)), c)
1264 })
1265 .collect();
1266 return QueryResult::Pairs(pairs);
1267 }
1268 }
1269 }
1270 QueryResult::Lines(cur)
1271}
1272
1273/// True if every op processes each line independently, so the pipeline can be
1274/// applied per-line and its results emitted incrementally (streaming). Reducers,
1275/// sorts/reorderers, and whole-document ops (`sel`/`csv`/`tsv`) are not.
1276/// Split lines into table `(headers, rows)` for the `table` widget. Each line is
1277/// split on whitespace into cells; `cols` (a comma-separated list, from `-cols`)
1278/// names the header row when present. Renderers pad short rows to the column
1279/// count. Shared by the ratatui TUI and the web dashboard so both agree.
1280pub fn table_data(lines: &[String], cols: Option<&str>) -> (Vec<String>, Vec<Vec<String>>) {
1281 let headers: Vec<String> = cols
1282 .map(|c| {
1283 c.split(',')
1284 .map(|s| s.trim().to_string())
1285 .filter(|s| !s.is_empty())
1286 .collect()
1287 })
1288 .unwrap_or_default();
1289 let rows: Vec<Vec<String>> = lines
1290 .iter()
1291 .map(|l| l.split_whitespace().map(str::to_string).collect())
1292 .collect();
1293 (headers, rows)
1294}
1295
1296/// Parse lines as a numeric series for the `spark` widget — each line's first
1297/// whitespace token that parses as a number; non-numeric lines are skipped.
1298pub fn numeric_series(lines: &[String]) -> Vec<f64> {
1299 lines
1300 .iter()
1301 .filter_map(|l| {
1302 l.split_whitespace()
1303 .next()
1304 .and_then(|t| t.parse::<f64>().ok())
1305 })
1306 .collect()
1307}
1308
1309/// Render a numeric series as a unicode sparkline (`▁▂▃▄▅▆▇█`), scaled between
1310/// the series min and max. Shared by the TUI and web so both draw the same shape.
1311/// A flat series renders as the lowest tick; an empty series is the empty string.
1312pub fn sparkline(values: &[f64]) -> String {
1313 const TICKS: [char; 8] = ['▁', '▂', '▃', '▄', '▅', '▆', '▇', '█'];
1314 if values.is_empty() {
1315 return String::new();
1316 }
1317 let min = values.iter().cloned().fold(f64::INFINITY, f64::min);
1318 let max = values.iter().cloned().fold(f64::NEG_INFINITY, f64::max);
1319 let range = max - min;
1320 values
1321 .iter()
1322 .map(|&v| {
1323 let idx = if range <= 0.0 {
1324 0
1325 } else {
1326 (((v - min) / range) * 7.0).round() as usize
1327 };
1328 TICKS[idx.min(7)]
1329 })
1330 .collect()
1331}
1332
1333/// The number of columns a table needs to hold `headers` and `rows` (at least 1).
1334pub fn table_ncols(headers: &[String], rows: &[Vec<String>]) -> usize {
1335 rows.iter()
1336 .map(Vec::len)
1337 .max()
1338 .unwrap_or(0)
1339 .max(headers.len())
1340 .max(1)
1341}
1342
1343pub fn is_line_streamable(ops: &[QueryOp]) -> bool {
1344 ops.iter().all(|op| {
1345 matches!(
1346 op,
1347 QueryOp::Match(_)
1348 | QueryOp::Reject(_)
1349 | QueryOp::Field(_)
1350 | QueryOp::Fields(_)
1351 | QueryOp::Each
1352 | QueryOp::Keys
1353 | QueryOp::Vals
1354 | QueryOp::NonNull
1355 | QueryOp::Pick(_)
1356 | QueryOp::Where(_)
1357 | QueryOp::Map(_)
1358 | QueryOp::Contains(_)
1359 | QueryOp::Starts(_)
1360 | QueryOp::Ends(_)
1361 | QueryOp::Nonempty
1362 | QueryOp::Numeric
1363 | QueryOp::Len
1364 | QueryOp::JsonLen
1365 | QueryOp::JsonSlice(_, _)
1366 | QueryOp::Wc
1367 | QueryOp::Abs
1368 | QueryOp::Round
1369 | QueryOp::Prepend(_)
1370 | QueryOp::Append(_)
1371 | QueryOp::Cut(_, _)
1372 | QueryOp::Upper
1373 | QueryOp::Lower
1374 | QueryOp::Trim
1375 | QueryOp::Replace(_, _)
1376 | QueryOp::B64
1377 | QueryOp::B64d
1378 | QueryOp::Hex
1379 | QueryOp::Unhex
1380 | QueryOp::Urlenc
1381 | QueryOp::Urldec
1382 | QueryOp::Extract(_)
1383 | QueryOp::Substr(_, _)
1384 | QueryOp::Title
1385 | QueryOp::Repeat(_)
1386 | QueryOp::Set(_, _)
1387 | QueryOp::Del(_)
1388 | QueryOp::Rename(_, _)
1389 | QueryOp::Default(_, _)
1390 | QueryOp::Floor
1391 | QueryOp::Ceil
1392 | QueryOp::Clamp(_, _)
1393 | QueryOp::Has(_)
1394 | QueryOp::HasKey(_)
1395 | QueryOp::Entries
1396 | QueryOp::Add
1397 | QueryOp::Over(_)
1398 | QueryOp::Under(_)
1399 | QueryOp::Between(_, _)
1400 | QueryOp::Pad(_)
1401 | QueryOp::Lpad(_)
1402 | QueryOp::Grepf(_, _)
1403 | QueryOp::Flip
1404 | QueryOp::Basename
1405 | QueryOp::Dirname
1406 | QueryOp::Commafy
1407 )
1408 })
1409}
1410
1411/// Extract a field from a line per the selector.
1412fn extract_field(line: &str, sel: &FieldSel) -> String {
1413 match sel {
1414 FieldSel::Col(n) => nth_col(line, *n).to_string(),
1415 FieldSel::Key(path) => serde_json::from_str::<Value>(line)
1416 .ok()
1417 .and_then(|v| walk(v, path))
1418 .map(|v| json_to_string(&v))
1419 .or_else(|| {
1420 if path.len() == 1 {
1421 logfmt_field(line, &path[0])
1422 } else {
1423 None
1424 }
1425 })
1426 .unwrap_or_default(),
1427 }
1428}
1429
1430/// Parse the stream as a YAML document (or multi-document) and emit each document
1431/// as a compact JSON line. Uses serde_yaml's document deserializer so every valid
1432/// YAML `---` document-start marker splits — including `--- # comment` and a
1433/// trailing-space `--- ` (a naive `split("\n---\n")` misses those and then feeds
1434/// the whole multi-doc stream to a single-doc parse that errors, dropping it all).
1435fn yaml_to_json(lines: &[String]) -> Vec<String> {
1436 use serde::Deserialize;
1437 let doc = lines.join("\n");
1438 serde_yaml::Deserializer::from_str(&doc)
1439 .filter_map(|de| Value::deserialize(de).ok())
1440 .filter(|v| !v.is_null())
1441 .map(|mut v| {
1442 apply_yaml_merge(&mut v);
1443 v.to_string()
1444 })
1445 .collect()
1446}
1447
1448/// Apply YAML merge keys (`<<`). serde_yaml expands anchors/aliases but leaves the
1449/// `<<` merge key literal, so `<<: *base` would otherwise surface a stray `<<`
1450/// key and omit the merged fields. Merge each `<<` source into its parent object
1451/// with correct precedence: an explicit key wins over any merged key, and among
1452/// several merged sources (`<<: [*a, *b]`) an earlier source wins.
1453fn apply_yaml_merge(v: &mut Value) {
1454 if let Value::Object(map) = v {
1455 if let Some(merge) = map.remove("<<") {
1456 let sources = match merge {
1457 Value::Array(a) => a,
1458 other => vec![other],
1459 };
1460 for src in sources {
1461 if let Value::Object(m) = src {
1462 for (k, val) in m {
1463 // `or_insert` keeps an already-present key, so explicit
1464 // keys and earlier merge sources both take precedence.
1465 map.entry(k).or_insert(val);
1466 }
1467 }
1468 }
1469 }
1470 }
1471 match v {
1472 Value::Object(map) => map.values_mut().for_each(apply_yaml_merge),
1473 Value::Array(a) => a.iter_mut().for_each(apply_yaml_merge),
1474 _ => {}
1475 }
1476}
1477
1478/// Parse the stream as one TOML document and emit it as a JSON object line
1479/// (empty if it does not parse). Goes through `toml::Value` and converts
1480/// explicitly so a TOML datetime becomes a clean scalar string — deserializing
1481/// straight into `serde_json::Value` instead leaks the toml crate's internal
1482/// `{"$__toml_private_datetime": …}` tagging map into the output.
1483fn toml_to_json(lines: &[String]) -> Vec<String> {
1484 match toml::from_str::<toml::Value>(&lines.join("\n")) {
1485 Ok(v) => vec![toml_value_to_json(&v).to_string()],
1486 Err(_) => Vec::new(),
1487 }
1488}
1489
1490/// Convert a `toml::Value` to a `serde_json::Value`, rendering datetimes as their
1491/// string form (RFC-3339 / date / time) rather than a tagged object.
1492fn toml_value_to_json(v: &toml::Value) -> Value {
1493 match v {
1494 toml::Value::String(s) => Value::String(s.clone()),
1495 toml::Value::Integer(i) => Value::Number((*i).into()),
1496 toml::Value::Float(f) => serde_json::Number::from_f64(*f)
1497 .map(Value::Number)
1498 .unwrap_or(Value::Null),
1499 toml::Value::Boolean(b) => Value::Bool(*b),
1500 toml::Value::Datetime(dt) => Value::String(dt.to_string()),
1501 toml::Value::Array(a) => Value::Array(a.iter().map(toml_value_to_json).collect()),
1502 toml::Value::Table(t) => Value::Object(
1503 t.iter()
1504 .map(|(k, v)| (k.clone(), toml_value_to_json(v)))
1505 .collect(),
1506 ),
1507 }
1508}
1509
1510/// Reassemble physical lines into logical CSV/TSV records. A quoted field may
1511/// contain newlines (RFC 4180), so a line whose double-quotes are unbalanced is
1512/// still inside a quoted field and continues on the next line until they balance
1513/// — otherwise a multi-line field is torn into corrupt phantom rows. A doubled
1514/// `""` (an escaped quote) contributes two quotes, so parity tracks correctly.
1515fn join_quoted_records(lines: &[String]) -> Vec<String> {
1516 let mut records = Vec::new();
1517 let mut buf = String::new();
1518 for line in lines {
1519 if !buf.is_empty() {
1520 buf.push('\n');
1521 }
1522 buf.push_str(line);
1523 if buf.matches('"').count().is_multiple_of(2) {
1524 records.push(std::mem::take(&mut buf));
1525 }
1526 }
1527 if !buf.is_empty() {
1528 records.push(buf); // trailing unbalanced record: emit what we have
1529 }
1530 records
1531}
1532
1533/// Parse a header + data rows of a delimited stream into JSON object strings
1534/// keyed by the header, so `field NAME` works over CSV/TSV.
1535fn to_json_records(lines: &[String], delim: char) -> Vec<String> {
1536 if lines.is_empty() {
1537 return Vec::new();
1538 }
1539 let records = join_quoted_records(lines);
1540 let Some(header_row) = records.first() else {
1541 return Vec::new();
1542 };
1543 let header = split_delim(header_row, delim);
1544 records[1..]
1545 .iter()
1546 .map(|row| {
1547 let vals = split_delim(row, delim);
1548 let mut obj = serde_json::Map::new();
1549 for (i, name) in header.iter().enumerate() {
1550 obj.insert(
1551 name.clone(),
1552 Value::String(vals.get(i).cloned().unwrap_or_default()),
1553 );
1554 }
1555 Value::Object(obj).to_string()
1556 })
1557 .collect()
1558}
1559
1560/// Split one delimited line into fields per RFC 4180 (line-oriented: no embedded
1561/// newlines). A field may be double-quoted; a quoted field may contain the
1562/// delimiter, and a doubled `""` inside it is one literal `"`. Unquoted fields
1563/// are trimmed (preserving prior behavior); quoted fields are returned verbatim.
1564fn split_delim(line: &str, delim: char) -> Vec<String> {
1565 let mut fields = Vec::new();
1566 let mut field = String::new();
1567 let mut in_quotes = false;
1568 let mut quoted = false; // this field opened with a quote
1569 let mut chars = line.chars().peekable();
1570
1571 // Unquoted fields are trimmed; a quoted field is returned exactly.
1572 let finish = |f: &str, q: bool| -> String {
1573 if q {
1574 f.to_string()
1575 } else {
1576 f.trim().to_string()
1577 }
1578 };
1579
1580 while let Some(c) = chars.next() {
1581 if in_quotes {
1582 if c == '"' {
1583 if chars.peek() == Some(&'"') {
1584 chars.next(); // "" -> one literal quote
1585 field.push('"');
1586 } else {
1587 in_quotes = false; // closing quote
1588 }
1589 } else {
1590 field.push(c);
1591 }
1592 } else if c == '"' && field.trim().is_empty() {
1593 in_quotes = true; // opening quote at field start
1594 quoted = true;
1595 field.clear(); // drop any leading whitespace
1596 } else if c == delim {
1597 fields.push(finish(&field, quoted));
1598 field.clear();
1599 quoted = false;
1600 } else {
1601 field.push(c);
1602 }
1603 }
1604 fields.push(finish(&field, quoted));
1605 fields
1606}
1607
1608/// Extract a `key=value` (logfmt) field. A value may be double-quoted and then
1609/// contain spaces (`msg="hello world"`), so this scans key=value pairs honoring
1610/// quotes rather than splitting on whitespace first (which would truncate a
1611/// quoted value at its first space).
1612fn logfmt_field(line: &str, key: &str) -> Option<String> {
1613 let cs: Vec<char> = line.chars().collect();
1614 let mut i = 0;
1615 while i < cs.len() {
1616 while i < cs.len() && cs[i].is_whitespace() {
1617 i += 1;
1618 }
1619 let kstart = i;
1620 while i < cs.len() && cs[i] != '=' && !cs[i].is_whitespace() {
1621 i += 1;
1622 }
1623 let k: String = cs[kstart..i].iter().collect();
1624 if i < cs.len() && cs[i] == '=' {
1625 i += 1; // consume '='
1626 let val = if cs.get(i) == Some(&'"') {
1627 i += 1;
1628 let mut v = String::new();
1629 while i < cs.len() && cs[i] != '"' {
1630 if cs[i] == '\\' && i + 1 < cs.len() {
1631 i += 1; // keep the escaped char verbatim
1632 }
1633 v.push(cs[i]);
1634 i += 1;
1635 }
1636 i += 1; // closing quote (or past end)
1637 v
1638 } else {
1639 let vstart = i;
1640 while i < cs.len() && !cs[i].is_whitespace() {
1641 i += 1;
1642 }
1643 cs[vstart..i].iter().collect()
1644 };
1645 if k == key {
1646 return Some(val);
1647 }
1648 }
1649 }
1650 None
1651}
1652
1653/// The 1-based whitespace column `n` of `line` ("" if absent; 0 = whole line).
1654fn nth_col(line: &str, n: usize) -> &str {
1655 if n == 0 {
1656 return line;
1657 }
1658 line.split_whitespace().nth(n - 1).unwrap_or("")
1659}
1660
1661/// Walk a JSON key/array-index path, consuming the value.
1662fn walk(mut cur: Value, path: &[String]) -> Option<Value> {
1663 for key in path {
1664 cur = match cur {
1665 Value::Object(mut m) => m.remove(key)?,
1666 Value::Array(mut a) => {
1667 // jq array index: a negative index counts from the end (`.[-1]`).
1668 let idx = key.parse::<i64>().ok()?;
1669 let i = if idx < 0 { a.len() as i64 + idx } else { idx };
1670 if i >= 0 && (i as usize) < a.len() {
1671 a.swap_remove(i as usize)
1672 } else {
1673 return None;
1674 }
1675 }
1676 _ => return None,
1677 };
1678 }
1679 Some(cur)
1680}
1681
1682/// Resolve jq slice bounds `[a:b]` to a clamped `lo..hi` over `len`. A negative
1683/// bound counts from the end; `None` is the start/end. Clamped into `0..=len`,
1684/// and `hi < lo` collapses to an empty slice.
1685fn slice_bounds(a: Option<i64>, b: Option<i64>, len: usize) -> (usize, usize) {
1686 let n = len as i64;
1687 let norm = |x: i64| (if x < 0 { x + n } else { x }).clamp(0, n);
1688 let lo = norm(a.unwrap_or(0));
1689 let hi = norm(b.unwrap_or(n));
1690 (lo as usize, hi.max(lo) as usize)
1691}
1692
1693/// Parse the numeric lines of a slice, ignoring non-numeric ones.
1694fn nums(lines: &[String]) -> Vec<f64> {
1695 lines
1696 .iter()
1697 .filter_map(|l| l.trim().parse::<f64>().ok())
1698 .collect()
1699}
1700
1701/// Format a computed number: integers without a decimal point, else default.
1702fn fmt_num(v: f64) -> String {
1703 // The fast `as i64` path is exact only inside i64's range; outside it an
1704 // `as` cast SATURATES to i64::MAX/MIN and silently corrupts the value
1705 // (1e19 -> 9.22e18). Beyond ~9.2e18 fall back to the float formatter, which
1706 // prints the full integer without scientific notation and without a `.0`.
1707 if v.is_finite() && v.fract() == 0.0 && v.abs() < 9.2e18 {
1708 format!("{}", v as i64)
1709 } else {
1710 format!("{v}")
1711 }
1712}
1713
1714/// Group the integer part of a numeric line with thousands separators; leaves a
1715/// non-numeric line (and any fractional/sign parts) intact.
1716fn commafy(line: &str) -> String {
1717 let s = line.trim();
1718 if s.parse::<f64>().is_err() {
1719 return line.to_string();
1720 }
1721 let (sign, rest) = match s.strip_prefix('-') {
1722 Some(r) => ("-", r),
1723 None => ("", s),
1724 };
1725 let (int, frac) = match rest.split_once('.') {
1726 Some((i, f)) => (i, Some(f)),
1727 None => (rest, None),
1728 };
1729 let mut grouped = String::new();
1730 for (idx, ch) in int.chars().enumerate() {
1731 if idx > 0 && (int.len() - idx) % 3 == 0 {
1732 grouped.push(',');
1733 }
1734 grouped.push(ch);
1735 }
1736 match frac {
1737 Some(f) => format!("{sign}{grouped}.{f}"),
1738 None => format!("{sign}{grouped}"),
1739 }
1740}
1741
1742/// Humanize a byte count (1024-based): `1536` -> `1.5 KB`, `1024` -> `1 KB`,
1743/// `500` -> `500 B`. One decimal, trailing `.0` trimmed. Negatives keep the sign.
1744fn humanize_bytes(v: f64) -> String {
1745 const UNITS: [&str; 6] = ["B", "KB", "MB", "GB", "TB", "PB"];
1746 let sign = if v < 0.0 { "-" } else { "" };
1747 let mut n = v.abs();
1748 let mut u = 0;
1749 while n >= 1024.0 && u < UNITS.len() - 1 {
1750 n /= 1024.0;
1751 u += 1;
1752 }
1753 // Bytes are whole; scaled values show one decimal (unless it rounds to .0).
1754 if u == 0 {
1755 format!("{sign}{} {}", n.round() as i64, UNITS[u])
1756 } else {
1757 let r = (n * 10.0).round() / 10.0;
1758 if (r.fract()).abs() < f64::EPSILON {
1759 format!("{sign}{} {}", r as i64, UNITS[u])
1760 } else {
1761 format!("{sign}{r:.1} {}", UNITS[u])
1762 }
1763 }
1764}
1765
1766/// Humanize a duration in seconds as the two largest non-zero units: `3661` ->
1767/// `1h 1m`, `45` -> `45s`, `90061` -> `1d 1h`, `0` -> `0s`. Negatives keep the sign.
1768fn humanize_duration(v: f64) -> String {
1769 let sign = if v < 0.0 { "-" } else { "" };
1770 let total = v.abs().round() as i64;
1771 if total == 0 {
1772 return "0s".to_string();
1773 }
1774 let units = [("d", 86400), ("h", 3600), ("m", 60), ("s", 1)];
1775 let mut rem = total;
1776 let mut parts = Vec::new();
1777 for (label, secs) in units {
1778 let q = rem / secs;
1779 if q > 0 {
1780 parts.push(format!("{q}{label}"));
1781 rem %= secs;
1782 }
1783 }
1784 parts.truncate(2); // two largest non-zero units
1785 format!("{sign}{}", parts.join(" "))
1786}
1787
1788/// Resolve a JSON field of `line` to a number for expression evaluation
1789/// (missing / non-numeric / non-JSON -> NaN, which fails numeric predicates).
1790fn field_num(line: &str, name: &str) -> f64 {
1791 if let Ok(Value::Object(mut m)) = serde_json::from_str::<Value>(line) {
1792 if let Some(v) = m.remove(name) {
1793 return value_to_f64(&v);
1794 }
1795 }
1796 logfmt_field(line, name)
1797 .and_then(|s| s.parse::<f64>().ok())
1798 .unwrap_or(f64::NAN)
1799}
1800
1801/// Public wrapper for `field_str` — used by the TUI facet control to derive
1802/// candidate values from a stream field.
1803pub fn field_str_pub(line: &str, name: &str) -> String {
1804 field_str(line, name)
1805}
1806
1807/// Public wrapper for `field_num` — used by the DAP `evaluate` request to
1808/// resolve `.field` references against the paused stream line (same resolver the
1809/// per-line `where`/`map` evaluation uses).
1810pub fn field_num_pub(line: &str, name: &str) -> f64 {
1811 field_num(line, name)
1812}
1813
1814/// Format a control scalar: integers without a decimal, else the shortest repr.
1815pub fn fmt_scalar(v: f64) -> String {
1816 fmt_num(v)
1817}
1818
1819/// A field's value as a string: a JSON object key, else a logfmt `key=value`,
1820/// else "". `x` (or `.` the whole line via Var) resolves to the whole line.
1821fn field_str(line: &str, name: &str) -> String {
1822 if name == "x" {
1823 return line.to_string();
1824 }
1825 if let Ok(Value::Object(m)) = serde_json::from_str::<Value>(line) {
1826 if let Some(v) = m.get(name) {
1827 return json_to_string(v);
1828 }
1829 }
1830 logfmt_field(line, name).unwrap_or_default()
1831}
1832
1833/// The string value of a substituted string node (`Str`), else "".
1834fn str_of(e: &crate::expr::Expr) -> String {
1835 match e {
1836 crate::expr::Expr::Str(s) => s.clone(),
1837 _ => String::new(),
1838 }
1839}
1840
1841/// Evaluate a string-bearing `where` predicate against one line (Rust, not the
1842/// numeric VM). `match`/`in .set` test strings; and/or/not compose; any purely
1843/// numeric subtree falls back to the fusevm predicate path.
1844fn eval_where(e: &crate::expr::Expr, line: &str) -> bool {
1845 use crate::expr::{BinOp, Expr};
1846 match e {
1847 Expr::Match(inner) => {
1848 let q = str_of(inner);
1849 q.is_empty() || line.to_lowercase().contains(&q.to_lowercase())
1850 }
1851 Expr::InSet(field, inner) => {
1852 let set = str_of(inner);
1853 let items: Vec<&str> = set
1854 .split(',')
1855 .map(str::trim)
1856 .filter(|s| !s.is_empty())
1857 .collect();
1858 if items.is_empty() {
1859 return true; // empty selection -> no filter
1860 }
1861 let val = field_str(line, field);
1862 items.iter().any(|it| *it == val)
1863 }
1864 Expr::Not(a) => !eval_where(a, line),
1865 Expr::Bin(BinOp::And, a, b) => eval_where(a, line) && eval_where(b, line),
1866 Expr::Bin(BinOp::Or, a, b) => eval_where(a, line) || eval_where(b, line),
1867 // A numeric subtree: evaluate it on fusevm as usual.
1868 _ => {
1869 let x = line.trim().parse::<f64>().unwrap_or(f64::NAN);
1870 let resolve = |n: &str| field_num(line, n);
1871 crate::expr::eval_pred_ctx(e, x, &resolve).unwrap_or(false)
1872 }
1873 }
1874}
1875
1876fn value_to_f64(v: &Value) -> f64 {
1877 match v {
1878 Value::Number(n) => n.as_f64().unwrap_or(f64::NAN),
1879 Value::String(s) => s.trim().parse().unwrap_or(f64::NAN),
1880 Value::Bool(b) => {
1881 if *b {
1882 1.0
1883 } else {
1884 0.0
1885 }
1886 }
1887 _ => f64::NAN,
1888 }
1889}
1890
1891/// Render a JSON scalar as a plain string; containers as compact JSON.
1892/// The first real element of a parsed fragment, skipping the synthetic
1893/// `html`/`head`/`body` wrappers that `Html::parse_fragment` inserts.
1894fn first_element(frag: &Html) -> Option<ElementRef<'_>> {
1895 let star = Selector::parse("*").ok()?;
1896 frag.select(&star)
1897 .find(|e| !matches!(e.value().name(), "html" | "head" | "body"))
1898}
1899
1900fn json_to_string(v: &Value) -> String {
1901 match v {
1902 Value::String(s) => s.clone(),
1903 Value::Null => String::new(),
1904 other => other.to_string(),
1905 }
1906}
1907
1908#[cfg(test)]
1909mod tests {
1910 use super::*;
1911
1912 #[test]
1913 fn split_plain() {
1914 assert_eq!(split_delim("a,b,c", ','), vec!["a", "b", "c"]);
1915 }
1916
1917 #[test]
1918 fn split_quoted_comma() {
1919 // A delimiter inside quotes is not a separator.
1920 assert_eq!(split_delim("\"a,b\",c", ','), vec!["a,b", "c"]);
1921 }
1922
1923 #[test]
1924 fn split_doubled_quote() {
1925 // "" inside a quoted field is one literal quote.
1926 assert_eq!(
1927 split_delim("\"she \"\"said\"\"\",x", ','),
1928 vec!["she \"said\"", "x"]
1929 );
1930 }
1931
1932 #[test]
1933 fn split_trailing_empty() {
1934 assert_eq!(split_delim("a,", ','), vec!["a", ""]);
1935 }
1936
1937 #[test]
1938 fn split_tsv() {
1939 assert_eq!(split_delim("a\tb\tc", '\t'), vec!["a", "b", "c"]);
1940 }
1941
1942 #[test]
1943 fn split_unquoted_trims_quoted_keeps_spaces() {
1944 // Unquoted fields are trimmed; a quoted field keeps its inner spaces.
1945 assert_eq!(split_delim(" a , b ", ','), vec!["a", "b"]);
1946 assert_eq!(split_delim("\" a \",b", ','), vec![" a ", "b"]);
1947 }
1948}