1use std::collections::{BTreeMap, HashSet};
9
10use regex::Regex;
11use scraper::{ElementRef, Html, Selector};
12use base64::{engine::general_purpose::STANDARD, Engine as _};
13use percent_encoding::{percent_decode_str, utf8_percent_encode, NON_ALPHANUMERIC};
14
15use serde_json::Value;
16
17use crate::expr::Expr;
18
19#[derive(Debug, Clone, PartialEq)]
21pub enum FieldSel {
22 Col(usize),
23 Key(Vec<String>),
24}
25
26#[derive(Debug, Clone)]
27pub enum QueryOp {
28 Match(Regex),
30 Reject(Regex),
32 Field(FieldSel),
34 Each,
37 Where(Expr),
40 Map(Expr),
43 Count,
45 Rate,
47 Tally,
49 Sum,
51 Min,
52 Max,
53 Avg,
54 Keys,
56 Vals,
57 Pick(Vec<String>),
61 Sort {
63 numeric: bool,
64 reverse: bool,
65 },
66 Uniq,
67 Rev,
68 First,
69 Last,
70 Take(usize),
71 Drop(usize),
72 Upper,
74 Lower,
75 Trim,
76 Replace(Regex, String),
78 Join(String),
80 Nth(usize),
82 Sel {
86 css: String,
87 attr: Option<String>,
88 },
89 Find(String),
93 Attr(String),
97 Text,
100 SortBy(String),
104 UniqueBy(String),
108 CountBy(String),
112 MinBy(String),
114 MaxBy(String),
116 Has(String),
118 Entries,
120 Flatten,
124 Add,
128 Over(f64),
131 Under(f64),
133 Between(f64, f64),
135 Enumerate,
137 Words,
139 Dedup,
141 Tailn(usize),
143 Pad(usize),
145 Lpad(usize),
147 Grepf(String, regex::Regex),
149 Flip,
151 Csv,
154 B64,
155 B64d,
158 Hex,
160 Unhex,
162 Urlenc,
164 Urldec,
166 Extract(Regex),
168 Split(String),
171 Substr(usize, usize),
173 Chars,
175 Title,
177 Repeat(usize),
179 Set(String, String),
181 Del(String),
183 Rename(String, String),
185 Default(String, String),
187 Merge,
189 Floor,
191 Ceil,
193 Clamp(f64, f64),
195 Tsv,
197 Yaml,
201 Toml,
204 Calc(Expr),
207 Contains(String),
209 Starts(String),
211 Ends(String),
213 Nonempty,
215 Numeric,
217 Len,
219 Wc,
221 Abs,
223 Round,
225 Prepend(String),
227 Append(String),
229 Cut(String, usize),
231 Median,
233 Stddev,
235 P95,
237 Range,
239 Product,
241 Distinct,
243 Sample(usize),
245 Slice(usize, usize),
247 Bins(usize),
249}
250
251#[derive(Debug, Clone, PartialEq)]
253pub enum QueryResult {
254 Lines(Vec<String>),
255 Scalar(f64),
256 Pairs(Vec<(String, u64)>),
257}
258
259pub fn eval(ops: &[QueryOp], lines: &[String], elapsed_secs: f64) -> QueryResult {
261 let mut cur: Vec<String> = lines.to_vec();
262 for op in ops {
263 match op {
264 QueryOp::Match(re) => cur.retain(|l| re.is_match(l)),
265 QueryOp::Reject(re) => cur.retain(|l| !re.is_match(l)),
266 QueryOp::Field(sel) => {
267 for l in cur.iter_mut() {
268 *l = extract_field(l, sel);
269 }
270 }
271 QueryOp::Each => {
272 let mut out = Vec::with_capacity(cur.len());
273 for l in &cur {
274 match serde_json::from_str::<Value>(l) {
275 Ok(Value::Array(arr)) => {
276 out.extend(arr.iter().map(json_to_string));
277 }
278 _ => out.push(l.clone()),
279 }
280 }
281 cur = out;
282 }
283 QueryOp::Where(e) => {
284 cur.retain(|l| {
285 let x = l.trim().parse::<f64>().unwrap_or(f64::NAN);
286 let resolve = |name: &str| field_num(l, name);
287 crate::expr::eval_pred_ctx(e, x, &resolve).unwrap_or(false)
288 });
289 }
290 QueryOp::Map(e) => {
291 for l in cur.iter_mut() {
292 let v = {
293 let x = l.trim().parse::<f64>().unwrap_or(f64::NAN);
294 let resolve = |name: &str| field_num(l, name);
295 crate::expr::eval_ctx(e, x, &resolve).unwrap_or(f64::NAN)
296 };
297 *l = fmt_num(v);
298 }
299 }
300 QueryOp::Count => return QueryResult::Scalar(cur.len() as f64),
301 QueryOp::Rate => {
302 return QueryResult::Scalar(cur.len() as f64 / elapsed_secs.max(0.001));
303 }
304 QueryOp::Tally => {
305 let mut counts: BTreeMap<String, u64> = BTreeMap::new();
306 for l in &cur {
307 *counts.entry(l.clone()).or_insert(0) += 1;
308 }
309 let mut pairs: Vec<(String, u64)> = counts.into_iter().collect();
310 pairs.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
311 return QueryResult::Pairs(pairs);
312 }
313 QueryOp::Calc(e) => {
314 let x = cur.len() as f64;
315 return QueryResult::Scalar(crate::expr::eval(e, x).unwrap_or(0.0));
316 }
317 QueryOp::Sum => return QueryResult::Scalar(nums(&cur).iter().sum()),
318 QueryOp::Min => {
319 let m = nums(&cur).into_iter().fold(f64::INFINITY, f64::min);
320 return QueryResult::Scalar(if m.is_finite() { m } else { 0.0 });
321 }
322 QueryOp::Max => {
323 let m = nums(&cur).into_iter().fold(f64::NEG_INFINITY, f64::max);
324 return QueryResult::Scalar(if m.is_finite() { m } else { 0.0 });
325 }
326 QueryOp::Avg => {
327 let ns = nums(&cur);
328 let a = if ns.is_empty() {
329 0.0
330 } else {
331 ns.iter().sum::<f64>() / ns.len() as f64
332 };
333 return QueryResult::Scalar(a);
334 }
335 QueryOp::Keys => {
336 let mut out = Vec::new();
337 for l in &cur {
338 match serde_json::from_str::<Value>(l) {
339 Ok(Value::Object(m)) => out.extend(m.keys().cloned()),
340 _ => out.push(l.clone()),
341 }
342 }
343 cur = out;
344 }
345 QueryOp::Vals => {
346 let mut out = Vec::new();
347 for l in &cur {
348 match serde_json::from_str::<Value>(l) {
349 Ok(Value::Object(m)) => out.extend(m.values().map(json_to_string)),
350 _ => out.push(l.clone()),
351 }
352 }
353 cur = out;
354 }
355 QueryOp::Pick(keys) => {
356 for l in cur.iter_mut() {
357 if let Ok(Value::Object(m)) = serde_json::from_str::<Value>(l) {
358 let parts: Vec<String> = keys
361 .iter()
362 .filter_map(|k| {
363 m.get(k).map(|v| format!("{}:{}", Value::String(k.clone()), v))
364 })
365 .collect();
366 *l = format!("{{{}}}", parts.join(","));
367 }
368 }
369 }
370 QueryOp::Csv => cur = to_json_records(&cur, ','),
371 QueryOp::Tsv => cur = to_json_records(&cur, '\t'),
372 QueryOp::Yaml => cur = yaml_to_json(&cur),
373 QueryOp::Toml => cur = toml_to_json(&cur),
374 QueryOp::Sort { numeric, reverse } => {
375 if *numeric {
376 cur.sort_by(|a, b| {
377 let na = a.trim().parse::<f64>().unwrap_or(f64::NAN);
378 let nb = b.trim().parse::<f64>().unwrap_or(f64::NAN);
379 na.partial_cmp(&nb).unwrap_or(std::cmp::Ordering::Equal)
380 });
381 } else {
382 cur.sort();
383 }
384 if *reverse {
385 cur.reverse();
386 }
387 }
388 QueryOp::Uniq => {
389 let mut seen = HashSet::new();
390 cur.retain(|l| seen.insert(l.clone()));
391 }
392 QueryOp::Rev => cur.reverse(),
393 QueryOp::First => cur.truncate(1),
394 QueryOp::Last => {
395 if let Some(l) = cur.pop() {
396 cur = vec![l];
397 } else {
398 cur.clear();
399 }
400 }
401 QueryOp::Take(n) => cur.truncate(*n),
402 QueryOp::Drop(n) => {
403 cur.drain(0..(*n).min(cur.len()));
404 }
405 QueryOp::Upper => {
406 for l in cur.iter_mut() {
407 *l = l.to_uppercase();
408 }
409 }
410 QueryOp::Lower => {
411 for l in cur.iter_mut() {
412 *l = l.to_lowercase();
413 }
414 }
415 QueryOp::Trim => {
416 for l in cur.iter_mut() {
417 *l = l.trim().to_string();
418 }
419 }
420 QueryOp::Replace(re, to) => {
421 for l in cur.iter_mut() {
422 *l = re.replace_all(l, to.as_str()).into_owned();
423 }
424 }
425 QueryOp::Join(sep) => {
426 cur = vec![cur.join(sep)];
427 }
428 QueryOp::Nth(n) => {
429 cur = cur
430 .get(n.saturating_sub(1))
431 .cloned()
432 .map(|l| vec![l])
433 .unwrap_or_default();
434 }
435 QueryOp::Sel { css, attr } => {
436 let doc = Html::parse_document(&cur.join("\n"));
437 cur = match Selector::parse(css) {
438 Ok(sel) => doc
439 .select(&sel)
440 .filter_map(|el| match attr {
441 Some(a) => el.value().attr(a).map(str::to_string),
442 None => Some(el.text().collect::<String>().trim().to_string()),
443 })
444 .collect(),
445 Err(_) => Vec::new(),
446 };
447 }
448 QueryOp::Find(css) => {
449 let doc = Html::parse_document(&cur.join("\n"));
450 cur = match Selector::parse(css) {
451 Ok(sel) => doc.select(&sel).map(|el| el.html()).collect(),
452 Err(_) => Vec::new(),
453 };
454 }
455 QueryOp::Attr(name) => {
456 let mut out = Vec::with_capacity(cur.len());
457 for l in &cur {
458 let frag = Html::parse_fragment(l);
459 if let Some(v) = first_element(&frag).and_then(|e| e.value().attr(name)) {
460 out.push(v.to_string());
461 }
462 }
463 cur = out;
464 }
465 QueryOp::Text => {
466 for l in cur.iter_mut() {
467 let frag = Html::parse_fragment(l);
468 if let Some(e) = first_element(&frag) {
469 *l = e.text().collect::<String>().trim().to_string();
470 }
471 }
472 }
473 QueryOp::SortBy(field) => {
474 let mut objs: Vec<(String, String)> = Vec::new();
477 let mut rest: Vec<String> = Vec::new();
478 for l in cur.drain(..) {
479 if let Ok(Value::Object(m)) = serde_json::from_str::<Value>(&l) {
480 let key = m.get(field).map(json_to_string).unwrap_or_default();
481 objs.push((key, l));
482 } else {
483 rest.push(l);
484 }
485 }
486 let all_numeric =
487 !objs.is_empty() && objs.iter().all(|(k, _)| k.trim().parse::<f64>().is_ok());
488 if all_numeric {
489 objs.sort_by(|a, b| {
491 let na = a.0.trim().parse::<f64>().unwrap_or(f64::NAN);
492 let nb = b.0.trim().parse::<f64>().unwrap_or(f64::NAN);
493 na.partial_cmp(&nb).unwrap_or(std::cmp::Ordering::Equal)
494 });
495 } else {
496 objs.sort_by(|a, b| a.0.cmp(&b.0));
497 }
498 cur = objs.into_iter().map(|(_, l)| l).collect();
499 cur.extend(rest);
500 }
501 QueryOp::UniqueBy(field) => {
502 let mut seen: HashSet<String> = HashSet::new();
503 cur.retain(|l| {
504 let key = match serde_json::from_str::<Value>(l) {
505 Ok(Value::Object(m)) => {
506 m.get(field).map(json_to_string).unwrap_or_default()
507 }
508 _ => l.clone(),
509 };
510 seen.insert(key)
511 });
512 }
513 QueryOp::CountBy(field) => {
514 let mut counts: BTreeMap<String, u64> = BTreeMap::new();
515 for l in &cur {
516 let key = match serde_json::from_str::<Value>(l) {
517 Ok(Value::Object(m)) => m.get(field).map(json_to_string).unwrap_or_default(),
518 _ => l.clone(),
519 };
520 *counts.entry(key).or_insert(0) += 1;
521 }
522 let mut pairs: Vec<(String, u64)> = counts.into_iter().collect();
523 pairs.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
524 return QueryResult::Pairs(pairs);
525 }
526 QueryOp::MinBy(field) => {
527 let best = cur
528 .iter()
529 .filter(|l| !field_num(l, field).is_nan())
530 .min_by(|a, b| {
531 field_num(a, field)
532 .partial_cmp(&field_num(b, field))
533 .unwrap_or(std::cmp::Ordering::Equal)
534 });
535 return QueryResult::Lines(best.into_iter().cloned().collect());
536 }
537 QueryOp::MaxBy(field) => {
538 let best = cur
542 .iter()
543 .filter(|l| !field_num(l, field).is_nan())
544 .max_by(|a, b| {
545 field_num(a, field)
546 .partial_cmp(&field_num(b, field))
547 .unwrap_or(std::cmp::Ordering::Equal)
548 })
549 .cloned();
550 return QueryResult::Lines(best.into_iter().collect());
551 }
552 QueryOp::Has(key) => {
553 cur.retain(|l| {
554 matches!(
555 serde_json::from_str::<Value>(l),
556 Ok(Value::Object(ref m)) if m.contains_key(key)
557 )
558 });
559 }
560 QueryOp::Entries => {
561 let mut out = Vec::with_capacity(cur.len());
562 for l in &cur {
563 match serde_json::from_str::<Value>(l) {
564 Ok(Value::Object(m)) => {
565 for (k, v) in &m {
566 out.push(format!(
567 "{{\"key\":{},\"value\":{}}}",
568 Value::String(k.clone()),
569 v
570 ));
571 }
572 }
573 _ => out.push(l.clone()),
574 }
575 }
576 cur = out;
577 }
578 QueryOp::Flatten => {
579 let mut out = Vec::with_capacity(cur.len());
580 for l in &cur {
581 match serde_json::from_str::<Value>(l) {
582 Ok(Value::Array(arr)) => {
583 for el in &arr {
584 match el {
585 Value::Array(inner) => {
586 out.extend(inner.iter().map(json_to_string));
587 }
588 other => out.push(json_to_string(other)),
589 }
590 }
591 }
592 _ => out.push(l.clone()),
593 }
594 }
595 cur = out;
596 }
597 QueryOp::Add => {
598 for l in cur.iter_mut() {
599 if let Ok(Value::Array(arr)) = serde_json::from_str::<Value>(l) {
600 if arr.is_empty() {
601 *l = String::new();
602 } else if arr.iter().all(Value::is_number) {
603 let sum: f64 = arr.iter().filter_map(Value::as_f64).sum();
604 *l = fmt_num(sum);
605 } else {
606 *l = arr.iter().map(json_to_string).collect::<String>();
607 }
608 }
609 }
610 }
611 QueryOp::Over(n) => {
612 cur.retain(|l| l.trim().parse::<f64>().map(|v| v > *n).unwrap_or(false));
613 }
614 QueryOp::Under(n) => {
615 cur.retain(|l| l.trim().parse::<f64>().map(|v| v < *n).unwrap_or(false));
616 }
617 QueryOp::Between(lo, hi) => {
618 cur.retain(|l| {
619 let x = l.trim().parse::<f64>().unwrap_or(f64::NAN);
620 x >= *lo && x <= *hi
621 });
622 }
623 QueryOp::Enumerate => {
624 for (i, l) in cur.iter_mut().enumerate() {
625 *l = format!("{}\t{}", i + 1, l);
626 }
627 }
628 QueryOp::Words => {
629 let mut out = Vec::with_capacity(cur.len());
630 for l in &cur {
631 out.extend(l.split_whitespace().map(str::to_string));
632 }
633 cur = out;
634 }
635 QueryOp::Dedup => {
636 cur.dedup();
637 }
638 QueryOp::Tailn(n) => {
639 let len = cur.len();
640 if *n < len {
641 cur.drain(0..len - *n);
642 }
643 }
644 QueryOp::Pad(n) => {
645 let n = *n;
646 for l in cur.iter_mut() {
647 *l = format!("{:<width$}", l, width = n);
648 }
649 }
650 QueryOp::Lpad(width) => {
651 let w = *width;
652 for l in cur.iter_mut() {
653 *l = format!("{:>width$}", l, width = w);
654 }
655 }
656 QueryOp::Grepf(field, re) => {
657 cur.retain(|l| {
658 let val = if let Ok(Value::Object(m)) = serde_json::from_str::<Value>(l) {
659 m.get(field).map(json_to_string).unwrap_or_default()
660 } else if let Ok(idx) = field.parse::<usize>() {
661 l.split_whitespace()
662 .nth(idx.saturating_sub(1))
663 .unwrap_or("")
664 .to_string()
665 } else {
666 String::new()
667 };
668 re.is_match(&val)
669 });
670 }
671 QueryOp::Flip => {
672 for l in cur.iter_mut() {
673 *l = l.chars().rev().collect();
674 }
675 }
676 QueryOp::B64 => {
677 for l in cur.iter_mut() {
678 *l = STANDARD.encode(l.as_bytes());
679 }
680 }
681 QueryOp::B64d => {
682 for l in cur.iter_mut() {
683 if let Some(s) = STANDARD.decode(l.as_bytes()).ok().and_then(|b| String::from_utf8(b).ok()) {
684 *l = s;
685 }
686 }
687 }
688 QueryOp::Hex => {
689 for l in cur.iter_mut() {
690 *l = l.bytes().map(|b| format!("{:02x}", b)).collect();
691 }
692 }
693 QueryOp::Unhex => {
694 for l in cur.iter_mut() {
695 let chars: Vec<char> = l.chars().collect();
696 if chars.is_empty() || chars.len() % 2 != 0 {
697 continue;
698 }
699 let mut bytes = Vec::with_capacity(chars.len() / 2);
700 let mut ok = true;
701 let mut i = 0;
702 while i < chars.len() {
703 let pair: String = chars[i..i + 2].iter().collect();
704 match u8::from_str_radix(&pair, 16) {
705 Ok(b) => bytes.push(b),
706 Err(_) => {
707 ok = false;
708 break;
709 }
710 }
711 i += 2;
712 }
713 if ok {
714 if let Ok(decoded) = String::from_utf8(bytes) {
715 *l = decoded;
716 }
717 }
718 }
719 }
720 QueryOp::Urlenc => {
721 for l in cur.iter_mut() {
722 *l = utf8_percent_encode(l, NON_ALPHANUMERIC).to_string();
723 }
724 }
725 QueryOp::Urldec => {
726 for l in cur.iter_mut() {
727 if let Ok(decoded) = percent_decode_str(l).decode_utf8() {
728 *l = decoded.into_owned();
729 }
730 }
731 }
732 QueryOp::Extract(re) => {
733 cur = cur
734 .iter()
735 .filter_map(|l| {
736 re.captures(l).map(|caps| {
737 caps.get(1)
738 .unwrap_or_else(|| caps.get(0).unwrap())
739 .as_str()
740 .to_string()
741 })
742 })
743 .collect();
744 }
745 QueryOp::Split(delim) => {
746 let mut out: Vec<String> = Vec::with_capacity(cur.len());
747 for l in cur.iter() {
748 for part in l.split(delim.as_str()) {
749 out.push(part.to_string());
750 }
751 }
752 cur = out;
753 }
754 QueryOp::Substr(a, b) => {
755 for l in cur.iter_mut() {
756 *l = l.chars().skip(*a).take(b.saturating_sub(*a)).collect();
757 }
758 }
759 QueryOp::Chars => {
760 let mut out: Vec<String> = Vec::new();
761 for l in cur.iter() {
762 for ch in l.chars() {
763 out.push(ch.to_string());
764 }
765 }
766 cur = out;
767 }
768 QueryOp::Title => {
769 for l in cur.iter_mut() {
770 *l = l
771 .split_whitespace()
772 .map(|w| {
773 let mut cs = w.chars();
774 match cs.next() {
775 Some(f) => f.to_uppercase().collect::<String>() + &cs.as_str().to_lowercase(),
776 None => String::new(),
777 }
778 })
779 .collect::<Vec<_>>()
780 .join(" ");
781 }
782 }
783 QueryOp::Repeat(n) => {
784 for l in cur.iter_mut() {
785 *l = l.repeat(*n);
786 }
787 }
788 QueryOp::Set(key, val) => {
789 for l in cur.iter_mut() {
790 if let Ok(Value::Object(mut m)) = serde_json::from_str::<Value>(l) {
791 m.insert(key.clone(), Value::String(val.clone()));
792 *l = Value::Object(m).to_string();
793 }
794 }
795 }
796 QueryOp::Del(key) => {
797 for l in cur.iter_mut() {
798 if let Ok(Value::Object(mut m)) = serde_json::from_str::<Value>(l) {
799 m.remove(key);
800 *l = Value::Object(m).to_string();
801 }
802 }
803 }
804 QueryOp::Rename(old, new) => {
805 for l in cur.iter_mut() {
806 if let Ok(Value::Object(mut m)) = serde_json::from_str::<Value>(l) {
807 if let Some(v) = m.remove(old) {
808 m.insert(new.clone(), v);
809 *l = Value::Object(m).to_string();
810 }
811 }
812 }
813 }
814 QueryOp::Default(key, val) => {
815 for l in cur.iter_mut() {
816 if let Ok(Value::Object(mut m)) = serde_json::from_str::<Value>(l) {
817 m.entry(key.clone()).or_insert(Value::String(val.clone()));
818 *l = Value::Object(m).to_string();
819 }
820 }
821 }
822 QueryOp::Merge => {
823 let mut acc = serde_json::Map::new();
824 let mut saw_object = false;
825 for l in cur.iter() {
826 if let Ok(Value::Object(m)) = serde_json::from_str::<Value>(l) {
827 saw_object = true;
828 for (k, v) in m {
829 acc.insert(k, v);
830 }
831 }
832 }
833 return if saw_object {
834 QueryResult::Lines(vec![Value::Object(acc).to_string()])
835 } else {
836 QueryResult::Lines(vec![])
837 };
838 }
839 QueryOp::Floor => {
840 for l in cur.iter_mut() {
841 if let Ok(x) = l.parse::<f64>() {
842 *l = fmt_num(x.floor());
843 }
844 }
845 }
846 QueryOp::Ceil => {
847 for l in cur.iter_mut() {
848 if let Ok(x) = l.trim().parse::<f64>() {
849 *l = fmt_num(x.ceil());
850 }
851 }
852 }
853 QueryOp::Clamp(lo, hi) => {
854 for l in cur.iter_mut() {
855 if let Ok(x) = l.trim().parse::<f64>() {
856 *l = fmt_num(x.clamp(*lo, *hi));
857 }
858 }
859 }
860 QueryOp::Contains(s) => cur.retain(|l| l.contains(s.as_str())),
861 QueryOp::Starts(p) => cur.retain(|l| l.starts_with(p.as_str())),
862 QueryOp::Ends(s) => cur.retain(|l| l.ends_with(s.as_str())),
863 QueryOp::Nonempty => cur.retain(|l| !l.trim().is_empty()),
864 QueryOp::Numeric => cur.retain(|l| l.trim().parse::<f64>().is_ok()),
865 QueryOp::Len => {
866 for l in cur.iter_mut() {
867 *l = l.chars().count().to_string();
868 }
869 }
870 QueryOp::Wc => {
871 for l in cur.iter_mut() {
872 *l = l.split_whitespace().count().to_string();
873 }
874 }
875 QueryOp::Abs => {
876 for l in cur.iter_mut() {
877 if let Ok(v) = l.trim().parse::<f64>() {
878 *l = fmt_num(v.abs());
879 }
880 }
881 }
882 QueryOp::Round => {
883 for l in cur.iter_mut() {
884 if let Ok(v) = l.trim().parse::<f64>() {
885 *l = fmt_num(v.round());
886 }
887 }
888 }
889 QueryOp::Prepend(pre) => {
890 for l in cur.iter_mut() {
891 *l = format!("{pre}{l}");
892 }
893 }
894 QueryOp::Append(suf) => {
895 for l in cur.iter_mut() {
896 l.push_str(suf);
897 }
898 }
899 QueryOp::Cut(delim, n) => {
900 for l in cur.iter_mut() {
901 *l = l
902 .split(delim.as_str())
903 .nth(n.saturating_sub(1))
904 .unwrap_or("")
905 .to_string();
906 }
907 }
908 QueryOp::Median => {
909 let mut ns = nums(&cur);
910 ns.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
911 let m = if ns.is_empty() {
912 0.0
913 } else if ns.len() % 2 == 1 {
914 ns[ns.len() / 2]
915 } else {
916 (ns[ns.len() / 2 - 1] + ns[ns.len() / 2]) / 2.0
917 };
918 return QueryResult::Scalar(m);
919 }
920 QueryOp::Stddev => {
921 let ns = nums(&cur);
922 let sd = if ns.is_empty() {
923 0.0
924 } else {
925 let mean = ns.iter().sum::<f64>() / ns.len() as f64;
926 let var =
927 ns.iter().map(|v| (v - mean).powi(2)).sum::<f64>() / ns.len() as f64;
928 var.sqrt()
929 };
930 return QueryResult::Scalar(sd);
931 }
932 QueryOp::P95 => {
933 let mut ns = nums(&cur);
934 ns.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
935 let v = if ns.is_empty() {
936 0.0
937 } else {
938 let rank = ((0.95 * ns.len() as f64).ceil() as usize).clamp(1, ns.len());
939 ns[rank - 1]
940 };
941 return QueryResult::Scalar(v);
942 }
943 QueryOp::Range => {
944 let ns = nums(&cur);
945 let r = if ns.is_empty() {
946 0.0
947 } else {
948 ns.iter().cloned().fold(f64::NEG_INFINITY, f64::max)
949 - ns.iter().cloned().fold(f64::INFINITY, f64::min)
950 };
951 return QueryResult::Scalar(r);
952 }
953 QueryOp::Product => return QueryResult::Scalar(nums(&cur).iter().product()),
954 QueryOp::Distinct => {
955 let set: std::collections::HashSet<&String> = cur.iter().collect();
956 return QueryResult::Scalar(set.len() as f64);
957 }
958 QueryOp::Sample(n) => {
959 if *n >= 1 {
960 let mut i = 0usize;
961 cur.retain(|_| {
962 i += 1;
963 i % *n == 0
964 });
965 }
966 }
967 QueryOp::Slice(a, b) => {
968 let lo = a.saturating_sub(1).min(cur.len());
969 let hi = (*b).min(cur.len());
970 cur = if lo < hi { cur[lo..hi].to_vec() } else { Vec::new() };
971 }
972 QueryOp::Bins(n) => {
973 let vals = nums(&cur);
974 let n = (*n).max(1);
975 if vals.is_empty() {
976 return QueryResult::Pairs(Vec::new());
977 }
978 let min = vals.iter().cloned().fold(f64::INFINITY, f64::min);
979 let max = vals.iter().cloned().fold(f64::NEG_INFINITY, f64::max);
980 let width = ((max - min) / n as f64).max(f64::MIN_POSITIVE);
981 let mut counts = vec![0u64; n];
982 for v in &vals {
983 let idx = (((v - min) / width) as usize).min(n - 1);
984 counts[idx] += 1;
985 }
986 let pairs = counts
987 .iter()
988 .enumerate()
989 .map(|(i, &c)| {
990 let lo = min + i as f64 * width;
991 (format!("{}-{}", fmt_num(lo), fmt_num(lo + width)), c)
992 })
993 .collect();
994 return QueryResult::Pairs(pairs);
995 }
996 }
997 }
998 QueryResult::Lines(cur)
999}
1000
1001pub fn is_line_streamable(ops: &[QueryOp]) -> bool {
1005 ops.iter().all(|op| {
1006 matches!(
1007 op,
1008 QueryOp::Match(_)
1009 | QueryOp::Reject(_)
1010 | QueryOp::Field(_)
1011 | QueryOp::Each
1012 | QueryOp::Keys
1013 | QueryOp::Vals
1014 | QueryOp::Pick(_)
1015 | QueryOp::Where(_)
1016 | QueryOp::Map(_)
1017 | QueryOp::Contains(_)
1018 | QueryOp::Starts(_)
1019 | QueryOp::Ends(_)
1020 | QueryOp::Nonempty
1021 | QueryOp::Numeric
1022 | QueryOp::Len
1023 | QueryOp::Wc
1024 | QueryOp::Abs
1025 | QueryOp::Round
1026 | QueryOp::Prepend(_)
1027 | QueryOp::Append(_)
1028 | QueryOp::Cut(_, _)
1029 | QueryOp::Upper
1030 | QueryOp::Lower
1031 | QueryOp::Trim
1032 | QueryOp::Replace(_, _)
1033 | QueryOp::B64
1034 | QueryOp::B64d
1035 | QueryOp::Hex
1036 | QueryOp::Unhex
1037 | QueryOp::Urlenc
1038 | QueryOp::Urldec
1039 | QueryOp::Extract(_)
1040 | QueryOp::Substr(_, _)
1041 | QueryOp::Title
1042 | QueryOp::Repeat(_)
1043 | QueryOp::Set(_, _)
1044 | QueryOp::Del(_)
1045 | QueryOp::Rename(_, _)
1046 | QueryOp::Default(_, _)
1047 | QueryOp::Floor
1048 | QueryOp::Ceil
1049 | QueryOp::Clamp(_, _)
1050
1051 | QueryOp::Has(_)
1052 | QueryOp::Entries
1053 | QueryOp::Add
1054 | QueryOp::Over(_)
1055 | QueryOp::Under(_)
1056 | QueryOp::Between(_, _)
1057 | QueryOp::Pad(_)
1058 | QueryOp::Lpad(_)
1059 | QueryOp::Grepf(_, _)
1060 | QueryOp::Flip
1061
1062 )
1063 })
1064}
1065
1066fn extract_field(line: &str, sel: &FieldSel) -> String {
1068 match sel {
1069 FieldSel::Col(n) => nth_col(line, *n).to_string(),
1070 FieldSel::Key(path) => serde_json::from_str::<Value>(line)
1071 .ok()
1072 .and_then(|v| walk(v, path))
1073 .map(|v| json_to_string(&v))
1074 .or_else(|| {
1075 if path.len() == 1 {
1076 logfmt_field(line, &path[0])
1077 } else {
1078 None
1079 }
1080 })
1081 .unwrap_or_default(),
1082 }
1083}
1084
1085fn yaml_to_json(lines: &[String]) -> Vec<String> {
1092 let doc = lines.join("\n");
1093 doc.split("\n---\n")
1094 .filter(|c| !c.trim().is_empty())
1095 .filter_map(|c| serde_yaml::from_str::<Value>(c).ok())
1096 .map(|v| v.to_string())
1097 .collect()
1098}
1099
1100fn toml_to_json(lines: &[String]) -> Vec<String> {
1103 match toml::from_str::<Value>(&lines.join("\n")) {
1104 Ok(v) => vec![v.to_string()],
1105 Err(_) => Vec::new(),
1106 }
1107}
1108
1109fn to_json_records(lines: &[String], delim: char) -> Vec<String> {
1110 if lines.is_empty() {
1111 return Vec::new();
1112 }
1113 let header = split_delim(&lines[0], delim);
1114 lines[1..]
1115 .iter()
1116 .map(|row| {
1117 let vals = split_delim(row, delim);
1118 let mut obj = serde_json::Map::new();
1119 for (i, name) in header.iter().enumerate() {
1120 obj.insert(
1121 name.clone(),
1122 Value::String(vals.get(i).cloned().unwrap_or_default()),
1123 );
1124 }
1125 Value::Object(obj).to_string()
1126 })
1127 .collect()
1128}
1129
1130fn split_delim(line: &str, delim: char) -> Vec<String> {
1131 line.split(delim).map(|s| s.trim().to_string()).collect()
1132}
1133
1134fn logfmt_field(line: &str, key: &str) -> Option<String> {
1136 line.split_whitespace().find_map(|tok| {
1137 tok.split_once('=')
1138 .filter(|(k, _)| *k == key)
1139 .map(|(_, v)| v.trim_matches('"').to_string())
1140 })
1141}
1142
1143fn nth_col(line: &str, n: usize) -> &str {
1145 if n == 0 {
1146 return line;
1147 }
1148 line.split_whitespace().nth(n - 1).unwrap_or("")
1149}
1150
1151fn walk(mut cur: Value, path: &[String]) -> Option<Value> {
1153 for key in path {
1154 cur = match cur {
1155 Value::Object(mut m) => m.remove(key)?,
1156 Value::Array(mut a) => {
1157 let i = key.parse::<usize>().ok()?;
1158 if i < a.len() {
1159 a.swap_remove(i)
1160 } else {
1161 return None;
1162 }
1163 }
1164 _ => return None,
1165 };
1166 }
1167 Some(cur)
1168}
1169
1170fn nums(lines: &[String]) -> Vec<f64> {
1172 lines
1173 .iter()
1174 .filter_map(|l| l.trim().parse::<f64>().ok())
1175 .collect()
1176}
1177
1178fn fmt_num(v: f64) -> String {
1180 if v.is_finite() && v.fract() == 0.0 {
1181 format!("{}", v as i64)
1182 } else {
1183 format!("{v}")
1184 }
1185}
1186
1187fn field_num(line: &str, name: &str) -> f64 {
1190 if let Ok(Value::Object(mut m)) = serde_json::from_str::<Value>(line) {
1191 if let Some(v) = m.remove(name) {
1192 return value_to_f64(&v);
1193 }
1194 }
1195 logfmt_field(line, name)
1196 .and_then(|s| s.parse::<f64>().ok())
1197 .unwrap_or(f64::NAN)
1198}
1199
1200fn value_to_f64(v: &Value) -> f64 {
1201 match v {
1202 Value::Number(n) => n.as_f64().unwrap_or(f64::NAN),
1203 Value::String(s) => s.trim().parse().unwrap_or(f64::NAN),
1204 Value::Bool(b) => {
1205 if *b {
1206 1.0
1207 } else {
1208 0.0
1209 }
1210 }
1211 _ => f64::NAN,
1212 }
1213}
1214
1215fn first_element(frag: &Html) -> Option<ElementRef<'_>> {
1219 let star = Selector::parse("*").ok()?;
1220 frag.select(&star)
1221 .find(|e| !matches!(e.value().name(), "html" | "head" | "body"))
1222}
1223
1224fn json_to_string(v: &Value) -> String {
1225 match v {
1226 Value::String(s) => s.clone(),
1227 Value::Null => String::new(),
1228 other => other.to_string(),
1229 }
1230}