1use crate::ast::{AltMode, AnchorKind, Atom, ByteClass, EmptyLoop, Greed, Look, Pattern};
12use crate::token::{BracketKind, TokenKind};
13
14#[derive(Clone, Debug, PartialEq, Eq)]
16pub struct ParseError {
17 pub pos: usize,
19 pub msg: String,
21}
22
23pub fn parse(src: &str) -> Result<Pattern, ParseError> {
30 let (rest, _) = split_empty_loop(src)?;
31 parse_with_shapes(rest, &crate::custom::ShapeSet::new())
32}
33
34pub fn parse_with_empty_loop(src: &str) -> Result<(Pattern, EmptyLoop), ParseError> {
52 let (rest, mode) = split_empty_loop(src)?;
53 Ok((parse_with_shapes(rest, &crate::custom::ShapeSet::new())?, mode))
54}
55
56pub(crate) fn split_empty_loop(src: &str) -> Result<(&str, EmptyLoop), ParseError> {
58 const OPEN: &str = "(?empty:";
59 let head = src.trim_start();
60 let Some(rest) = head.strip_prefix(OPEN) else {
61 return Ok((src, EmptyLoop::Thompson));
62 };
63 let Some(close) = rest.find(')') else {
64 return Err(ParseError { pos: src.len(), msg: "unterminated (?empty:...) directive".into() });
65 };
66 let name = rest[..close].trim();
67 let mode = match name {
68 "thompson" => EmptyLoop::Thompson,
69 "perl" => EmptyLoop::Perl,
70 other => {
71 return Err(ParseError {
72 pos: src.len() - rest.len(),
73 msg: format!(
74 "unknown empty-loop reading (?empty:{other}); the readings are `thompson` and `perl`"
75 ),
76 });
77 }
78 };
79 Ok((&rest[close + 1..], mode))
80}
81
82pub fn parse_with_shapes(
94 src: &str,
95 shapes: &crate::custom::ShapeSet,
96) -> Result<Pattern, ParseError> {
97 parse_with_shapes_to_depth(src, shapes, NEST_LIMIT)
98}
99
100pub fn parse_with_shapes_to_depth(
107 src: &str,
108 shapes: &crate::custom::ShapeSet,
109 limit: u32,
110) -> Result<Pattern, ParseError> {
111 parse_full(src, shapes, &[], limit)
112}
113
114pub fn parse_with_inputs(
122 src: &str,
123 shapes: &crate::custom::ShapeSet,
124 inputs: &[(&str, &[u8])],
125) -> Result<Pattern, ParseError> {
126 parse_full(src, shapes, inputs, NEST_LIMIT)
127}
128
129fn parse_full(
130 src: &str,
131 shapes: &crate::custom::ShapeSet,
132 inputs: &[(&str, &[u8])],
133 limit: u32,
134) -> Result<Pattern, ParseError> {
135 let mut p = Parser {
136 s: src.as_bytes(),
137 i: 0,
138 shapes,
139 inputs,
140 others: std::collections::HashMap::new(),
141 depth: 0,
142 limit,
143 orbit: None,
144 };
145 let pat = p.parse_alt()?;
146 p.skip_spaces();
147 if p.i != p.s.len() {
148 return Err(p.err("unexpected trailing input"));
149 }
150 if pat.resume_is_misplaced() {
151 return Err(p.err("\\G may only begin a pattern, where it says a match abuts the previous one"));
152 }
153 if pat.mentions_reset_start() && crate::nfa::needs_set_engine(&pat) {
159 return Err(p.err(
160 "\\K cannot be combined with a balanced group, a field anchor, a property anchor, \
161 a sub-pattern assertion or a whitespace atom",
162 ));
163 }
164 Ok(crate::ast::normalize(pat))
168}
169
170#[cfg(debug_assertions)]
200pub const NEST_LIMIT: u32 = 32;
201
202#[cfg(not(debug_assertions))]
204pub const NEST_LIMIT: u32 = 250;
205
206struct Parser<'a> {
207 s: &'a [u8],
208 i: usize,
209 shapes: &'a crate::custom::ShapeSet,
210 inputs: &'a [(&'a str, &'a [u8])],
212 others: std::collections::HashMap<String, std::sync::Arc<crate::ast::OtherInput>>,
215 depth: u32,
218 limit: u32,
219 orbit: Option<crate::orbit::OrbitGroup>,
225}
226
227pub(crate) fn set_orbit(pat: &mut Pattern, group: crate::orbit::OrbitGroup) {
234 match pat {
235 Pattern::Atom(a) => a.set_orbit(group),
236 Pattern::Within(v, _) => {
237 for e in v {
238 e.atom.set_orbit(group);
239 }
240 }
241 Pattern::Anchor(crate::ast::AnchorKind::Echo(_, g)) => *g = group,
244 Pattern::Anchor(crate::ast::AnchorKind::Joined { group: g, .. }) => *g = group,
245 Pattern::Empty | Pattern::Guard(..) | Pattern::Anchor(_) => {}
246 Pattern::Star(p, _) | Pattern::Plus(p, _) | Pattern::Opt(p, _) | Pattern::Bind(_, _, p) => {
247 set_orbit(p, group);
248 }
249 Pattern::Repeat(p, _, _, _)
250 | Pattern::Balanced(_, p)
251 | Pattern::Field(_, p)
252 | Pattern::Atomic(p)
253 | Pattern::Assert(p, _, _) => set_orbit(p, group),
254 Pattern::Concat(v) | Pattern::Alt(v, _) => {
255 for p in v {
256 set_orbit(p, group);
257 }
258 }
259 }
260}
261
262impl Parser<'_> {
263 fn err(&self, msg: &str) -> ParseError {
264 ParseError { pos: self.i, msg: msg.to_string() }
265 }
266
267 fn peek(&self) -> Option<u8> {
268 self.s.get(self.i).copied()
269 }
270
271 fn bump(&mut self) -> Option<u8> {
272 let b = self.peek();
273 if b.is_some() {
274 self.i += 1;
275 }
276 b
277 }
278
279 fn skip_spaces(&mut self) {
280 while self.peek() == Some(b' ') {
281 self.i += 1;
282 }
283 }
284
285 fn parse_alt(&mut self) -> Result<Pattern, ParseError> {
286 self.depth += 1;
289 if self.depth > self.limit {
290 let limit = self.limit;
291 self.depth -= 1;
294 return Err(self.err(&format!("pattern nests deeper than {limit} groups")));
295 }
296 let out = self.parse_alt_inner();
297 self.depth -= 1;
298 out
299 }
300
301 fn parse_alt_inner(&mut self) -> Result<Pattern, ParseError> {
302 let mut alts = vec![self.parse_concat()?];
303 let mut mode: Option<AltMode> = None;
304 loop {
305 self.skip_spaces();
306 let Some(m) = self.peek_alt_op() else { break };
307 let at = self.i;
308 self.bump();
309 if m != AltMode::First {
310 self.bump();
311 }
312 if let Some(prev) = mode
316 && prev != m
317 {
318 return Err(ParseError {
319 pos: at,
320 msg: format!(
321 "mixed alternation kinds ({} then {}) at one level; parenthesize to say which binds tighter",
322 prev.glyph(),
323 m.glyph()
324 ),
325 });
326 }
327 mode = Some(m);
328 alts.push(self.parse_concat()?);
329 }
330 if alts.len() == 1 {
331 Ok(alts.pop().expect("one alternative"))
332 } else {
333 Ok(Pattern::Alt(alts, mode.unwrap_or(AltMode::First)))
334 }
335 }
336
337 fn peek_alt_op(&self) -> Option<AltMode> {
341 if self.peek() != Some(b'|') {
342 return None;
343 }
344 match self.s.get(self.i + 1) {
345 Some(b'|') => Some(AltMode::Longest),
346 Some(b'>') => Some(AltMode::Committed),
347 _ => Some(AltMode::First),
348 }
349 }
350
351 fn parse_concat(&mut self) -> Result<Pattern, ParseError> {
352 let mut items: Vec<Pattern> = Vec::new();
353 loop {
354 self.skip_spaces();
355 match self.peek() {
356 None | Some(b'|') | Some(b')') | Some(b']') | Some(b'}') => break,
357 _ => items.push(self.parse_postfix()?),
358 }
359 }
360 if items.is_empty() {
361 Ok(Pattern::Empty)
362 } else if items.len() == 1 {
363 Ok(items.pop().unwrap())
364 } else {
365 Ok(Pattern::Concat(items))
366 }
367 }
368
369 fn parse_postfix(&mut self) -> Result<Pattern, ParseError> {
370 let mut atom = self.parse_atom()?;
371 let kind = match &atom {
376 Pattern::Atom(Atom::Kind(k)) => Some(*k),
377 _ => None,
378 };
379 if let Some(k) = kind
380 && self.peek() == Some(b'{')
381 && self.peek_is_kind_predicate()
382 {
383 atom = self.parse_kind_predicate(k)?;
384 } else if let Pattern::Atom(Atom::Class(c)) = &atom
385 && Self::is_quantity_class(c)
386 && self.peek() == Some(b'{')
387 && self.peek_is_kind_predicate()
388 {
389 let class = c.clone();
392 atom = self.parse_class_predicate(class)?;
393 }
394 let mut possessive = false;
396 match self.peek() {
397 Some(b'*') => {
398 self.bump();
399 let g = self.lean();
400 possessive = self.possessive();
401 atom = Pattern::Star(atom.boxed(), g);
402 }
403 Some(b'+') => {
404 self.bump();
405 let g = self.lean();
406 possessive = self.possessive();
407 atom = Pattern::Plus(atom.boxed(), g);
408 }
409 Some(b'?') => {
410 self.bump();
411 let g = self.lean();
412 possessive = self.possessive();
413 atom = Pattern::Opt(atom.boxed(), g);
414 }
415 Some(b'{') => {
416 atom = self.parse_repeat(atom)?;
417 possessive = self.possessive();
418 }
419 _ => {}
420 }
421 if possessive {
422 atom = Pattern::Atomic(atom.boxed());
423 }
424 if self.peek() == Some(b':') {
427 self.bump();
428 let scoped = self.peek() == Some(b':');
429 if scoped {
430 self.bump();
431 }
432 let name = self.parse_name()?;
433 if atom.binds_anything() {
437 atom.nest_registers(&name);
438 }
439 atom = Pattern::Bind(name, scoped, atom.boxed());
440 }
441 Ok(atom)
442 }
443
444 fn lean(&mut self) -> Greed {
449 if self.peek() == Some(b'?') {
450 self.bump();
451 Greed::Lazy
452 } else {
453 Greed::Greedy
454 }
455 }
456
457 fn possessive(&mut self) -> bool {
465 if self.peek() == Some(b'+') {
466 self.bump();
467 true
468 } else {
469 false
470 }
471 }
472
473 fn peek_is_kind_predicate(&self) -> bool {
477 let mut j = self.i + 1;
478 while j < self.s.len() && self.s[j] != b'}' {
479 if !(self.s[j].is_ascii_digit() || self.s[j] == b',' || self.s[j] == b' ') {
480 return true;
481 }
482 j += 1;
483 }
484 false
485 }
486
487 fn parse_kind_predicate(&mut self, kind: TokenKind) -> Result<Pattern, ParseError> {
495 self.expect(b'{')?;
496 let start = self.i;
497 while !matches!(self.peek(), None | Some(b'}')) {
498 self.bump();
499 }
500 if self.peek() != Some(b'}') {
501 return Err(self.err("unterminated predicate"));
502 }
503 let raw = &self.s[start..self.i];
504 self.bump();
505 let body = String::from_utf8_lossy(raw).into_owned();
506 let trimmed = body.trim();
507 let after_op = trimmed.trim_start_matches(['>', '<', '=']).trim_start();
508 let magnitude = trimmed.starts_with("mag")
509 || (trimmed.starts_with(['>', '<']) && after_op.starts_with(['+', '-']));
510 if kind == TokenKind::Timestamp
514 && trimmed.starts_with(['>', '<'])
515 && after_op.starts_with(['+', '-'])
516 && let Some((body, reg)) = trimmed.rsplit_once(':')
517 {
518 return Self::since_atom(body, reg).map_err(|m| ParseError { pos: start, msg: m });
519 }
520 if magnitude {
521 let pred = Self::magnitude_pred(raw).map_err(|m| self.err(&m))?;
522 return Ok(Pattern::Atom(Atom::KindMag(kind, pred)));
523 }
524 match crate::typed::TypedPred::parse_in(kind, trimmed, self.shapes) {
525 Ok(pred) => Ok(Pattern::Atom(Atom::KindPred(kind, pred))),
526 Err(msg) => Err(ParseError { pos: start, msg }),
527 }
528 }
529
530 fn is_quantity_class(c: &crate::ast::TokenClass) -> bool {
534 !c.negated
535 && c.all.is_empty()
536 && c.none.is_empty()
537 && c.any.iter().all(|m| {
538 matches!(
539 m,
540 Atom::Kind(
541 TokenKind::Quantity
542 | TokenKind::ByteSize
543 | TokenKind::Duration
544 | TokenKind::Percent
545 | TokenKind::Custom(_)
546 )
547 )
548 })
549 }
550
551 fn parse_class_predicate(
554 &mut self,
555 mut class: Box<crate::ast::TokenClass>,
556 ) -> Result<Pattern, ParseError> {
557 let on_quantity = self.parse_kind_predicate(TokenKind::Quantity)?;
558 for member in &mut class.any {
559 let Atom::Kind(k) = *member else {
560 continue;
561 };
562 *member = match &on_quantity {
563 Pattern::Atom(Atom::KindPred(_, pred)) => Atom::KindPred(k, pred.clone()),
564 Pattern::Atom(Atom::KindMag(_, mag)) => Atom::KindMag(k, mag.clone()),
565 _ => continue,
566 };
567 }
568 Ok(Pattern::Atom(Atom::Class(class)))
569 }
570
571 fn quantity_class(&self) -> crate::ast::TokenClass {
575 let mut any = vec![
576 Atom::Kind(TokenKind::Quantity),
577 Atom::Kind(TokenKind::ByteSize),
578 Atom::Kind(TokenKind::Duration),
579 Atom::Kind(TokenKind::Percent),
580 ];
581 for name in crate::library::context_kind_names() {
582 let id = match self.shapes.id_of(name) {
583 Some(id) => Some(id),
584 None if self.shapes.consults_library() => crate::library::id_of(name),
585 None => None,
586 };
587 if let Some(id) = id {
588 any.push(Atom::Kind(TokenKind::Custom(id)));
589 }
590 }
591 crate::ast::TokenClass { any, all: Vec::new(), none: Vec::new(), negated: false }
592 }
593
594 fn since_atom(body: &str, reg: &str) -> Result<Pattern, String> {
597 use crate::ast::{Cmp, Signed};
598 let body = body.trim();
599 let (op, rest) = if let Some(r) = body.strip_prefix(">=") {
600 (Cmp::Ge, r)
601 } else if let Some(r) = body.strip_prefix("<=") {
602 (Cmp::Le, r)
603 } else if let Some(r) = body.strip_prefix('>') {
604 (Cmp::Gt, r)
605 } else if let Some(r) = body.strip_prefix('<') {
606 (Cmp::Lt, r)
607 } else {
608 return Err("a stream-relative instant needs an ordering: \\T{>+1h:t}".to_string());
609 };
610 let rest = rest.trim();
611 let (negative, mag) = match rest.strip_prefix('-') {
612 Some(m) => (true, m),
613 None => (false, rest.strip_prefix('+').unwrap_or(rest)),
614 };
615 let Some(nanos) = crate::typed::parse_duration(mag) else {
616 return Err(format!("{mag:?} is not a duration; write the unit: 500ms, 2s, 1h30m"));
617 };
618 if reg.is_empty() || !reg.bytes().all(|b| b == b'_' || b.is_ascii_alphanumeric()) {
619 return Err(format!("{reg:?} is not a register name"));
620 }
621 Ok(Pattern::Atom(Atom::Since(op, Signed { negative, nanos }, reg.to_string())))
622 }
623
624 fn parse_repeat(&mut self, inner: Pattern) -> Result<Pattern, ParseError> {
625 self.bump(); let m = self.parse_number()?;
628 let n = if self.peek() == Some(b',') {
629 self.bump();
630 if self.peek() == Some(b'}') {
631 None
632 } else {
633 Some(self.parse_number()?)
634 }
635 } else {
636 Some(m)
637 };
638 if self.peek() != Some(b'}') {
639 return Err(self.err("expected '}' to close a quantifier"));
640 }
641 self.bump();
642 let g = self.lean();
643 Ok(Pattern::Repeat(inner.boxed(), m, n, g))
644 }
645
646 fn parse_atom(&mut self) -> Result<Pattern, ParseError> {
647 self.skip_spaces();
648 match self.peek() {
649 Some(b'\\') => {
650 self.bump();
651 let c = self.bump().ok_or_else(|| self.err("dangling backslash"))?;
652 self.atom_from_escape(c)
653 }
654 Some(b'.') => {
655 self.bump();
656 Ok(Pattern::Atom(Atom::Any))
657 }
658 Some(b'(') => {
659 self.bump();
660 if self.peek() == Some(b'?') {
663 return self.parse_modifier_group();
664 }
665 let inner = self.parse_alt()?;
666 self.expect(b')')?;
667 if self.peek() == Some(b'~') && self.s.get(self.i + 1).is_some_and(u8::is_ascii_digit) {
672 self.bump();
673 let k = self.parse_number()?;
674 let k = u8::try_from(k)
675 .map_err(|e| self.err(&format!("an edit count fits one byte: {e}")))?;
676 return self.edit_group(inner, k);
677 }
678 Ok(inner)
679 }
680 Some(b'"') => {
681 let lit = self.read_quoted()?;
682 if self.peek() == Some(b'~') && self.s.get(self.i + 1).is_some_and(u8::is_ascii_digit) {
686 self.bump();
687 let k = self.parse_number()?;
688 let k = u8::try_from(k)
689 .map_err(|e| self.err(&format!("an edit count fits one byte: {e}")))?;
690 return Ok(Pattern::Atom(Atom::LiteralWithin(
691 lit,
692 k,
693 crate::orbit::OrbitGroup::Identity,
694 )));
695 }
696 Ok(Pattern::Atom(Atom::literal(&lit)))
697 }
698 Some(b'=') => {
699 self.bump();
700 let first = self.parse_name()?;
701 if let Some(digits) = first.strip_prefix("edit")
705 && !digits.is_empty()
706 && digits.bytes().all(|b| b.is_ascii_digit())
707 {
708 let save = self.i;
709 self.skip_spaces();
710 if matches!(self.peek(), Some(c) if c == b'_' || c.is_ascii_alphabetic()) {
711 let name = self.parse_name()?;
712 let k = digits
713 .parse::<u8>()
714 .map_err(|e| self.err(&format!("an edit count fits one byte: {e}")))?;
715 return Ok(Pattern::Atom(Atom::RegisterWithin(
716 name,
717 k,
718 crate::orbit::OrbitGroup::Identity,
719 )));
720 }
721 self.i = save;
722 }
723 if first == "kin" {
729 let save = self.i;
730 let grain = self.gravity_grain()?;
731 self.skip_spaces();
732 if matches!(self.peek(), Some(c) if c == b'_' || c.is_ascii_alphabetic()) {
733 let name = self.parse_name()?;
734 return Ok(Pattern::Atom(Atom::RegisterKin(name, grain)));
735 }
736 self.i = save;
737 }
738 if let Some(mut relation) = crate::typed::Relation::parse(&first) {
751 let save = self.i;
752 if self.peek() == Some(b'/') {
753 self.bump();
754 let bits = self.parse_number()?;
755 if bits > 128 {
756 return Err(self.err("a prefix length is at most 128"));
757 }
758 relation = relation.with_prefix(bits as u8).ok_or_else(|| {
759 self.err("only =subnet takes a prefix length after a slash")
760 })?;
761 }
762 self.skip_spaces();
763 if matches!(self.peek(), Some(c) if c == b'_' || c.is_ascii_alphabetic()) {
764 let name = self.parse_name()?;
765 return Ok(Pattern::Atom(Atom::RegisterRelated(name, relation)));
766 }
767 self.i = save;
768 }
769 if let Some(group) = crate::orbit::OrbitGroup::parse(&first) {
775 let save = self.i;
776 self.skip_spaces();
777 if matches!(self.peek(), Some(c) if c == b'_' || c.is_ascii_alphabetic()) {
778 let name = self.parse_name()?;
779 return Ok(Pattern::Atom(Atom::RegisterEq(name, group)));
780 }
781 self.i = save;
782 }
783 Ok(Pattern::Atom(Atom::RegisterEq(
787 first,
788 self.orbit.unwrap_or(crate::orbit::OrbitGroup::Identity),
789 )))
790 }
791 Some(b'~') => {
792 self.bump();
793 self.parse_assertion(false)
794 }
795 Some(b'!') => {
796 self.bump();
799 if self.peek() != Some(b'~') {
800 return Err(self.err("expected '~' after '!' for a negative assertion"));
801 }
802 self.bump();
803 self.parse_assertion(true)
804 }
805 Some(b'`') => {
806 self.bump();
807 let start = self.i;
808 while !matches!(self.peek(), None | Some(b'`')) {
809 self.bump();
810 }
811 if self.peek() != Some(b'`') {
812 return Err(self.err("unterminated byte-pattern"));
813 }
814 let raw = &self.s[start..self.i];
815 self.bump();
816 match crate::bytepat::parse(raw) {
817 Ok(bp) => Ok(Pattern::Atom(Atom::BytePattern(bp))),
818 Err(msg) => Err(self.err(&msg)),
819 }
820 }
821 Some(b'[') => {
822 self.bump();
823 self.parse_class()
824 }
825 Some(b'#') => {
826 self.bump();
832 if self.peek() == Some(b'"') {
833 let tmpl = self.read_quoted()?;
834 self.expand_silhouette(&tmpl)
835 } else {
836 Ok(Pattern::Atom(Atom::literal("#")))
837 }
838 }
839 Some(b'@') => {
840 self.bump();
841 if matches!(self.peek(), Some(c) if c.is_ascii_digit()) {
844 let k = self.parse_number()?;
845 let inner = self.parse_atom()?;
849 Ok(Pattern::Field(k, inner.boxed()))
850 } else {
851 self.parse_lens()
852 }
853 }
854 Some(b'^') => {
858 self.bump();
859 Ok(Pattern::Anchor(AnchorKind::LineStart))
860 }
861 Some(b'$') => {
862 self.bump();
863 Ok(Pattern::Anchor(AnchorKind::LineEnd))
864 }
865 Some(c) => {
866 self.bump();
867 Ok(Pattern::Atom(Atom::literal(&(c as char).to_string())))
868 }
869 None => Err(self.err("expected a pattern atom")),
870 }
871 }
872
873 fn expand_silhouette(&self, tmpl: &str) -> Result<Pattern, ParseError> {
880 let mut items: Vec<Pattern> = Vec::new();
881 for c in tmpl.chars() {
882 if c == ' ' {
883 continue;
884 }
885 let atom = match c {
886 'W' => Atom::Kind(TokenKind::Word),
887 'N' => Atom::Kind(TokenKind::Number),
888 'Q' => Atom::Kind(TokenKind::Quoted),
889 'I' => Atom::Kind(TokenKind::Ip),
890 'U' => Atom::Kind(TokenKind::Url),
891 'E' => Atom::Kind(TokenKind::Email),
892 'T' => Atom::Kind(TokenKind::Timestamp),
893 'P' => Atom::Kind(TokenKind::Punct),
894 'V' => Atom::Kind(TokenKind::Version),
895 'A' => Atom::Kind(TokenKind::Mac),
896 'H' => Atom::Kind(TokenKind::HexColor),
897 'C' => Atom::Kind(TokenKind::Cidr),
898 '.' => Atom::Any,
899 other => Atom::literal(&other.to_string()),
900 };
901 items.push(Pattern::Atom(atom));
902 }
903 match items.len() {
904 0 => Err(self.err("empty silhouette template")),
905 1 => Ok(items.pop().unwrap()),
906 _ => Ok(Pattern::Concat(items)),
907 }
908 }
909
910 fn atom_from_escape(&mut self, c: u8) -> Result<Pattern, ParseError> {
911 let kind = match c {
912 b'N' => Some(TokenKind::Number),
913 b'W' => Some(TokenKind::Word),
914 b'Q' => Some(TokenKind::Quoted),
915 b'I' => Some(TokenKind::Ip),
916 b'U' => Some(TokenKind::Url),
917 b'E' => Some(TokenKind::Email),
918 b'T' => Some(TokenKind::Timestamp),
919 b'P' => Some(TokenKind::Punct),
920 b'S' => Some(TokenKind::Whitespace),
921 b'V' => Some(TokenKind::Version),
922 b'H' => Some(TokenKind::HexColor),
923 b'C' => Some(TokenKind::Cidr),
924 b'Z' => Some(TokenKind::ByteSize),
925 b'%' => Some(TokenKind::Percent),
926 b'$' => Some(TokenKind::Money),
927 b'D' => Some(TokenKind::HashDigest),
928 b'R' => Some(TokenKind::Duration),
929 b'L' => Some(TokenKind::Path),
930 _ => None,
931 };
932 if let Some(k) = kind {
933 return Ok(Pattern::Atom(Atom::Kind(k)));
934 }
935 match c {
936 b'B' => self.parse_balanced(),
937 b'A' => Ok(Pattern::Anchor(AnchorKind::InputStart)),
942 b'z' => Ok(Pattern::Anchor(AnchorKind::InputEnd)),
943 b'G' => Ok(Pattern::Anchor(AnchorKind::Resume)),
951 b'K' => Ok(Pattern::Anchor(AnchorKind::ResetStart)),
952 b'd' => Ok(Pattern::Atom(Atom::Byte(ByteClass::Digit))),
953 b'w' => Ok(Pattern::Atom(Atom::Byte(ByteClass::Word))),
954 b's' => Ok(Pattern::Atom(Atom::Byte(ByteClass::Space))),
955 b'h' => Ok(Pattern::Atom(Atom::Byte(ByteClass::Hex))),
956 b'a' => Ok(Pattern::Atom(Atom::Byte(ByteClass::Alpha))),
957 b'u' => Ok(Pattern::Atom(Atom::Byte(ByteClass::Upper))),
958 b'l' => Ok(Pattern::Atom(Atom::Byte(ByteClass::Lower))),
959 b'F' => self.parse_spectral(),
960 b'M' => self.parse_magnitude(),
961 b'{' => self.parse_named_atom(),
962 other => Ok(Pattern::Atom(Atom::literal(&(other as char).to_string()))),
964 }
965 }
966
967 fn parse_named_atom(&mut self) -> Result<Pattern, ParseError> {
972 fn kind_by_name(name: &str) -> Option<TokenKind> {
973 Some(match name {
974 "number" => TokenKind::Number,
975 "word" => TokenKind::Word,
976 "quoted" => TokenKind::Quoted,
977 "ip" => TokenKind::Ip,
978 "url" => TokenKind::Url,
979 "email" => TokenKind::Email,
980 "timestamp" => TokenKind::Timestamp,
981 "punct" => TokenKind::Punct,
982 "whitespace" => TokenKind::Whitespace,
983 "version" => TokenKind::Version,
984 "uuid" => TokenKind::Uuid,
985 "mac" => TokenKind::Mac,
986 "hexcolor" => TokenKind::HexColor,
987 "cidr" => TokenKind::Cidr,
988 "bytesize" => TokenKind::ByteSize,
989 "percent" => TokenKind::Percent,
990 "money" => TokenKind::Money,
991 "hash" | "hashdigest" => TokenKind::HashDigest,
992 "duration" => TokenKind::Duration,
993 "path" => TokenKind::Path,
994 "jwt" => TokenKind::Jwt,
995 "creditcard" | "card" => TokenKind::CreditCard,
996 "base64" | "b64" => TokenKind::Base64,
997 "geo" | "coord" => TokenKind::Geo,
998 "phone" | "tel" => TokenKind::Phone,
999 "quantity" => TokenKind::Quantity,
1000 _ => return None,
1001 })
1002 }
1003 let start = self.i;
1004 let mut name = String::new();
1005 while let Some(c) = self.peek() {
1006 if c == b'}' {
1007 break;
1008 }
1009 name.push(c as char);
1010 self.bump();
1011 }
1012 if self.peek() != Some(b'}') {
1013 return Err(ParseError { pos: start, msg: "unterminated \\{name}".to_string() });
1014 }
1015 self.bump();
1016 if let Some(k) = kind_by_name(&name) {
1017 return Ok(Pattern::Atom(Atom::Kind(k)));
1018 }
1019 if name == "qty" {
1020 return Ok(Pattern::Atom(Atom::Class(Box::new(self.quantity_class()))));
1021 }
1022 if let Some(id) = self.shapes.id_of(&name) {
1026 return Ok(Pattern::Atom(Atom::Kind(TokenKind::Custom(id))));
1027 }
1028 if let Some(p) = self.shapes.let_of(&name) {
1029 return Ok(p.clone());
1030 }
1031 if self.shapes.consults_library() {
1032 if let Some(p) = crate::library::let_of(&name) {
1033 return Ok(p.clone());
1034 }
1035 if let Some(id) = crate::library::id_of(&name) {
1036 return Ok(Pattern::Atom(Atom::Kind(TokenKind::Custom(id))));
1037 }
1038 }
1039 Err(ParseError {
1040 pos: start,
1041 msg: format!(
1042 "unknown named atom \\{{{name}}}; it is neither a built-in, a declared shape, kind or sub-pattern, nor a library entry"
1043 ),
1044 })
1045 }
1046
1047 fn parse_spectral(&mut self) -> Result<Pattern, ParseError> {
1050 self.expect(b'{')?;
1051 let start = self.i;
1052 while !matches!(self.peek(), None | Some(b'}')) {
1053 self.bump();
1054 }
1055 if self.peek() != Some(b'}') {
1056 return Err(self.err("unterminated \\F{...} spectral predicate"));
1057 }
1058 let raw = &self.s[start..self.i];
1059 self.bump(); let pred = Self::spectral_pred(raw).map_err(|m| self.err(&m))?;
1061 Ok(Pattern::Atom(Atom::Spectral(pred)))
1062 }
1063
1064 fn spectral_pred(raw: &[u8]) -> Result<crate::ast::SpectralPred, String> {
1066 use crate::ast::{SpecTexture, SpectralPred};
1067 let s = std::str::from_utf8(raw)
1068 .map_err(|_| "spectral predicate is not UTF-8".to_string())?
1069 .trim();
1070 if let Some(rest) = s.strip_prefix("entropy") {
1071 let rest = rest.trim();
1072 let (ge, num) = if let Some(n) = rest.strip_prefix(">=").or_else(|| rest.strip_prefix('>')) {
1073 (true, n)
1074 } else if let Some(n) = rest.strip_prefix("<=").or_else(|| rest.strip_prefix('<')) {
1075 (false, n)
1076 } else {
1077 return Err("entropy needs a > or < threshold, e.g. entropy>0.8".to_string());
1078 };
1079 let v: f32 = num.trim().parse().map_err(|_| format!("bad entropy threshold {num:?}"))?;
1080 let pct = (v * 100.0).round().clamp(0.0, 100.0) as u8;
1081 return Ok(if ge { SpectralPred::EntropyGe(pct) } else { SpectralPred::EntropyLe(pct) });
1082 }
1083 if let Some(rest) = s.strip_prefix("period") {
1084 let n = rest.trim().trim_start_matches([':', '=']).trim();
1085 if n.is_empty() || n == "any" || n == "line" {
1088 return Ok(SpectralPred::PeriodAny);
1089 }
1090 let p: u16 = n.parse().map_err(|_| format!("bad period {n:?}"))?;
1091 return Ok(SpectralPred::PeriodEq(p));
1092 }
1093 if let Some(rest) = s.strip_prefix("texture:").or_else(|| s.strip_prefix("texture=")) {
1094 return match rest.trim() {
1095 "prose" => Ok(SpectralPred::Texture(SpecTexture::Prose)),
1096 "code" => Ok(SpectralPred::Texture(SpecTexture::Code)),
1097 "math" => Ok(SpectralPred::Texture(SpecTexture::Math)),
1098 "data" => Ok(SpectralPred::Texture(SpecTexture::Data)),
1099 other => Err(format!("unknown texture {other:?} (prose|code|math|data)")),
1100 };
1101 }
1102 if s == "onset" {
1103 return Ok(SpectralPred::Onset);
1104 }
1105 Err(format!("unknown spectral predicate {s:?} (entropy|period|texture|onset)"))
1106 }
1107
1108 fn parse_magnitude(&mut self) -> Result<Pattern, ParseError> {
1114 self.expect(b'{')?;
1115 let start = self.i;
1116 while !matches!(self.peek(), None | Some(b'}')) {
1117 self.bump();
1118 }
1119 if self.peek() != Some(b'}') {
1120 return Err(self.err("unterminated \\M{...} magnitude predicate"));
1121 }
1122 let raw = &self.s[start..self.i];
1123 self.bump(); let pred = Self::magnitude_pred(raw).map_err(|m| self.err(&m))?;
1125 Ok(Pattern::Atom(Atom::Magnitude(pred)))
1126 }
1127
1128 fn magnitude_pred(raw: &[u8]) -> Result<crate::ast::MagPred, String> {
1138 use crate::ast::{Delta, MagPred, Scope};
1139 let s = std::str::from_utf8(raw)
1140 .map_err(|e| format!("magnitude predicate is not UTF-8: {e}"))?
1141 .trim();
1142 let s = s.strip_prefix("mag").map_or(s, str::trim_start).trim();
1144 let (ge, num) = if let Some(n) = s.strip_prefix(">=").or_else(|| s.strip_prefix('>')) {
1145 (true, n)
1146 } else if let Some(n) = s.strip_prefix("<=").or_else(|| s.strip_prefix('<')) {
1147 (false, n)
1148 } else {
1149 return Err("magnitude needs a > or < threshold, e.g. \\M{>6}".to_string());
1150 };
1151 let num = num.trim();
1152 if num.starts_with('+') || num.starts_with('-') {
1153 let (body, scope) = match num.split_once(':') {
1154 Some((b, name)) => {
1155 let name = name.trim();
1156 if name.is_empty()
1157 || !name.chars().all(|c| c == '_' || c.is_ascii_alphanumeric())
1158 {
1159 return Err(format!(
1160 "bad context {name:?} after the relative threshold (use window, phase, regime, echo, enclosing, or a register name)"
1161 ));
1162 }
1163 (b.trim(), Scope::parse(name))
1164 }
1165 None => (num, Scope::Window),
1166 };
1167 let (value, sigmas) = match body.strip_suffix('s') {
1168 Some(v) => (v.trim(), true),
1169 None => (body, false),
1170 };
1171 let v: f32 = value
1172 .parse()
1173 .map_err(|e| format!("bad relative magnitude threshold {body:?}: {e}"))?;
1174 let centi = (v * 100.0).round() as i32;
1175 let delta = if sigmas { Delta::Sigmas(centi) } else { Delta::Orders(centi) };
1176 return Ok(if ge { MagPred::Above(scope, delta) } else { MagPred::Below(scope, delta) });
1177 }
1178 let v: f32 = num.parse().map_err(|e| format!("bad magnitude threshold {num:?}: {e}"))?;
1179 let centi = (v * 100.0).round() as i32;
1180 Ok(if ge { MagPred::Ge(centi) } else { MagPred::Le(centi) })
1181 }
1182
1183 fn edit_group(&mut self, inner: Pattern, k: u8) -> Result<Pattern, ParseError> {
1192 let parts: Vec<&Pattern> = match &inner {
1193 Pattern::Concat(v) => v.iter().collect(),
1194 one => vec![one],
1195 };
1196 let mut atoms = Vec::with_capacity(parts.len());
1197 for part in parts {
1198 let (bind, body) = match part {
1199 Pattern::Bind(name, scoped, p) => (Some((name.clone(), *scoped)), p.as_ref()),
1200 p => (None, p),
1201 };
1202 let Pattern::Atom(atom) = body else {
1203 return Err(self.err(
1204 "a ~k group holds single-token atoms - a kind, a literal, a class, a \
1205 predicate or a byte pattern, each optionally bound - and this one holds \
1206 something that matches a run",
1207 ));
1208 };
1209 atoms.push(crate::ast::EditAtom { atom: atom.clone(), bind });
1210 }
1211 if usize::from(k) >= atoms.len() {
1212 return Err(self.err(&format!(
1213 "~{k} over {} atom(s) matches a run that resembles none of them; the count \
1214 stands under the number of atoms",
1215 atoms.len()
1216 )));
1217 }
1218 Ok(Pattern::Within(atoms, k))
1219 }
1220
1221 fn parse_modifier_group(&mut self) -> Result<Pattern, ParseError> {
1229 self.bump();
1230 if self.peek() == Some(b'>') {
1233 self.bump();
1234 let inner = self.parse_alt()?;
1235 self.expect(b')')?;
1236 return Ok(Pattern::Atomic(inner.boxed()));
1237 }
1238 let name = self.parse_name()?;
1239 if name != "orbit" {
1240 return Err(self.err(&format!(
1241 "unknown group modifier (?{name}...); the modifier axis is `orbit`"
1242 )));
1243 }
1244 self.expect(b':')?;
1245 let group_name = self.parse_name()?;
1246 let Some(mut group) = crate::orbit::OrbitGroup::parse(&group_name) else {
1247 return Err(self.err(&format!("unknown orbit group {group_name:?}")));
1248 };
1249 if self.peek() == Some(b'/') {
1252 self.bump();
1253 let bits = self.parse_number()?;
1254 if bits > 128 {
1255 return Err(self.err("a prefix length is at most 128"));
1256 }
1257 group = group.with_prefix(bits as u8).ok_or_else(|| {
1258 self.err("only the subnet rung takes a prefix length after a slash")
1259 })?;
1260 }
1261 if self.orbit.is_some() {
1266 return Err(self.err("an orbit scope cannot stand inside another"));
1267 }
1268 self.orbit = Some(group);
1269 let inner = self.parse_alt();
1270 self.orbit = None;
1271 let mut inner = inner?;
1272 self.expect(b')')?;
1273 set_orbit(&mut inner, group);
1274 Ok(inner)
1275 }
1276
1277 fn parse_cmp(&mut self) -> Option<crate::ast::Cmp> {
1280 use crate::ast::Cmp;
1281 let two = |p: &mut Self, c| {
1282 p.bump();
1283 p.bump();
1284 Some(c)
1285 };
1286 match (self.peek(), self.s.get(self.i + 1)) {
1287 (Some(b'>'), Some(b'=')) => two(self, Cmp::Ge),
1288 (Some(b'<'), Some(b'=')) => two(self, Cmp::Le),
1289 (Some(b'!'), Some(b'=')) => two(self, Cmp::Ne),
1290 (Some(b'>'), _) => {
1291 self.bump();
1292 Some(Cmp::Gt)
1293 }
1294 (Some(b'<'), _) => {
1295 self.bump();
1296 Some(Cmp::Lt)
1297 }
1298 (Some(b'='), _) => {
1299 self.bump();
1300 Some(Cmp::Eq)
1301 }
1302 _ => None,
1303 }
1304 }
1305
1306 fn echo_anchor(&mut self, recurs: bool) -> Result<Pattern, ParseError> {
1314 if self.peek() != Some(b':') {
1315 return Ok(Pattern::Anchor(if recurs { AnchorKind::Echoed } else { AnchorKind::Novel }));
1316 }
1317 let at = self.i;
1318 self.bump();
1319 if self.peek() != Some(b'@') {
1320 return Err(ParseError {
1321 pos: at,
1322 msg: "a second input is named after the colon: @echoed:@other.log".to_string(),
1323 });
1324 }
1325 self.bump();
1326 let name = if self.peek() == Some(b'"') {
1327 self.read_quoted()?
1328 } else {
1329 let start = self.i;
1330 while matches!(self.peek(), Some(c) if !c.is_ascii_whitespace() && c != b')') {
1331 self.bump();
1332 }
1333 String::from_utf8_lossy(&self.s[start..self.i]).into_owned()
1334 };
1335 if name.is_empty() {
1336 return Err(self.err("a second input needs a name: @echoed:@other.log"));
1337 }
1338 let other = self.other_input(&name).map_err(|m| ParseError { pos: at, msg: m })?;
1339 Ok(Pattern::Anchor(AnchorKind::Joined {
1340 other,
1341 recurs,
1342 group: crate::orbit::OrbitGroup::Identity,
1343 }))
1344 }
1345
1346 fn other_input(
1351 &mut self,
1352 name: &str,
1353 ) -> Result<std::sync::Arc<crate::ast::OtherInput>, String> {
1354 use std::sync::Arc;
1355 if let Some(found) = self.others.get(name) {
1356 return Ok(Arc::clone(found));
1357 }
1358 let bytes: Vec<u8> = match self.inputs.iter().find(|(n, _)| *n == name) {
1359 Some((_, bytes)) => bytes.to_vec(),
1360 None => {
1361 let path = match self.shapes.base_dir() {
1362 Some(dir) if std::path::Path::new(name).is_relative() => dir.join(name),
1363 _ => std::path::PathBuf::from(name),
1364 };
1365 match std::fs::read(&path) {
1366 Ok(b) => b,
1367 Err(e) => {
1368 return Err(format!(
1369 "cannot read the second input {}: {e}",
1370 path.display()
1371 ));
1372 }
1373 }
1374 }
1375 };
1376 let tokens = crate::lexer::lex(&bytes);
1377 let other = Arc::new(crate::ast::OtherInput { name: name.to_string(), bytes, tokens });
1378 self.others.insert(name.to_string(), Arc::clone(&other));
1379 Ok(other)
1380 }
1381
1382 fn parse_echo_anchor(&mut self) -> Result<Pattern, ParseError> {
1383 use crate::ast::{AnchorKind, Cmp, EchoPred};
1384 use crate::orbit::OrbitGroup;
1385 let pred = if self.peek() == Some(b':') {
1386 self.bump();
1387 let name = self.parse_name()?;
1388 match name.as_str() {
1389 "nth" => {
1390 let Some(op) = self.parse_cmp() else {
1391 return Err(self.err("@echo:nth needs an operator, e.g. @echo:nth=3"));
1392 };
1393 let negative = self.peek() == Some(b'-');
1394 if negative {
1395 self.bump();
1396 }
1397 let k = self.parse_number()?;
1398 if k == 0 {
1399 return Err(self.err(
1400 "@echo:nth counts from one, or from the last backwards as -1; there is no zeroth occurrence",
1401 ));
1402 }
1403 let k = i32::try_from(k)
1404 .map_err(|e| self.err(&format!("an occurrence index fits four bytes: {e}")))?;
1405 EchoPred::Nth(op, if negative { -k } else { k })
1406 }
1407 "period" => match self.parse_cmp() {
1408 Some(op) => EchoPred::PeriodAt(op, self.parse_number()? as u32),
1409 None => EchoPred::Period,
1410 },
1411 other => {
1412 return Err(self.err(&format!(
1413 "unknown echo reading {other:?} (use @echo, @echo>k, @echo:nth=k, @echo:period)"
1414 )));
1415 }
1416 }
1417 } else {
1418 match self.parse_cmp() {
1419 Some(op) => EchoPred::Count(op, self.parse_number()? as u32),
1420 None => EchoPred::Count(Cmp::Ge, 2),
1423 }
1424 };
1425 Ok(Pattern::Anchor(AnchorKind::Echo(pred, OrbitGroup::Identity)))
1426 }
1427
1428 fn parse_assertion(&mut self, neg: bool) -> Result<Pattern, ParseError> {
1438 let look = if self.peek() == Some(b'<') {
1439 self.bump();
1440 Look::Behind
1441 } else if self.peek() == Some(b'>') {
1442 self.bump();
1443 let window = self.parse_number()?;
1444 if window == 0 {
1445 return Err(self.err("a proximity window of zero tokens can never hold a match"));
1446 }
1447 let (at_least, at_most) = self.parse_count_range()?;
1448 if at_least > window {
1449 return Err(
1450 self.err("a window cannot hold more occurrences than it holds tokens")
1451 );
1452 }
1453 Look::Within { window, at_least, at_most }
1454 } else if self.peek() == Some(b'#') {
1455 self.bump();
1456 let (at_least, at_most) = self.parse_count_range()?;
1457 Look::InGroup { at_least, at_most }
1458 } else {
1459 Look::Ahead
1460 };
1461 if self.peek() == Some(b'"') {
1462 match look {
1463 Look::Ahead => {
1464 let lit = self.read_quoted()?;
1465 return Ok(Pattern::Guard(lit, neg));
1466 }
1467 Look::Behind => {
1468 return Err(self.err(
1469 "a literal guard reads the forward window; use ~<(\"lit\") for a backward assertion",
1470 ));
1471 }
1472 Look::Within { .. } | Look::InGroup { .. } => {
1477 return Err(self.err(
1478 "a counted assertion takes a parenthesised sub-pattern; write ~>3{2}(\"lit\"), not ~>3{2}\"lit\"",
1479 ));
1480 }
1481 }
1482 }
1483 self.expect(b'(')?;
1484 let inner = self.parse_alt()?;
1485 self.expect(b')')?;
1486 if look == Look::Behind && crate::nfa::bounded_max_len(&inner).is_none() {
1490 return Err(self.err(
1491 "a backward assertion needs a bounded sub-pattern; `*`, `+` and `{m,}` have no fixed length",
1492 ));
1493 }
1494 Ok(Pattern::Assert(inner.boxed(), neg, look))
1495 }
1496
1497 fn parse_count_range(&mut self) -> Result<(usize, Option<usize>), ParseError> {
1504 if self.peek() != Some(b'{') {
1505 return Ok((1, None));
1506 }
1507 self.bump();
1508 let lo = self.parse_number()?;
1509 let hi = if self.peek() == Some(b',') {
1510 self.bump();
1511 if self.peek() == Some(b'}') { None } else { Some(self.parse_number()?) }
1512 } else {
1513 Some(lo)
1514 };
1515 self.expect(b'}')?;
1516 if let Some(hi) = hi
1517 && hi < lo
1518 {
1519 return Err(self.err("a count whose top is below its bottom can never be met"));
1520 }
1521 Ok((lo, hi))
1522 }
1523
1524 fn parse_class(&mut self) -> Result<Pattern, ParseError> {
1532 let mut cls = crate::ast::TokenClass {
1533 any: Vec::new(),
1534 all: Vec::new(),
1535 none: Vec::new(),
1536 negated: false,
1537 };
1538 self.skip_spaces();
1539 if self.peek() == Some(b'^') {
1540 self.bump();
1541 cls.negated = true;
1542 }
1543 let mut target = 0u8;
1545 loop {
1546 self.skip_spaces();
1547 match self.peek() {
1548 None => return Err(self.err("unterminated token class")),
1549 Some(b']') => {
1550 self.bump();
1551 break;
1552 }
1553 Some(b'&') if self.s.get(self.i + 1) == Some(&b'&') => {
1554 self.bump();
1555 self.bump();
1556 target = 1;
1557 continue;
1558 }
1559 Some(b'-') if self.s.get(self.i + 1) == Some(&b'-') => {
1560 self.bump();
1561 self.bump();
1562 target = 2;
1563 continue;
1564 }
1565 _ => {}
1566 }
1567 let at = self.i;
1568 let member = match self.parse_atom()? {
1569 Pattern::Atom(a) => a,
1570 other => {
1571 return Err(ParseError {
1572 pos: at,
1573 msg: format!(
1574 "a token class holds single-token atoms; {other:?} is not one"
1575 ),
1576 });
1577 }
1578 };
1579 match target {
1580 1 => cls.all.push(member),
1581 2 => cls.none.push(member),
1582 _ => cls.any.push(member),
1583 }
1584 }
1585 if cls.any.is_empty() {
1586 return Err(self.err("a token class needs at least one member before && or --"));
1587 }
1588 Ok(Pattern::Atom(Atom::Class(Box::new(cls))))
1589 }
1590
1591 fn parse_balanced(&mut self) -> Result<Pattern, ParseError> {
1592 match self.peek() {
1593 Some(b'(') => {
1594 self.bump();
1595 let inner = self.parse_alt()?;
1596 self.expect(b')')?;
1597 Ok(Pattern::Balanced(Some(BracketKind::Paren), inner.boxed()))
1598 }
1599 Some(b'[') => {
1600 self.bump();
1601 let inner = self.parse_alt()?;
1602 self.expect(b']')?;
1603 Ok(Pattern::Balanced(Some(BracketKind::Square), inner.boxed()))
1604 }
1605 Some(b'{') => {
1606 self.bump();
1607 let inner = self.parse_alt()?;
1608 self.expect(b'}')?;
1609 Ok(Pattern::Balanced(Some(BracketKind::Brace), inner.boxed()))
1610 }
1611 _ => Ok(Pattern::Balanced(
1613 None,
1614 Pattern::Star(Pattern::Atom(Atom::Any).boxed(), Greed::Greedy).boxed(),
1615 )),
1616 }
1617 }
1618
1619 fn parse_lens(&mut self) -> Result<Pattern, ParseError> {
1625 let any_interior = || Pattern::Star(Pattern::Atom(Atom::Any).boxed(), Greed::Greedy).boxed();
1626 let name = self.parse_name()?;
1627 match name.as_str() {
1628 "call" => Ok(Pattern::Concat(vec![
1630 Pattern::Atom(Atom::Kind(TokenKind::Word)),
1631 Pattern::Balanced(Some(BracketKind::Paren), any_interior()),
1632 ])),
1633 "block" => Ok(Pattern::Balanced(Some(BracketKind::Brace), any_interior())),
1635 "nesting" => Ok(Pattern::Balanced(None, any_interior())),
1637 "string" => Ok(Pattern::Atom(Atom::Kind(TokenKind::Quoted))),
1639 "number" => Ok(Pattern::Atom(Atom::Kind(TokenKind::Number))),
1641 "ident" => Ok(Pattern::Atom(Atom::Kind(TokenKind::Word))),
1643 "assignment" => Ok(Pattern::Concat(vec![
1647 Pattern::Atom(Atom::Kind(TokenKind::Word)),
1648 Pattern::Atom(Atom::literal("=")),
1649 ])),
1650 "kv" => Ok(Pattern::Concat(vec![
1654 Pattern::Atom(Atom::Kind(TokenKind::Word)),
1655 Pattern::Alt(
1656 vec![
1657 Pattern::Atom(Atom::literal(":")),
1658 Pattern::Atom(Atom::literal("=")),
1659 ],
1660 AltMode::First,
1661 ),
1662 ])),
1663 "flag" => Ok(Pattern::Concat(vec![
1666 Pattern::Atom(Atom::literal("-")),
1667 Pattern::Opt(Pattern::Atom(Atom::literal("-")).boxed(), Greed::Greedy),
1668 Pattern::Atom(Atom::Kind(TokenKind::Word)),
1669 ])),
1670 "list" => Ok(Pattern::Concat(vec![
1673 Pattern::Atom(Atom::Any),
1674 Pattern::Plus(
1675 Pattern::Concat(vec![
1676 Pattern::Atom(Atom::literal(",")),
1677 Pattern::Atom(Atom::Any),
1678 ])
1679 .boxed(),
1680 Greed::Greedy,
1681 ),
1682 ])),
1683 "range" => Ok(Pattern::Concat(vec![
1688 Pattern::Atom(Atom::Kind(TokenKind::Number)),
1689 Pattern::Alt(
1690 vec![
1691 Pattern::Concat(vec![
1692 Pattern::Atom(Atom::literal(".")),
1693 Pattern::Atom(Atom::literal(".")),
1694 ]),
1695 Pattern::Atom(Atom::literal("-")),
1696 Pattern::Atom(Atom::literal(":")),
1697 ],
1698 AltMode::First,
1699 ),
1700 Pattern::Atom(Atom::Kind(TokenKind::Number)),
1701 ])),
1702 "seam" => {
1705 if self.peek() != Some(b':') {
1719 return Ok(Pattern::Anchor(AnchorKind::Seam(crate::ast::Grain::Token)));
1720 }
1721 self.bump();
1722 let name = self.parse_name()?;
1723 match crate::ast::Grain::parse(&name) {
1724 Some(g) => Ok(Pattern::Anchor(AnchorKind::Seam(g))),
1725 None => Err(self
1726 .err(&format!("unknown grain {name:?} (use byte, token, super)"))),
1727 }
1728 }
1729 "strain" | "bound" => {
1734 use crate::ast::{GravityReading, Level, Real};
1735 let reading = if name == "strain" { GravityReading::Strain } else { GravityReading::Bound };
1736 let grain = self.gravity_grain()?;
1737 let cmp = self.parse_cmp().ok_or_else(|| {
1738 self.err(&format!(
1739 "@{name} needs a comparison: a percentile of the input, e.g. @{name}>90, or bits, e.g. @{name}<1.5b"
1740 ))
1741 })?;
1742 let v = self.read_real()?;
1743 let level = if self.peek() == Some(b'b') {
1744 self.bump();
1745 Level::Bits(Real::new(v))
1746 } else if (0.0..=100.0).contains(&v) {
1747 Level::Percentile(Real::new(v))
1748 } else {
1749 return Err(self.err(&format!(
1750 "{v} is not a percentile, which runs 0 to 100; write b after a value in bits, e.g. @{name}>{v}b"
1751 )));
1752 };
1753 Ok(Pattern::Anchor(AnchorKind::Gravity(reading, grain, cmp, level)))
1754 }
1755 "kin" => {
1758 let grain = self.gravity_grain()?;
1759 self.expect(b'(')?;
1760 let example = self.read_quoted()?;
1761 self.expect(b')')?;
1762 if crate::gravity::example_key(grain, example.as_bytes()).is_none() {
1763 return Err(self.err(&format!(
1764 "@kin example {example:?} names no type at this grain: one byte at :byte, a token otherwise"
1765 )));
1766 }
1767 Ok(Pattern::Anchor(AnchorKind::Kin(grain, example)))
1768 }
1769 "nested" => {
1773 if self.peek() != Some(b'>') {
1774 return Err(self.err("@nested needs a > threshold, e.g. @nested>2"));
1775 }
1776 self.bump();
1777 let ge = self.peek() == Some(b'=');
1778 if ge {
1779 self.bump();
1780 }
1781 let k = self.parse_number()? as u16;
1782 let min = if ge { k } else { k.saturating_add(1) };
1783 Ok(Pattern::Anchor(AnchorKind::Nested(min)))
1784 }
1785 "ambiguous" => {
1788 if self.peek() != Some(b':') {
1789 return Ok(Pattern::Anchor(AnchorKind::Ambiguous(crate::ast::Grain::Byte)));
1790 }
1791 self.bump();
1792 let name = self.parse_name()?;
1793 match crate::ast::Grain::parse(&name) {
1794 Some(g) => Ok(Pattern::Anchor(AnchorKind::Ambiguous(g))),
1795 None => Err(self
1796 .err(&format!("unknown grain {name:?} (use byte, token, super)"))),
1797 }
1798 }
1799 "novel" => self.echo_anchor(false),
1803 "echoed" => self.echo_anchor(true),
1804 "echo" => self.parse_echo_anchor(),
1809 "shape" => {
1812 if self.peek() != Some(b':') {
1813 return Err(self.err("@shape needs a reading, e.g. @shape:rare"));
1814 }
1815 self.bump();
1816 let name = self.parse_name()?;
1817 if name != "rare" {
1818 return Err(self.err(&format!("unknown shape reading {name:?} (use rare)")));
1819 }
1820 let cut = if self.peek() == Some(b'<') {
1821 self.bump();
1822 let start = self.i;
1823 while matches!(self.peek(), Some(c) if c.is_ascii_digit() || c == b'.' || c == b'%')
1824 {
1825 self.bump();
1826 }
1827 let written = String::from_utf8_lossy(&self.s[start..self.i]).into_owned();
1828 crate::templates::Rarity::parse(&written).map_err(|m| self.err(&m))?
1829 } else {
1830 crate::templates::Rarity::Mean
1831 };
1832 Ok(Pattern::Anchor(AnchorKind::Rare(cut)))
1833 }
1834 "order" => {
1837 if self.peek() != Some(b':') {
1838 return Err(self.err("@order needs a direction, e.g. @order:desc"));
1839 }
1840 self.bump();
1841 let name = self.parse_name()?;
1842 match crate::ast::TimeOrder::parse(&name) {
1843 Some(o) => Ok(Pattern::Anchor(AnchorKind::Order(o))),
1844 None => {
1845 Err(self.err(&format!("unknown order {name:?} (use asc, desc)")))
1846 }
1847 }
1848 }
1849 "phase" => {
1852 if self.peek() != Some(b':') {
1853 return Err(self.err("@phase needs a column, e.g. @phase:2"));
1854 }
1855 self.bump();
1856 let max = crate::context::PHASE_MAX_PERIOD;
1857 let column = self.parse_number()?;
1858 let k = match u16::try_from(column) {
1859 Ok(k) if k < max => k,
1860 Ok(_) => {
1861 return Err(self.err(&format!("@phase:{column} names no column: a period is at most {max} tokens")));
1862 }
1863 Err(e) => return Err(self.err(&format!("@phase:{column} names no column ({e})"))),
1864 };
1865 let named = match self.peek() {
1868 Some(b'/') => {
1869 self.bump();
1870 let p = self.parse_number()?;
1871 match u16::try_from(p) {
1872 Ok(p) if (2..=max).contains(&p) && k < p => Some(crate::ast::PeriodRef::Length(p)),
1873 Ok(_) => {
1874 return Err(self.err(&format!(
1875 "@phase:{k}/{p} names no column: a period runs 2 to {max} tokens and its columns 0 to one less"
1876 )));
1877 }
1878 Err(e) => return Err(self.err(&format!("@phase:{k}/{p} names no period ({e})"))),
1879 }
1880 }
1881 Some(b'#') => {
1882 self.bump();
1883 let n = self.parse_number()?;
1884 match u16::try_from(n) {
1885 Ok(n) if n >= 1 => Some(crate::ast::PeriodRef::Rank(n)),
1886 Ok(_) => return Err(self.err(&format!("@phase:{k}#{n} names no period: the strongest is #1"))),
1887 Err(e) => return Err(self.err(&format!("@phase:{k}#{n} names no period ({e})"))),
1888 }
1889 }
1890 Some(_) | None => None,
1891 };
1892 Ok(Pattern::Anchor(match named {
1893 Some(period) => AnchorKind::PhaseIn(k, period),
1894 None => AnchorKind::Phase(k),
1895 }))
1896 }
1897 "super" => {
1902 if self.peek() != Some(b':') {
1903 return Ok(Pattern::Anchor(AnchorKind::SuperStart));
1904 }
1905 self.bump();
1906 let name = self.parse_name()?;
1907 match crate::supertoken::Role::parse(&name) {
1908 Some(role) => Ok(Pattern::Anchor(AnchorKind::SuperRole(role))),
1909 None => Err(self.err(&format!(
1910 "unknown supertoken role {name:?} (use call, assign, kv, list, numeric, plain)"
1911 ))),
1912 }
1913 }
1914 other => Err(self.err(&format!(
1915 "unknown lens @{other} (use call, block, nesting, string, number, ident, assignment, kv, flag, list, range, seam, nested, ambiguous, novel, echoed, super, phase)"
1916 ))),
1917 }
1918 }
1919
1920 fn expect(&mut self, b: u8) -> Result<(), ParseError> {
1921 self.skip_spaces();
1922 if self.peek() == Some(b) {
1923 self.bump();
1924 Ok(())
1925 } else {
1926 Err(self.err(&format!("expected '{}'", b as char)))
1927 }
1928 }
1929
1930 fn parse_name(&mut self) -> Result<String, ParseError> {
1935 let start = self.i;
1936 if !matches!(self.peek(), Some(c) if c == b'_' || c.is_ascii_alphabetic()) {
1937 return Err(self.err("expected a register name"));
1938 }
1939 loop {
1940 while matches!(self.peek(), Some(c) if c == b'_' || c.is_ascii_alphanumeric()) {
1941 self.bump();
1942 }
1943 let joins = self.peek() == Some(b'.')
1944 && matches!(self.s.get(self.i + 1), Some(&c) if c == b'_' || c.is_ascii_alphabetic());
1945 if !joins {
1946 break;
1947 }
1948 self.bump();
1949 }
1950 Ok(String::from_utf8_lossy(&self.s[start..self.i]).into_owned())
1951 }
1952
1953 fn parse_number(&mut self) -> Result<usize, ParseError> {
1954 let start = self.i;
1955 while matches!(self.peek(), Some(c) if c.is_ascii_digit()) {
1956 self.bump();
1957 }
1958 if self.i == start {
1959 return Err(self.err("expected a number"));
1960 }
1961 String::from_utf8_lossy(&self.s[start..self.i])
1962 .parse::<usize>()
1963 .map_err(|_| self.err("invalid number"))
1964 }
1965
1966 fn read_real(&mut self) -> Result<f64, ParseError> {
1968 let start = self.i;
1969 if self.peek() == Some(b'-') {
1970 self.bump();
1971 }
1972 while matches!(self.peek(), Some(c) if c.is_ascii_digit()) {
1973 self.bump();
1974 }
1975 if self.peek() == Some(b'.') {
1976 self.bump();
1977 while matches!(self.peek(), Some(c) if c.is_ascii_digit()) {
1978 self.bump();
1979 }
1980 }
1981 let text = String::from_utf8_lossy(&self.s[start..self.i]).into_owned();
1982 text.parse::<f64>().map_err(|e| self.err(&format!("expected a number, read {text:?} ({e})")))
1983 }
1984
1985 fn gravity_grain(&mut self) -> Result<crate::ast::Grain, ParseError> {
1988 if self.peek() != Some(b':') {
1989 return Ok(crate::ast::Grain::Token);
1990 }
1991 self.bump();
1992 let name = self.parse_name()?;
1993 crate::ast::Grain::parse(&name)
1994 .ok_or_else(|| self.err(&format!("unknown grain {name:?} (use byte, token, super)")))
1995 }
1996
1997 fn read_quoted(&mut self) -> Result<String, ParseError> {
1998 if self.peek() != Some(b'"') {
1999 return Err(self.err("expected a quoted literal"));
2000 }
2001 self.bump();
2002 let mut out = String::new();
2003 loop {
2004 match self.bump() {
2005 Some(b'\\') => {
2006 if let Some(c) = self.bump() {
2007 out.push(c as char);
2008 } else {
2009 return Err(self.err("dangling backslash in literal"));
2010 }
2011 }
2012 Some(b'"') => break,
2013 Some(c) => out.push(c as char),
2014 None => return Err(self.err("unterminated quoted literal")),
2015 }
2016 }
2017 Ok(out)
2018 }
2019}
2020
2021#[cfg(test)]
2022mod tests {
2023 use super::*;
2024
2025 #[test]
2026 fn parses_number_atom() {
2027 assert_eq!(parse("\\N").unwrap(), Pattern::Atom(Atom::Kind(TokenKind::Number)));
2028 }
2029
2030 #[test]
2031 fn parses_named_atom() {
2032 assert_eq!(parse("\\{jwt}").unwrap(), Pattern::Atom(Atom::Kind(TokenKind::Jwt)));
2033 assert_eq!(parse("\\{creditcard}").unwrap(), Pattern::Atom(Atom::Kind(TokenKind::CreditCard)));
2034 assert_eq!(parse("\\{ip}").unwrap(), Pattern::Atom(Atom::Kind(TokenKind::Ip)));
2036 assert!(parse("\\{bogus}").is_err());
2037 assert!(parse("\\{jwt").is_err());
2038 }
2039
2040 #[test]
2041 fn parses_lowercase_byte_classes() {
2042 for (src, bc) in [
2043 ("\\h", ByteClass::Hex),
2044 ("\\a", ByteClass::Alpha),
2045 ("\\u", ByteClass::Upper),
2046 ("\\l", ByteClass::Lower),
2047 ] {
2048 assert_eq!(parse(src).unwrap(), Pattern::Atom(Atom::Byte(bc)));
2049 }
2050 }
2051
2052 #[test]
2053 fn parses_matched_tag_pattern() {
2054 let p = parse("<\\W:t>.*</=t>").unwrap();
2056 let Pattern::Concat(items) = p else {
2057 panic!("expected a concatenation, got {p:?}");
2058 };
2059 assert_eq!(items.len(), 8);
2061 assert_eq!(items[0], Pattern::Atom(Atom::literal("<")));
2062 assert_eq!(
2063 items[1],
2064 Pattern::Bind("t".to_string(), false, Pattern::Atom(Atom::Kind(TokenKind::Word)).boxed())
2065 );
2066 assert_eq!(items[3], Pattern::Star(Pattern::Atom(Atom::Any).boxed(), Greed::Greedy));
2067 assert_eq!(
2068 items[6],
2069 Pattern::Atom(Atom::RegisterEq("t".to_string(), crate::orbit::OrbitGroup::Identity))
2070 );
2071 }
2072
2073 #[test]
2074 fn parses_balanced_with_interior() {
2075 let p = parse("\\W\\B(.*)").unwrap();
2077 let Pattern::Concat(items) = p else {
2078 panic!("expected a concatenation, got {p:?}");
2079 };
2080 assert_eq!(items.len(), 2);
2081 assert_eq!(
2082 items[1],
2083 Pattern::Balanced(
2084 Some(BracketKind::Paren),
2085 Pattern::Star(Pattern::Atom(Atom::Any).boxed(), Greed::Greedy).boxed()
2086 )
2087 );
2088 }
2089
2090 #[test]
2091 fn reports_position_on_dangling_backslash() {
2092 let e = parse("\\").unwrap_err();
2093 assert_eq!(e.pos, 1);
2094 }
2095
2096 #[test]
2097 fn a_brace_body_on_a_kind_atom_is_a_repeat_a_magnitude_or_a_typed_predicate() {
2098 assert!(matches!(parse("\\W{2,3}").unwrap(), Pattern::Repeat(_, 2, Some(3), _)));
2100 assert!(matches!(parse("\\N{2}").unwrap(), Pattern::Repeat(_, 2, Some(2), _)));
2101 assert!(matches!(parse("\\N{>500}").unwrap(), Pattern::Atom(Atom::KindPred(TokenKind::Number, _))));
2103 assert!(matches!(parse("\\N{500..599}").unwrap(), Pattern::Atom(Atom::KindPred(TokenKind::Number, _))));
2104 assert!(matches!(parse("\\I{in:10.0.0.0/8}").unwrap(), Pattern::Atom(Atom::KindPred(TokenKind::Ip, _))));
2105 assert!(matches!(parse("\\{creditcard}{issuer:visa}").unwrap(), Pattern::Atom(Atom::KindPred(TokenKind::CreditCard, _))));
2106 assert!(matches!(parse("\\N{mag>3}").unwrap(), Pattern::Atom(Atom::KindMag(TokenKind::Number, _))));
2108 assert!(matches!(parse("\\N{>+1}").unwrap(), Pattern::Atom(Atom::KindMag(TokenKind::Number, _))));
2109 assert!(matches!(parse("\\N{<-1:k}").unwrap(), Pattern::Atom(Atom::KindMag(TokenKind::Number, _))));
2110 assert!(matches!(parse("\\N{>+2s}").unwrap(), Pattern::Atom(Atom::KindMag(TokenKind::Number, _))));
2111 let e = parse("\\I{host:x}").unwrap_err();
2114 assert!(e.msg.contains("no field `host`"), "{}", e.msg);
2115 let e = parse("\\R{>500}").unwrap_err();
2116 assert!(e.msg.contains("unit"), "{}", e.msg);
2117 let e = parse("\\W{>5}").unwrap_err();
2118 assert!(e.msg.contains("name a field"), "{}", e.msg);
2119 assert!(parse("\\N{>500").is_err());
2120 assert!(matches!(parse("\\N{>500}+").unwrap(), Pattern::Plus(..)));
2122 assert!(matches!(parse("\\N{>500}:big").unwrap(), Pattern::Bind(..)));
2123 }
2124
2125 #[test]
2126 fn a_back_reference_reads_under_a_group_a_relation_or_neither() {
2127 use crate::orbit::OrbitGroup;
2128 use crate::typed::Relation;
2129 let atom = |src: &str| match parse(src).unwrap_or_else(|e| panic!("{src}: {e:?}")) {
2130 Pattern::Atom(a) => a,
2131 other => panic!("{src}: {other:?}"),
2132 };
2133 assert_eq!(atom("=ip a"), Atom::RegisterEq("a".to_string(), OrbitGroup::Ip));
2134 assert_eq!(atom("=fold w"), Atom::RegisterEq("w".to_string(), OrbitGroup::Fold));
2135 assert_eq!(atom("=subnet a"), Atom::RegisterRelated("a".to_string(), Relation::Subnet(None)));
2136 assert_eq!(atom("=subnet/16 a"), Atom::RegisterRelated("a".to_string(), Relation::Subnet(Some(16))));
2137 assert_eq!(atom("=domain e"), Atom::RegisterRelated("e".to_string(), Relation::Domain));
2138 assert_eq!(atom("=day t"), Atom::RegisterRelated("t".to_string(), Relation::Day));
2139 assert_eq!(atom("=domain"), Atom::RegisterEq("domain".to_string(), OrbitGroup::Identity));
2141 assert_eq!(atom("=ip"), Atom::RegisterEq("ip".to_string(), OrbitGroup::Identity));
2142 assert!(parse("=domain/8 e").is_err());
2143 assert!(parse("=subnet/200 a").is_err());
2144 assert_eq!(atom("(?orbit:numeric \"1000\")"), Atom::Literal("1000".to_string(), OrbitGroup::Numeric));
2146 }
2147
2148 #[test]
2149 fn an_orbit_scope_names_a_typed_rung_and_reaches_a_plain_back_reference() {
2150 use crate::orbit::OrbitGroup;
2151 use crate::typed::Relation;
2152 let atoms = |src: &str| match parse(src).unwrap_or_else(|e| panic!("{src}: {e:?}")) {
2153 Pattern::Concat(v) => v,
2154 other => panic!("{src}: {other:?}"),
2155 };
2156 let subnet = OrbitGroup::Typed(Relation::Subnet(Some(24)));
2157 assert_eq!(
2158 atoms("(?orbit:subnet/24 \"10.0.0.1\" =a)"),
2159 vec![
2160 Pattern::Atom(Atom::Literal("10.0.0.1".to_string(), subnet)),
2161 Pattern::Atom(Atom::RegisterEq("a".to_string(), subnet)),
2162 ]
2163 );
2164 assert_eq!(
2167 atoms("(?orbit:domain =a =case b =subnet c)"),
2168 vec![
2169 Pattern::Atom(Atom::RegisterEq(
2170 "a".to_string(),
2171 OrbitGroup::Typed(Relation::Domain)
2172 )),
2173 Pattern::Atom(Atom::RegisterEq("b".to_string(), OrbitGroup::Case)),
2174 Pattern::Atom(Atom::RegisterRelated("c".to_string(), Relation::Subnet(None))),
2175 ]
2176 );
2177 assert_eq!(
2179 parse("=a").unwrap_or_else(|e| panic!("{e:?}")),
2180 Pattern::Atom(Atom::RegisterEq("a".to_string(), OrbitGroup::Identity))
2181 );
2182 let e = parse("(?orbit:case (?orbit:numeric \"1\"))").unwrap_err();
2183 assert!(e.msg.contains("cannot stand inside another"), "{}", e.msg);
2184 let e = parse("(?orbit:domain/24 \\E)").unwrap_err();
2185 assert!(e.msg.contains("only the subnet rung"), "{}", e.msg);
2186 let e = parse("(?orbit:nosuch \\W)").unwrap_err();
2187 assert!(e.msg.contains("unknown orbit group"), "{}", e.msg);
2188 }
2189
2190 #[test]
2191 fn an_edit_group_holds_single_token_atoms_and_a_count_under_their_number() {
2192 use crate::ast::EditAtom;
2193 use crate::token::TokenKind;
2194 let plain = |a: Atom| EditAtom { atom: a, bind: None };
2195 assert_eq!(
2196 parse("(\\W \\N)~1").unwrap_or_else(|e| panic!("{e:?}")),
2197 Pattern::Within(
2198 vec![plain(Atom::Kind(TokenKind::Word)), plain(Atom::Kind(TokenKind::Number))],
2199 1
2200 )
2201 );
2202 assert_eq!(
2205 parse("(\\W:who \\N)~1").unwrap_or_else(|e| panic!("{e:?}")),
2206 Pattern::Within(
2207 vec![
2208 EditAtom {
2209 atom: Atom::Kind(TokenKind::Word),
2210 bind: Some(("who".to_string(), false)),
2211 },
2212 plain(Atom::Kind(TokenKind::Number)),
2213 ],
2214 1
2215 )
2216 );
2217 assert_eq!(parse("(\\W \\N)~1").unwrap_or_else(|e| panic!("{e:?}")).capture_names(), Vec::<String>::new());
2218 assert_eq!(
2219 parse("(\\W:who \\N)~1").unwrap_or_else(|e| panic!("{e:?}")).capture_names(),
2220 vec!["who".to_string()]
2221 );
2222 assert!(matches!(
2225 parse("(\\W \\N) ~\"end\"").unwrap_or_else(|e| panic!("{e:?}")),
2226 Pattern::Concat(_)
2227 ));
2228 for (src, said) in [
2229 ("(\\W \\N*)~1", "holds single-token atoms"),
2230 ("(\\W (\\N \\W))~1", "holds single-token atoms"),
2231 ("(\\W \\N \\W)~3", "the count stands under the number of atoms"),
2232 ("(\\W)~1", "the count stands under the number of atoms"),
2233 ] {
2234 let e = parse(src).unwrap_err();
2235 assert!(e.msg.contains(said), "{src}: {}", e.msg);
2236 }
2237 }
2238}