clitest_lib/
output.rs

1use std::{
2    collections::{HashMap, HashSet},
3    sync::{Arc, Mutex},
4};
5
6use grok::Grok;
7use serde::Serialize;
8
9use crate::script::{IfCondition, ScriptLocation, ScriptRunContext};
10
11#[derive(Clone, Debug, PartialEq, Eq)]
12pub struct Line {
13    pub number: usize,
14    pub text: String,
15}
16
17#[derive(Clone)]
18pub struct Lines {
19    lines: Arc<Vec<String>>,
20    current_line: usize,
21    ignored_patterns: Vec<Arc<Vec<OutputPattern>>>,
22    negative_disabled: bool,
23    rejected_patterns: Vec<Arc<Vec<OutputPattern>>>,
24}
25
26impl<'s> IntoIterator for &'s Lines {
27    type Item = &'s String;
28    type IntoIter = std::slice::Iter<'s, String>;
29
30    fn into_iter(self) -> Self::IntoIter {
31        self.lines.iter()
32    }
33}
34
35impl std::fmt::Display for Lines {
36    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
37        write!(f, "{}", self.lines[self.current_line..].join("\n"))
38    }
39}
40
41impl Lines {
42    pub fn new(lines: Vec<String>) -> Self {
43        Self {
44            lines: Arc::new(lines),
45            current_line: 0,
46            ignored_patterns: Default::default(),
47            negative_disabled: false,
48            rejected_patterns: Default::default(),
49        }
50    }
51
52    pub fn is_exhausted(&self) -> bool {
53        self.current_line >= self.lines.len()
54    }
55
56    pub fn next(
57        &self,
58        context: OutputMatchContext,
59    ) -> Result<(Option<Line>, Lines), OutputPatternMatchFailure> {
60        let mut next = self.clone();
61        'outer: while next.current_line < next.lines.len() {
62            if !self.negative_disabled {
63                let ignore_check = next.without_negatives();
64                for ignored in &next.ignored_patterns {
65                    for ignored_pattern in ignored.iter() {
66                        if let Ok(next_next) =
67                            ignored_pattern.matches(context.ignore(), ignore_check.clone())
68                        {
69                            next = next_next.with_negatives();
70                            continue 'outer;
71                        }
72                    }
73                }
74                for rejected in &next.rejected_patterns {
75                    for rejected_pattern in rejected.iter() {
76                        if let Ok(_) =
77                            rejected_pattern.matches(context.ignore(), ignore_check.clone())
78                        {
79                            return Err(OutputPatternMatchFailure {
80                                location: rejected_pattern.location.clone(),
81                                pattern_type: "reject",
82                                output_line: None,
83                            });
84                        }
85                    }
86                }
87            }
88            let line = Line {
89                number: next.current_line,
90                text: next.lines[next.current_line].clone(),
91            };
92            next.current_line += 1;
93            return Ok((Some(line), next));
94        }
95        Ok((None, next))
96    }
97
98    pub fn with_ignore(&self, ignore: &Arc<Vec<OutputPattern>>) -> Self {
99        let mut ignored_patterns = self.ignored_patterns.clone();
100        ignored_patterns.push(ignore.clone());
101        Self {
102            ignored_patterns,
103            ..self.clone()
104        }
105    }
106
107    pub fn with_reject(&self, reject: &Arc<Vec<OutputPattern>>) -> Self {
108        let mut rejected_patterns = self.rejected_patterns.clone();
109        rejected_patterns.push(reject.clone());
110        Self {
111            rejected_patterns,
112            ..self.clone()
113        }
114    }
115
116    fn without_negatives(&self) -> Self {
117        Self {
118            negative_disabled: true,
119            ..self.clone()
120        }
121    }
122
123    fn with_negatives(&self) -> Self {
124        Self {
125            negative_disabled: false,
126            ..self.clone()
127        }
128    }
129
130    pub fn into_inner(self) -> Vec<String> {
131        Arc::unwrap_or_clone(self.lines).split_off(self.current_line)
132    }
133
134    pub fn is_empty(&self) -> bool {
135        self.lines.is_empty()
136    }
137}
138
139#[derive(Clone)]
140pub struct OutputPattern {
141    pub location: ScriptLocation,
142    pub pattern: OutputPatternType,
143    pub ignore: Arc<Vec<OutputPattern>>,
144    pub reject: Arc<Vec<OutputPattern>>,
145}
146
147impl Serialize for OutputPattern {
148    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
149    where
150        S: serde::Serializer,
151    {
152        self.pattern.serialize(serializer)
153    }
154}
155
156impl std::fmt::Debug for OutputPattern {
157    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
158        write!(f, "{:?}", self.pattern)
159    }
160}
161
162impl OutputPattern {
163    pub fn new_sequence(location: ScriptLocation, mut patterns: Vec<OutputPattern>) -> Self {
164        if patterns.len() == 1 {
165            patterns.remove(0)
166        } else {
167            Self {
168                pattern: OutputPatternType::Sequence(patterns),
169                ignore: Default::default(),
170                reject: Default::default(),
171                location: location.clone(),
172            }
173        }
174    }
175
176    pub fn matches(
177        &self,
178        context: OutputMatchContext,
179        output: Lines,
180    ) -> Result<Lines, OutputPatternMatchFailure> {
181        if self.ignore.is_empty() && self.reject.is_empty() {
182            self.pattern.matches(&self.location, context, output)
183        } else {
184            let output = output.with_ignore(&self.ignore).with_reject(&self.reject);
185            self.pattern.matches(&self.location, context, output)
186        }
187    }
188
189    /// The minimum number of lines this pattern will match.
190    pub fn min_matches(&self) -> usize {
191        self.pattern.min_matches()
192    }
193
194    /// The maximum number of lines this pattern will match (or usize::MAX if unbounded).
195    pub fn max_matches(&self) -> usize {
196        self.pattern.max_matches()
197    }
198}
199
200#[derive(Clone)]
201pub enum OutputPatternType {
202    /// The end of the output
203    End,
204    /// Matches no lines of output, always succeeds
205    None,
206    /// Any lines, followed by a pattern.
207    Any(Box<OutputPattern>),
208    /// A literal string
209    Literal(String),
210    /// A grok pattern
211    Pattern(Arc<GrokPattern>),
212    /// A pattern that matches one or more of the given pattern
213    Repeat(Box<OutputPattern>),
214    /// A pattern that matches zero or one of the given pattern
215    Optional(Box<OutputPattern>),
216    /// A pattern that all of its subpatterns, but in any order
217    Unordered(Vec<OutputPattern>),
218    /// A pattern that matches one of the given patterns
219    Choice(Vec<OutputPattern>),
220    /// A pattern that matches a sequence of patterns
221    Sequence(Vec<OutputPattern>),
222    /// A pattern that matches a condition
223    If(IfCondition, Box<OutputPattern>),
224}
225
226impl Serialize for OutputPatternType {
227    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
228    where
229        S: serde::Serializer,
230    {
231        match self {
232            OutputPatternType::Literal(literal) => {
233                serializer.serialize_str(&format!("! {literal}"))
234            }
235            OutputPatternType::Pattern(pattern) => {
236                serializer.serialize_str(&format!("? {}", pattern.pattern))
237            }
238            OutputPatternType::Repeat(pattern) => {
239                HashMap::from([("repeat", &pattern)]).serialize(serializer)
240            }
241            OutputPatternType::Optional(pattern) => {
242                HashMap::from([("optional", &pattern)]).serialize(serializer)
243            }
244            OutputPatternType::Unordered(patterns) => {
245                HashMap::from([("unordered", &patterns)]).serialize(serializer)
246            }
247            OutputPatternType::Choice(patterns) => {
248                HashMap::from([("choice", &patterns)]).serialize(serializer)
249            }
250            OutputPatternType::Sequence(patterns) => {
251                HashMap::from([("sequence", &patterns)]).serialize(serializer)
252            }
253            OutputPatternType::Any(pattern) => {
254                HashMap::from([("any", &pattern)]).serialize(serializer)
255            }
256            OutputPatternType::If(condition, pattern) => {
257                #[derive(Serialize)]
258                struct If<'a> {
259                    condition: &'a IfCondition,
260                    pattern: &'a OutputPattern,
261                }
262                If { condition, pattern }.serialize(serializer)
263            }
264            OutputPatternType::End => serializer.serialize_str("end"),
265            OutputPatternType::None => serializer.serialize_str("none"),
266        }
267    }
268}
269
270impl OutputPatternType {
271    /// The minimum number of lines this pattern will match.
272    pub fn min_matches(&self) -> usize {
273        match self {
274            OutputPatternType::None => 0,
275            OutputPatternType::Literal(_) => 1,
276            OutputPatternType::Pattern(_) => 1,
277            OutputPatternType::Repeat(pattern) => pattern.min_matches(),
278            OutputPatternType::Optional(_) => 0,
279            OutputPatternType::Unordered(patterns) => {
280                patterns.iter().map(|p| p.min_matches()).sum()
281            }
282            OutputPatternType::Choice(patterns) => {
283                patterns.iter().map(|p| p.min_matches()).min().unwrap_or(0)
284            }
285            OutputPatternType::Sequence(patterns) => patterns.iter().map(|p| p.min_matches()).sum(),
286            OutputPatternType::Any(pattern) => pattern.min_matches(),
287            OutputPatternType::If(_, _) => 0,
288            OutputPatternType::End => 0,
289        }
290    }
291
292    /// The maximum number of lines this pattern will match (or usize::MAX if unbounded).
293    pub fn max_matches(&self) -> usize {
294        fn saturating_iter_sum<I>(iter: I) -> usize
295        where
296            I: IntoIterator<Item = usize>,
297        {
298            iter.into_iter()
299                .reduce(|n, i| n.saturating_add(i))
300                .unwrap_or(0)
301        }
302
303        match self {
304            OutputPatternType::None => 0,
305            OutputPatternType::Literal(_) => 1,
306            OutputPatternType::Pattern(_) => 1,
307            OutputPatternType::Repeat(pattern) => {
308                if pattern.max_matches() == 0 {
309                    0
310                } else {
311                    usize::MAX
312                }
313            }
314            OutputPatternType::Optional(pattern) => pattern.max_matches(),
315            OutputPatternType::Unordered(patterns) => {
316                saturating_iter_sum(patterns.iter().map(|p| p.max_matches()))
317            }
318            OutputPatternType::Choice(patterns) => {
319                patterns.iter().map(|p| p.max_matches()).max().unwrap_or(0)
320            }
321            OutputPatternType::Sequence(patterns) => {
322                saturating_iter_sum(patterns.iter().map(|p| p.max_matches()))
323            }
324            OutputPatternType::Any(_) => usize::MAX,
325            OutputPatternType::If(_, pattern) => pattern.max_matches(),
326            OutputPatternType::End => 0,
327        }
328    }
329}
330
331impl Default for OutputPatternType {
332    fn default() -> Self {
333        Self::Sequence(vec![])
334    }
335}
336
337impl std::fmt::Debug for OutputPatternType {
338    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
339        match self {
340            OutputPatternType::Literal(literal) => write!(f, "Literal({literal})"),
341            OutputPatternType::Pattern(pattern) => write!(f, "Pattern({pattern:?})"),
342            OutputPatternType::Repeat(pattern) => write!(f, "Repeat({pattern:?})"),
343            OutputPatternType::Optional(pattern) => write!(f, "Optional({pattern:?})"),
344            OutputPatternType::Unordered(patterns) => write!(f, "Unordered({patterns:?})"),
345            OutputPatternType::Choice(patterns) => write!(f, "Choice({patterns:?})"),
346            OutputPatternType::Sequence(patterns) => write!(f, "Sequence({patterns:?})"),
347            OutputPatternType::Any(until) => write!(f, "Any({until:?})"),
348            OutputPatternType::If(condition, pattern) => {
349                write!(f, "If({condition:?}, {pattern:?})")
350            }
351            OutputPatternType::End => write!(f, "End"),
352            OutputPatternType::None => write!(f, "None"),
353        }
354    }
355}
356
357#[derive(Serialize, derive_more::Debug)]
358#[debug("/{pattern:?}/")]
359pub struct GrokPattern {
360    pattern: String,
361    #[serde(skip)]
362    grok: grok::Pattern,
363}
364
365impl GrokPattern {
366    pub fn compile(
367        grok: &mut Grok,
368        line: &str,
369        escape_non_grok: bool,
370    ) -> Result<Self, grok::Error> {
371        if escape_non_grok {
372            // Borrowed from grok crate
373            const GROK_PATTERN: &str = r"%\{(?<name>(?<pattern>[A-z0-9]+)(?::(?<alias>[A-z0-9_:;\/\s\.]+))?)(?:=(?<definition>(?:(?:[^{}]+|\.+)+)+))?\}";
374            let re = onig::Regex::new(GROK_PATTERN).expect("Failed to compile Grok metapattern");
375            let mut prev_start = 0;
376
377            // Escape the text between grok pattern matches to make it easier to write
378            // literals-with-grok patterns.
379            let mut escaped_string = String::with_capacity(line.len() * 2);
380            for re_match in re.find_iter(line) {
381                let text = &line[prev_start..re_match.0];
382                text.chars().for_each(|c| {
383                    if c.is_ascii() && !c.is_alphanumeric() {
384                        escaped_string.push('\\');
385                        escaped_string.push(c);
386                    } else {
387                        escaped_string.push(c);
388                    }
389                });
390                escaped_string.push_str(&line[re_match.0..re_match.1]);
391                prev_start = re_match.1;
392            }
393            let text = &line[prev_start..];
394            text.chars().for_each(|c| {
395                if c.is_ascii() && !c.is_alphanumeric() {
396                    escaped_string.push('\\');
397                    escaped_string.push(c);
398                } else {
399                    escaped_string.push(c);
400                }
401            });
402            let eol = format!("{escaped_string}$");
403            Ok(Self {
404                pattern: escaped_string,
405                grok: grok.compile(&eol, false)?,
406            })
407        } else {
408            let eol = format!("{line}$");
409            Ok(Self {
410                pattern: line.to_string(),
411                grok: grok.compile(&eol, false)?,
412            })
413        }
414    }
415}
416
417#[derive(Clone, Debug, thiserror::Error, derive_more::Display, PartialEq, Eq)]
418#[display("pattern {pattern_type} at line {location} does not match output line {:?}", output_line.as_ref().map(|l| l.text.clone()).unwrap_or("<eof>".to_string()))]
419pub struct OutputPatternMatchFailure {
420    pub location: ScriptLocation,
421    pub pattern_type: &'static str,
422    pub output_line: Option<Line>,
423}
424
425#[derive(Debug, Clone)]
426pub struct OutputMatchContext<'s> {
427    depth: usize,
428    trace: Arc<Mutex<Vec<String>>>,
429    ignore: bool,
430    script_context: &'s ScriptRunContext,
431}
432
433impl<'s> OutputMatchContext<'s> {
434    pub fn new(script_context: &'s ScriptRunContext) -> Self {
435        Self {
436            depth: 0,
437            trace: Default::default(),
438            ignore: false,
439            script_context,
440        }
441    }
442
443    pub fn descend(&self) -> Self {
444        Self {
445            depth: self.depth + 1,
446            trace: self.trace.clone(),
447            ignore: self.ignore,
448            script_context: self.script_context,
449        }
450    }
451
452    pub fn ignore(&self) -> Self {
453        Self {
454            depth: self.depth,
455            trace: self.trace.clone(),
456            ignore: true,
457            script_context: self.script_context,
458        }
459    }
460
461    pub fn trace(&self, line: &str) {
462        let ignore = if self.ignore { "-" } else { "" };
463        self.trace.lock().unwrap().push(format!(
464            "{:indent$}{ignore}{}",
465            "",
466            line,
467            indent = self.depth * 2
468        ));
469    }
470
471    pub fn traces(&self) -> Vec<String> {
472        std::mem::take(&mut self.trace.lock().unwrap())
473    }
474}
475
476impl OutputPatternType {
477    pub fn matches<'s>(
478        &self,
479        location: &ScriptLocation,
480        context: OutputMatchContext,
481        mut output: Lines,
482    ) -> Result<Lines, OutputPatternMatchFailure> {
483        context.trace(&format!("matching {:?}", self));
484        match self {
485            OutputPatternType::None => Ok(output),
486            OutputPatternType::Literal(literal) => {
487                let (line, next) = output.next(context.clone())?;
488                if let Some(line) = line {
489                    if &line.text.trim_end() == literal {
490                        context.trace(&format!("literal match: {:?} == {literal:?}", line.text));
491                        Ok(next)
492                    } else {
493                        context.trace(&format!(
494                            "literal FAILED match: {:?} == {literal:?}",
495                            line.text
496                        ));
497                        Err(OutputPatternMatchFailure {
498                            location: location.clone(),
499                            pattern_type: "literal",
500                            output_line: Some(line),
501                        })
502                    }
503                } else {
504                    Err(OutputPatternMatchFailure {
505                        location: location.clone(),
506                        pattern_type: "literal",
507                        output_line: None,
508                    })
509                }
510            }
511            OutputPatternType::Pattern(pattern) => {
512                let (line, next) = output.next(context.clone())?;
513                if let Some(line) = line {
514                    let text = line.text.clone();
515                    // Don't print panic backtraces
516                    let res = match std::panic::catch_unwind(|| pattern.grok.match_against(&text)) {
517                        Ok(res) => res,
518                        Err(_) => {
519                            return Err(OutputPatternMatchFailure {
520                                location: location.clone(),
521                                pattern_type: "pattern",
522                                output_line: Some(line),
523                            });
524                        }
525                    };
526                    if let Some(_matches) = res {
527                        context.trace(&format!("pattern match: {:?} =~ {pattern:?}", line.text));
528                        Ok(next)
529                    } else {
530                        context.trace(&format!(
531                            "pattern FAILED match: {:?} =~ {pattern:?}",
532                            line.text
533                        ));
534                        Err(OutputPatternMatchFailure {
535                            location: location.clone(),
536                            pattern_type: "pattern",
537                            output_line: Some(line),
538                        })
539                    }
540                } else {
541                    Err(OutputPatternMatchFailure {
542                        location: location.clone(),
543                        pattern_type: "pattern",
544                        output_line: None,
545                    })
546                }
547            }
548            OutputPatternType::Sequence(patterns) => {
549                for pattern in patterns {
550                    match pattern.matches(context.descend(), output) {
551                        Ok(v) => {
552                            output = v;
553                        }
554                        Err(e) => {
555                            return Err(e);
556                        }
557                    }
558                }
559                Ok(output)
560            }
561            OutputPatternType::Repeat(pattern) => {
562                // Mandatory first match
563                let mut output = pattern.matches(context.descend(), output)?;
564                // Any number of additional matches, greedy
565                loop {
566                    match pattern.matches(context.descend(), output.clone()) {
567                        Ok(new_rest) => {
568                            output = new_rest;
569                        }
570                        Err(_) => break Ok(output),
571                    }
572                }
573            }
574            OutputPatternType::Optional(pattern) => {
575                // Never fails
576                match pattern.matches(context.descend(), output.clone()) {
577                    Ok(v) => Ok(v),
578                    Err(_) => Ok(output),
579                }
580            }
581            OutputPatternType::Unordered(patterns) => {
582                // Found is initialized with 0..patterns.len()
583                let mut not_found = (0..patterns.len()).collect::<HashSet<_>>();
584                'outer: while !not_found.is_empty() {
585                    for pattern in &not_found {
586                        let pattern = *pattern;
587                        match patterns[pattern].matches(context.descend(), output.clone()) {
588                            Ok(v) => {
589                                not_found.remove(&pattern);
590                                output = v;
591                                continue 'outer;
592                            }
593                            Err(_) => {
594                                continue;
595                            }
596                        }
597                    }
598                    return Err(OutputPatternMatchFailure {
599                        location: location.clone(),
600                        pattern_type: "unordered",
601                        output_line: None,
602                    });
603                }
604                Ok(output)
605            }
606            OutputPatternType::Choice(patterns) => {
607                for pattern in patterns {
608                    if let Ok(v) = pattern.matches(context.descend(), output.clone()) {
609                        return Ok(v);
610                    }
611                }
612                Err(OutputPatternMatchFailure {
613                    location: location.clone(),
614                    pattern_type: "choice",
615                    output_line: None,
616                })
617            }
618            OutputPatternType::Any(until) => {
619                loop {
620                    match until.matches(context.descend(), output.clone()) {
621                        Ok(v) => {
622                            output = v;
623                            break Ok(output);
624                        }
625                        Err(e) => {
626                            // Eat one line and try again
627                            let (line, next) = output.next(context.clone())?;
628                            if line.is_some() {
629                                output = next;
630                                continue;
631                            } else {
632                                break Err(e);
633                            }
634                        }
635                    }
636                }
637            }
638            OutputPatternType::If(condition, pattern) => {
639                if condition.matches(context.script_context) {
640                    context.trace(&format!("if match: {:?}", condition));
641                    pattern.matches(context.clone(), output.clone())
642                } else {
643                    context.trace(&format!("if FAILED match: {:?}", condition));
644                    Ok(output)
645                }
646            }
647            OutputPatternType::End => {
648                let (line, next) = output.next(context)?;
649                if let Some(line) = line {
650                    Err(OutputPatternMatchFailure {
651                        location: location.clone(),
652                        pattern_type: "end",
653                        output_line: Some(line),
654                    })
655                } else {
656                    Ok(next)
657                }
658            }
659        }
660    }
661}
662
663#[derive(Debug)]
664pub enum PatternResult {
665    Matches,
666    MatchesFailure,
667    ExpectedFailure,
668    Mismatch(OutputPatternMatchFailure, String),
669}
clitest_lib/output.rs

clitest_lib/
output.rs
