drain_flow/record/
tokens.rs

1// Copyright Nicholas Harring. All rights reserved.
2//
3// This program is free software: you can redistribute it and/or modify it under
4// the terms of the Server Side Public License, version 1, as published by MongoDB, Inc.
5// This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
6// without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
7// See the Server Side Public License for more details. You should have received a copy of the
8// Server Side Public License along with this program.
9// If not, see <http://www.mongodb.com/licensing/server-side-public-license>.
10
11use std::{
12    collections::HashMap,
13    fmt::{self, Display},
14};
15
16use itertools::Itertools;
17use joinery::JoinableIterator;
18use lazy_static::lazy_static;
19use regex::RegexSet;
20use string_interner::DefaultSymbol;
21use tracing::{debug, instrument};
22
23pub use super::ASTERISK; // Made ASTERISK re-export public
24use crate::drains::simple::INTERNER;
25
26lazy_static! {
27    static ref MATCHERS: RegexSet = Grokker::build_pattern_set();
28    static ref GROKKER_COUNT: usize = Grokker::iter_variants().count() - 1;
29    static ref GROKKER_SYMS: HashMap<Grokker, DefaultSymbol> = symbolize_grokker();
30    static ref GROKKER_VARIANTS: HashMap<usize, Grokker> = Grokker::iter_variants()
31        .enumerate()
32        .collect::<HashMap<usize, Grokker>>();
33}
34
35fn symbolize_grokker() -> HashMap<Grokker, DefaultSymbol> {
36    Grokker::iter_variants()
37        .map(|v| (v, INTERNER.write().get_or_intern(v.to_string())))
38        .collect::<HashMap<Grokker, DefaultSymbol>>()
39}
40
41/// An enumeration of various data patterns (groks) that can be identified in log tokens.
42///
43/// Each variant represents a specific type of data, such as integers, floats, UUIDs,
44/// MAC addresses, IP addresses, hostnames, months, and days.
45custom_derive! {
46    #[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, IterVariants(GrokkerVariants), EnumDisplay)]
47    pub enum Grokker {
48        /// Matches base-10 integer numbers.
49        Base10Integer,
50        /// Matches base-10 floating-point numbers.
51        Base10Float,
52        /// Matches base-16 (hexadecimal) integer numbers.
53        Base16Integer,
54        /// Matches base-16 (hexadecimal) floating-point numbers.
55        Base16Float,
56        /// Matches UUIDs (Universally Unique Identifiers).
57        UUID,
58        /// Matches MAC addresses.
59        MAC,
60        /// Matches IPv6 addresses.
61        IPv6,
62        /// Matches IPv4 addresses.
63        IPv4,
64        /// Matches hostnames.
65        Hostname,
66        /// Matches month names (e.g., Jan, January).
67        Month,
68        /// Matches day names (e.g., Mon, Monday).
69        Day,
70    }
71}
72
73impl Grokker {
74    /// Returns the regular expression pattern string for the given `Grokker` variant.
75    ///
76    /// # Returns
77    ///
78    /// A `String` containing the regular expression pattern.
79    #[must_use]
80    pub fn to_pattern(self) -> String {
81        match self {
82            Grokker::Base10Integer => r"^(?:[+-]?(?:[0-9]+))$".to_string(),
83            Grokker::Base10Float => {
84                r"^(?:[+-]?(?:(?:[0-9]+(?:\.[0-9]+))|(?:\.[0-9]+)))$".to_string()
85            }
86            Grokker::Base16Integer => r"^(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+))$".to_string(),
87            Grokker::Base16Float => {
88                r"^(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+)(?:\.[0-9A-Fa-f]+))$".to_string()
89            }
90            Grokker::UUID => r"^[A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}$".to_string(),
91            Grokker::MAC => r"^(?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})$".to_string(),
92            Grokker::IPv6 => {
93                r"^((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?$".to_string()
94            }
95            Grokker::IPv4 => {
96                r"^(?:(?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5]))$".to_string()
97            }
98            Grokker::Hostname => {
99                r"^(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b)$".to_string()
100            }
101            Grokker::Month => {
102                r"^(?:[Jj]an(?:uary|uar)?|[Ff]eb(?:ruary|ruar)?|[Mm](?:a|ä)?r(?:ch|z)?|[Aa]pr(?:il)?|[Mm]a(?:y|i)?|[Jj]un(?:e|i)?|[Jj]ul(?:y)?|[Aa]ug(?:ust)?|[Ss]ep(?:tember)?|[Oo](?:c|k)?t(?:ober)?|[Nn]ov(?:ember)?|[Dd]e(?:c|z)(?:ember)?)$".to_string()
103            }
104            Grokker::Day => {
105                r"^(?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)$".to_string()
106            }
107        }
108    }
109
110    /// Builds a `RegexSet` containing all patterns for `Grokker` variants.
111    ///
112    /// This set is used for efficient matching of a string against multiple patterns.
113    ///
114    /// # Returns
115    ///
116    /// A `RegexSet` containing all `Grokker` patterns.
117    ///
118    /// # Panics
119    ///
120    /// Panics if any of the `Grokker` patterns are invalid regular expressions.
121    fn build_pattern_set() -> RegexSet {
122        let variants = Grokker::iter_variants()
123            .map(Grokker::to_pattern)
124            .collect::<Vec<String>>();
125        RegexSet::new(variants).expect("valid regular expressions compile")
126    }
127
128    /// Converts a match index from a `RegexSet` into a corresponding `Grokker` variant.
129    ///
130    /// # Arguments
131    ///
132    /// * `idx` - The index of the matched pattern in the `RegexSet`.
133    ///
134    /// # Returns
135    ///
136    /// An `Option<Grokker>` containing the `Grokker` variant if the index is valid,
137    /// otherwise `None`.
138    #[instrument(level = "trace")]
139    pub fn from_match_index(idx: usize) -> Option<Grokker> {
140        if idx > *GROKKER_COUNT {
141            return None;
142        }
143        Some(GROKKER_VARIANTS[&idx])
144    }
145}
146
147/// A convenience wrapper over `regex::RegexSet::matches` and `Grokker` variants.
148///
149/// `GrokSet` provides methods to check if a string matches certain predefined
150/// data patterns (groks).
151#[derive(Debug, Clone)]
152pub struct GrokSet {
153    match_types: Vec<Grokker>,
154}
155
156impl GrokSet {
157    /// Creates a new `GrokSet` by analyzing the provided string `value`.
158    ///
159    /// It uses a pre-compiled `RegexSet` to determine which `Grokker` patterns
160    /// the `value` matches.
161    ///
162    /// # Arguments
163    ///
164    /// * `value` - The string to analyze.
165    ///
166    /// # Returns
167    ///
168    /// A new `GrokSet` instance containing the types of `Grokker` patterns matched.
169    #[must_use]
170    pub fn new(value: &str) -> Self {
171        let matches = MATCHERS.matches(value);
172        let match_types: Vec<_> = matches
173            .iter()
174            .filter_map(Grokker::from_match_index)
175            .collect();
176        Self { match_types }
177    }
178
179    /// Checks if any of the matched `Grokker` types are numeric (integers or floats).
180    ///
181    /// # Returns
182    ///
183    /// `true` if the `GrokSet` contains any numeric `Grokker` type, `false` otherwise.
184    #[must_use]
185    pub fn is_numeric(&self) -> bool {
186        self.match_types.iter().any(|i| {
187            matches!(
188                i,
189                Grokker::Base10Integer
190                    | Grokker::Base16Integer
191                    | Grokker::Base16Float
192                    | Grokker::Base10Float
193            )
194        })
195    }
196
197    /// Checks if any of the matched `Grokker` types are integers (base-10 or base-16).
198    ///
199    /// # Returns
200    ///
201    /// `true` if the `GrokSet` contains any integer `Grokker` type, `false` otherwise.
202    #[must_use]
203    pub fn is_integer(&self) -> bool {
204        self.match_types
205            .iter()
206            .any(|i| matches!(i, Grokker::Base10Integer | Grokker::Base16Integer))
207    }
208}
209
210/// Represents a token within a log line, which can be a wildcard, a typed match, or a specific value.
211#[derive(Debug, Clone, PartialEq)]
212pub enum Token {
213    /// A wildcard token, matching any other token. Represented as "<*>" in display.
214    Wildcard,
215    /// A token that matches any value of a specific predefined type (e.g., UUID, IPv4).
216    TypedMatch(Grokker),
217    /// A token containing a specific, non-wildcard value.
218    Value(TypedToken),
219}
220
221impl Token {
222    /// Parses an input string and attempts to classify it into a `Token` type.
223    ///
224    /// This function uses a set of regular expressions (`MATCHERS`) to determine
225    /// if the input string matches any known `Grokker` patterns. It prioritizes
226    /// more specific matches and handles ambiguities (e.g., a UUID also matching
227    /// a hostname pattern).
228    ///
229    /// # Arguments
230    ///
231    /// * `input` - The string slice to parse.
232    ///
233    /// # Returns
234    ///
235    /// A `Token` representing the classification of the input string.
236    #[instrument(level = "trace")]
237    pub fn from_parse(input: &str) -> Token {
238        let matches = MATCHERS.matches(input);
239        let match_types: Vec<_> = matches
240            .iter()
241            .filter_map(Grokker::from_match_index)
242            .collect();
243
244        debug!("comparing {} tokens", match_types.len());
245
246        let tok = match match_types.len() {
247            0 => Token::Value(TypedToken::from_parse(input)),
248            1 => {
249                let idx = matches.iter().collect::<Vec<usize>>()[0];
250                let grokker = Grokker::from_match_index(idx).unwrap();
251                debug!(%grokker, "single match");
252                Token::TypedMatch(grokker)
253            }
254            2 => {
255                debug!(?match_types, "2 match arm");
256                // UUID and hostname can overlap, if they do its 99.999% a UUID
257                if match_types.contains(&Grokker::UUID) && match_types.contains(&Grokker::Hostname)
258                {
259                    debug!("uuid & hostname");
260                    return Token::TypedMatch(Grokker::UUID);
261                }
262                // All base10 ints match base16 ints
263                if match_types.contains(&Grokker::Base10Integer)
264                    && match_types.contains(&Grokker::Base16Integer)
265                {
266                    return Token::TypedMatch(Grokker::Base10Integer);
267                }
268                // All base10 floats match base16 floats
269                if match_types.contains(&Grokker::Base10Float)
270                    && match_types.contains(&Grokker::Base16Float)
271                {
272                    debug!("base10 & base16 float");
273                    return Token::TypedMatch(Grokker::Base10Float);
274                }
275                // base16 numbers and hostname can overlap, if they do its 99.999% a number
276                if match_types.contains(&Grokker::Base16Integer)
277                    && match_types.contains(&Grokker::Hostname)
278                {
279                    debug!("base16 int & hostname");
280                    return Token::TypedMatch(Grokker::Base16Integer);
281                }
282                if match_types.contains(&Grokker::Base16Float)
283                    && match_types.contains(&Grokker::Hostname)
284                {
285                    debug!("base16 float & hostname");
286                    return Token::TypedMatch(Grokker::Base16Float);
287                }
288                debug!("fallback to wildcard");
289                Token::Wildcard
290            }
291            3 => {
292                debug!(?match_types, "3 match arm");
293                // All base10 integers also match as base16 and weirdly as hostnames
294                if match_types.contains(&Grokker::Base10Integer)
295                    && match_types.contains(&Grokker::Base16Integer)
296                    && match_types.contains(&Grokker::Hostname)
297                {
298                    debug!("base10 int mistaken for hostname");
299                    return Token::TypedMatch(Grokker::Base10Integer);
300                }
301
302                if match_types.contains(&Grokker::Base10Float)
303                    && match_types.contains(&Grokker::Base16Float)
304                    && match_types.contains(&Grokker::Hostname)
305                {
306                    debug!("base 10 float mistaken for hostname");
307                    return Token::TypedMatch(Grokker::Base10Float);
308                }
309                debug!("fallback to wildcard");
310                Token::Wildcard
311            }
312            // Todo: Explore if there is a way to figure out a "best match"
313            _ => Token::Wildcard,
314        };
315        tok
316    }
317}
318
319impl fmt::Display for Token {
320    /// Formats the `Token` for display.
321    ///
322    /// This implementation provides a string representation of the token,
323    /// using "<*>" for `Wildcard` tokens, the `Grokker`'s display for `TypedMatch`,
324    /// and the underlying value's string representation for `Value` tokens.
325    ///
326    /// # Arguments
327    ///
328    /// * `f` - The formatter to write into.
329    ///
330    /// # Returns
331    ///
332    /// A `fmt::Result` indicating success or failure of the formatting operation.
333    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
334        let out: String = match self {
335            Token::Wildcard => "<*>".to_string(),
336            Token::TypedMatch(t) => t.to_string(),
337            Token::Value(v) => match v {
338                TypedToken::String(sym) => INTERNER
339                    .read()
340                    .resolve(*sym)
341                    .expect("symbols must resolve")
342                    .to_string(),
343                TypedToken::Int(i) => format!("{}", i),
344                TypedToken::Float(f) => f.to_string(),
345            },
346        };
347        write!(f, "{}", out)
348    }
349}
350
351impl From<Token> for DefaultSymbol {
352    /// Converts a `Token` into its `DefaultSymbol` representation.
353    ///
354    /// This conversion uses the global string interner to obtain a symbol
355    /// for the token's string value or a predefined symbol for `Wildcard`
356    /// and `TypedMatch` tokens.
357    ///
358    /// # Arguments
359    ///
360    /// * `tok` - The `Token` to convert.
361    ///
362    /// # Returns
363    ///
364    /// The `DefaultSymbol` corresponding to the `Token`.
365    fn from(tok: Token) -> DefaultSymbol {
366        match tok {
367            Token::Wildcard => *ASTERISK,
368            Token::TypedMatch(t) => *GROKKER_SYMS
369                .get(&t)
370                .expect("every grokker must have a symbol"),
371            Token::Value(v) => match v {
372                TypedToken::String(s) => s,
373                TypedToken::Int(i) => INTERNER.write().get_or_intern(i.to_string()),
374                TypedToken::Float(f) => INTERNER.write().get_or_intern(f.to_string()),
375            },
376        }
377    }
378}
379
380/// Represents a typed token value.
381///
382/// This enum distinguishes between string, integer, and floating-point token values.
383#[derive(PartialEq, Debug, Clone)]
384pub enum TypedToken {
385    /// A token containing a string value. This is typically used for words or phrases
386    /// that do not match any specific numeric or other structured patterns.
387    String(DefaultSymbol),
388    /// A token containing a whole number.
389    Int(i64),
390    /// A token containing a floating-point number.
391    Float(f64),
392}
393
394impl TypedToken {
395    /// Parses a supplied string and returns a `TypedToken::String`.
396    ///
397    /// This function currently only interns the input string as a `DefaultSymbol`
398    /// and wraps it in a `TypedToken::String`.
399    ///
400    /// # Arguments
401    ///
402    /// * `input` - The string slice to parse.
403    ///
404    /// # Returns
405    ///
406    /// A `TypedToken::String` containing the interned representation of the input.
407    #[must_use]
408    pub fn from_parse(input: &str) -> TypedToken {
409        TypedToken::String(INTERNER.write().get_or_intern(input))
410    }
411}
412
413/// Represents an offset within a string, indicating the start and end byte positions.
414#[derive(Copy, Clone, Debug, PartialEq, Eq)]
415pub struct Offset {
416    /// The starting byte position of the token.
417    start: usize,
418    /// The ending byte position of the token.
419    end: usize,
420}
421
422impl Display for Offset {
423    /// Formats the `Offset` for display.
424    ///
425    /// # Arguments
426    ///
427    /// * `f` - The formatter to write into.
428    ///
429    /// # Returns
430    ///
431    /// A `fmt::Result` indicating success or failure of the formatting operation.
432    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
433        write!(f, "Offset(start: {}, end: {})", self.start, self.end)
434    }
435}
436
437/// Represents a stream of tokens, typically derived from a log line.
438///
439/// A `TokenStream` stores a sequence of `(Offset, Token)` pairs, preserving
440/// the original position and type of each token within the source string.
441#[derive(Clone, Debug, PartialEq)]
442pub struct TokenStream {
443    pub(crate) inner: Vec<(Offset, Token)>,
444}
445
446impl TokenStream {
447    /// Creates a `TokenStream` from a Unicode log line.
448    ///
449    /// This function splits the input line by ASCII whitespace and attempts to
450    /// identify and intern each word as a `Token::Value(TypedToken::String)`. It also
451    /// calculates and stores the `Offset` for each token.
452    ///
453    /// # Arguments
454    ///
455    /// * `line` - The input log line as a string slice.
456    ///
457    /// # Returns
458    ///
459    /// A new `TokenStream` instance.
460    #[instrument(skip(line))]
461    pub fn from_unicode_line(line: &str) -> Self {
462        let mut interner = INTERNER.write();
463        let mut progress = 0usize;
464        let words = line
465            .split_ascii_whitespace()
466            .filter_map(|w| {
467                debug!(%w, %progress, "got");
468                let start = line.match_indices(w).find(|(i, _w)| {
469                    debug!(%progress, %i, "found");
470                    i >= &progress
471                })?;
472                let end = start.0 + start.1.len();
473                progress = end;
474                let token = (
475                    Offset {
476                        start: start.0,
477                        end,
478                    },
479                    Token::Value(TypedToken::String(interner.get_or_intern(w))),
480                );
481                debug!(?token, %w, ?start, "built");
482                Some(token)
483            })
484            .collect::<Vec<(Offset, Token)>>();
485        Self { inner: words }
486    }
487
488    /// Returns the first `Token` in the stream.
489    ///
490    /// # Returns
491    ///
492    /// An `Option<Token>` containing a clone of the first token if the stream is not empty,
493    /// otherwise `None`.
494    #[instrument(level = "trace", skip(self))]
495    pub fn first(&self) -> Option<Token> {
496        match self.inner.len() {
497            0 => None,
498            _ => Some(self.inner[0].1.clone()),
499        }
500    }
501
502    /// Returns the number of tokens in the stream.
503    ///
504    /// # Returns
505    ///
506    /// The length of the token stream as a `usize`.
507    #[instrument(level = "trace", skip(self))]
508    pub fn len(&self) -> usize {
509        self.inner.len()
510    }
511
512    /// Checks if the token stream is empty.
513    ///
514    /// # Returns
515    ///
516    /// `true` if the token stream contains no tokens, `false` otherwise.
517    #[instrument(level = "trace", skip(self))]
518    pub fn is_empty(&self) -> bool {
519        self.inner.is_empty()
520    }
521
522    /// Returns a clone of the `Token` at the specified index.
523    ///
524    /// # Arguments
525    ///
526    /// * `idx` - The zero-based index of the token to retrieve.
527    ///
528    /// # Returns
529    ///
530    /// An `Option<Token>` containing a clone of the token if the index is valid,
531    /// otherwise `None`.
532    #[instrument(skip(self))]
533    pub fn get_token_at_index(&self, idx: usize) -> Option<Token> {
534        if idx < self.inner.len() {
535            Some(self.inner[idx].1.clone())
536        } else {
537            None
538        }
539    }
540}
541
542impl fmt::Display for TokenStream {
543    /// Formats the `TokenStream` for display.
544    ///
545    /// This implementation reconstructs the original string from the tokens and their
546    /// offsets, preserving the original whitespace between tokens.
547    ///
548    /// # Arguments
549    ///
550    /// * `f` - The formatter to write into.
551    ///
552    /// # Returns
553    ///
554    /// A `fmt::Result` indicating success or failure of the formatting operation.
555    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
556        let words = self
557            .inner
558            .iter()
559            .map(|(_, t)| t.to_string())
560            .collect::<Vec<String>>();
561        let whitespace = self
562            .inner
563            .iter()
564            .tuple_windows()
565            .map(|(first, second)| (first.0.end, second.0.start))
566            .map(|t| " ".repeat(t.1 - t.0))
567            .collect::<Vec<String>>();
568        write!(
569            f,
570            "{}",
571            words.iter().interleave(whitespace.iter()).join_concat()
572        )
573    }
574}
575#[cfg(test)]
576mod should {
577    use proptest::prelude::*;
578
579    use crate::record::tokens::{GrokSet, Grokker, Token};
580
581    // The below makes debugging tests much easier
582    // use tracing_test::traced_test;
583
584    prop_compose! {
585        fn gen_uuid()(s in "[A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}") -> String {
586            s
587        }
588    }
589    prop_compose! {
590        fn gen_mac()(s in "(?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})") -> String {
591            s
592        }
593    }
594    prop_compose! {
595        fn gen_int10()(s in "(?:[+-]?(?:[1-9]{2,3})(?:[0-9]{2,}))") -> String {
596            s
597        }
598    }
599    prop_compose! {
600        fn gen_int16()(s in "(?:[+-]?(?:0x)(?:[0-9A-Fa-f]+))") -> String {
601            s
602        }
603    }
604    prop_compose! {
605        fn gen_float10()(s in r"(?:[+-]?(?:(?:[0-9]+(?:\.[0-9]+))|(?:\.[0-9]+)))") -> String {
606            s
607        }
608    }
609    prop_compose! {
610        fn gen_float16()(s in r"(?:[+-]?(?:0x)(?:[0-9A-Fa-f]+)(?:\.[0-9A-Fa-f]+))") -> String {
611            s
612        }
613    }
614
615    proptest! {
616        #[test]
617        fn test_token_from_parse_uuid(u in gen_uuid()) {
618            let token = Token::from_parse(&u);
619            prop_assert!({
620                match token {
621                    Token::Wildcard=>false,
622                    Token::TypedMatch(Grokker::UUID)=>true,
623                    Token::TypedMatch(_) => false,
624                    Token::Value(_) => false,
625                }
626            }, "Token should be a uuid");
627        }
628
629        #[test]
630        fn test_token_from_parse_mac(u in gen_mac()) {
631            let token = Token::from_parse(&u);
632            prop_assert!({
633                match token {
634                    Token::Wildcard=>false,
635                    Token::TypedMatch(Grokker::MAC)=>true,
636                    Token::TypedMatch(_) => false,
637                    Token::Value(_) => false,
638                }
639            }, "Token should be a MAC address");
640        }
641
642        #[test]
643        fn test_token_from_parse_int10(u in gen_int10()) {
644            let token = Token::from_parse(&u);
645            prop_assert!({
646                match token {
647                    Token::Wildcard=>false,
648                    Token::TypedMatch(Grokker::Base10Integer)=>true,
649                    Token::TypedMatch(_) => false,
650                    Token::Value(_) => false,
651                }
652            }, "Token should be a base 10 integer");
653        }
654
655        #[test]
656        fn test_token_from_parse_int16(u in gen_int16()) {
657            let token = Token::from_parse(&u);
658            prop_assert!({
659                match token {
660                    Token::Wildcard=>false,
661                    Token::TypedMatch(Grokker::Base16Integer)=>true,
662                    Token::TypedMatch(_) => false,
663                    Token::Value(_) => false,
664                }
665            }, "Token should be a base 16 integer");
666        }
667
668        #[test]
669        fn test_token_from_parse_float16(u in gen_float16()) {
670            let token = Token::from_parse(&u);
671            prop_assert!({
672                match token {
673                    Token::Wildcard=>false,
674                    Token::TypedMatch(Grokker::Base16Float)=>true,
675                    Token::TypedMatch(_) => false,
676                    Token::Value(_) => false,
677                }
678            }, "Token should be a base 16 float");
679        }
680
681        #[test]
682        fn test_token_from_parse_float10(u in gen_float10()) {
683            let token = Token::from_parse(&u);
684            prop_assert!({
685                match token {
686                    Token::Wildcard=>false,
687                    Token::TypedMatch(Grokker::Base10Float)=>true,
688                    Token::TypedMatch(_) => false,
689                    Token::Value(_) => false,
690                }
691            }, "Token should be a base 10 float");
692        }
693
694        #[test]
695        fn test_grokset_isnumeric_float10(u in gen_float10()) {
696            let line = u.to_string();
697            let grokset = GrokSet::new(&line);
698            prop_assert!(grokset.is_numeric(), "GrokSet should indicate is_numeric");
699        }
700
701        #[test]
702        fn test_grokset_isnumeric_in10(u in gen_int10()) {
703            let line = u.to_string();
704            let grokset = GrokSet::new(&line);
705            prop_assert!(grokset.is_numeric(), "GrokSet should indicate is_numeric");
706        }
707
708        #[test]
709        fn test_grokset_isnumeric_float16(u in gen_float16()) {
710            let line = u.to_string();
711            let grokset = GrokSet::new(&line);
712            prop_assert!(grokset.is_numeric(), "GrokSet should indicate is_numeric");
713        }
714
715        #[test]
716        fn test_grokset_isnumeric_int16(u in gen_int16()) {
717            let line = u.to_string();
718            let grokset = GrokSet::new(&line);
719            prop_assert!(grokset.is_numeric(), "GrokSet should indicate is_numeric");
720        }
721    }
722}
drain_flow/record/tokens.rs

drain_flow/record/
tokens.rs
