drain3 0.1.6

Fast log template extraction via fixed-depth prefix trees (Rust port of logpai/Drain3)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65

use std::sync::Arc;

pub fn tokenize_whitespace_count(
    content: &str,
    dst: &mut Vec<Arc<str>>,
    max_tokens: usize,
) -> usize {
    if content.is_empty() || max_tokens == 0 {
        return 0;
    }
    dst.clear();
    let bytes = content.as_bytes();
    let mut start = 0;
    let mut count = 1;
    for i in 0..bytes.len() {
        if bytes[i] != b' ' {
            continue;
        }
        dst.push(Arc::from(std::str::from_utf8(&bytes[start..i]).unwrap()));
        start = i + 1;
        if count >= max_tokens {
            return count + 1;
        }
        count += 1;
    }
    dst.push(Arc::from(std::str::from_utf8(&bytes[start..]).unwrap()));
    count
}

pub fn tokenize(
    content: &str,
    extra_delimiters: &[String],
    max_tokens: usize,
    dst: &mut Vec<Arc<str>>,
) {
    dst.clear();
    let trimmed = content.trim();
    if trimmed.is_empty() {
        return;
    }

    // Fast path: no extra delimiters - use split_whitespace (zero allocation)
    if extra_delimiters.is_empty() {
        dst.reserve(16.min(max_tokens));
        for t in trimmed.split_whitespace().take(max_tokens) {
            dst.push(Arc::from(t));
        }
        return;
    }

    // Slow path: need delimiter replacement
    let mut s = trimmed.to_string();
    for d in extra_delimiters {
        if !d.is_empty() {
            s = s.replace(d, " ");
        }
    }
    for t in s.split(' ').filter(|t| !t.is_empty()).take(max_tokens) {
        dst.push(Arc::from(t));
    }
}

pub fn has_numbers(s: &str) -> bool {
    s.bytes().any(|b| b.is_ascii_digit())
}