tokf 0.1.4

Config-driven CLI tool that compresses command output before it reaches an LLM context
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129

use regex::Regex;

use crate::config::types::ExtractRule;

/// Apply an extract rule across lines — first match wins.
///
/// Returns the interpolated template on match. On invalid regex or no match,
/// returns all lines joined with newlines (passthrough).
pub fn apply_extract(rule: &ExtractRule, lines: &[&str]) -> String {
    let Ok(re) = Regex::new(&rule.pattern) else {
        return lines.join("\n");
    };

    for line in lines {
        if let Some(caps) = re.captures(line) {
            return interpolate(&rule.output, &caps);
        }
    }

    lines.join("\n")
}

/// Replace `{0}`, `{1}`, `{2}`, ... placeholders with capture groups.
///
/// Iterates in reverse order so `{10}` is replaced before `{1}`.
/// Missing groups become empty strings.
pub(super) fn interpolate(template: &str, caps: &regex::Captures<'_>) -> String {
    let mut result = template.to_string();
    let max_group = caps.len().saturating_sub(1);

    for i in (0..=max_group).rev() {
        let placeholder = format!("{{{i}}}");
        let value = caps.get(i).map_or("", |m| m.as_str());
        result = result.replace(&placeholder, value);
    }

    result
}

#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used)]
mod tests {
    use super::*;

    fn rule(pattern: &str, output: &str) -> ExtractRule {
        ExtractRule {
            pattern: pattern.to_string(),
            output: output.to_string(),
        }
    }

    #[test]
    fn extract_first_match_wins() {
        let r = rule(r"(\S+)\s*->\s*(\S+)", "ok \u{2713} {2}");
        let lines = vec![
            "Enumerating objects: 5",
            "abc1234..def5678 main -> main",
            "another -> branch",
        ];
        assert_eq!(apply_extract(&r, &lines), "ok \u{2713} main");
    }

    #[test]
    fn extract_no_match_passthrough() {
        let r = rule(r"NOMATCH", "{1}");
        let lines = vec!["line one", "line two"];
        assert_eq!(apply_extract(&r, &lines), "line one\nline two");
    }

    #[test]
    fn extract_invalid_regex_passthrough() {
        let r = rule(r"[invalid", "{1}");
        let lines = vec!["line one", "line two"];
        assert_eq!(apply_extract(&r, &lines), "line one\nline two");
    }

    #[test]
    fn extract_empty_lines_no_match() {
        let r = rule(r"(\d+)", "{1}");
        let lines: Vec<&str> = vec![];
        assert_eq!(apply_extract(&r, &lines), "");
    }

    #[test]
    fn interpolate_replaces_numbered_groups() {
        let re = Regex::new(r"^\[(\S+)\s+(\w+)\]").unwrap();
        let caps = re.captures("[main abc1234] Add feature X").unwrap();
        assert_eq!(interpolate("ok \u{2713} {2}", &caps), "ok \u{2713} abc1234");
    }

    #[test]
    fn interpolate_group_zero_is_full_match() {
        let re = Regex::new(r"(hello) (world)").unwrap();
        let caps = re.captures("hello world").unwrap();
        assert_eq!(interpolate("{0}", &caps), "hello world");
    }

    #[test]
    fn interpolate_missing_group_becomes_empty() {
        let re = Regex::new(r"(a)(b)?").unwrap();
        let caps = re.captures("a").unwrap();
        assert_eq!(interpolate("{1}-{2}", &caps), "a-");
    }

    #[test]
    fn interpolate_reverse_order_prevents_partial_replace() {
        // Ensure {10} doesn't get mangled by {1} replacement first
        let re = Regex::new(r"(a)(b)(c)(d)(e)(f)(g)(h)(i)(j)(k)").unwrap();
        let caps = re.captures("abcdefghijk").unwrap();
        assert_eq!(interpolate("{10}", &caps), "j");
    }

    #[test]
    fn extract_git_commit_pattern() {
        let r = rule(r"^\[(\S+)\s+(\w+)\]", "ok \u{2713} {2}");
        let lines = vec![
            "[main abc1234] Add feature X",
            " 1 file changed, 10 insertions(+), 2 deletions(-)",
        ];
        assert_eq!(apply_extract(&r, &lines), "ok \u{2713} abc1234");
    }

    #[test]
    fn extract_git_push_pattern() {
        let r = rule(r"(\S+)\s*->\s*(\S+)", "ok \u{2713} {2}");
        let lines = vec!["   abc1234..def5678 main -> main"];
        assert_eq!(apply_extract(&r, &lines), "ok \u{2713} main");
    }
}