dejavu/reduce/
normalize.rs

1//! Output normalization (spec §13): strip volatile noise so equivalent runs
2//! hash identically, without hiding useful information.
3
4use regex::Regex;
5use std::sync::LazyLock;
6
7static ISO_TIMESTAMP: LazyLock<Regex> = LazyLock::new(|| {
8    Regex::new(r"\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?").unwrap()
9});
10static CLOCK_TIME: LazyLock<Regex> =
11    LazyLock::new(|| Regex::new(r"\b\d{2}:\d{2}:\d{2}(?:\.\d+)?\b").unwrap());
12static DURATION: LazyLock<Regex> =
13    LazyLock::new(|| Regex::new(r"\b\d+(?:\.\d+)?\s?(?:ms|µs|us|ns|s|m)\b").unwrap());
14static TMP_PATH: LazyLock<Regex> =
15    LazyLock::new(|| Regex::new(r"(?:/private/tmp|/tmp|/var/folders)/[^\s:]*").unwrap());
16static PID: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"(?i)\bpid\s+\d+").unwrap());
17static PORT: LazyLock<Regex> =
18    LazyLock::new(|| Regex::new(r"\b(?:localhost|127\.0\.0\.1):\d{2,5}\b").unwrap());
19static BLANKS: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"\n{3,}").unwrap());
20/// Package-manager self-update banners: appear on some runs and not others,
21/// breaking `unchanged` detection for otherwise-identical output (spec §1
22/// "warnings package manager repetes"). Update notices only — never warnings.
23static PM_NOTICE: LazyLock<Regex> = LazyLock::new(|| {
24    Regex::new(
25        r"(?i)^\s*(npm notice\b.*|.*update available!?\s.*→.*|\[?yarn\]? .*new version .*available.*)$",
26    )
27    .unwrap()
28});
29
30const MAX_LINE_CHARS: usize = 2000;
31const LINE_KEEP: usize = 1000;
32
33/// Normalize captured output. Ordering is load-bearing: ANSI first, then time
34/// placeholders before durations.
35pub fn normalize(input: &str) -> String {
36    // 0. expand tabs to spaces first: the VTE-based ANSI stripper drops tab
37    // control chars, which would destroy tab-based indentation (e.g. git status).
38    let detabbed = input.replace('\t', "    ");
39
40    // 1. strip ANSI escapes.
41    let stripped = strip_ansi_escapes::strip(detabbed.as_bytes());
42    let text = String::from_utf8_lossy(&stripped);
43
44    // 2. CRLF / lone CR -> LF.
45    let text = text.replace("\r\n", "\n").replace('\r', "\n");
46
47    // 5-9. volatile-token placeholders.
48    let text = ISO_TIMESTAMP.replace_all(&text, "<TIMESTAMP>");
49    let text = CLOCK_TIME.replace_all(&text, "<TIME>");
50    let text = DURATION.replace_all(&text, "<DURATION>");
51    let text = TMP_PATH.replace_all(&text, "<TMP_PATH>");
52    let text = PID.replace_all(&text, "pid <PID>");
53    let text = PORT.replace_all(&text, "localhost:<PORT>");
54
55    // 3, 10, 11. line pass: trailing whitespace, drop progress bars, truncate
56    // very long lines.
57    let mut lines: Vec<String> = Vec::new();
58    for line in text.split('\n') {
59        if is_progress_bar(line) || PM_NOTICE.is_match(line) {
60            continue;
61        }
62        lines.push(truncate_long_line(line.trim_end()));
63    }
64    let joined = lines.join("\n");
65
66    // 4. collapse 2+ blank lines into one.
67    BLANKS.replace_all(&joined, "\n\n").into_owned()
68}
69
70/// A progress-bar / spinner line to drop. Conservative: always drop lines with
71/// block-drawing chars; drop ASCII bars only when a percentage is present (so
72/// markdown tables and separators survive).
73fn is_progress_bar(line: &str) -> bool {
74    let t = line.trim();
75    if t.chars().count() < 8 {
76        return false;
77    }
78    if t.chars().any(|c| ('\u{2588}'..='\u{258F}').contains(&c)) {
79        return true;
80    }
81    if !t.contains('%') {
82        return false;
83    }
84    let total = t.chars().count();
85    let barish = t
86        .chars()
87        .filter(|c| matches!(c, '=' | '>' | '#' | '-' | '.' | ' ' | '[' | ']'))
88        .count();
89    barish * 10 >= total * 7
90}
91
92fn truncate_long_line(line: &str) -> String {
93    if line.chars().count() <= MAX_LINE_CHARS {
94        return line.to_string();
95    }
96    let head: String = line.chars().take(LINE_KEEP).collect();
97    let tail_rev: Vec<char> = line.chars().rev().take(LINE_KEEP).collect();
98    let tail: String = tail_rev.into_iter().rev().collect();
99    format!("{head}…<TRUNCATED>…{tail}")
100}
101
102#[cfg(test)]
103mod tests {
104    use super::*;
105
106    #[test]
107    fn strips_ansi_and_normalizes_newlines() {
108        let out = normalize("\x1b[31mred\x1b[0m\r\ntext\r\n");
109        assert_eq!(out, "red\ntext\n");
110    }
111
112    #[test]
113    fn replaces_volatile_tokens() {
114        let out =
115            normalize("done in 843ms at 2026-07-06T10:41:22.123Z pid 12345 on localhost:5173");
116        assert!(out.contains("<DURATION>"));
117        assert!(out.contains("<TIMESTAMP>"));
118        assert!(out.contains("pid <PID>"));
119        assert!(out.contains("localhost:<PORT>"));
120        assert!(!out.contains("843ms"));
121        assert!(!out.contains("12345"));
122    }
123
124    #[test]
125    fn tmp_paths_and_clock_times() {
126        let out = normalize("wrote /tmp/foo-839201/bar.txt at 12:41:22");
127        assert!(out.contains("<TMP_PATH>"));
128        assert!(out.contains("<TIME>"));
129    }
130
131    #[test]
132    fn collapses_blank_lines_and_trims() {
133        let out = normalize("a   \n\n\n\nb");
134        assert_eq!(out, "a\n\nb");
135    }
136
137    #[test]
138    fn equivalent_runs_normalize_identically() {
139        let a = normalize("PASS in 120ms at 10:00:01");
140        let b = normalize("PASS in 456ms at 11:22:33");
141        assert_eq!(a, b);
142    }
143
144    #[test]
145    fn npm_update_notice_is_dropped() {
146        let with_notice = "ok\nnpm notice\nnpm notice New minor version of npm available! 11.16.0 -> 11.18.0\nnpm notice To update run: npm install -g npm@11.18.0\ndone";
147        let without = "ok\ndone";
148        assert_eq!(normalize(with_notice), normalize(without));
149        // Warnings survive.
150        assert!(normalize("npm warn deprecated foo@1").contains("npm warn"));
151    }
152}
dejavu/reduce/normalize.rs

dejavu/reduce/
normalize.rs
