use std::sync::LazyLock;
use regex::Regex;
use serde_json::Value;
#[derive(Debug, Clone)]
pub struct CustomMask {
pub regex: Regex,
pub placeholder: String,
}
impl CustomMask {
pub fn from_cli(pattern: &str) -> Result<Self, regex::Error> {
Ok(CustomMask {
regex: Regex::new(pattern)?,
placeholder: "<CUSTOM>".to_string(),
})
}
pub fn from_config_line(line: &str) -> Result<Self, regex::Error> {
if let Some((name, pattern)) = line.split_once('=') {
let name = name.trim();
if !name.is_empty() && name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') {
return Ok(CustomMask {
regex: Regex::new(pattern.trim())?,
placeholder: format!("<{}>", name.to_uppercase()),
});
}
}
Self::from_cli(line)
}
}
macro_rules! lazy_re {
($name:ident, $pat:expr) => {
static $name: LazyLock<Regex> = LazyLock::new(|| Regex::new($pat).expect("valid regex"));
};
}
lazy_re!(
ANSI,
r"\x1b\[[0-9;?]*[ -/]*[@-~]|\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)?"
);
pub fn strip_escapes(line: &str) -> std::borrow::Cow<'_, str> {
ANSI.replace_all(line, "")
}
lazy_re!(
TS_ISO,
r"\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:[.,]\d+)?(?:Z|[+-]\d{2}:?\d{2})?"
);
lazy_re!(
TS_SYSLOG,
r"\b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d{1,2}\s+\d{2}:\d{2}:\d{2}\b"
);
lazy_re!(
TS_SLASH,
r"\b\d{1,2}/\d{1,2}/\d{4}[ T]\d{2}:\d{2}:\d{2}(?:[.,]\d+)?\b"
);
lazy_re!(TS_EPOCH, r"\b1[0-9]{9}(?:[0-9]{3})?\b");
lazy_re!(
UUID,
r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b"
);
lazy_re!(EMAIL, r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b");
lazy_re!(
URL,
r#"\b(?:https?|postgres(?:ql)?|mysql|mongodb(?:\+srv)?|redis|rediss|amqps?|ftp|sftp|ssh|s3)://[^\s"'<>()\[\]]+"#
);
lazy_re!(
IPV4,
r"\b(?:(?:25[0-5]|2[0-4]\d|1?\d?\d)\.){3}(?:25[0-5]|2[0-4]\d|1?\d?\d)\b(?::\d{1,5})?"
);
lazy_re!(
IPV6,
concat!(
r"\[[0-9a-fA-F]*:[0-9a-fA-F:]*\](?::\d{1,5})?",
r"|\b(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}(?:%[0-9a-zA-Z]+)?\b",
)
);
lazy_re!(HEX_PREFIXED, r"\b0[xX][0-9a-fA-F]+\b");
lazy_re!(HEX_ID, r"\b[0-9a-fA-F]{7,}\b");
lazy_re!(
TEMP_PATH,
r"(?:/tmp/|/var/tmp/|/var/folders/|\\AppData\\Local\\Temp\\|\\Temp\\)[A-Za-z0-9_.=-]*(?:[/\\][A-Za-z0-9_.=-]*)*"
);
lazy_re!(CRI_PREFIX, r"^\S+\s+(stdout|stderr)\s+[FP]\s+(.*)$");
lazy_re!(
JOURNALD_PREFIX,
r"^[A-Z][a-z]{2}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2}\s+\S+\s+([\w.-]+(?:\[\d+\])?):\s?(.*)$"
);
lazy_re!(PID_BRACKET, r"\[\d+\]");
fn strip_docker_wrapper(s: &str) -> (Vec<String>, String) {
let trimmed = s.trim_start();
if !trimmed.starts_with('{') {
return (Vec::new(), s.to_string());
}
let Ok(Value::Object(map)) = serde_json::from_str::<Value>(trimmed) else {
return (Vec::new(), s.to_string());
};
let has_envelope_shape = map.contains_key("stream") || map.contains_key("time");
match map.get("log") {
Some(Value::String(log)) if has_envelope_shape => {
let prefix = map
.get("stream")
.and_then(Value::as_str)
.map(|s| vec![s.to_string()])
.unwrap_or_default();
(prefix, log.trim_end_matches('\n').to_string())
}
_ => (Vec::new(), s.to_string()),
}
}
fn strip_leading_timestamp(s: &str) -> Option<String> {
for re in [&*TS_ISO, &*TS_SYSLOG, &*TS_SLASH] {
if let Some(m) = re.find(s) {
if m.start() == 0 {
return Some(
s[m.end()..]
.trim_start_matches([' ', '\t', ':', '-'])
.to_string(),
);
}
}
}
None
}
fn strip_structural_prefix(s: &str) -> (Vec<String>, String) {
if let Some(caps) = CRI_PREFIX.captures(s) {
return (vec![caps[1].to_string()], caps[2].to_string());
}
if let Some(caps) = JOURNALD_PREFIX.captures(s) {
let process = PID_BRACKET.replace(&caps[1], "[<NUM>]").into_owned();
return (vec![process], caps[2].to_string());
}
if let Some(rest) = strip_leading_timestamp(s) {
return (Vec::new(), rest);
}
(Vec::new(), s.to_string())
}
fn mask_json_value(v: &Value) -> String {
match v {
Value::String(s) => format!("\"{}\"", mask_body(s)),
Value::Number(_) => "<NUM>".to_string(),
Value::Bool(b) => b.to_string(),
Value::Null => "null".to_string(),
Value::Array(items) => {
let cap = 8;
let inner: Vec<String> = items.iter().take(cap).map(mask_json_value).collect();
let more = if items.len() > cap { ",…" } else { "" };
format!("[{}{more}]", inner.join(","))
}
Value::Object(map) => {
let cap = 8;
let inner: Vec<String> = map
.iter()
.take(cap)
.map(|(k, v)| format!("{k}={}", mask_json_value(v)))
.collect();
let more = if map.len() > cap { ",…" } else { "" };
format!("{{{}{more}}}", inner.join(","))
}
}
}
fn flatten_json_line(payload: &str) -> Option<Vec<String>> {
let trimmed = payload.trim();
if !trimmed.starts_with('{') {
return None;
}
let Value::Object(map) = serde_json::from_str::<Value>(trimmed).ok()? else {
return None;
};
if map.is_empty() {
return None;
}
let mut tokens = Vec::with_capacity(map.len() * 2);
for (key, value) in &map {
tokens.push(format!("{key}="));
tokens.push(mask_json_value(value));
}
Some(tokens)
}
lazy_re!(
QUANTITY,
r"\b\d+(?:\.\d+)?\s?(?:ns|[uµ]s|ms|s|m|h|d|B|KB|KiB|MB|MiB|GB|GiB|TB|TiB|bps|Kbps|Mbps|Gbps)\b"
);
lazy_re!(DURATION_CLOCK, r"\b\d{1,2}:\d{2}:\d{2}(?:\.\d+)?\b");
lazy_re!(VERSION_NUM, r"\b\d+(?:\.\d+){1,3}\b");
lazy_re!(BARE_NUM, r"\b\d{4,}\b");
fn mask_path_segment(seg: &str) -> String {
if seg.is_empty() {
return seg.to_string();
}
let looks_numeric = seg.chars().all(|c| c.is_ascii_digit());
let looks_hex = seg.len() >= 6
&& seg.chars().all(|c| c.is_ascii_hexdigit())
&& seg.chars().any(|c| c.is_ascii_alphabetic());
if UUID.is_match(seg) {
return "<UUID>".to_string();
}
if TS_ISO.is_match(seg) || TS_EPOCH.is_match(seg) {
return "<TS>".to_string();
}
if looks_numeric || looks_hex {
"<ID>".to_string()
} else {
seg.to_string()
}
}
fn mask_url_internals(url: &str) -> String {
let Some((scheme, rest)) = url.split_once("://") else {
return url.to_string();
};
let (before_query, query) = match rest.split_once('?') {
Some((p, q)) => (p, Some(q)),
None => (rest, None),
};
let (authority, path) = match before_query.split_once('/') {
Some((a, p)) => (a, Some(p)),
None => (before_query, None),
};
let masked_authority = match authority.rsplit_once('@') {
Some((_credentials, host)) => format!("<CRED>@{host}"),
None => authority.to_string(),
};
let mut out = format!("{scheme}://{masked_authority}");
if let Some(p) = path {
out.push('/');
out.push_str(
&p.split('/')
.map(mask_path_segment)
.collect::<Vec<_>>()
.join("/"),
);
}
if let Some(q) = query {
if !q.is_empty() {
out.push_str("?<QUERY>");
}
}
out
}
fn progress_placeholder(token: &str) -> Option<&'static str> {
if token.len() < 4 {
return None;
}
let mut dots = 0;
let mut failed = false;
for b in token.bytes() {
match b {
b'.' => dots += 1,
b'F' | b'E' => failed = true,
b's' | b'S' | b'x' | b'X' | b'*' | b'U' | b'R' | b'I' | b'f' | b'e' => {}
_ => return None,
}
}
if dots < 3 || dots * 2 < token.len() {
return None;
}
Some(if failed { "<PROGRESS!>" } else { "<PROGRESS>" })
}
fn progress_counter(token: &str) -> Option<String> {
let counter = token
.bytes()
.all(|b| b.is_ascii_digit() || b"%/[]()".contains(&b));
if !counter || !token.bytes().any(|b| b.is_ascii_digit()) {
return None;
}
let mut out = String::with_capacity(token.len() + 4);
let mut in_digits = false;
for ch in token.chars() {
if ch.is_ascii_digit() {
if !in_digits {
out.push_str("<NUM>");
}
in_digits = true;
} else {
in_digits = false;
out.push(ch);
}
}
Some(out)
}
fn mask_progress_runs(s: &str) -> Option<String> {
if s.bytes().filter(|&b| b == b'.').count() < 3 {
return None;
}
let mut out = String::with_capacity(s.len());
let mut seen_progress = false;
let mut push_token = |out: &mut String, token: &str| {
if let Some(placeholder) = progress_placeholder(token) {
seen_progress = true;
out.push_str(placeholder);
} else if let Some(counter) = seen_progress.then(|| progress_counter(token)).flatten() {
out.push_str(&counter);
} else {
out.push_str(token);
}
};
let mut token_start: Option<usize> = None;
for (i, ch) in s.char_indices() {
if ch.is_whitespace() {
if let Some(start) = token_start.take() {
push_token(&mut out, &s[start..i]);
}
out.push(ch);
} else if token_start.is_none() {
token_start = Some(i);
}
}
if let Some(start) = token_start {
push_token(&mut out, &s[start..]);
}
Some(out)
}
fn replace_in(s: &mut String, re: &Regex, with: &str) {
if let std::borrow::Cow::Owned(replaced) = re.replace_all(s, with) {
*s = replaced;
}
}
fn mask_body(s: &str) -> String {
let mut s = s.to_string();
if URL.is_match(&s) {
let mut out = String::with_capacity(s.len());
let mut last = 0;
for m in URL.find_iter(&s) {
out.push_str(&s[last..m.start()]);
let masked = mask_url_internals(m.as_str());
out.push_str(&masked);
last = m.end();
}
out.push_str(&s[last..]);
s = out;
}
if let Some(masked) = mask_progress_runs(&s) {
s = masked;
}
replace_in(&mut s, &TS_ISO, "<TS>");
replace_in(&mut s, &TS_SYSLOG, "<TS>");
replace_in(&mut s, &TS_SLASH, "<TS>");
replace_in(&mut s, &TS_EPOCH, "<TS>");
replace_in(&mut s, &DURATION_CLOCK, "<DUR>");
replace_in(&mut s, &UUID, "<UUID>");
replace_in(&mut s, &EMAIL, "<EMAIL>");
replace_in(&mut s, &IPV4, "<IP>");
replace_in(&mut s, &IPV6, "<IP>");
replace_in(&mut s, &TEMP_PATH, "<TMPPATH>");
replace_in(&mut s, &HEX_PREFIXED, "<HEX>");
let hex_ids = HEX_ID.replace_all(&s, |caps: ®ex::Captures| {
let m = &caps[0];
if m.chars()
.any(|c| c.is_ascii_hexdigit() && !c.is_ascii_digit())
{
"<HEX>".to_string()
} else {
m.to_string()
}
});
if let std::borrow::Cow::Owned(replaced) = hex_ids {
s = replaced;
}
replace_in(&mut s, &QUANTITY, "<QTY>");
replace_in(&mut s, &VERSION_NUM, "<NUM>");
replace_in(&mut s, &BARE_NUM, "<NUM>");
s
}
pub fn mask_line(line: &str, custom: &[CustomMask]) -> String {
let mut s = strip_escapes(line).into_owned();
for cm in custom {
s = cm
.regex
.replace_all(&s, cm.placeholder.as_str())
.into_owned();
}
mask_body(&s)
}
pub fn tokenize_line(line: &str, custom: &[CustomMask]) -> Vec<String> {
let mut s = strip_escapes(line).into_owned();
for cm in custom {
s = cm
.regex
.replace_all(&s, cm.placeholder.as_str())
.into_owned();
}
let (docker_prefix, s) = strip_docker_wrapper(&s);
let (struct_prefix, payload) = strip_structural_prefix(&s);
let mut tokens = Vec::with_capacity(4);
tokens.extend(docker_prefix);
tokens.extend(struct_prefix);
if let Some(json_tokens) = flatten_json_line(&payload) {
tokens.extend(json_tokens);
} else {
tokens.extend(mask_body(&payload).split_whitespace().map(str::to_owned));
}
tokens
}
pub fn is_placeholder(token: &str) -> bool {
token == "<*>" || (token.starts_with('<') && token.ends_with('>') && token.len() > 2)
}
#[cfg(test)]
mod tests {
use super::*;
fn m(s: &str) -> String {
mask_line(s, &[])
}
#[test]
fn masks_test_runner_progress_lines() {
assert_eq!(
m("....s..........................x......................................... [ 6%]"),
"<PROGRESS> [ <NUM>%]"
);
assert_eq!(
m("s....................................................................... [ 7%]"),
"<PROGRESS> [ <NUM>%]"
);
assert_eq!(m("........ [100%]"), "<PROGRESS> [<NUM>%]");
assert_eq!(m(".s.s.s.s"), "<PROGRESS>");
assert_eq!(m("......F......E... [ 42%]"), "<PROGRESS!> [ <NUM>%]");
assert_eq!(
m("............................................................... 63 / 500 ( 12%)"),
"<PROGRESS> <NUM> / <NUM> ( <NUM>%)"
);
assert_eq!(m("..*..F..."), "<PROGRESS!>");
assert_eq!(
m("\x1b[32m.\x1b[0m\x1b[32m.\x1b[0m\x1b[33ms\x1b[0m\x1b[32m.\x1b[0m\x1b[32m.\x1b[0m [ 50%]"),
"<PROGRESS> [ <NUM>%]"
);
assert_eq!(
tokenize_line("....s....................x.............. [ 6%]", &[]),
tokenize_line(
"s.......................................................... [ 93%]",
&[]
)
);
}
#[test]
fn leaves_ordinary_dots_and_words_alone() {
assert_eq!(m("waiting for the server..."), "waiting for the server...");
assert_eq!(m("retrying ... done"), "retrying ... done");
assert_eq!(m("FEES and EXXES"), "FEES and EXXES");
assert_eq!(m("see ../../src/lib.rs"), "see ../../src/lib.rs");
assert_eq!(m("sss..x"), "sss..x");
assert_eq!(m("Fs.E"), "Fs.E");
assert_eq!(
m("test_login ............ PASSED"),
"test_login <PROGRESS> PASSED"
);
assert_eq!(
m("test_logout .................... PASSED"),
"test_logout <PROGRESS> PASSED"
);
assert_eq!(
m("suite ........ 12 passed in 0.12s"),
"suite <PROGRESS> <NUM> passed in <QTY>"
);
}
#[test]
fn strips_ansi() {
assert_eq!(m("\x1b[31mERROR\x1b[0m boom"), "ERROR boom");
assert_eq!(
m("warning: \x1b]8;;file:///src/main.rs\x1b\\src/main.rs\x1b]8;;\x1b\\ unused"),
"warning: src/main.rs unused"
);
assert_eq!(m("\x1b]0;building\x07done"), "done");
}
#[test]
fn masks_iso8601_and_rfc3339() {
assert_eq!(m("2024-01-15T10:23:45.123456Z connected"), "<TS> connected");
assert_eq!(m("2024-01-15 10:23:45,123 INFO start"), "<TS> INFO start");
assert_eq!(m("2024-01-15T10:23:45+02:00 tick"), "<TS> tick");
}
#[test]
fn masks_syslog_timestamp() {
assert_eq!(m("Jan 15 10:23:45 host sshd: ok"), "<TS> host sshd: ok");
}
#[test]
fn masks_epoch() {
assert_eq!(m("ts=1700000000 event=boot"), "ts=<TS> event=boot");
assert_eq!(m("ts=1700000000123 event=boot"), "ts=<TS> event=boot");
}
#[test]
fn masks_uuid() {
assert_eq!(
m("request 123e4567-e89b-12d3-a456-426614174000 done"),
"request <UUID> done"
);
}
#[test]
fn masks_email() {
assert_eq!(
m("user alice@example.com logged in"),
"user <EMAIL> logged in"
);
}
#[test]
fn masks_ipv4_with_port() {
assert_eq!(m("connect 10.0.0.5:8443 failed"), "connect <IP> failed");
}
#[test]
fn masks_ipv6_uncompressed_and_bracketed() {
assert_eq!(
m("peer fe80:0000:0000:0000:0000:0000:0000:0001 joined"),
"peer <IP> joined"
);
assert_eq!(m("dial [2001:db8::1]:8443 now"), "dial <IP> now");
assert_eq!(m("dial [::1]:9000 now"), "dial <IP> now");
}
#[test]
fn does_not_mask_rust_style_paths_as_ipv6() {
assert_eq!(
m("called std::io::Read::read_to_string"),
"called std::io::Read::read_to_string"
);
assert_eq!(
m("panic in module a::b::c::d"),
"panic in module a::b::c::d"
);
}
#[test]
fn masks_hex_hash_not_decimal() {
assert_eq!(m("commit abc1234 pushed"), "commit <HEX> pushed");
assert_eq!(m("retry 1234567 times"), "retry <NUM> times");
}
#[test]
fn masks_hex_hash_regardless_of_length() {
assert_eq!(
m("commit 1234567890abcdef1234567890abcdef12345678 pushed"),
"commit <HEX> pushed"
);
assert_eq!(
m("layer sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 pulled"),
"layer sha256:<HEX> pulled"
);
}
#[test]
fn masks_url_query_and_ids() {
assert_eq!(
m("GET https://api.example.com/users/4200/orders/9f1c2b?token=xyz&x=1 200"),
"GET https://api.example.com/users/<ID>/orders/<ID>?<QUERY> 200"
);
}
#[test]
fn masks_timestamp_and_uuid_inside_url_path() {
assert_eq!(
m("GET https://api.example.com/events/2024-01-15T10:23:45Z 200"),
"GET https://api.example.com/events/<TS> 200"
);
assert_eq!(
m("GET https://api.example.com/jobs/123e4567-e89b-12d3-a456-426614174000 200"),
"GET https://api.example.com/jobs/<UUID> 200"
);
}
#[test]
fn masks_url_credentials_and_non_http_schemes() {
assert_eq!(
m("connecting to postgres://admin:hunter2@db.internal:5432/orders"),
"connecting to postgres://<CRED>@db.internal:<NUM>/orders"
);
assert_eq!(
m("mongo mongodb+srv://svc:s3cr3t@cluster0.example.net/mydb"),
"mongo mongodb+srv://<CRED>@cluster0.example.net/mydb"
);
assert_eq!(
m("cache redis://cache.internal:6379/0"),
"cache redis://cache.internal:<NUM>/<ID>"
);
}
#[test]
fn masks_quantities() {
assert_eq!(m("uploaded 512KiB in 12ms"), "uploaded <QTY> in <QTY>");
assert_eq!(m("elapsed 3.4s"), "elapsed <QTY>");
}
#[test]
fn masks_clock_duration() {
assert_eq!(m("job finished in 01:23:45"), "job finished in <DUR>");
}
#[test]
fn masks_temp_path() {
assert_eq!(
m("writing /tmp/pytest-of-root/pytest-12/test_foo0/data.json"),
"writing <TMPPATH>"
);
}
#[test]
fn temp_path_does_not_swallow_trailing_punctuation() {
assert_eq!(
m("(see /tmp/pytest-of-root/pytest-12/data.json) done"),
"(see <TMPPATH>) done"
);
assert_eq!(
m("wrote /tmp/out.bin, then exited"),
"wrote <TMPPATH>, then exited"
);
}
#[test]
fn leaves_short_numbers_and_wordlike_tokens_alone() {
assert_eq!(m("exit code 1"), "exit code 1");
assert_eq!(m("utf8 sha256 log4j http2"), "utf8 sha256 log4j http2");
}
#[test]
fn custom_mask_from_cli() {
let custom = vec![CustomMask::from_cli(r"job-\d+").unwrap()];
assert_eq!(
mask_line("running job-42 now", &custom),
"running <CUSTOM> now"
);
}
#[test]
fn custom_mask_from_config_named() {
let custom = vec![CustomMask::from_config_line("TRACE=trace-[a-f0-9]+").unwrap()];
assert_eq!(
mask_line("span trace-deadbeef ok", &custom),
"span <TRACE> ok"
);
}
#[test]
fn is_placeholder_detects_variables() {
assert!(is_placeholder("<TS>"));
assert!(is_placeholder("<*>"));
assert!(!is_placeholder("plain"));
assert!(!is_placeholder("<>"));
}
fn tok(s: &str) -> Vec<String> {
tokenize_line(s, &[])
}
#[test]
fn strips_cri_prefix_and_keeps_stream_name() {
assert_eq!(
tok("2024-03-02T08:15:00.123456789Z stdout F server ready"),
vec!["stdout", "server", "ready"]
);
assert_eq!(
tok("2024-03-02T08:15:00.123456789Z stderr F oops"),
vec!["stderr", "oops"]
);
}
#[test]
fn strips_journald_prefix_keeps_process_genericizes_pid() {
assert_eq!(
tok("Jan 15 10:23:45 host sshd[1234]: Accepted password"),
vec!["sshd[<NUM>]", "Accepted", "password"]
);
}
#[test]
fn strips_bare_leading_timestamp() {
assert_eq!(
tok("2024-01-15T10:23:45.1234567Z Run actions/checkout@v7"),
vec!["Run", "actions/checkout@v7"]
);
}
#[test]
fn unwraps_docker_json_file_envelope() {
let line =
r#"{"log":"listening on :8080\n","stream":"stdout","time":"2024-01-15T10:23:45Z"}"#;
assert_eq!(tok(line), vec!["stdout", "listening", "on", ":<NUM>"]);
}
#[test]
fn flattens_json_payload_into_key_value_token_pairs() {
let line = r#"2024-03-02T08:15:00Z stdout F {"level":"info","msg":"server listening","addr":"0.0.0.0:8080"}"#;
assert_eq!(
tok(line),
vec![
"stdout",
"addr=",
"\"<IP>\"",
"level=",
"\"info\"",
"msg=",
"\"server listening\"",
]
);
}
#[test]
fn json_numbers_are_always_masked_even_when_short() {
let line = r#"{"retries":3}"#;
assert_eq!(tok(line), vec!["retries=", "<NUM>"]);
}
#[test]
fn non_object_json_falls_back_to_plain_text_tokenizing() {
assert_eq!(tok("[1,2,3]"), vec!["[1,2,3]"]);
}
#[test]
fn plain_text_with_no_recognized_prefix_is_unaffected() {
assert_eq!(
tok("user alice logged in"),
vec!["user", "alice", "logged", "in"]
);
}
}