use std::sync::LazyLock;
use regex::Regex;
use serde_json::Value;
#[derive(Debug, Clone)]
pub struct CustomMask {
pub regex: Regex,
pub placeholder: String,
}
impl CustomMask {
pub fn from_cli(pattern: &str) -> Result<Self, regex::Error> {
Ok(CustomMask {
regex: Regex::new(pattern)?,
placeholder: "<CUSTOM>".to_string(),
})
}
pub fn from_config_line(line: &str) -> Result<Self, regex::Error> {
if let Some((name, pattern)) = line.split_once('=') {
let name = name.trim();
if !name.is_empty() && name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') {
return Ok(CustomMask {
regex: Regex::new(pattern.trim())?,
placeholder: format!("<{}>", name.to_uppercase()),
});
}
}
Self::from_cli(line)
}
}
macro_rules! lazy_re {
($name:ident, $pat:expr) => {
static $name: LazyLock<Regex> = LazyLock::new(|| Regex::new($pat).expect("valid regex"));
};
}
lazy_re!(ANSI, r"\x1b\[[0-9;?]*[ -/]*[@-~]");
lazy_re!(
TS_ISO,
r"\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:[.,]\d+)?(?:Z|[+-]\d{2}:?\d{2})?"
);
lazy_re!(
TS_SYSLOG,
r"\b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d{1,2}\s+\d{2}:\d{2}:\d{2}\b"
);
lazy_re!(
TS_SLASH,
r"\b\d{1,2}/\d{1,2}/\d{4}[ T]\d{2}:\d{2}:\d{2}(?:[.,]\d+)?\b"
);
lazy_re!(TS_EPOCH, r"\b1[0-9]{9}(?:[0-9]{3})?\b");
lazy_re!(
UUID,
r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b"
);
lazy_re!(EMAIL, r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b");
lazy_re!(
URL,
r#"\b(?:https?|postgres(?:ql)?|mysql|mongodb(?:\+srv)?|redis|rediss|amqps?|ftp|sftp|ssh|s3)://[^\s"'<>()\[\]]+"#
);
lazy_re!(
IPV4,
r"\b(?:(?:25[0-5]|2[0-4]\d|1?\d?\d)\.){3}(?:25[0-5]|2[0-4]\d|1?\d?\d)\b(?::\d{1,5})?"
);
lazy_re!(
IPV6,
concat!(
r"\[[0-9a-fA-F]*:[0-9a-fA-F:]*\](?::\d{1,5})?",
r"|\b(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}(?:%[0-9a-zA-Z]+)?\b",
)
);
lazy_re!(HEX_PREFIXED, r"\b0[xX][0-9a-fA-F]+\b");
lazy_re!(HEX_ID, r"\b[0-9a-fA-F]{7,}\b");
lazy_re!(
TEMP_PATH,
r"(?:/tmp/|/var/tmp/|/var/folders/|\\AppData\\Local\\Temp\\|\\Temp\\)[A-Za-z0-9_.=-]*(?:[/\\][A-Za-z0-9_.=-]*)*"
);
lazy_re!(CRI_PREFIX, r"^\S+\s+(stdout|stderr)\s+[FP]\s+(.*)$");
lazy_re!(
JOURNALD_PREFIX,
r"^[A-Z][a-z]{2}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2}\s+\S+\s+([\w.-]+(?:\[\d+\])?):\s?(.*)$"
);
lazy_re!(PID_BRACKET, r"\[\d+\]");
fn strip_docker_wrapper(s: &str) -> (Vec<String>, String) {
let trimmed = s.trim_start();
if !trimmed.starts_with('{') {
return (Vec::new(), s.to_string());
}
let Ok(Value::Object(map)) = serde_json::from_str::<Value>(trimmed) else {
return (Vec::new(), s.to_string());
};
let has_envelope_shape = map.contains_key("stream") || map.contains_key("time");
match map.get("log") {
Some(Value::String(log)) if has_envelope_shape => {
let prefix = map
.get("stream")
.and_then(Value::as_str)
.map(|s| vec![s.to_string()])
.unwrap_or_default();
(prefix, log.trim_end_matches('\n').to_string())
}
_ => (Vec::new(), s.to_string()),
}
}
fn strip_leading_timestamp(s: &str) -> Option<String> {
for re in [&*TS_ISO, &*TS_SYSLOG, &*TS_SLASH] {
if let Some(m) = re.find(s) {
if m.start() == 0 {
return Some(
s[m.end()..]
.trim_start_matches([' ', '\t', ':', '-'])
.to_string(),
);
}
}
}
None
}
fn strip_structural_prefix(s: &str) -> (Vec<String>, String) {
if let Some(caps) = CRI_PREFIX.captures(s) {
return (vec![caps[1].to_string()], caps[2].to_string());
}
if let Some(caps) = JOURNALD_PREFIX.captures(s) {
let process = PID_BRACKET.replace(&caps[1], "[<NUM>]").into_owned();
return (vec![process], caps[2].to_string());
}
if let Some(rest) = strip_leading_timestamp(s) {
return (Vec::new(), rest);
}
(Vec::new(), s.to_string())
}
fn mask_json_value(v: &Value) -> String {
match v {
Value::String(s) => format!("\"{}\"", mask_body(s)),
Value::Number(_) => "<NUM>".to_string(),
Value::Bool(b) => b.to_string(),
Value::Null => "null".to_string(),
Value::Array(items) => {
let cap = 8;
let inner: Vec<String> = items.iter().take(cap).map(mask_json_value).collect();
let more = if items.len() > cap { ",…" } else { "" };
format!("[{}{more}]", inner.join(","))
}
Value::Object(map) => {
let cap = 8;
let inner: Vec<String> = map
.iter()
.take(cap)
.map(|(k, v)| format!("{k}={}", mask_json_value(v)))
.collect();
let more = if map.len() > cap { ",…" } else { "" };
format!("{{{}{more}}}", inner.join(","))
}
}
}
fn flatten_json_line(payload: &str) -> Option<Vec<String>> {
let trimmed = payload.trim();
if !trimmed.starts_with('{') {
return None;
}
let Value::Object(map) = serde_json::from_str::<Value>(trimmed).ok()? else {
return None;
};
if map.is_empty() {
return None;
}
let mut tokens = Vec::with_capacity(map.len() * 2);
for (key, value) in &map {
tokens.push(format!("{key}="));
tokens.push(mask_json_value(value));
}
Some(tokens)
}
lazy_re!(
QUANTITY,
r"\b\d+(?:\.\d+)?\s?(?:ns|[uµ]s|ms|s|m|h|d|B|KB|KiB|MB|MiB|GB|GiB|TB|TiB|bps|Kbps|Mbps|Gbps)\b"
);
lazy_re!(DURATION_CLOCK, r"\b\d{1,2}:\d{2}:\d{2}(?:\.\d+)?\b");
lazy_re!(VERSION_NUM, r"\b\d+(?:\.\d+){1,3}\b");
lazy_re!(BARE_NUM, r"\b\d{4,}\b");
fn mask_path_segment(seg: &str) -> String {
if seg.is_empty() {
return seg.to_string();
}
let looks_numeric = seg.chars().all(|c| c.is_ascii_digit());
let looks_hex = seg.len() >= 6
&& seg.chars().all(|c| c.is_ascii_hexdigit())
&& seg.chars().any(|c| c.is_ascii_alphabetic());
if UUID.is_match(seg) {
return "<UUID>".to_string();
}
if TS_ISO.is_match(seg) || TS_EPOCH.is_match(seg) {
return "<TS>".to_string();
}
if looks_numeric || looks_hex {
"<ID>".to_string()
} else {
seg.to_string()
}
}
fn mask_url_internals(url: &str) -> String {
let Some((scheme, rest)) = url.split_once("://") else {
return url.to_string();
};
let (before_query, query) = match rest.split_once('?') {
Some((p, q)) => (p, Some(q)),
None => (rest, None),
};
let (authority, path) = match before_query.split_once('/') {
Some((a, p)) => (a, Some(p)),
None => (before_query, None),
};
let masked_authority = match authority.rsplit_once('@') {
Some((_credentials, host)) => format!("<CRED>@{host}"),
None => authority.to_string(),
};
let mut out = format!("{scheme}://{masked_authority}");
if let Some(p) = path {
out.push('/');
out.push_str(
&p.split('/')
.map(mask_path_segment)
.collect::<Vec<_>>()
.join("/"),
);
}
if let Some(q) = query {
if !q.is_empty() {
out.push_str("?<QUERY>");
}
}
out
}
fn mask_body(s: &str) -> String {
let mut s = s.to_string();
s = {
let mut out = String::with_capacity(s.len());
let mut last = 0;
for m in URL.find_iter(&s) {
out.push_str(&s[last..m.start()]);
let masked = mask_url_internals(m.as_str());
out.push_str(&masked);
last = m.end();
}
out.push_str(&s[last..]);
out
};
s = TS_ISO.replace_all(&s, "<TS>").into_owned();
s = TS_SYSLOG.replace_all(&s, "<TS>").into_owned();
s = TS_SLASH.replace_all(&s, "<TS>").into_owned();
s = TS_EPOCH.replace_all(&s, "<TS>").into_owned();
s = DURATION_CLOCK.replace_all(&s, "<DUR>").into_owned();
s = UUID.replace_all(&s, "<UUID>").into_owned();
s = EMAIL.replace_all(&s, "<EMAIL>").into_owned();
s = IPV4.replace_all(&s, "<IP>").into_owned();
s = IPV6.replace_all(&s, "<IP>").into_owned();
s = TEMP_PATH.replace_all(&s, "<TMPPATH>").into_owned();
s = HEX_PREFIXED.replace_all(&s, "<HEX>").into_owned();
s = HEX_ID
.replace_all(&s, |caps: ®ex::Captures| {
let m = &caps[0];
if m.chars()
.any(|c| c.is_ascii_hexdigit() && !c.is_ascii_digit())
{
"<HEX>".to_string()
} else {
m.to_string()
}
})
.into_owned();
s = QUANTITY.replace_all(&s, "<QTY>").into_owned();
s = VERSION_NUM.replace_all(&s, "<NUM>").into_owned();
s = BARE_NUM.replace_all(&s, "<NUM>").into_owned();
s
}
pub fn mask_line(line: &str, custom: &[CustomMask]) -> String {
let mut s = ANSI.replace_all(line, "").into_owned();
for cm in custom {
s = cm
.regex
.replace_all(&s, cm.placeholder.as_str())
.into_owned();
}
mask_body(&s)
}
pub fn tokenize_line(line: &str, custom: &[CustomMask]) -> Vec<String> {
let mut s = ANSI.replace_all(line, "").into_owned();
for cm in custom {
s = cm
.regex
.replace_all(&s, cm.placeholder.as_str())
.into_owned();
}
let (docker_prefix, s) = strip_docker_wrapper(&s);
let (struct_prefix, payload) = strip_structural_prefix(&s);
let mut tokens = Vec::with_capacity(4);
tokens.extend(docker_prefix);
tokens.extend(struct_prefix);
if let Some(json_tokens) = flatten_json_line(&payload) {
tokens.extend(json_tokens);
} else {
tokens.extend(mask_body(&payload).split_whitespace().map(str::to_owned));
}
tokens
}
pub fn is_placeholder(token: &str) -> bool {
token == "<*>" || (token.starts_with('<') && token.ends_with('>') && token.len() > 2)
}
#[cfg(test)]
mod tests {
use super::*;
fn m(s: &str) -> String {
mask_line(s, &[])
}
#[test]
fn strips_ansi() {
assert_eq!(m("\x1b[31mERROR\x1b[0m boom"), "ERROR boom");
}
#[test]
fn masks_iso8601_and_rfc3339() {
assert_eq!(m("2024-01-15T10:23:45.123456Z connected"), "<TS> connected");
assert_eq!(m("2024-01-15 10:23:45,123 INFO start"), "<TS> INFO start");
assert_eq!(m("2024-01-15T10:23:45+02:00 tick"), "<TS> tick");
}
#[test]
fn masks_syslog_timestamp() {
assert_eq!(m("Jan 15 10:23:45 host sshd: ok"), "<TS> host sshd: ok");
}
#[test]
fn masks_epoch() {
assert_eq!(m("ts=1700000000 event=boot"), "ts=<TS> event=boot");
assert_eq!(m("ts=1700000000123 event=boot"), "ts=<TS> event=boot");
}
#[test]
fn masks_uuid() {
assert_eq!(
m("request 123e4567-e89b-12d3-a456-426614174000 done"),
"request <UUID> done"
);
}
#[test]
fn masks_email() {
assert_eq!(
m("user alice@example.com logged in"),
"user <EMAIL> logged in"
);
}
#[test]
fn masks_ipv4_with_port() {
assert_eq!(m("connect 10.0.0.5:8443 failed"), "connect <IP> failed");
}
#[test]
fn masks_ipv6_uncompressed_and_bracketed() {
assert_eq!(
m("peer fe80:0000:0000:0000:0000:0000:0000:0001 joined"),
"peer <IP> joined"
);
assert_eq!(m("dial [2001:db8::1]:8443 now"), "dial <IP> now");
assert_eq!(m("dial [::1]:9000 now"), "dial <IP> now");
}
#[test]
fn does_not_mask_rust_style_paths_as_ipv6() {
assert_eq!(
m("called std::io::Read::read_to_string"),
"called std::io::Read::read_to_string"
);
assert_eq!(
m("panic in module a::b::c::d"),
"panic in module a::b::c::d"
);
}
#[test]
fn masks_hex_hash_not_decimal() {
assert_eq!(m("commit abc1234 pushed"), "commit <HEX> pushed");
assert_eq!(m("retry 1234567 times"), "retry <NUM> times");
}
#[test]
fn masks_hex_hash_regardless_of_length() {
assert_eq!(
m("commit 1234567890abcdef1234567890abcdef12345678 pushed"),
"commit <HEX> pushed"
);
assert_eq!(
m("layer sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 pulled"),
"layer sha256:<HEX> pulled"
);
}
#[test]
fn masks_url_query_and_ids() {
assert_eq!(
m("GET https://api.example.com/users/4200/orders/9f1c2b?token=xyz&x=1 200"),
"GET https://api.example.com/users/<ID>/orders/<ID>?<QUERY> 200"
);
}
#[test]
fn masks_timestamp_and_uuid_inside_url_path() {
assert_eq!(
m("GET https://api.example.com/events/2024-01-15T10:23:45Z 200"),
"GET https://api.example.com/events/<TS> 200"
);
assert_eq!(
m("GET https://api.example.com/jobs/123e4567-e89b-12d3-a456-426614174000 200"),
"GET https://api.example.com/jobs/<UUID> 200"
);
}
#[test]
fn masks_url_credentials_and_non_http_schemes() {
assert_eq!(
m("connecting to postgres://admin:hunter2@db.internal:5432/orders"),
"connecting to postgres://<CRED>@db.internal:<NUM>/orders"
);
assert_eq!(
m("mongo mongodb+srv://svc:s3cr3t@cluster0.example.net/mydb"),
"mongo mongodb+srv://<CRED>@cluster0.example.net/mydb"
);
assert_eq!(
m("cache redis://cache.internal:6379/0"),
"cache redis://cache.internal:<NUM>/<ID>"
);
}
#[test]
fn masks_quantities() {
assert_eq!(m("uploaded 512KiB in 12ms"), "uploaded <QTY> in <QTY>");
assert_eq!(m("elapsed 3.4s"), "elapsed <QTY>");
}
#[test]
fn masks_clock_duration() {
assert_eq!(m("job finished in 01:23:45"), "job finished in <DUR>");
}
#[test]
fn masks_temp_path() {
assert_eq!(
m("writing /tmp/pytest-of-root/pytest-12/test_foo0/data.json"),
"writing <TMPPATH>"
);
}
#[test]
fn temp_path_does_not_swallow_trailing_punctuation() {
assert_eq!(
m("(see /tmp/pytest-of-root/pytest-12/data.json) done"),
"(see <TMPPATH>) done"
);
assert_eq!(
m("wrote /tmp/out.bin, then exited"),
"wrote <TMPPATH>, then exited"
);
}
#[test]
fn leaves_short_numbers_and_wordlike_tokens_alone() {
assert_eq!(m("exit code 1"), "exit code 1");
assert_eq!(m("utf8 sha256 log4j http2"), "utf8 sha256 log4j http2");
}
#[test]
fn custom_mask_from_cli() {
let custom = vec![CustomMask::from_cli(r"job-\d+").unwrap()];
assert_eq!(
mask_line("running job-42 now", &custom),
"running <CUSTOM> now"
);
}
#[test]
fn custom_mask_from_config_named() {
let custom = vec![CustomMask::from_config_line("TRACE=trace-[a-f0-9]+").unwrap()];
assert_eq!(
mask_line("span trace-deadbeef ok", &custom),
"span <TRACE> ok"
);
}
#[test]
fn is_placeholder_detects_variables() {
assert!(is_placeholder("<TS>"));
assert!(is_placeholder("<*>"));
assert!(!is_placeholder("plain"));
assert!(!is_placeholder("<>"));
}
fn tok(s: &str) -> Vec<String> {
tokenize_line(s, &[])
}
#[test]
fn strips_cri_prefix_and_keeps_stream_name() {
assert_eq!(
tok("2024-03-02T08:15:00.123456789Z stdout F server ready"),
vec!["stdout", "server", "ready"]
);
assert_eq!(
tok("2024-03-02T08:15:00.123456789Z stderr F oops"),
vec!["stderr", "oops"]
);
}
#[test]
fn strips_journald_prefix_keeps_process_genericizes_pid() {
assert_eq!(
tok("Jan 15 10:23:45 host sshd[1234]: Accepted password"),
vec!["sshd[<NUM>]", "Accepted", "password"]
);
}
#[test]
fn strips_bare_leading_timestamp() {
assert_eq!(
tok("2024-01-15T10:23:45.1234567Z Run actions/checkout@v7"),
vec!["Run", "actions/checkout@v7"]
);
}
#[test]
fn unwraps_docker_json_file_envelope() {
let line =
r#"{"log":"listening on :8080\n","stream":"stdout","time":"2024-01-15T10:23:45Z"}"#;
assert_eq!(tok(line), vec!["stdout", "listening", "on", ":<NUM>"]);
}
#[test]
fn flattens_json_payload_into_key_value_token_pairs() {
let line = r#"2024-03-02T08:15:00Z stdout F {"level":"info","msg":"server listening","addr":"0.0.0.0:8080"}"#;
assert_eq!(
tok(line),
vec![
"stdout",
"addr=",
"\"<IP>\"",
"level=",
"\"info\"",
"msg=",
"\"server listening\"",
]
);
}
#[test]
fn json_numbers_are_always_masked_even_when_short() {
let line = r#"{"retries":3}"#;
assert_eq!(tok(line), vec!["retries=", "<NUM>"]);
}
#[test]
fn non_object_json_falls_back_to_plain_text_tokenizing() {
assert_eq!(tok("[1,2,3]"), vec!["[1,2,3]"]);
}
#[test]
fn plain_text_with_no_recognized_prefix_is_unaffected() {
assert_eq!(
tok("user alice logged in"),
vec!["user", "alice", "logged", "in"]
);
}
}