use fancy_regex::{Regex, RegexBuilder};
use super::extended;
use super::js;
use super::position::locate_all;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum Notation {
Iso,
Rfc2822,
Unix,
Utc,
Local,
Simple,
Week,
Ordinal,
Basic,
Custom,
}
impl Notation {
pub(crate) fn as_str(self) -> &'static str {
match self {
Self::Iso => "iso",
Self::Rfc2822 => "rfc2822",
Self::Unix => "unix",
Self::Utc => "utc",
Self::Local => "local",
Self::Simple => "simple",
Self::Week => "week",
Self::Ordinal => "ordinal",
Self::Basic => "basic",
Self::Custom => "custom",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Resolver {
DateParse,
Unix,
Syslog,
Apache,
Week,
Ordinal,
Basic,
}
pub(crate) struct Pattern {
regex: Regex,
notation: Notation,
resolver: Resolver,
}
#[derive(Debug, Clone)]
pub(crate) struct Found {
pub(crate) value: String,
pub(crate) notation: Notation,
pub(crate) timestamp: i64,
pub(crate) line: usize,
pub(crate) column: usize,
}
#[derive(Debug, Clone)]
struct Candidate {
value: String,
notation: Notation,
timestamp: i64,
start: usize,
end: usize,
order: usize,
}
fn base_patterns() -> Vec<Pattern> {
vec![
Pattern {
regex: build(
r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]{3})?(?:Z|[+-][0-9]{2}:[0-9]{2})?",
),
notation: Notation::Iso,
resolver: Resolver::DateParse,
},
Pattern {
regex: build(
r"[A-Za-z]{3},\s[0-9]{1,2}\s[A-Za-z]{3}\s[0-9]{4}\s[0-9]{2}:[0-9]{2}:[0-9]{2}\s[A-Za-z]{3,4}",
),
notation: Notation::Rfc2822,
resolver: Resolver::DateParse,
},
Pattern {
regex: build(r"(?<![0-9.])(?:[0-9]{19}|[0-9]{16}|[0-9]{13}|[0-9]{10})(?![0-9])"),
notation: Notation::Unix,
resolver: Resolver::Unix,
},
Pattern {
regex: build(
r"[A-Za-z]{3}\s[A-Za-z]{3}\s[0-9]{2}\s[0-9]{4}\s[0-9]{2}:[0-9]{2}:[0-9]{2}\sGMT[+-][0-9]{4}",
),
notation: Notation::Utc,
resolver: Resolver::DateParse,
},
Pattern {
regex: build(
r"(?<![0-9])[0-9]{1,2}/[0-9]{1,2}/[0-9]{4}\s[0-9]{1,2}:[0-9]{2}:[0-9]{2}(?![0-9])",
),
notation: Notation::Local,
resolver: Resolver::DateParse,
},
Pattern {
regex: build(r"(?<![0-9])[0-9]{4}-[0-9]{2}-[0-9]{2}(?![0-9])"),
notation: Notation::Simple,
resolver: Resolver::DateParse,
},
Pattern {
regex: build(r"(?<![0-9])[0-9]{4}-W[0-9]{2}(?:-[1-7])?(?![0-9])"),
notation: Notation::Week,
resolver: Resolver::Week,
},
Pattern {
regex: build(r"(?<![0-9])[0-9]{4}-[0-9]{3}(?![0-9])"),
notation: Notation::Ordinal,
resolver: Resolver::Ordinal,
},
Pattern {
regex: build(
r"(?<![0-9.])[0-9]{8}(?:T[0-9]{6}(?:\.[0-9]{1,3})?(?:Z|[+-][0-9]{4}|[+-][0-9]{2})?)?(?![0-9])",
),
notation: Notation::Basic,
resolver: Resolver::Basic,
},
]
}
fn log_patterns() -> Vec<Pattern> {
vec![
Pattern {
regex: build(
r"(?<![0-9])[0-9]{4}-[0-9]{2}-[0-9]{2}\s[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]{3})?(?![0-9])",
),
notation: Notation::Iso,
resolver: Resolver::DateParse,
},
Pattern {
regex: build(
r"(?<![A-Za-z])[A-Za-z]{3}\s+[0-9]{1,2}\s[0-9]{2}:[0-9]{2}:[0-9]{2}(?![0-9])",
),
notation: Notation::Custom,
resolver: Resolver::Syslog,
},
Pattern {
regex: build(
r"\[([0-9]{2}/[A-Za-z]{3}/[0-9]{4}:[0-9]{2}:[0-9]{2}:[0-9]{2}\s[+-][0-9]{4})\]",
),
notation: Notation::Custom,
resolver: Resolver::Apache,
},
]
}
fn javascript_patterns() -> Vec<Pattern> {
[
"new\\s+Date",
"Date\\.parse",
"moment",
"dayjs",
"DateTime\\.fromISO",
]
.into_iter()
.map(|callee| Pattern {
regex: build(&format!(
r#"(?<![A-Za-z0-9_]){callee}\s*\(\s*(['"`])([^'"`\n]+)\1\s*,?\s*\)"#
)),
notation: Notation::Custom,
resolver: Resolver::DateParse,
})
.collect()
}
fn html_patterns() -> Vec<Pattern> {
vec![
Pattern {
regex: build(r#"(?i)(?<![A-Za-z0-9_])datetime\s*=\s*(['"`])([^'"`]+)\1"#),
notation: Notation::Custom,
resolver: Resolver::DateParse,
},
Pattern {
regex: build(
r#"(?i)<meta[^>]*(?:property|name)\s*=\s*(['"`])(?:date|published|modified|created)\1[^>]*content\s*=\s*(['"`])([^'"`]+)\2"#,
),
notation: Notation::Custom,
resolver: Resolver::DateParse,
},
Pattern {
regex: build(r#"(?i)"date(?:Published|Modified)"\s*:\s*(['"`])([^'"`]+)\1"#),
notation: Notation::Custom,
resolver: Resolver::DateParse,
},
]
}
fn build(pattern: &str) -> Regex {
let pattern = pattern.replace(r"\s", &format!("[{}]", js::JS_SPACE_CLASS));
RegexBuilder::new(&pattern)
.backtrack_limit(usize::MAX)
.build()
.expect("every pattern in this file is a literal and is tested")
}
pub(crate) fn patterns_for(language: &str) -> Vec<Pattern> {
let mut patterns = base_patterns();
match language {
"log" | "plaintext" => patterns.extend(log_patterns()),
"javascript" | "typescript" => patterns.extend(javascript_patterns()),
"html" => patterns.extend(html_patterns()),
_ => {}
}
patterns
}
pub(crate) fn scan(haystack: &str, original: &str, patterns: &[Pattern], year: i64) -> Vec<Found> {
debug_assert_eq!(
haystack.len(),
original.len(),
"the haystack and the document it is located against must agree byte for byte"
);
let content = haystack;
let mut candidates = Vec::new();
for (order, pattern) in patterns.iter().enumerate() {
let mut from = 0;
while let Some(captures) = pattern
.regex
.captures_from_pos(content, from)
.expect("the patterns in this file have no backtrack limit and cannot fail at runtime")
{
let group = captures.len() - 1;
let Some(matched) = captures.get(group) else {
break;
};
let whole = captures.get(0).expect("group 0 always participates");
from = whole.end().max(whole.start() + 1);
let value = matched.as_str();
let Some(timestamp) = resolve(value, pattern.resolver, year) else {
continue;
};
candidates.push(Candidate {
value: value.to_string(),
notation: pattern.notation,
timestamp,
start: matched.start(),
end: matched.end(),
order,
});
}
}
attach_positions(original, dedupe_contained(candidates))
}
fn resolve(value: &str, resolver: Resolver, year: i64) -> Option<i64> {
match resolver {
Resolver::DateParse => extended::instant(value),
Resolver::Unix => unix_timestamp(value),
Resolver::Syslog => extended::instant(&format!("{value} {year}")),
Resolver::Apache => extended::instant(&apache_shape(value)),
Resolver::Week => extended::week_date(value),
Resolver::Ordinal => extended::ordinal_date(value),
Resolver::Basic => extended::basic_format(value),
}
}
const PLAUSIBLE_FROM: i64 = 1_000_000_000_000;
const PLAUSIBLE_UNTIL: i64 = 4_102_444_800_000;
fn unix_timestamp(value: &str) -> Option<i64> {
match value.len() {
10 => {
let number: i64 = value.parse().ok()?;
(number > 1_000_000_000).then_some(number * 1000)
}
13 | 16 | 19 => {
let milliseconds: i64 = value.get(..13)?.parse().ok()?;
(milliseconds > PLAUSIBLE_FROM && milliseconds < PLAUSIBLE_UNTIL)
.then_some(milliseconds)
}
_ => None,
}
}
fn apache_shape(value: &str) -> String {
let spaced = value.replace('/', " ");
match spaced.find(':') {
Some(index) => format!("{} {}", &spaced[..index], &spaced[index + 1..]),
None => spaced,
}
}
fn dedupe_contained(mut candidates: Vec<Candidate>) -> Vec<Candidate> {
candidates.sort_by(|a, b| {
a.start
.cmp(&b.start)
.then(b.end.cmp(&a.end))
.then(a.order.cmp(&b.order))
});
let mut kept: Vec<Candidate> = Vec::new();
let mut covering_end = 0usize;
let mut started = false;
for candidate in candidates {
if started && candidate.end <= covering_end {
continue;
}
covering_end = covering_end.max(candidate.end);
started = true;
kept.push(candidate);
}
kept
}
fn attach_positions(content: &str, candidates: Vec<Candidate>) -> Vec<Found> {
let offsets: Vec<usize> = candidates.iter().map(|candidate| candidate.start).collect();
let located = locate_all(content, &offsets);
candidates
.into_iter()
.zip(located)
.map(|(candidate, (line, column))| Found {
value: candidate.value,
notation: candidate.notation,
timestamp: candidate.timestamp,
line,
column,
})
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
fn find(content: &str, language: &str) -> Vec<Found> {
scan(content, content, &patterns_for(language), 2026)
}
fn values(content: &str, language: &str) -> Vec<String> {
find(content, language)
.into_iter()
.map(|found| found.value)
.collect()
}
#[test]
fn each_of_the_nine_shapes_is_classified() {
for (text, expected) in [
("2024-01-15T10:30:45Z", Notation::Iso),
("Mon, 15 Jan 2024 10:30:45 GMT", Notation::Rfc2822),
("1705314645", Notation::Unix),
("Mon Jan 15 2024 10:30:45 GMT+0000", Notation::Utc),
("1/15/2024 10:30:45", Notation::Local),
("2024-01-15", Notation::Simple),
("2024-W03-1", Notation::Week),
("2024-015", Notation::Ordinal),
("20240115T103045Z", Notation::Basic),
] {
let found = find(text, "json");
assert_eq!(found.len(), 1, "{text}");
assert_eq!(found[0].notation, expected, "{text}");
}
}
#[test]
fn the_iso_shapes_v8_refuses_land_on_the_same_instant() {
let extended = find("2024-01-15", "json")[0].timestamp;
for text in ["2024-W03-1", "2024-015", "20240115"] {
let found = find(text, "json");
assert_eq!(found.len(), 1, "{text}");
assert_eq!(found[0].timestamp, extended, "{text}");
}
}
#[test]
fn an_eight_digit_identifier_is_not_a_date() {
assert!(values("98765432", "json").is_empty(), "the year 9876");
assert!(values("12345678", "json").is_empty(), "month 56");
}
#[test]
fn a_zone_v8_refuses_is_still_read() {
let found = find("Mon, 15 Jan 2024 10:30:45 CEST", "json");
assert_eq!(found.len(), 1);
assert_eq!(found[0].notation, Notation::Rfc2822);
assert_eq!(found[0].timestamp, 1_705_307_445_000);
}
#[test]
fn a_date_inside_an_iso_one_is_dropped_but_a_repeat_is_not() {
assert_eq!(
values("2024-01-15T10:30:45Z", "json"),
["2024-01-15T10:30:45Z"]
);
assert_eq!(
values("2024-01-15T10:30:45Z and 2024-01-15", "json"),
["2024-01-15T10:30:45Z", "2024-01-15"]
);
assert_eq!(
values("2024-01-15 and 2024-01-15", "json"),
["2024-01-15", "2024-01-15"]
);
}
#[test]
fn a_digit_run_that_is_not_an_epoch_is_not_a_date() {
assert!(values("999999999", "json").is_empty(), "nine digits");
assert!(values("0000000001", "json").is_empty(), "below the floor");
assert!(
values("12345678901234567", "json").is_empty(),
"seventeen digits is no unit at all"
);
}
#[test]
fn a_ten_digit_phone_number_is_a_false_positive() {
assert_eq!(values("5551234567", "json"), ["5551234567"]);
}
#[test]
fn a_finer_epoch_outside_the_plausible_window_is_not_a_date() {
assert!(
values("4532015112830366", "json").is_empty(),
"a card number"
);
assert!(
values("9007199254740991", "json").is_empty(),
"Number.MAX_SAFE_INTEGER"
);
assert!(
values("9999999999999999999", "json").is_empty(),
"nineteen nines"
);
assert!(
values("1000000000000000", "json").is_empty(),
"on the floor"
);
}
#[test]
fn the_window_ends_at_the_year_twenty_one_hundred() {
assert_eq!(
values("4102444799999123", "json"),
["4102444799999123"],
"the last instant inside"
);
assert!(
values("4102444800000123", "json").is_empty(),
"the first instant outside"
);
}
#[test]
fn a_run_of_one_digit_inside_the_window_is_still_a_date() {
assert_eq!(
values("1111111111111111111", "json"),
["1111111111111111111"]
);
}
#[test]
fn the_fraction_of_a_float_is_not_an_epoch() {
assert!(values("RATIO = 1.2345678901234567", "json").is_empty());
assert!(values("ratio = 0.1705314645123", "json").is_empty());
assert!(values("share = 0.20240115", "json").is_empty());
assert_eq!(values("1705314645123456", "json"), ["1705314645123456"]);
}
#[test]
fn the_finer_epoch_units_truncate_to_the_millisecond() {
for value in [
"1705314645123456", "1705314645123456789", ] {
let found = find(value, "json");
assert_eq!(found.len(), 1, "{value}");
assert_eq!(found[0].timestamp, 1_705_314_645_123, "{value}");
assert_eq!(found[0].notation, Notation::Unix, "{value}");
}
}
#[test]
fn only_javascript_reads_a_constructor_argument() {
assert_eq!(
values("new Date('March 5, 2024')", "typescript"),
["March 5, 2024"]
);
assert!(values("new Date('March 5, 2024')", "json").is_empty());
}
#[test]
fn a_constructor_split_across_lines_is_still_found() {
assert_eq!(
values("new Date(\n 'January 15 2024',\n)", "javascript"),
["January 15 2024"]
);
}
#[test]
fn an_argument_that_is_not_a_date_is_not_emitted() {
assert!(values("new Date('sometime next week')", "typescript").is_empty());
}
#[test]
fn a_recognisable_argument_keeps_its_base_classification() {
let found = find("moment('2024-01-15')", "javascript");
assert_eq!(found[0].notation, Notation::Simple);
}
#[test]
fn html_reads_an_attribute_a_bare_scan_would_miss() {
assert_eq!(
values("<time datetime=\"March 5, 2024\">then</time>", "html"),
["March 5, 2024"]
);
}
#[test]
fn apache_loses_its_brackets_and_keeps_its_instant() {
let found = find("[15/Jan/2024:10:30:08 +0000]", "log");
assert_eq!(found[0].value, "15/Jan/2024:10:30:08 +0000");
assert_eq!(found[0].timestamp, 1_705_314_608_000);
}
#[test]
fn a_syslog_line_takes_the_year_it_is_given() {
let found = scan(
"Jan 15 10:30:47",
"Jan 15 10:30:47",
&patterns_for("log"),
2026,
);
assert_eq!(found.len(), 1);
let other = scan(
"Jan 15 10:30:47",
"Jan 15 10:30:47",
&patterns_for("log"),
2020,
);
assert_ne!(found[0].timestamp, other[0].timestamp);
}
#[test]
fn log_patterns_belong_to_log_and_plaintext_only() {
assert!(!values("Jan 15 10:30:47", "log").is_empty());
assert!(!values("Jan 15 10:30:47", "plaintext").is_empty());
assert!(values("Jan 15 10:30:47", "json").is_empty());
}
#[test]
fn non_ascii_digits_are_not_digits() {
assert!(values("٢٠٢٤-٠١-١٥", "json").is_empty());
}
#[test]
fn the_separator_class_is_javascripts_whitespace() {
assert_eq!(
values("<time datetime\u{feff}=\"March 5, 2024\">x</time>", "html"),
["March 5, 2024"]
);
assert_eq!(
values("new\u{feff}Date('March 5, 2024')", "javascript"),
["March 5, 2024"]
);
assert!(values("<time datetime\u{85}=\"March 5, 2024\">x</time>", "html").is_empty());
assert!(values("new\u{85}Date('March 5, 2024')", "javascript").is_empty());
}
#[test]
fn the_epoch_pattern_reads_milliseconds_too() {
let found = find("1705314645123", "json");
assert_eq!(found[0].timestamp, 1_705_314_645_123);
}
}