pagesift_diff/
noise.rs

1//! 噪声分类：把数字、价格、日期、相对时间、计数换成占位符后比较，
2//! 占位符之外的文字完全一致才算噪声。
3
4use std::sync::OnceLock;
5
6use regex::Regex;
7use serde::{Deserialize, Serialize};
8
9#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
10#[serde(rename_all = "camelCase")]
11pub enum NoiseCategory {
12    Date,
13    RelativeTime,
14    Price,
15    Count,
16    Number,
17}
18
19const MONTHS: &str = "jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec";
20const UNITS: &str = "seconds?|secs?|minutes?|mins?|hours?|hrs?|days?|weeks?|months?|years?|yrs?";
21const COUNT_CN: &str = "次阅读|次浏览|次播放|次下载|条评论|条回复|个赞|人关注|人在看|阅读|浏览|评论|点赞|收藏|播放|转发|分享|下载|回复|关注|赞";
22const COUNT_EN: &str = "views?|comments?|likes?|reads?|shares?|replies|reply|retweets?|stars?|downloads?|followers?|subscribers?|votes?|points?|visits?|viewers?|reviews?|ratings?";
23
24struct Patterns {
25    any: Regex,
26    kinds: [(NoiseCategory, Regex); 4],
27}
28
29fn patterns() -> &'static Patterns {
30    static RE: OnceLock<Patterns> = OnceLock::new();
31    RE.get_or_init(|| {
32        let date = format!(
33            r"(?i)(?:[0-9]{{4}}[-/.][0-9]{{1,2}}[-/.][0-9]{{1,2}}(?:[T ][0-9]{{1,2}}:[0-9]{{2}}(?::[0-9]{{2}}(?:\.[0-9]+)?)?(?:Z|[+-][0-9]{{2}}:?[0-9]{{2}})?)?|[0-9]{{4}}年[0-9]{{1,2}}月(?:[0-9]{{1,2}}[日号])?|[0-9]{{1,2}}月[0-9]{{1,2}}[日号]|[0-9]{{1,2}}:[0-9]{{2}}(?::[0-9]{{2}})?|\b(?:{m})[a-z]*\.? [0-9]{{1,2}}(?:st|nd|rd|th)?(?:, [0-9]{{4}})?\b|\b[0-9]{{1,2}}(?:st|nd|rd|th)? (?:{m})[a-z]*\.?(?:,? [0-9]{{4}})?\b)",
34            m = MONTHS
35        );
36        let rel = format!(
37            r"(?:(?i:\b(?:[0-9]+|an?|one)\s?(?:{u})\s+ago\b|\bin\s+(?:[0-9]+|an?)\s?(?:{u})\b|\bjust now\b|\byesterday\b)|[0-9]+\s?(?:个)?(?:秒钟?|分钟|小时|天|周|星期|个月|年)[前后]|刚刚|昨天|前天)",
38            u = UNITS
39        );
40        let price = r"(?:(?:US\$|[$€£¥￥₹₩])\s?[0-9][0-9,]*(?:\.[0-9]+)?(?:[kKmM万亿]\b)?|(?:USD|EUR|GBP|CNY|RMB|JPY)\s?[0-9][0-9,]*(?:\.[0-9]+)?|[0-9][0-9,]*(?:\.[0-9]+)?\s?(?:USD|EUR|GBP|CNY|RMB|JPY|元|块钱|美元|欧元|日元|港元))";
41        let count = format!(
42            r"(?:[0-9][0-9,]*(?:\.[0-9]+)?[kKmM万]?\s?(?:{cn}|(?i:(?:{en})\b))|(?:{cn}|(?i:(?:{en})\b))[:：]?\s?[0-9][0-9,]*(?:\.[0-9]+)?)",
43            cn = COUNT_CN,
44            en = COUNT_EN
45        );
46        let num = r"(?:[0-9][0-9,]*(?:\.[0-9]+)?%?)";
47        let full = |p: &str| Regex::new(&format!("^(?:{p})$")).expect("noise regex");
48        Patterns {
49            any: Regex::new(&format!("{date}|{rel}|{price}|{count}|{num}")).expect("noise regex"),
50            kinds: [
51                (NoiseCategory::Date, full(&date)),
52                (NoiseCategory::RelativeTime, full(&rel)),
53                (NoiseCategory::Price, full(price)),
54                (NoiseCategory::Count, full(&count)),
55            ],
56        }
57    })
58}
59
60struct Entity<'a> {
61    kind: NoiseCategory,
62    text: &'a str,
63}
64
65fn mask(s: &str) -> (String, Vec<Entity<'_>>) {
66    let mut out = String::with_capacity(s.len());
67    let mut ents = Vec::new();
68    let mut last = 0;
69    let pats = patterns();
70    for m in pats.any.find_iter(s) {
71        // 联合正则按优先级取最左匹配，按同样的顺序用锚定正则回查类别
72        let kind = match pats.kinds.iter().find(|(_, re)| re.is_match(m.as_str())) {
73            Some(&(k, _)) => k,
74            None => {
75                // 紧跟在字母后的数字（哈希、标识符）不当作数字
76                if s[..m.start()]
77                    .chars()
78                    .next_back()
79                    .is_some_and(|c| c.is_ascii_alphabetic() || c == '_')
80                {
81                    continue;
82                }
83                NoiseCategory::Number
84            }
85        };
86        out.push_str(&s[last..m.start()]);
87        out.push('\u{E000}');
88        out.push(match kind {
89            NoiseCategory::Date => 'd',
90            NoiseCategory::RelativeTime => 'r',
91            NoiseCategory::Price => 'p',
92            NoiseCategory::Count => 'c',
93            NoiseCategory::Number => 'n',
94        });
95        last = m.end();
96        ents.push(Entity {
97            kind,
98            text: m.as_str(),
99        });
100    }
101    out.push_str(&s[last..]);
102    (out, ents)
103}
104
105/// 返回 `Some(变化的类别)` 表示只有噪声实体变了；`None` 表示有实质变化。
106pub(crate) fn classify(
107    old: &str,
108    new: &str,
109    enabled: &[NoiseCategory],
110) -> Option<Vec<NoiseCategory>> {
111    let (mo, eo) = mask(old);
112    let (mn, en) = mask(new);
113    if mo != mn {
114        return None;
115    }
116    let mut cats: Vec<NoiseCategory> = eo
117        .iter()
118        .zip(&en)
119        .filter(|(a, b)| a.text != b.text)
120        .map(|(a, _)| a.kind)
121        .collect();
122    cats.sort();
123    cats.dedup();
124    if cats.is_empty() || cats.iter().any(|c| !enabled.contains(c)) {
125        return None;
126    }
127    Some(cats)
128}
129
130#[cfg(test)]
131mod tests {
132    use super::*;
133    use NoiseCategory::*;
134
135    const ALL: [NoiseCategory; 5] = [Date, RelativeTime, Price, Count, Number];
136
137    fn c(a: &str, b: &str) -> Option<Vec<NoiseCategory>> {
138        classify(a, b, &ALL)
139    }
140
141    #[test]
142    fn categories() {
143        assert_eq!(
144            c("Now $1,299.00 only", "Now $1,199.00 only"),
145            Some(vec![Price])
146        );
147        assert_eq!(
148            c("发布于 2024年1月5日", "发布于 2024年2月9日"),
149            Some(vec![Date])
150        );
151        assert_eq!(
152            c("Updated 3 hours ago", "Updated 5 hours ago"),
153            Some(vec![RelativeTime])
154        );
155        assert_eq!(c("1,234 次阅读", "1,301 次阅读"), Some(vec![Count]));
156        assert_eq!(c("12 条评论", "15 条评论"), Some(vec![Count]));
157        assert_eq!(
158            c("2024-01-05T10:00:00Z", "2024-01-06T10:00:00Z"),
159            Some(vec![Date])
160        );
161        assert_eq!(
162            c("version has 3 items", "version has 4 items"),
163            Some(vec![Number])
164        );
165    }
166
167    #[test]
168    fn substantive() {
169        assert_eq!(c("Price $5", "Cost $5"), None);
170        assert_eq!(c("Now $5", "Now 5 views"), None);
171        assert_eq!(c("a", "a b"), None);
172        assert_eq!(classify("Now $5", "Now $6", &[Date]), None);
173    }
174}
pagesift_diff/noise.rs

pagesift_diff/
noise.rs
