Skip to main content

pagesift_diff/
noise.rs

1//! 噪声分类:把数字、价格、日期、相对时间、计数换成占位符后比较,
2//! 占位符之外的文字完全一致才算噪声。
3
4use std::sync::OnceLock;
5
6use regex::Regex;
7use serde::{Deserialize, Serialize};
8
9#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
10#[serde(rename_all = "camelCase")]
11pub enum NoiseCategory {
12    Date,
13    RelativeTime,
14    Price,
15    Count,
16    Number,
17    /// 广告位:新旧两块都以广告标注开头(“Sponsored”“Advertisement”“广告”……),轮换的广告内容。
18    Advertisement,
19}
20
21/// 广告标注词:各地法规要求的广告披露标签(美国 FTC 背书指南的 “Sponsored”/“Ad”、欧盟《数字服务法》
22/// 第 26 条的 “Advertisement”、中国《互联网广告管理办法》第九条的“广告”)及常见的写法。
23const AD_LABELS: &[&str] = &[
24    "sponsored",
25    "advertisement",
26    "advertising",
27    "ad:",
28    "ads by",
29    "promoted",
30    "anzeige",
31    "publicidad",
32    "publicité",
33    "広告",
34    "スポンサー",
35    "广告",
36    "推广",
37    "赞助",
38];
39
40pub(crate) fn is_ad(s: &str) -> bool {
41    let t = s.trim_start().to_lowercase();
42    AD_LABELS.iter().any(|l| {
43        // 是标注才算:后面是结尾或分隔符。“Promoted to CEO”“广告费用上涨”是正文
44        t.strip_prefix(l).is_some_and(|rest| {
45            matches!(*l, "ad:" | "ads by")
46                || rest
47                    .trim_start()
48                    .chars()
49                    .next()
50                    .is_none_or(|c| matches!(c, ':' | ':' | '-' | '–' | '—' | '|' | '·' | '•'))
51        })
52    })
53}
54
55const MONTHS: &str = "jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec";
56const UNITS: &str = "seconds?|secs?|minutes?|mins?|hours?|hrs?|days?|weeks?|months?|years?|yrs?";
57const COUNT_CN: &str = "次阅读|次浏览|次播放|次下载|条评论|条回复|个赞|人关注|人在看|阅读|浏览|评论|点赞|收藏|播放|转发|分享|下载|回复|关注|赞";
58const COUNT_EN: &str = "views?|comments?|likes?|reads?|shares?|replies|reply|retweets?|stars?|downloads?|followers?|subscribers?|votes?|points?|visits?|viewers?|reviews?|ratings?";
59
60struct Patterns {
61    any: Regex,
62    kinds: [(NoiseCategory, Regex); 4],
63}
64
65fn patterns() -> &'static Patterns {
66    static RE: OnceLock<Patterns> = OnceLock::new();
67    RE.get_or_init(|| {
68        let date = format!(
69            r"(?i)(?:[0-9]{{4}}[-/.][0-9]{{1,2}}[-/.][0-9]{{1,2}}(?:[T ][0-9]{{1,2}}:[0-9]{{2}}(?::[0-9]{{2}}(?:\.[0-9]+)?)?(?:Z|[+-][0-9]{{2}}:?[0-9]{{2}})?)?|[0-9]{{4}}年[0-9]{{1,2}}月(?:[0-9]{{1,2}}[日号])?|[0-9]{{1,2}}月[0-9]{{1,2}}[日号]|[0-9]{{1,2}}:[0-9]{{2}}(?::[0-9]{{2}})?|\b(?:{m})[a-z]*\.? [0-9]{{1,2}}(?:st|nd|rd|th)?(?:, [0-9]{{4}})?\b|\b[0-9]{{1,2}}(?:st|nd|rd|th)? (?:{m})[a-z]*\.?(?:,? [0-9]{{4}})?\b)",
70            m = MONTHS
71        );
72        let rel = format!(
73            r"(?:(?i:\b(?:[0-9]+|an?|one)\s?(?:{u})\s+ago\b|\bin\s+(?:[0-9]+|an?)\s?(?:{u})\b|\bjust now\b|\byesterday\b)|[0-9]+\s?(?:个)?(?:秒钟?|分钟|小时|天|周|星期|个月|年)[前后]|刚刚|昨天|前天)",
74            u = UNITS
75        );
76        let price = r"(?:(?:US\$|[$€£¥¥₹₩])\s?[0-9][0-9,]*(?:\.[0-9]+)?(?:[kKmM万亿]\b)?|(?:USD|EUR|GBP|CNY|RMB|JPY)\s?[0-9][0-9,]*(?:\.[0-9]+)?|[0-9][0-9,]*(?:\.[0-9]+)?\s?(?:USD|EUR|GBP|CNY|RMB|JPY|元|块钱|美元|欧元|日元|港元))";
77        let count = format!(
78            r"(?:[0-9][0-9,]*(?:\.[0-9]+)?[kKmM万]?\s?(?:{cn}|(?i:(?:{en})\b))|(?:{cn}|(?i:(?:{en})\b))[::]?\s?[0-9][0-9,]*(?:\.[0-9]+)?)",
79            cn = COUNT_CN,
80            en = COUNT_EN
81        );
82        let num = r"(?:[0-9][0-9,]*(?:\.[0-9]+)?%?)";
83        let full = |p: &str| Regex::new(&format!("^(?:{p})$")).expect("noise regex");
84        Patterns {
85            any: Regex::new(&format!("{date}|{rel}|{price}|{count}|{num}")).expect("noise regex"),
86            kinds: [
87                (NoiseCategory::Date, full(&date)),
88                (NoiseCategory::RelativeTime, full(&rel)),
89                (NoiseCategory::Price, full(price)),
90                (NoiseCategory::Count, full(&count)),
91            ],
92        }
93    })
94}
95
96struct Entity<'a> {
97    kind: NoiseCategory,
98    text: &'a str,
99}
100
101fn mask(s: &str) -> (String, Vec<Entity<'_>>) {
102    let mut out = String::with_capacity(s.len());
103    let mut ents = Vec::new();
104    let mut last = 0;
105    let pats = patterns();
106    for m in pats.any.find_iter(s) {
107        // 联合正则按优先级取最左匹配,按同样的顺序用锚定正则回查类别
108        let kind = match pats.kinds.iter().find(|(_, re)| re.is_match(m.as_str())) {
109            Some(&(k, _)) => k,
110            None => {
111                // 紧跟在字母后的数字(哈希、标识符)不当作数字
112                if s[..m.start()]
113                    .chars()
114                    .next_back()
115                    .is_some_and(|c| c.is_ascii_alphabetic() || c == '_')
116                {
117                    continue;
118                }
119                NoiseCategory::Number
120            }
121        };
122        out.push_str(&s[last..m.start()]);
123        out.push('\u{E000}');
124        out.push(match kind {
125            NoiseCategory::Date => 'd',
126            NoiseCategory::RelativeTime => 'r',
127            NoiseCategory::Price => 'p',
128            NoiseCategory::Count => 'c',
129            NoiseCategory::Number => 'n',
130            // 广告位按整块判断(`is_ad`),不是行内实体
131            NoiseCategory::Advertisement => 'a',
132        });
133        last = m.end();
134        ents.push(Entity {
135            kind,
136            text: m.as_str(),
137        });
138    }
139    out.push_str(&s[last..]);
140    (out, ents)
141}
142
143/// 返回 `Some(变化的类别)` 表示只有噪声实体变了;`None` 表示有实质变化。
144pub(crate) fn classify(
145    old: &str,
146    new: &str,
147    enabled: &[NoiseCategory],
148) -> Option<Vec<NoiseCategory>> {
149    if is_ad(old) && is_ad(new) {
150        return enabled
151            .contains(&NoiseCategory::Advertisement)
152            .then(|| vec![NoiseCategory::Advertisement]);
153    }
154    let (mo, eo) = mask(old);
155    let (mn, en) = mask(new);
156    if mo != mn {
157        return None;
158    }
159    let mut cats: Vec<NoiseCategory> = eo
160        .iter()
161        .zip(&en)
162        .filter(|(a, b)| a.text != b.text)
163        .map(|(a, _)| a.kind)
164        .collect();
165    cats.sort();
166    cats.dedup();
167    if cats.is_empty() || cats.iter().any(|c| !enabled.contains(c)) {
168        return None;
169    }
170    Some(cats)
171}
172
173#[cfg(test)]
174mod tests {
175    use super::*;
176    use NoiseCategory::*;
177
178    const ALL: [NoiseCategory; 6] = [Date, RelativeTime, Price, Count, Number, Advertisement];
179
180    #[test]
181    fn timestamps_with_clock_times_are_dates() {
182        let got = classify(
183            "Updated 2026-11-02 03:00 · 1111 views",
184            "Updated 2026-11-03 04:00 · 1148 views",
185            &[Date, RelativeTime, Count, Advertisement],
186        );
187        assert!(
188            got.is_some(),
189            "{got:?} {:?}",
190            mask("Updated 2026-11-03 04:00 · 1148 views")
191                .1
192                .iter()
193                .map(|e| (e.kind, e.text))
194                .collect::<Vec<_>>()
195        );
196    }
197
198    #[test]
199    fn rotating_ad_slots_are_noise() {
200        assert_eq!(
201            c(
202                "Sponsored: Spring sale on tools",
203                "Sponsored: New phones in stock"
204            ),
205            Some(vec![Advertisement])
206        );
207        assert_eq!(
208            c("广告:新品上市", "广告:限时优惠"),
209            Some(vec![Advertisement])
210        );
211        assert_eq!(c("Our new product", "Sponsored: x"), None);
212    }
213
214    fn c(a: &str, b: &str) -> Option<Vec<NoiseCategory>> {
215        classify(a, b, &ALL)
216    }
217
218    #[test]
219    fn categories() {
220        assert_eq!(
221            c("Now $1,299.00 only", "Now $1,199.00 only"),
222            Some(vec![Price])
223        );
224        assert_eq!(
225            c("发布于 2024年1月5日", "发布于 2024年2月9日"),
226            Some(vec![Date])
227        );
228        assert_eq!(
229            c("Updated 3 hours ago", "Updated 5 hours ago"),
230            Some(vec![RelativeTime])
231        );
232        assert_eq!(c("1,234 次阅读", "1,301 次阅读"), Some(vec![Count]));
233        assert_eq!(c("12 条评论", "15 条评论"), Some(vec![Count]));
234        assert_eq!(
235            c("2024-01-05T10:00:00Z", "2024-01-06T10:00:00Z"),
236            Some(vec![Date])
237        );
238        assert_eq!(
239            c("version has 3 items", "version has 4 items"),
240            Some(vec![Number])
241        );
242    }
243
244    #[test]
245    fn substantive() {
246        assert_eq!(c("Price $5", "Cost $5"), None);
247        assert_eq!(c("Now $5", "Now 5 views"), None);
248        assert_eq!(c("a", "a b"), None);
249        assert_eq!(classify("Now $5", "Now $6", &[Date]), None);
250    }
251}