1use std::sync::OnceLock;
5
6use regex::Regex;
7use serde::{Deserialize, Serialize};
8
9#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
10#[serde(rename_all = "camelCase")]
11pub enum NoiseCategory {
12 Date,
13 RelativeTime,
14 Price,
15 Count,
16 Number,
17 Advertisement,
19}
20
21const AD_LABELS: &[&str] = &[
24 "sponsored",
25 "advertisement",
26 "advertising",
27 "ad:",
28 "ads by",
29 "promoted",
30 "anzeige",
31 "publicidad",
32 "publicité",
33 "広告",
34 "スポンサー",
35 "广告",
36 "推广",
37 "赞助",
38];
39
40pub(crate) fn is_ad(s: &str) -> bool {
41 let t = s.trim_start().to_lowercase();
42 AD_LABELS.iter().any(|l| {
43 t.strip_prefix(l).is_some_and(|rest| {
45 matches!(*l, "ad:" | "ads by")
46 || rest
47 .trim_start()
48 .chars()
49 .next()
50 .is_none_or(|c| matches!(c, ':' | ':' | '-' | '–' | '—' | '|' | '·' | '•'))
51 })
52 })
53}
54
55const MONTHS: &str = "jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec";
56const UNITS: &str = "seconds?|secs?|minutes?|mins?|hours?|hrs?|days?|weeks?|months?|years?|yrs?";
57const COUNT_CN: &str = "次阅读|次浏览|次播放|次下载|条评论|条回复|个赞|人关注|人在看|阅读|浏览|评论|点赞|收藏|播放|转发|分享|下载|回复|关注|赞";
58const COUNT_EN: &str = "views?|comments?|likes?|reads?|shares?|replies|reply|retweets?|stars?|downloads?|followers?|subscribers?|votes?|points?|visits?|viewers?|reviews?|ratings?";
59
60struct Patterns {
61 any: Regex,
62 kinds: [(NoiseCategory, Regex); 4],
63}
64
65fn patterns() -> &'static Patterns {
66 static RE: OnceLock<Patterns> = OnceLock::new();
67 RE.get_or_init(|| {
68 let date = format!(
69 r"(?i)(?:[0-9]{{4}}[-/.][0-9]{{1,2}}[-/.][0-9]{{1,2}}(?:[T ][0-9]{{1,2}}:[0-9]{{2}}(?::[0-9]{{2}}(?:\.[0-9]+)?)?(?:Z|[+-][0-9]{{2}}:?[0-9]{{2}})?)?|[0-9]{{4}}年[0-9]{{1,2}}月(?:[0-9]{{1,2}}[日号])?|[0-9]{{1,2}}月[0-9]{{1,2}}[日号]|[0-9]{{1,2}}:[0-9]{{2}}(?::[0-9]{{2}})?|\b(?:{m})[a-z]*\.? [0-9]{{1,2}}(?:st|nd|rd|th)?(?:, [0-9]{{4}})?\b|\b[0-9]{{1,2}}(?:st|nd|rd|th)? (?:{m})[a-z]*\.?(?:,? [0-9]{{4}})?\b)",
70 m = MONTHS
71 );
72 let rel = format!(
73 r"(?:(?i:\b(?:[0-9]+|an?|one)\s?(?:{u})\s+ago\b|\bin\s+(?:[0-9]+|an?)\s?(?:{u})\b|\bjust now\b|\byesterday\b)|[0-9]+\s?(?:个)?(?:秒钟?|分钟|小时|天|周|星期|个月|年)[前后]|刚刚|昨天|前天)",
74 u = UNITS
75 );
76 let price = r"(?:(?:US\$|[$€£¥¥₹₩])\s?[0-9][0-9,]*(?:\.[0-9]+)?(?:[kKmM万亿]\b)?|(?:USD|EUR|GBP|CNY|RMB|JPY)\s?[0-9][0-9,]*(?:\.[0-9]+)?|[0-9][0-9,]*(?:\.[0-9]+)?\s?(?:USD|EUR|GBP|CNY|RMB|JPY|元|块钱|美元|欧元|日元|港元))";
77 let count = format!(
78 r"(?:[0-9][0-9,]*(?:\.[0-9]+)?[kKmM万]?\s?(?:{cn}|(?i:(?:{en})\b))|(?:{cn}|(?i:(?:{en})\b))[::]?\s?[0-9][0-9,]*(?:\.[0-9]+)?)",
79 cn = COUNT_CN,
80 en = COUNT_EN
81 );
82 let num = r"(?:[0-9][0-9,]*(?:\.[0-9]+)?%?)";
83 let full = |p: &str| Regex::new(&format!("^(?:{p})$")).expect("noise regex");
84 Patterns {
85 any: Regex::new(&format!("{date}|{rel}|{price}|{count}|{num}")).expect("noise regex"),
86 kinds: [
87 (NoiseCategory::Date, full(&date)),
88 (NoiseCategory::RelativeTime, full(&rel)),
89 (NoiseCategory::Price, full(price)),
90 (NoiseCategory::Count, full(&count)),
91 ],
92 }
93 })
94}
95
96struct Entity<'a> {
97 kind: NoiseCategory,
98 text: &'a str,
99}
100
101fn mask(s: &str) -> (String, Vec<Entity<'_>>) {
102 let mut out = String::with_capacity(s.len());
103 let mut ents = Vec::new();
104 let mut last = 0;
105 let pats = patterns();
106 for m in pats.any.find_iter(s) {
107 let kind = match pats.kinds.iter().find(|(_, re)| re.is_match(m.as_str())) {
109 Some(&(k, _)) => k,
110 None => {
111 if s[..m.start()]
113 .chars()
114 .next_back()
115 .is_some_and(|c| c.is_ascii_alphabetic() || c == '_')
116 {
117 continue;
118 }
119 NoiseCategory::Number
120 }
121 };
122 out.push_str(&s[last..m.start()]);
123 out.push('\u{E000}');
124 out.push(match kind {
125 NoiseCategory::Date => 'd',
126 NoiseCategory::RelativeTime => 'r',
127 NoiseCategory::Price => 'p',
128 NoiseCategory::Count => 'c',
129 NoiseCategory::Number => 'n',
130 NoiseCategory::Advertisement => 'a',
132 });
133 last = m.end();
134 ents.push(Entity {
135 kind,
136 text: m.as_str(),
137 });
138 }
139 out.push_str(&s[last..]);
140 (out, ents)
141}
142
143pub(crate) fn classify(
145 old: &str,
146 new: &str,
147 enabled: &[NoiseCategory],
148) -> Option<Vec<NoiseCategory>> {
149 if is_ad(old) && is_ad(new) {
150 return enabled
151 .contains(&NoiseCategory::Advertisement)
152 .then(|| vec![NoiseCategory::Advertisement]);
153 }
154 let (mo, eo) = mask(old);
155 let (mn, en) = mask(new);
156 if mo != mn {
157 return None;
158 }
159 let mut cats: Vec<NoiseCategory> = eo
160 .iter()
161 .zip(&en)
162 .filter(|(a, b)| a.text != b.text)
163 .map(|(a, _)| a.kind)
164 .collect();
165 cats.sort();
166 cats.dedup();
167 if cats.is_empty() || cats.iter().any(|c| !enabled.contains(c)) {
168 return None;
169 }
170 Some(cats)
171}
172
173#[cfg(test)]
174mod tests {
175 use super::*;
176 use NoiseCategory::*;
177
178 const ALL: [NoiseCategory; 6] = [Date, RelativeTime, Price, Count, Number, Advertisement];
179
180 #[test]
181 fn timestamps_with_clock_times_are_dates() {
182 let got = classify(
183 "Updated 2026-11-02 03:00 · 1111 views",
184 "Updated 2026-11-03 04:00 · 1148 views",
185 &[Date, RelativeTime, Count, Advertisement],
186 );
187 assert!(
188 got.is_some(),
189 "{got:?} {:?}",
190 mask("Updated 2026-11-03 04:00 · 1148 views")
191 .1
192 .iter()
193 .map(|e| (e.kind, e.text))
194 .collect::<Vec<_>>()
195 );
196 }
197
198 #[test]
199 fn rotating_ad_slots_are_noise() {
200 assert_eq!(
201 c(
202 "Sponsored: Spring sale on tools",
203 "Sponsored: New phones in stock"
204 ),
205 Some(vec![Advertisement])
206 );
207 assert_eq!(
208 c("广告:新品上市", "广告:限时优惠"),
209 Some(vec![Advertisement])
210 );
211 assert_eq!(c("Our new product", "Sponsored: x"), None);
212 }
213
214 fn c(a: &str, b: &str) -> Option<Vec<NoiseCategory>> {
215 classify(a, b, &ALL)
216 }
217
218 #[test]
219 fn categories() {
220 assert_eq!(
221 c("Now $1,299.00 only", "Now $1,199.00 only"),
222 Some(vec![Price])
223 );
224 assert_eq!(
225 c("发布于 2024年1月5日", "发布于 2024年2月9日"),
226 Some(vec![Date])
227 );
228 assert_eq!(
229 c("Updated 3 hours ago", "Updated 5 hours ago"),
230 Some(vec![RelativeTime])
231 );
232 assert_eq!(c("1,234 次阅读", "1,301 次阅读"), Some(vec![Count]));
233 assert_eq!(c("12 条评论", "15 条评论"), Some(vec![Count]));
234 assert_eq!(
235 c("2024-01-05T10:00:00Z", "2024-01-06T10:00:00Z"),
236 Some(vec![Date])
237 );
238 assert_eq!(
239 c("version has 3 items", "version has 4 items"),
240 Some(vec![Number])
241 );
242 }
243
244 #[test]
245 fn substantive() {
246 assert_eq!(c("Price $5", "Cost $5"), None);
247 assert_eq!(c("Now $5", "Now 5 views"), None);
248 assert_eq!(c("a", "a b"), None);
249 assert_eq!(classify("Now $5", "Now $6", &[Date]), None);
250 }
251}