crawlkit_parser/
rules.rs

1use regex::Regex;
2
3/// URL 匹配规则，用于判断一个链接是否为文章/新闻链接。
4///
5/// 内部维护两套正则模式列表：
6/// - **包含模式（include）**：命中任意一条即视为文章链接
7/// - **排除模式（exclude）**：命中任意一条即跳过，优先级高于包含模式
8///
9/// 默认内置 30+ 条常用文章路径规则（日期路径、关键词、CMS 模式等），
10/// 覆盖绝大多数政府、智库、新闻类站点。对于规则未能覆盖的站点，
11/// 可通过链式调用 `with_include` / `with_exclude` 扩展。
12///
13/// # 示例
14///
15/// ```rust
16/// use crawlkit_parser::UrlRule;
17///
18/// let rule = UrlRule::default();
19///
20/// // 日期路径匹配
21/// assert!(rule.is_article_url("https://example.com/2024/01/15/hello-world"));
22/// // 关键词路径匹配
23/// assert!(rule.is_article_url("https://example.com/news/breaking-story"));
24/// // 静态资源被排除
25/// assert!(!rule.is_article_url("https://example.com/image.jpg"));
26/// // 标签页面被排除
27/// assert!(!rule.is_article_url("https://example.com/tag/rust"));
28///
29/// // 扩展自定义规则
30/// let rule = UrlRule::default()
31///     .with_include(r"/my-section/")
32///     .with_exclude(r"print=true")
33///     .with_selector("div.content a");
34/// assert!(rule.is_article_url("https://example.com/my-section/foo"));
35/// assert!(!rule.is_article_url("https://example.com/my-section/foo?print=true"));
36/// ```
37#[derive(Debug, Clone)]
38pub struct UrlRule {
39    #[allow(dead_code)]
40    pub name: String,
41    /// 包含模式列表 —— 命中任意一条即为文章链接
42    pub include_patterns: Vec<Regex>,
43    /// 排除模式列表 —— 命中任意一条即跳过
44    pub exclude_patterns: Vec<Regex>,
45    /// 可选的 CSS 选择器，限制只在页面特定区域内查找链接（如 "div.news-list a"）
46    pub link_selector: Option<String>,
47}
48
49#[allow(dead_code)]
50impl UrlRule {
51    /// 判断一个 URL 是否符合文章规则。
52    ///
53    /// 先检查所有排除模式，任一匹配则返回 `false`；
54    /// 再检查所有包含模式，任一匹配则返回 `true`；
55    /// 均不匹配则返回 `false`。
56    ///
57    /// ```rust
58    /// use crawlkit_parser::UrlRule;
59    ///
60    /// let rule = UrlRule::default();
61    /// assert!(rule.is_article_url("https://example.com/news/story"));
62    /// assert!(!rule.is_article_url("https://example.com/style.css"));
63    /// assert!(!rule.is_article_url("https://example.com/login"));
64    /// ```
65    pub fn is_article_url(&self, url: &str) -> bool {
66        for ex in &self.exclude_patterns {
67            if ex.is_match(url) {
68                return false;
69            }
70        }
71        for pat in &self.include_patterns {
72            if pat.is_match(url) {
73                return true;
74            }
75        }
76        false
77    }
78
79    /// 添加一条包含模式（链式调用）。
80    ///
81    /// ```rust
82    /// use crawlkit_parser::UrlRule;
83    ///
84    /// let rule = UrlRule::default()
85    ///     .with_include(r"/custom-section/");
86    /// assert!(rule.is_article_url("https://example.com/custom-section/foo"));
87    /// ```
88    pub fn with_include(mut self, pattern: &str) -> Self {
89        self.include_patterns.push(Regex::new(pattern).expect("invalid regex"));
90        self
91    }
92
93    /// 添加一条排除模式（链式调用）。
94    ///
95    /// ```rust
96    /// use crawlkit_parser::UrlRule;
97    ///
98    /// let rule = UrlRule::default()
99    ///     .with_exclude(r"print=true");
100    /// assert!(!rule.is_article_url("https://example.com/news/story?print=true"));
101    /// ```
102    pub fn with_exclude(mut self, pattern: &str) -> Self {
103        self.exclude_patterns.push(Regex::new(pattern).expect("invalid regex"));
104        self
105    }
106
107    /// 设置 CSS 选择器，限定链接查找范围（链式调用）。
108    ///
109    /// 设置后 `Extractor` 仅在匹配该选择器的元素内部查找 `<a>` 标签。
110    ///
111    /// ```rust
112    /// use crawlkit_parser::UrlRule;
113    ///
114    /// let rule = UrlRule::default()
115    ///     .with_selector("div.article-list");
116    /// assert_eq!(rule.link_selector.as_deref(), Some("div.article-list"));
117    /// ```
118    pub fn with_selector(mut self, selector: &str) -> Self {
119        self.link_selector = Some(selector.to_string());
120        self
121    }
122}
123
124fn default_include_patterns() -> Vec<Regex> {
125    vec![
126        Regex::new(r"/\d{4}/\d{2}/\d{2}/").unwrap(),
127        Regex::new(r"/\d{4}/\d{2}/").unwrap(),
128        Regex::new(r"\.(html|shtml|htm|cfm|asp|aspx|php)(?:\?|$)").unwrap(),
129        Regex::new(r"(?i)/news").unwrap(),
130        Regex::new(r"(?i)/articles?/").unwrap(),
131        Regex::new(r"(?i)/story/").unwrap(),
132        Regex::new(r"(?i)/blog").unwrap(),
133        Regex::new(r"(?i)/post").unwrap(),
134        Regex::new(r"(?i)/press").unwrap(),
135        Regex::new(r"(?i)/releases?/").unwrap(),
136        Regex::new(r"(?i)/hearings?").unwrap(),
137        Regex::new(r"(?i)/events?/").unwrap(),
138        Regex::new(r"(?i)/video/").unwrap(),
139        Regex::new(r"(?i)/media/").unwrap(),
140        Regex::new(r"(?i)/download/").unwrap(),
141        Regex::new(r"(?i)/document/").unwrap(),
142        Regex::new(r"(?i)/publication").unwrap(),
143        Regex::new(r"(?i)/research/").unwrap(),
144        Regex::new(r"(?i)/analysis/").unwrap(),
145        Regex::new(r"(?i)/commentary/").unwrap(),
146        Regex::new(r"(?i)/report").unwrap(),
147        Regex::new(r"(?i)/briefing").unwrap(),
148        Regex::new(r"(?i)/archives/").unwrap(),
149        Regex::new(r"(?i)/content/").unwrap(),
150        Regex::new(r"(?i)/detail/").unwrap(),
151        Regex::new(r"(?i)/show/").unwrap(),
152        Regex::new(r"(?i)/highlights?/").unwrap(),
153        Regex::new(r"(?i)/statuses/").unwrap(),
154        Regex::new(r"/p/\d+").unwrap(),
155        Regex::new(r"/a/\d+").unwrap(),
156        Regex::new(r"(?i)/node/").unwrap(),
157        Regex::new(r"(?i)/opa/").unwrap(),
158        Regex::new(r"(?i)/news-events/").unwrap(),
159        Regex::new(r"(?i)/profiles/").unwrap(),
160    ]
161}
162
163fn default_exclude_patterns() -> Vec<Regex> {
164    vec![
165        Regex::new(r"\.(css|js|json|xml|rss|ico|png|jpg|jpeg|gif|svg|webp|pdf|zip|rar|exe|dmg|mp4|mp3)$").unwrap(),
166        Regex::new(r"(?i)/(tag|tags|category|categories|author|page|login|register|logout|search|wp-content|wp-includes)/").unwrap(),
167        Regex::new(r"(javascript:|mailto:|tel:)").unwrap(),
168        Regex::new(r"#comment").unwrap(),
169    ]
170}
171
172impl Default for UrlRule {
173    fn default() -> Self {
174        Self {
175            name: "default".into(),
176            include_patterns: default_include_patterns(),
177            exclude_patterns: default_exclude_patterns(),
178            link_selector: None,
179        }
180    }
181}
182
183#[cfg(test)]
184mod tests {
185    use super::*;
186
187    #[test]
188    fn test_typical_article_urls() {
189        let rule = UrlRule::default();
190        let cases = vec![
191            "https://example.com/2024/01/15/hello-world",
192            "https://example.com/news/breaking-story",
193            "https://example.com/article/some-title",
194            "https://example.com/story/something",
195            "https://example.com/2024/01/15/hello-world.html",
196            "https://example.com/p/12345",
197            "https://example.com/a/67890",
198            "https://example.com/archives/123",
199            "https://example.com/detail/xyz",
200            "https://example.com/content/abc",
201            "https://example.com/show/abc",
202
203        ];
204        for url in cases {
205            assert!(rule.is_article_url(url), "should match: {url}");
206        }
207    }
208
209    #[test]
210    fn test_non_article_urls() {
211        let rule = UrlRule::default();
212        let cases = vec![
213            "https://example.com/style.css",
214            "https://example.com/app.js",
215            "https://example.com/image.jpg",
216            "https://example.com/tag/rust",
217            "https://example.com/categories/tech",
218            "https://example.com/login",
219            "https://example.com/register",
220            "https://example.com/search?q=news",
221            "javascript:void(0)",
222            "mailto:test@example.com",
223        ];
224        for url in cases {
225            assert!(!rule.is_article_url(url), "should reject: {url}");
226        }
227    }
228
229    #[test]
230    fn test_real_world_unmatched_urls() {
231        let rule = UrlRule::default();
232        let cases = vec![
233            "https://www.heritage.org/progressivism/commentary/the-state-escape",
234            "https://www.hoover.org/research/irans-nightmares",
235            "https://www.csis.org/analysis/great-sudan-policy-reset",
236            "https://www.cnas.org/publications/reports/russia-china-relations",
237            "https://fas.org/publication/talent-pipeline-for-the-clean-energy-transition/",
238            "https://www.nextgov.com/defense/2023/08/army-gets-new-prototypes/389037/",
239            "https://www.defense.gov/News/Releases/Release/Article/3496391/",
240            "https://www.justice.gov/opa/pr/final-defendant-sentenced",
241            "https://www.fcc.gov/document/cgb-announces-second-round-acp",
242            "https://www.transportation.gov/briefing-room/icymi-usdot-launches",
243            "https://www.state.gov/reports/country-reports-on-terrorism-2021/",
244            "https://www.finance.senate.gov/hearings/open-executive-session",
245            "https://www.judiciary.senate.gov/press/releases/durbin-statement",
246            "https://energycommerce.house.gov/posts/chair-rodgers-statement",
247            "https://trumpstruth.org/statuses/32029",
248            "https://www.darpa.mil/news-events/2023-08-15",
249            "https://cset.georgetown.edu/publication/china-science-ethics-guiding/",
250            "https://www.navy.mil/Press-Office/Press-Briefings/Article/3357084/",
251            "https://www.dol.gov/newsroom/releases/eta/eta20230817-0",
252            "https://foreignaffairs.house.gov/press-release/mccaul-requests-interview/",
253        ];
254        for url in cases {
255            assert!(rule.is_article_url(url), "should match: {url}");
256        }
257    }
258
259    #[test]
260    fn test_new_patterns_coverage() {
261        let rule = UrlRule::default();
262        let cases = vec![
263            "https://www.energy.gov/articles/biden-harris-announces",
264            "https://cyberscoop.com/video/understanding-the-economic-impact/",
265            "https://cyberscoop.com/event/microsoft-federal-innovation-series/",
266            "https://www.stimson.org/event/taiwans-economic-security/",
267            "https://www.atlanticcouncil.org/in-depth-research-reports/report/sanctioning-china/",
268            "https://www.disa.mil/en/NewsandEvents/2023/AbilityOne-Base-Supply-Center",
269            "https://science.osti.gov/bes/Highlights/2023/BES-2023-08-b",
270            "https://www.finance.senate.gov/download/122222-letter",
271            "https://www.afcea.org/signal/resources/linkreq.cfm?id=491",
272            "https://www.usda.gov/media/radio/daily-newsline/2023-08-16/actuality-opportunities",
273        ];
274        for url in cases {
275            assert!(rule.is_article_url(url), "should match: {url}");
276        }
277    }
278
279    #[test]
280    fn test_year_month_without_day() {
281        let rule = UrlRule::default();
282        assert!(rule.is_article_url("https://example.com/2024/03/some-article"));
283        assert!(rule.is_article_url("https://example.com/2023/08/another-one"));
284    }
285
286    #[test]
287    fn test_with_include_extension() {
288        let rule = UrlRule::default();
289        assert!(rule.is_article_url("https://example.com/blog/my-post"));
290    }
291
292    #[test]
293    fn test_with_exclude_override() {
294        let rule = UrlRule::default().with_exclude(r"print=true");
295        assert!(!rule.is_article_url("https://example.com/news/foo?print=true"));
296    }
297}
crawlkit_parser/rules.rs

crawlkit_parser/
rules.rs
