spider_lib/utils/
mod.rs

1//! General utility functions and helper traits for the `spider-lib` framework.
2//!
3//! This module provides a collection of miscellaneous functions and extensions
4//! that are used across different components of the `spider-lib`. These utilities
5//! aim to simplify common tasks such as URL manipulation, file system operations,
6//! and HTML selector parsing.
7//!
8//! Key functionalities include:
9//! - Normalizing URL origins and checking same-site policies.
10//! - Ensuring the existence of directories for output files.
11//! - Conveniently converting strings into `scraper::Selector` instances.
12
13use crate::{Request, SpiderError};
14use psl::{List, Psl};
15use scraper::Selector;
16use std::fs;
17use std::path::Path;
18use url::Url;
19
20mod bloom_filter;
21pub use bloom_filter::BloomFilter;
22
23/// Normalizes the origin of a request's URL.
24pub fn normalize_origin(request: &Request) -> String {
25    let url = &request.url;
26    let scheme = url.scheme();
27    let host = url.host_str().unwrap_or("");
28    let port = url.port_or_known_default().unwrap_or(0);
29
30    format!("{scheme}://{host}:{port}")
31}
32/// Checks if two URLs belong to the same site.
33pub fn is_same_site(a: &Url, b: &Url) -> bool {
34    a.host_str().and_then(|h| List.domain(h.as_bytes()))
35        == b.host_str().and_then(|h| List.domain(h.as_bytes()))
36}
37
38/// Validates that the parent directory of a given file path exists, creating it if necessary.
39pub fn validate_output_dir(file_path: impl AsRef<Path>) -> Result<(), SpiderError> {
40    let Some(parent_dir) = file_path.as_ref().parent() else {
41        return Ok(());
42    };
43
44    if !parent_dir.as_os_str().is_empty() && !parent_dir.exists() {
45        fs::create_dir_all(parent_dir)?;
46    }
47
48    Ok(())
49}
50
51/// Creates a directory and all of its parent components if they are missing.
52pub fn create_dir(dir_path: impl AsRef<Path>) -> Result<(), SpiderError> {
53    fs::create_dir_all(dir_path)?;
54    Ok(())
55}
56
57pub trait ToSelector {
58    /// Parses a string slice into a `scraper::Selector`, returning a `SpiderError` on failure.
59    fn to_selector(&self) -> Result<Selector, SpiderError>;
60}
61
62impl ToSelector for &str {
63    fn to_selector(&self) -> Result<Selector, SpiderError> {
64        Selector::parse(self).map_err(|e| SpiderError::HtmlParseError(e.to_string()))
65    }
66}
67
68impl ToSelector for String {
69    fn to_selector(&self) -> Result<Selector, SpiderError> {
70        Selector::parse(self).map_err(|e| SpiderError::HtmlParseError(e.to_string()))
71    }
72}
spider_lib/utils/mod.rs

spider_lib/utils/
mod.rs
