spider_util/
utils.rs

1//! Utility functions for the `spider-lib` framework.
2//!
3//! This module provides utility functions that are used across different
4//! components of the framework.
5
6use psl::{List, Psl};
7use scraper::Selector;
8use std::fs;
9use std::path::Path;
10use url::Url;
11
12use crate::error::SpiderError;
13use crate::request::Request;
14
15/// Checks if two URLs belong to the same site.
16pub fn is_same_site(a: &Url, b: &Url) -> bool {
17    a.host_str().and_then(|h| List.domain(h.as_bytes()))
18        == b.host_str().and_then(|h| List.domain(h.as_bytes()))
19}
20
21/// Normalizes the origin of a request's URL.
22pub fn normalize_origin(request: &Request) -> String {
23    let url = &request.url;
24    let scheme = url.scheme();
25    let host = url.host_str().unwrap_or("");
26    let port = url.port_or_known_default().unwrap_or(0);
27
28    format!("{scheme}://{host}:{port}")
29}
30
31/// Validates that the parent directory of a given file path exists, creating it if necessary.
32pub fn validate_output_dir(file_path: impl AsRef<Path>) -> Result<(), SpiderError> {
33    let Some(parent_dir) = file_path.as_ref().parent() else {
34        return Ok(());
35    };
36
37    if !parent_dir.as_os_str().is_empty() && !parent_dir.exists() {
38        fs::create_dir_all(parent_dir)?;
39    }
40
41    Ok(())
42}
43
44/// Creates a directory and all of its parent components if they are missing.
45pub fn create_dir(dir_path: impl AsRef<Path>) -> Result<(), SpiderError> {
46    fs::create_dir_all(dir_path)?;
47    Ok(())
48}
49
50pub trait ToSelector {
51    /// Parses a string slice into a `scraper::Selector`, returning a `SpiderError` on failure.
52    fn to_selector(&self) -> Result<Selector, SpiderError>;
53}
54
55impl ToSelector for &str {
56    fn to_selector(&self) -> Result<Selector, SpiderError> {
57        Selector::parse(self).map_err(|e| SpiderError::HtmlParseError(e.to_string()))
58    }
59}
60
61impl ToSelector for String {
62    fn to_selector(&self) -> Result<Selector, SpiderError> {
63        Selector::parse(self).map_err(|e| SpiderError::HtmlParseError(e.to_string()))
64    }
65}
66
spider_util/utils.rs

spider_util/
utils.rs
