docbox_web_scraper/
lib.rs

1#![forbid(unsafe_code)]
2#![warn(missing_docs)]
3
4//! # Docbox Web Scraper
5//!
6//! Web-scraping client for getting website metadata, favicon, ...etc and
7//! maintaining an internal cache
8//!
9//! ## Environment Variables
10//!
11//! * `DOCBOX_WEB_SCRAPE_HTTP_PROXY` - Proxy server address to use for HTTP requests
12//! * `DOCBOX_WEB_SCRAPE_HTTPS_PROXY` - Proxy server address to use for HTTPS requests
13//! * `DOCBOX_WEB_SCRAPE_METADATA_CACHE_DURATION` - Time before cached metadata is considered expired
14//! * `DOCBOX_WEB_SCRAPE_METADATA_CACHE_CAPACITY` - Maximum amount of metadata to cache at once
15//! * `DOCBOX_WEB_SCRAPE_METADATA_CONNECT_TIMEOUT` - Timeout when connecting while scraping
16//! * `DOCBOX_WEB_SCRAPE_METADATA_READ_TIMEOUT` - Timeout when reading responses from scraping
17
18use document::{determine_best_favicon, get_website_metadata};
19use download_image::{download_image_href, resolve_full_url};
20use mime::Mime;
21use reqwest::{Proxy, redirect::Policy};
22use serde::{Deserialize, Serialize};
23use std::{str::FromStr, time::Duration};
24use thiserror::Error;
25use url_validation::TokioDomainResolver;
26
27mod data_uri;
28mod document;
29mod download_image;
30mod request;
31mod url_validation;
32
33pub use document::Favicon;
34pub use reqwest::Url;
35
36use crate::{document::is_allowed_robots_txt, download_image::ImageStream};
37
38/// Configuration for the website metadata service
39#[derive(Debug, Deserialize, Serialize)]
40#[serde(default)]
41pub struct WebsiteMetaServiceConfig {
42    /// HTTP proxy to use when making HTTP metadata requests
43    pub http_proxy: Option<String>,
44    /// HTTPS proxy to use when making HTTPS metadata requests
45    pub https_proxy: Option<String>,
46    /// Time to wait when attempting to fetch resource before timing out
47    ///
48    /// This option is ignored if you manually provide a [`reqwest::Client`]
49    ///
50    /// Default: 5s
51    pub metadata_connect_timeout: Duration,
52    /// Time to wait while downloading a resource before timing out (between each read of data)
53    ///
54    /// This option is ignored if you manually provide a [`reqwest::Client`]
55    ///
56    /// Default: 10s
57    pub metadata_read_timeout: Duration,
58}
59
60/// Errors that could occur when loading the configuration
61#[derive(Debug, Error)]
62pub enum WebsiteMetaServiceConfigError {
63    /// Provided connect timeout was an invalid number
64    #[error("DOCBOX_WEB_SCRAPE_METADATA_CONNECT_TIMEOUT must be a number in seconds: {0}")]
65    InvalidMetadataConnectTimeout(<u64 as FromStr>::Err),
66    /// Provided read timeout was an invalid number
67    #[error("DOCBOX_WEB_SCRAPE_METADATA_READ_TIMEOUT must be a number in seconds")]
68    InvalidMetadataReadTimeout(<u64 as FromStr>::Err),
69}
70
71impl Default for WebsiteMetaServiceConfig {
72    fn default() -> Self {
73        Self {
74            http_proxy: None,
75            https_proxy: None,
76            metadata_connect_timeout: Duration::from_secs(5),
77            metadata_read_timeout: Duration::from_secs(10),
78        }
79    }
80}
81
82impl WebsiteMetaServiceConfig {
83    /// Load a website meta service config from its environment variables
84    pub fn from_env() -> Result<WebsiteMetaServiceConfig, WebsiteMetaServiceConfigError> {
85        let mut config = WebsiteMetaServiceConfig {
86            http_proxy: std::env::var("DOCBOX_WEB_SCRAPE_HTTP_PROXY").ok(),
87            https_proxy: std::env::var("DOCBOX_WEB_SCRAPE_HTTPS_PROXY").ok(),
88            ..Default::default()
89        };
90
91        if let Ok(metadata_connect_timeout) =
92            std::env::var("DOCBOX_WEB_SCRAPE_METADATA_CONNECT_TIMEOUT")
93        {
94            let metadata_connect_timeout = metadata_connect_timeout
95                .parse::<u64>()
96                .map_err(WebsiteMetaServiceConfigError::InvalidMetadataConnectTimeout)?;
97
98            config.metadata_connect_timeout = Duration::from_secs(metadata_connect_timeout);
99        }
100
101        if let Ok(metadata_read_timeout) = std::env::var("DOCBOX_WEB_SCRAPE_METADATA_READ_TIMEOUT")
102        {
103            let metadata_read_timeout = metadata_read_timeout
104                .parse::<u64>()
105                .map_err(WebsiteMetaServiceConfigError::InvalidMetadataReadTimeout)?;
106
107            config.metadata_read_timeout = Duration::from_secs(metadata_read_timeout);
108        }
109
110        Ok(config)
111    }
112}
113
114/// Service for looking up website metadata and storing a cached value
115pub struct WebsiteMetaService {
116    client: reqwest::Client,
117}
118
119/// Metadata resolved from a scraped website
120#[derive(Debug, Clone, Serialize, Deserialize)]
121pub struct ResolvedWebsiteMetadata {
122    /// Title of the website from the `<title/>` element
123    pub title: Option<String>,
124
125    /// OGP title of the website
126    pub og_title: Option<String>,
127
128    /// OGP metadata description of the website
129    pub og_description: Option<String>,
130
131    /// Best determined image
132    #[serde(skip)]
133    pub og_image: Option<String>,
134
135    /// Best determined favicon
136    #[serde(skip)]
137    pub best_favicon: Option<String>,
138}
139
140/// Represents an image that has been resolved where the
141/// contents are now know and the content type as well
142#[derive(Debug)]
143pub struct ResolvedImage {
144    /// Content type of the image
145    pub content_type: Mime,
146    /// Stream for the image bytes
147    pub stream: ImageStream,
148}
149
150impl WebsiteMetaService {
151    /// Creates a new instance of the service, this initializes the HTTP
152    /// client and creates the cache
153    pub fn new() -> reqwest::Result<Self> {
154        Self::from_config(Default::default())
155    }
156
157    /// Create a web scraper from the provided client
158    ///
159    /// Using a custom client can cause security measures to not
160    /// be fully applied leading to gaps in security its recommended
161    /// you use [WebsiteMetaService::from_config] unless you have a
162    /// specific use case which is prevented by it
163    #[deprecated]
164    pub fn from_client(client: reqwest::Client) -> Self {
165        Self { client }
166    }
167
168    /// Create a web scraper from the provided config
169    pub fn from_config(config: WebsiteMetaServiceConfig) -> reqwest::Result<Self> {
170        let mut builder = reqwest::Client::builder();
171
172        if let Some(http_proxy) = config.http_proxy.clone() {
173            builder = builder.proxy(Proxy::http(http_proxy)?);
174        }
175
176        if let Some(https_proxy) = config.https_proxy.clone() {
177            builder = builder.proxy(Proxy::https(https_proxy)?);
178        }
179
180        let client = builder
181            .user_agent("DocboxLinkBot")
182            .connect_timeout(config.metadata_connect_timeout)
183            .read_timeout(config.metadata_read_timeout)
184            // Redirect logic is handled internally by the scraper for security
185            .redirect(Policy::none())
186            // Enforce our custom tokio lookup_host based resolver to ensure
187            // consistency when blocking requests
188            .dns_resolver(TokioDomainResolver)
189            .build()?;
190
191        Ok(Self { client })
192    }
193
194    /// Resolves the metadata for the website at the provided URL
195    pub async fn resolve_website(&self, url: &Url) -> Option<ResolvedWebsiteMetadata> {
196        // Check that the site allows scraping based on its robots.txt
197        let is_allowed_scraping = is_allowed_robots_txt::<TokioDomainResolver>(&self.client, url)
198            .await
199            .unwrap_or(false);
200
201        if !is_allowed_scraping {
202            return None;
203        }
204
205        // Get the website metadata
206        let res = match get_website_metadata::<TokioDomainResolver>(&self.client, url).await {
207            Ok(value) => value,
208            Err(error) => {
209                tracing::error!(?error, "failed to get website metadata");
210                return None;
211            }
212        };
213
214        let best_favicon = determine_best_favicon(&res.favicons).cloned();
215
216        Some(ResolvedWebsiteMetadata {
217            title: res.title,
218            og_title: res.og_title,
219            og_description: res.og_description,
220            og_image: res.og_image,
221            best_favicon: best_favicon.map(|value| value.href),
222        })
223    }
224
225    /// Resolve the favicon image at the provided URL
226    pub async fn resolve_website_favicon(&self, url: &Url) -> Option<ResolvedImage> {
227        let website = self.resolve_website(url).await?;
228
229        self.resolve_favicon(url, website.best_favicon).await
230    }
231
232    /// Resolve the OGP metadata image from the provided URL
233    pub async fn resolve_website_image(&self, url: &Url) -> Option<ResolvedImage> {
234        let website = self.resolve_website(url).await?;
235        let og_image = website.og_image?;
236
237        self.resolve_image(url, &og_image).await
238    }
239
240    /// Resolve the favicon using the best favicon. Used by wrapping services to provide
241    /// the default favicon fallback URL for favicons and resolve favicon images without
242    /// internally calling [`Self::resolve_website`]
243    pub async fn resolve_favicon(
244        &self,
245        url: &Url,
246        best_favicon: Option<String>,
247    ) -> Option<ResolvedImage> {
248        let favicon = match best_favicon {
249            Some(best) => best,
250
251            // No favicon from document? Fallback and try to use the default path
252            None => {
253                let mut url = url.clone();
254                url.set_path("/favicon.ico");
255                url.to_string()
256            }
257        };
258
259        self.resolve_image(url, &favicon).await
260    }
261
262    /// Resolve an image content type and provide a stream to download the image
263    pub async fn resolve_image(&self, url: &Url, image: &str) -> Option<ResolvedImage> {
264        let image_url = resolve_full_url(url, image).ok()?;
265
266        let (stream, content_type) =
267            download_image_href::<TokioDomainResolver>(&self.client, image_url)
268                .await
269                .ok()?;
270
271        Some(ResolvedImage {
272            content_type,
273            stream,
274        })
275    }
276}
docbox_web_scraper/lib.rs

docbox_web_scraper/
lib.rs
