web_capture/
html.rs

1//! HTML processing module
2//!
3//! This module provides functions for fetching, parsing, and processing HTML content.
4
5use std::collections::BTreeMap;
6
7use crate::{Result, WebCaptureError};
8use regex::Regex;
9use tracing::{debug, info};
10use url::Url;
11
12/// Default user agent string
13const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36";
14
15/// Fetch HTML content from a URL
16///
17/// This function makes a simple HTTP GET request to fetch the HTML content.
18///
19/// # Arguments
20///
21/// * `url` - The URL to fetch
22///
23/// # Returns
24///
25/// The HTML content as a string
26///
27/// # Errors
28///
29/// Returns an error if the fetch fails or the response cannot be decoded
30pub async fn fetch_html(url: &str) -> Result<String> {
31    info!("Fetching HTML from URL: {}", url);
32
33    if crate::stackoverflow::is_stackoverflow_question_url(url) {
34        return crate::stackoverflow::fetch_stackoverflow_html(url).await;
35    }
36
37    let client = reqwest::Client::builder()
38        .user_agent(USER_AGENT)
39        .build()
40        .map_err(|error| WebCaptureError::FetchError(error.to_string()))?;
41
42    let response = client
43        .get(url)
44        .header("Accept-Language", "en-US,en;q=0.9")
45        .header("Accept-Charset", "utf-8")
46        .send()
47        .await
48        .map_err(|error| WebCaptureError::FetchError(error.to_string()))?;
49
50    let html = response
51        .text()
52        .await
53        .map_err(|error| WebCaptureError::FetchError(error.to_string()))?;
54
55    info!("Successfully fetched HTML ({} bytes)", html.len());
56    Ok(html)
57}
58
59/// Fetch an undecoded response through caller-owned transport.
60pub async fn fetch_html_receipt_with_transport(
61    url: &str,
62    transport: &dyn crate::transport::Transport,
63) -> std::result::Result<crate::transport::ResponseReceipt, crate::transport::TransportError> {
64    let request = crate::transport::TransportRequest {
65        url: url.to_string(),
66        method: "GET".to_string(),
67        headers: BTreeMap::from([
68            ("user-agent".to_string(), USER_AGENT.to_string()),
69            ("accept-encoding".to_string(), "identity".to_string()),
70            ("accept-language".to_string(), "en-US,en;q=0.9".to_string()),
71            ("accept-charset".to_string(), "utf-8".to_string()),
72        ]),
73    };
74    crate::transport::capture_response_with_transport(request, transport).await
75}
76
77/// Fetch an undecoded response through the default reqwest transport.
78pub async fn fetch_html_receipt(
79    url: &str,
80) -> std::result::Result<crate::transport::ResponseReceipt, crate::transport::TransportError> {
81    fetch_html_receipt_with_transport(url, &crate::transport::ReqwestTransport::default()).await
82}
83
84/// Convert relative URLs to absolute URLs in HTML content
85///
86/// Processes various HTML attributes that contain URLs and converts
87/// relative URLs to absolute URLs using the provided base URL.
88///
89/// # Arguments
90///
91/// * `html` - The HTML content to process
92/// * `base_url` - The base URL to use for resolving relative URLs
93///
94/// # Returns
95///
96/// The HTML content with absolute URLs
97pub fn convert_relative_urls(html: &str, base_url: &str) -> String {
98    debug!(
99        "Converting relative URLs to absolute using base: {}",
100        base_url
101    );
102
103    let Ok(base) = Url::parse(base_url) else {
104        return html.to_string();
105    };
106
107    let mut result = html.to_string();
108
109    // List of tag/attribute combinations to process
110    let attributes = [
111        ("a", "href"),
112        ("img", "src"),
113        ("script", "src"),
114        ("link", "href"),
115        ("form", "action"),
116        ("video", "src"),
117        ("audio", "src"),
118        ("source", "src"),
119        ("track", "src"),
120        ("embed", "src"),
121        ("object", "data"),
122        ("iframe", "src"),
123    ];
124
125    for (tag, attr) in &attributes {
126        let pattern = format!(r#"<{tag}[^>]*{attr}=["']([^"']+)["'][^>]*>"#);
127        if let Ok(regex) = Regex::new(&pattern) {
128            result = regex
129                .replace_all(&result, |caps: &regex::Captures| {
130                    let full_match = caps.get(0).map_or("", |m| m.as_str());
131                    let url_match = caps.get(1).map_or("", |m| m.as_str());
132
133                    let absolute_url = to_absolute_url(url_match, &base);
134                    full_match.replace(url_match, &absolute_url)
135                })
136                .to_string();
137        }
138    }
139
140    // Handle inline styles with url()
141    if let Ok(url_regex) = Regex::new(r#"url\(['"]?([^'"()]+)['"]?\)"#) {
142        result = url_regex
143            .replace_all(&result, |caps: &regex::Captures| {
144                let url_match = caps.get(1).map_or("", |m| m.as_str());
145                let absolute_url = to_absolute_url(url_match, &base);
146                format!(r#"url("{absolute_url}")"#)
147            })
148            .to_string();
149    }
150
151    debug!("URL conversion complete");
152    result
153}
154
155/// Convert a potentially relative URL to an absolute URL
156fn to_absolute_url(url: &str, base: &Url) -> String {
157    // Skip data:, blob:, and javascript: URLs
158    if url.is_empty()
159        || url.starts_with("data:")
160        || url.starts_with("blob:")
161        || url.starts_with("javascript:")
162    {
163        return url.to_string();
164    }
165
166    // Try to resolve the URL against the base
167    base.join(url)
168        .map_or_else(|_| url.to_string(), |absolute| absolute.to_string())
169}
170
171/// Convert HTML content to UTF-8 encoding
172///
173/// Detects the current encoding from meta tags and ensures UTF-8 encoding.
174///
175/// # Arguments
176///
177/// * `html` - The HTML content to convert
178///
179/// # Returns
180///
181/// The UTF-8 encoded HTML content
182pub fn convert_to_utf8(html: &str) -> String {
183    debug!("Converting HTML to UTF-8");
184
185    // Check for charset meta tag
186    let charset_regex = Regex::new(r#"<meta[^>]+charset=["']?([^"'>\s]+)"#).ok();
187
188    let current_charset = charset_regex
189        .as_ref()
190        .and_then(|re| re.captures(html))
191        .and_then(|caps| caps.get(1))
192        .map_or_else(|| "utf-8".to_string(), |m| m.as_str().to_lowercase());
193
194    // If already UTF-8, ensure the meta tag is present
195    if current_charset == "utf-8" || current_charset == "utf8" {
196        // Add meta charset if not present
197        if !html.to_lowercase().contains("charset") {
198            if let Ok(head_regex) = Regex::new(r"<head[^>]*>") {
199                return head_regex
200                    .replace(html, r#"$0<meta charset="utf-8">"#)
201                    .to_string();
202            }
203        }
204        return html.to_string();
205    }
206
207    // For other charsets, try to convert and update the meta tag
208    let charset_update_regex = Regex::new(r#"<meta[^>]+charset=["']?[^"'>\s]+["']?"#).ok();
209
210    charset_update_regex.map_or_else(
211        || html.to_string(),
212        |regex| regex.replace(html, r#"<meta charset="utf-8""#).to_string(),
213    )
214}
215
216/// Check if HTML content contains JavaScript
217///
218/// # Arguments
219///
220/// * `html` - The HTML content to check
221///
222/// # Returns
223///
224/// True if the HTML contains JavaScript
225#[must_use]
226pub fn has_javascript(html: &str) -> bool {
227    let pattern = r"<script[^>]*>[\s\S]*?</script>|<script[^>]*/\s*>|javascript:";
228    Regex::new(pattern).is_ok_and(|re| re.is_match(html))
229}
230
231/// Check if content is valid HTML
232///
233/// # Arguments
234///
235/// * `html` - The content to check
236///
237/// # Returns
238///
239/// True if the content appears to be valid HTML
240#[must_use]
241pub fn is_html(html: &str) -> bool {
242    let pattern = r"<html[^>]*>[\s\S]*?</html>";
243    Regex::new(pattern).is_ok_and(|re| re.is_match(html))
244}
245
246/// Decode HTML entities to unicode characters.
247///
248/// Converts HTML entities like `&amp;`, `&lt;`, `&#39;`, `&#x27;` etc.
249/// to their actual unicode character equivalents.
250///
251/// # Arguments
252///
253/// * `html` - The HTML content containing entities to decode
254///
255/// # Returns
256///
257/// The content with all HTML entities decoded to unicode
258#[must_use]
259pub fn decode_html_entities(html: &str) -> String {
260    html_escape::decode_html_entities(html).into_owned()
261}
262
263/// Pretty-print HTML with indentation.
264///
265/// Adds newlines and indentation to make HTML human-readable.
266/// Void elements (br, hr, img, input, meta, link) are not indented as blocks.
267///
268/// # Arguments
269///
270/// * `html` - The HTML content to format
271///
272/// # Returns
273///
274/// The pretty-printed HTML content
275#[must_use]
276pub fn pretty_print_html(html: &str) -> String {
277    use std::sync::OnceLock;
278
279    static TAG_RE: OnceLock<Regex> = OnceLock::new();
280    static VOID_RE: OnceLock<Regex> = OnceLock::new();
281
282    let re = TAG_RE.get_or_init(|| Regex::new(r"(</?[a-zA-Z][^>]*?>)").unwrap());
283    let void_pat = VOID_RE.get_or_init(|| {
284        Regex::new(
285            r"(?i)^<(area|base|br|col|embed|hr|img|input|link|meta|param|source|track|wbr)\b",
286        )
287        .unwrap()
288    });
289    let mut result = String::with_capacity(html.len() * 2);
290    let mut indent: usize = 0;
291    let indent_str = "  ";
292    let mut last_end = 0;
293    let mut parts: Vec<(bool, &str)> = Vec::new();
294
295    for m in re.find_iter(html) {
296        let before = &html[last_end..m.start()];
297        if !before.trim().is_empty() {
298            parts.push((false, before));
299        }
300        parts.push((true, m.as_str()));
301        last_end = m.end();
302    }
303    let trailing = &html[last_end..];
304    if !trailing.trim().is_empty() {
305        parts.push((false, trailing));
306    }
307
308    for (is_tag, content) in &parts {
309        if *is_tag {
310            let tag = *content;
311            let is_closing = tag.starts_with("</");
312            let is_void = void_pat.is_match(tag);
313            let is_self_closing = tag.ends_with("/>");
314
315            if is_closing {
316                indent = indent.saturating_sub(1);
317            }
318            for _ in 0..indent {
319                result.push_str(indent_str);
320            }
321            result.push_str(tag);
322            result.push('\n');
323            if !is_closing && !is_void && !is_self_closing {
324                indent += 1;
325            }
326        } else {
327            let text = content.trim();
328            if !text.is_empty() {
329                for _ in 0..indent {
330                    result.push_str(indent_str);
331                }
332                result.push_str(text);
333                result.push('\n');
334            }
335        }
336    }
337
338    result
339}
340
341/// Normalize URL to ensure it's absolute.
342///
343/// Prepends `https://` if no scheme is present and validates the URL.
344///
345/// # Errors
346///
347/// Returns an error string if the URL is empty or invalid.
348pub fn normalize_url(url: &str) -> std::result::Result<String, String> {
349    if url.is_empty() {
350        return Err("Missing url parameter".to_string());
351    }
352
353    let absolute_url = if url.starts_with("http://") || url.starts_with("https://") {
354        url.to_string()
355    } else {
356        format!("https://{url}")
357    };
358
359    // Validate the URL
360    Url::parse(&absolute_url).map_err(|e| format!("Invalid URL: {e}"))?;
361
362    Ok(absolute_url)
363}
web_capture/html.rs

web_capture/
html.rs
