spider_lib/
response.rs

1//! Data structures and utilities for handling HTTP responses in `spider-lib`.
2//!
3//! This module defines the `Response` struct, which represents an HTTP response
4//! received from a web server. It encapsulates crucial information such as
5//! the URL, status code, headers, and body of the response, along with any
6//! associated metadata.
7//!
8//! Additionally, this module provides:
9//! - Helper methods for `Response` to facilitate common tasks like parsing
10//!   the body as HTML or JSON, and reconstructing the original `Request`.
11//! - `Link` and `LinkType` enums for structured representation and extraction
12//!   of hyperlinks found within the response content.
13use crate::{request::Request, utils};
14use bytes::Bytes;
15use dashmap::{DashMap, DashSet};
16use linkify::{LinkFinder, LinkKind};
17use reqwest::StatusCode;
18use reqwest::header::HeaderMap;
19use scraper::{Html, Selector};
20use serde::de::DeserializeOwned;
21use serde_json;
22use std::borrow::Cow;
23use url::Url;
24
25/// Represents the type of a discovered link.
26#[derive(Debug, Clone, PartialEq, Eq, Hash)]
27pub enum LinkType {
28    /// A link to another web page.
29    Page,
30    /// A link to a script file.
31    Script,
32    /// A link to a stylesheet.
33    Stylesheet,
34    /// A link to an image.
35    Image,
36    /// A link to a media file (audio/video).
37    Media,
38    /// A link to another type of resource.
39    Other(String),
40}
41
42/// Represents a link discovered on a web page.
43#[derive(Debug, Clone, PartialEq, Eq, Hash)]
44pub struct Link {
45    /// The URL of the discovered link.
46    pub url: Url,
47    /// The type of the discovered link.
48    pub link_type: LinkType,
49}
50
51/// Represents an HTTP response received from a server.
52#[derive(Debug, Clone)]
53pub struct Response {
54    /// The final URL of the response after any redirects.
55    pub url: Url,
56    /// The HTTP status code of the response.
57    pub status: StatusCode,
58    /// The headers of the response.
59    pub headers: HeaderMap,
60    /// The body of the response.
61    pub body: Bytes,
62    /// The original URL of the request that led to this response.
63    pub request_url: Url,
64    /// Metadata associated with the response, carried over from the request.
65    pub meta: DashMap<Cow<'static, str>, serde_json::Value>,
66    /// Indicates if the response was served from a cache.
67    pub cached: bool,
68}
69
70impl Response {
71    /// Reconstructs the original `Request` that led to this response.
72    pub fn request_from_response(&self) -> Request {
73        let mut request = Request::new(self.request_url.clone());
74        request.meta = self.meta.clone();
75        request
76    }
77
78    /// Deserializes the response body as JSON.
79    pub fn json<T: DeserializeOwned>(&self) -> Result<T, serde_json::Error> {
80        serde_json::from_slice(&self.body)
81    }
82
83    /// Parses the response body as HTML.
84    pub fn to_html(&self) -> Result<Html, std::str::Utf8Error> {
85        let body_str = std::str::from_utf8(&self.body)?;
86        Ok(Html::parse_document(body_str))
87    }
88
89    /// Extracts all unique, same-site links from the response body.
90    pub fn links(&self) -> DashSet<Link> {
91        let links = DashSet::new();
92
93        if let Ok(html) = self.to_html() {
94            let selectors = vec![
95                ("a[href]", "href"),
96                ("link[href]", "href"),
97                ("script[src]", "src"),
98                ("img[src]", "src"),
99                ("audio[src]", "src"),
100                ("video[src]", "src"),
101                ("source[src]", "src"),
102            ];
103
104            for (selector_str, attr_name) in selectors {
105                if let Ok(selector) = Selector::parse(selector_str) {
106                    for element in html.select(&selector) {
107                        if let Some(attr_value) = element.value().attr(attr_name)
108                            && let Ok(url) = self.url.join(attr_value)
109                            && utils::is_same_site(&url, &self.url)
110                        {
111                            let link_type = match element.value().name() {
112                                "a" => LinkType::Page,
113                                "link" => {
114                                    if let Some(rel) = element.value().attr("rel") {
115                                        if rel.eq_ignore_ascii_case("stylesheet") {
116                                            LinkType::Stylesheet
117                                        } else {
118                                            LinkType::Other(rel.to_string())
119                                        }
120                                    } else {
121                                        LinkType::Other("link".to_string())
122                                    }
123                                }
124                                "script" => LinkType::Script,
125                                "img" => LinkType::Image,
126                                "audio" | "video" | "source" => LinkType::Media,
127                                _ => LinkType::Other(element.value().name().to_string()),
128                            };
129                            links.insert(Link { url, link_type });
130                        }
131                    }
132                }
133            }
134
135            let finder = LinkFinder::new();
136            for text_node in html.tree.values().filter_map(|node| node.as_text()) {
137                for link in finder.links(text_node) {
138                    if link.kind() == &LinkKind::Url
139                        && let Ok(url) = self.url.join(link.as_str())
140                        && utils::is_same_site(&url, &self.url)
141                    {
142                        links.insert(Link {
143                            url,
144                            link_type: LinkType::Page,
145                        });
146                    }
147                }
148            }
149        }
150
151        links
152    }
153}
spider_lib/response.rs

spider_lib/
response.rs
