halldyll-core 0.1.0

Core scraping engine for Halldyll - high-performance async web scraper for AI agents
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85

//! Request - HTTP request building

use reqwest::header::{HeaderMap, HeaderName, HeaderValue, ACCEPT, ACCEPT_ENCODING, ACCEPT_LANGUAGE, IF_MODIFIED_SINCE, IF_NONE_MATCH};
use url::Url;

use crate::types::Config;

/// HTTP request builder
pub struct RequestBuilder {
    url: Url,
    headers: HeaderMap,
    etag: Option<String>,
    last_modified: Option<String>,
}

impl RequestBuilder {
    /// New request for a URL
    pub fn new(url: Url) -> Self {
        Self {
            url,
            headers: HeaderMap::new(),
            etag: None,
            last_modified: None,
        }
    }

    /// Apply default configuration
    pub fn with_config(mut self, config: &Config) -> Self {
        // Accept headers
        if let Ok(v) = HeaderValue::from_str(&config.fetch.accept_headers.accept) {
            self.headers.insert(ACCEPT, v);
        }
        if let Ok(v) = HeaderValue::from_str(&config.fetch.accept_headers.accept_language) {
            self.headers.insert(ACCEPT_LANGUAGE, v);
        }
        if let Ok(v) = HeaderValue::from_str(&config.fetch.accept_headers.accept_encoding) {
            self.headers.insert(ACCEPT_ENCODING, v);
        }
        self
    }

    /// Add a custom header
    pub fn header(mut self, name: &str, value: &str) -> Self {
        if let (Ok(n), Ok(v)) = (
            HeaderName::try_from(name),
            HeaderValue::from_str(value),
        ) {
            self.headers.insert(n, v);
        }
        self
    }

    /// Configure conditional request with ETag
    pub fn if_none_match(mut self, etag: &str) -> Self {
        self.etag = Some(etag.to_string());
        if let Ok(v) = HeaderValue::from_str(etag) {
            self.headers.insert(IF_NONE_MATCH, v);
        }
        self
    }

    /// Configure conditional request with Last-Modified
    pub fn if_modified_since(mut self, date: &str) -> Self {
        self.last_modified = Some(date.to_string());
        if let Ok(v) = HeaderValue::from_str(date) {
            self.headers.insert(IF_MODIFIED_SINCE, v);
        }
        self
    }

    /// Returns the URL
    pub fn url(&self) -> &Url {
        &self.url
    }

    /// Returns the headers
    pub fn headers(&self) -> &HeaderMap {
        &self.headers
    }

    /// Consume and return components
    pub fn build(self) -> (Url, HeaderMap) {
        (self.url, self.headers)
    }
}