Skip to main content

fetchkit/
client.rs

1//! HTTP client for Fetchkit
2//!
3//! This module provides the main entry points for fetching URLs.
4//! The actual fetch logic is implemented by fetchers in the [`fetchers`](crate::fetchers) module.
5
6#[cfg(feature = "bot-auth")]
7use crate::bot_auth::BotAuthConfig;
8use crate::content::ContentProcessor;
9use crate::dns::DnsPolicy;
10use crate::documents::DocumentConversion;
11use crate::error::FetchError;
12use crate::fetchers::FetcherRegistry;
13use crate::transport::{HttpTransport, ReqwestTransport};
14use crate::types::{FetchRequest, FetchResponse};
15use std::sync::Arc;
16use url::Url;
17
18/// Fetch options that can be configured via tool builder
19#[derive(Clone, Default)]
20pub struct FetchOptions {
21    /// Custom User-Agent
22    pub user_agent: Option<String>,
23    /// Allow list of URL prefixes
24    pub allow_prefixes: Vec<String>,
25    /// Block list of URL prefixes
26    pub block_prefixes: Vec<String>,
27    /// Enable as_markdown option
28    pub enable_markdown: bool,
29    /// Enable as_text option
30    pub enable_text: bool,
31    /// DNS resolution policy for SSRF prevention
32    pub dns_policy: DnsPolicy,
33    /// Maximum response body size in bytes (default: 10 MB)
34    /// Protects against TM-DOS-001 (unbounded body) and TM-DOS-003 (gzip bombs)
35    pub max_body_size: Option<usize>,
36    /// Enable save_to_file parameter in requests
37    pub enable_save_to_file: bool,
38    /// Whether to respect HTTP_PROXY/HTTPS_PROXY/NO_PROXY from the environment
39    pub respect_proxy_env: bool,
40    /// Restrict outbound requests to these ports. Empty means any port.
41    pub allowed_ports: Vec<u16>,
42    /// Block exact hosts and suffix rules. Leading '.' means suffix match.
43    pub blocked_hosts: Vec<String>,
44    /// Restrict redirects to the original host only.
45    pub same_host_redirects_only: bool,
46    /// Internal crawl boundary: redirects must stay on this exact origin.
47    #[doc(hidden)]
48    pub redirect_origin: Option<Url>,
49    /// Enable rakers-rendered HTML fetching. The request must still opt in.
50    pub enable_render_rakers: bool,
51    /// Web Bot Authentication config (draft-meunier-web-bot-auth-architecture).
52    /// When set, outgoing requests are signed with Ed25519.
53    #[cfg(feature = "bot-auth")]
54    pub bot_auth: Option<BotAuthConfig>,
55    /// Pluggable HTTP transport. When `None`, the default [`ReqwestTransport`] is used.
56    ///
57    /// A host application can supply its own transport to route fetchkit's outbound
58    /// HTTP through a dedicated egress boundary. fetchkit still performs all URL
59    /// validation, DNS policy resolution, redirect following, and content handling;
60    /// only the socket-level send is delegated.
61    pub transport: Option<Arc<dyn HttpTransport>>,
62    /// Document (PDF, Office) conversion config. `None` uses
63    /// [`DocumentConversion::default`]: built-in local converters only.
64    pub document_conversion: Option<DocumentConversion>,
65    /// Custom content processors, consulted before the built-in ones.
66    pub content_processors: Vec<Arc<dyn ContentProcessor>>,
67}
68
69// Manual Debug: `transport` is a trait object that does not implement Debug.
70impl std::fmt::Debug for FetchOptions {
71    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
72        let mut d = f.debug_struct("FetchOptions");
73        d.field("user_agent", &self.user_agent)
74            .field("allow_prefixes", &self.allow_prefixes)
75            .field("block_prefixes", &self.block_prefixes)
76            .field("enable_markdown", &self.enable_markdown)
77            .field("enable_text", &self.enable_text)
78            .field("dns_policy", &self.dns_policy)
79            .field("max_body_size", &self.max_body_size)
80            .field("enable_save_to_file", &self.enable_save_to_file)
81            .field("respect_proxy_env", &self.respect_proxy_env)
82            .field("allowed_ports", &self.allowed_ports)
83            .field("blocked_hosts", &self.blocked_hosts)
84            .field("same_host_redirects_only", &self.same_host_redirects_only)
85            .field("redirect_origin", &self.redirect_origin)
86            .field("enable_render_rakers", &self.enable_render_rakers);
87        #[cfg(feature = "bot-auth")]
88        d.field("bot_auth", &self.bot_auth);
89        d.field(
90            "transport",
91            &self
92                .transport
93                .as_ref()
94                .map(|_| "<custom>")
95                .unwrap_or("None"),
96        )
97        .field("document_conversion", &self.document_conversion)
98        .field(
99            "content_processors",
100            &self
101                .content_processors
102                .iter()
103                .map(|processor| processor.name())
104                .collect::<Vec<_>>(),
105        );
106        d.finish()
107    }
108}
109
110impl FetchOptions {
111    /// Resolve the active transport, falling back to [`ReqwestTransport`] when none
112    /// was supplied.
113    pub(crate) fn transport(&self) -> Arc<dyn HttpTransport> {
114        match &self.transport {
115            Some(t) => Arc::clone(t),
116            None => Arc::new(ReqwestTransport::new()),
117        }
118    }
119
120    pub(crate) fn validate_url(&self, url: &Url) -> Result<(), FetchError> {
121        self.validate_host(url)?;
122        self.validate_port(url)?;
123        Ok(())
124    }
125
126    pub(crate) fn validate_redirect_target(
127        &self,
128        current_url: &Url,
129        next_url: &Url,
130    ) -> Result<(), FetchError> {
131        self.validate_url(next_url)?;
132
133        if self.same_host_redirects_only
134            && normalized_host(current_url) != normalized_host(next_url)
135        {
136            return Err(FetchError::BlockedUrl);
137        }
138
139        if let Some(origin) = &self.redirect_origin {
140            let same_origin = origin.scheme() == next_url.scheme()
141                && normalized_host(origin) == normalized_host(next_url)
142                && origin.port_or_known_default() == next_url.port_or_known_default();
143            if !same_origin {
144                return Err(FetchError::BlockedUrl);
145            }
146        }
147
148        Ok(())
149    }
150
151    fn validate_host(&self, url: &Url) -> Result<(), FetchError> {
152        let Some(host) = normalized_host(url) else {
153            return Ok(());
154        };
155
156        if self
157            .blocked_hosts
158            .iter()
159            .any(|rule| host_matches_rule(&host, rule))
160        {
161            return Err(FetchError::BlockedUrl);
162        }
163
164        Ok(())
165    }
166
167    fn validate_port(&self, url: &Url) -> Result<(), FetchError> {
168        if self.allowed_ports.is_empty() {
169            return Ok(());
170        }
171
172        let port = url.port_or_known_default().ok_or(FetchError::BlockedUrl)?;
173        if self.allowed_ports.contains(&port) {
174            Ok(())
175        } else {
176            Err(FetchError::BlockedUrl)
177        }
178    }
179}
180
181fn normalized_host(url: &Url) -> Option<String> {
182    url.host_str()
183        .map(|host| host.trim_end_matches('.').to_ascii_lowercase())
184}
185
186fn host_matches_rule(host: &str, rule: &str) -> bool {
187    let normalized_rule = rule.trim_end_matches('.').to_ascii_lowercase();
188
189    if let Some(suffix) = normalized_rule.strip_prefix('.') {
190        host == suffix || host.ends_with(&format!(".{suffix}"))
191    } else {
192        host == normalized_rule
193    }
194}
195
196/// Fetch a URL and return the response
197///
198/// Uses the default fetcher registry with all built-in fetchers.
199/// Markdown and text conversions are enabled by default.
200/// For custom options, use [`fetch_with_options`].
201///
202/// # Examples
203///
204/// ```no_run
205/// use fetchkit::{FetchRequest, fetch};
206///
207/// # async fn example() -> Result<(), fetchkit::FetchError> {
208/// let response = fetch(FetchRequest::new("https://example.com")).await?;
209/// println!("Status: {}", response.status_code);
210/// # Ok(())
211/// # }
212/// ```
213pub async fn fetch(req: FetchRequest) -> Result<FetchResponse, FetchError> {
214    let options = FetchOptions {
215        enable_markdown: true,
216        enable_text: true,
217        ..Default::default()
218    };
219    fetch_with_options(req, options).await
220}
221
222/// Fetch a URL with custom options
223///
224/// Uses all built-in fetchers, with content processors and document conversion
225/// taken from `options`. For custom fetchers, use [`FetcherRegistry`] directly.
226pub async fn fetch_with_options(
227    mut req: FetchRequest,
228    options: FetchOptions,
229) -> Result<FetchResponse, FetchError> {
230    // Validate URL early
231    if req.url.is_empty() {
232        return Err(FetchError::MissingUrl);
233    }
234    req.normalize_url_for_fetch()?;
235
236    if req.wants_crawl() {
237        return crate::crawl::crawl_fetch_with_options(req, options).await;
238    }
239
240    let registry = FetcherRegistry::for_options(&options);
241    registry.fetch(req, options).await
242}
243
244/// Default concurrency limit for batch fetching.
245const DEFAULT_BATCH_CONCURRENCY: usize = 5;
246/// Maximum allowed concurrency for batch fetching to prevent resource exhaustion.
247const MAX_BATCH_CONCURRENCY: usize = 20;
248
249/// Fetch multiple URLs concurrently.
250///
251/// Each URL is processed independently — one failure doesn't affect others.
252/// Results are returned in the same order as the input requests.
253/// Concurrency is limited to `concurrency` (default: 5).
254///
255/// # Examples
256///
257/// ```no_run
258/// use fetchkit::{FetchRequest, batch_fetch};
259///
260/// # async fn example() -> Result<(), fetchkit::FetchError> {
261/// let requests = vec![
262///     FetchRequest::new("https://example.com").as_markdown(),
263///     FetchRequest::new("https://example.org").as_markdown(),
264/// ];
265/// let results = batch_fetch(requests, None).await;
266/// for result in &results {
267///     match result {
268///         Ok(resp) => println!("{}: {}", resp.url, resp.status_code),
269///         Err(e) => println!("Error: {}", e),
270///     }
271/// }
272/// # Ok(())
273/// # }
274/// ```
275pub async fn batch_fetch(
276    requests: Vec<FetchRequest>,
277    concurrency: Option<usize>,
278) -> Vec<Result<FetchResponse, FetchError>> {
279    let options = FetchOptions {
280        enable_markdown: true,
281        enable_text: true,
282        ..Default::default()
283    };
284    batch_fetch_with_options(requests, options, concurrency).await
285}
286
287/// Fetch multiple URLs concurrently with custom options.
288///
289/// Each URL is processed independently with the shared options.
290/// Returns results in the same order as input requests.
291pub async fn batch_fetch_with_options(
292    requests: Vec<FetchRequest>,
293    options: FetchOptions,
294    concurrency: Option<usize>,
295) -> Vec<Result<FetchResponse, FetchError>> {
296    use futures::stream::{self, StreamExt};
297    use std::sync::Arc;
298
299    let concurrency = concurrency
300        .unwrap_or(DEFAULT_BATCH_CONCURRENCY)
301        .clamp(1, MAX_BATCH_CONCURRENCY);
302    let num_requests = requests.len();
303    let options = Arc::new(options);
304
305    let mut indexed_results: Vec<(usize, Result<FetchResponse, FetchError>)> =
306        stream::iter(requests.into_iter().enumerate())
307            .map(|(idx, req)| {
308                let options = Arc::clone(&options);
309                async move {
310                    let registry = FetcherRegistry::for_options(&options);
311                    let result = registry.fetch(req, (*options).clone()).await;
312                    (idx, result)
313                }
314            })
315            .buffer_unordered(concurrency)
316            .collect()
317            .await;
318
319    // Sort by original index to preserve request order
320    indexed_results.sort_by_key(|(idx, _)| *idx);
321
322    // Pre-fill with errors, then replace with actual results
323    let mut results: Vec<Result<FetchResponse, FetchError>> = (0..num_requests)
324        .map(|_| Err(FetchError::MissingUrl))
325        .collect();
326    for (idx, result) in indexed_results {
327        results[idx] = result;
328    }
329
330    results
331}
332
333#[cfg(test)]
334mod tests {
335    use super::*;
336
337    #[tokio::test]
338    async fn test_fetch_empty_url() {
339        let req = FetchRequest::new("");
340        let result = fetch(req).await;
341        assert!(matches!(result, Err(FetchError::MissingUrl)));
342    }
343
344    #[tokio::test]
345    async fn test_fetch_invalid_scheme() {
346        let req = FetchRequest::new("ftp://example.com");
347        let result = fetch(req).await;
348        assert!(matches!(result, Err(FetchError::InvalidUrlScheme)));
349    }
350
351    #[tokio::test]
352    async fn test_fetch_options_default() {
353        let options = FetchOptions::default();
354        assert!(options.user_agent.is_none());
355        assert!(options.allow_prefixes.is_empty());
356        assert!(options.block_prefixes.is_empty());
357        assert!(!options.enable_markdown);
358        assert!(!options.enable_text);
359        // Safe by default: private IPs blocked
360        assert!(options.dns_policy.block_private);
361        assert!(options.max_body_size.is_none());
362        assert!(!options.enable_save_to_file);
363        assert!(!options.respect_proxy_env);
364        assert!(options.allowed_ports.is_empty());
365        assert!(options.blocked_hosts.is_empty());
366        assert!(!options.same_host_redirects_only);
367    }
368
369    #[test]
370    fn test_validate_url_blocks_configured_host_and_port() {
371        let options = FetchOptions {
372            allowed_ports: vec![443],
373            blocked_hosts: vec!["localhost".to_string(), ".internal".to_string()],
374            ..Default::default()
375        };
376
377        assert!(matches!(
378            options.validate_url(&Url::parse("https://api.internal").unwrap()),
379            Err(FetchError::BlockedUrl)
380        ));
381        assert!(matches!(
382            options.validate_url(&Url::parse("https://example.com:8443").unwrap()),
383            Err(FetchError::BlockedUrl)
384        ));
385        assert!(options
386            .validate_url(&Url::parse("https://example.com").unwrap())
387            .is_ok());
388    }
389
390    #[test]
391    fn test_validate_redirect_target_blocks_cross_host_when_enabled() {
392        let options = FetchOptions {
393            same_host_redirects_only: true,
394            ..Default::default()
395        };
396
397        let current = Url::parse("https://example.com/start").unwrap();
398        let next = Url::parse("https://www.example.com/end").unwrap();
399
400        assert!(matches!(
401            options.validate_redirect_target(&current, &next),
402            Err(FetchError::BlockedUrl)
403        ));
404    }
405
406    #[tokio::test]
407    async fn test_batch_fetch_multiple_urls() {
408        use wiremock::matchers::{method, path};
409        use wiremock::{Mock, MockServer, ResponseTemplate};
410
411        let server = MockServer::start().await;
412        Mock::given(method("GET"))
413            .and(path("/page1"))
414            .respond_with(
415                ResponseTemplate::new(200)
416                    .set_body_string("Page 1")
417                    .insert_header("content-type", "text/plain"),
418            )
419            .mount(&server)
420            .await;
421        Mock::given(method("GET"))
422            .and(path("/page2"))
423            .respond_with(
424                ResponseTemplate::new(200)
425                    .set_body_string("Page 2")
426                    .insert_header("content-type", "text/plain"),
427            )
428            .mount(&server)
429            .await;
430
431        let requests = vec![
432            FetchRequest::new(format!("{}/page1", server.uri())),
433            FetchRequest::new(format!("{}/page2", server.uri())),
434        ];
435
436        let options = FetchOptions {
437            enable_markdown: true,
438            dns_policy: DnsPolicy::allow_all(),
439            ..Default::default()
440        };
441        let results = batch_fetch_with_options(requests, options, None).await;
442
443        assert_eq!(results.len(), 2);
444        assert!(results[0]
445            .as_ref()
446            .unwrap()
447            .content
448            .as_deref()
449            .unwrap()
450            .contains("Page 1"));
451        assert!(results[1]
452            .as_ref()
453            .unwrap()
454            .content
455            .as_deref()
456            .unwrap()
457            .contains("Page 2"));
458    }
459
460    #[tokio::test]
461    async fn test_batch_fetch_partial_failure() {
462        use wiremock::matchers::{method, path};
463        use wiremock::{Mock, MockServer, ResponseTemplate};
464
465        let server = MockServer::start().await;
466        Mock::given(method("GET"))
467            .and(path("/ok"))
468            .respond_with(
469                ResponseTemplate::new(200)
470                    .set_body_string("OK")
471                    .insert_header("content-type", "text/plain"),
472            )
473            .mount(&server)
474            .await;
475
476        let requests = vec![
477            FetchRequest::new(format!("{}/ok", server.uri())),
478            FetchRequest::new(""), // Will fail with MissingUrl
479        ];
480
481        let options = FetchOptions {
482            dns_policy: DnsPolicy::allow_all(),
483            ..Default::default()
484        };
485        let results = batch_fetch_with_options(requests, options, None).await;
486
487        assert_eq!(results.len(), 2);
488        assert!(results[0].is_ok());
489        assert!(results[1].is_err());
490    }
491
492    #[tokio::test]
493    async fn test_batch_fetch_respects_concurrency_limit() {
494        // Just verify it works with concurrency=1
495        let requests = vec![
496            FetchRequest::new(""), // Will fail
497            FetchRequest::new(""), // Will fail
498        ];
499
500        let results = batch_fetch(requests, Some(1)).await;
501        assert_eq!(results.len(), 2);
502        assert!(results[0].is_err());
503        assert!(results[1].is_err());
504    }
505
506    #[tokio::test]
507    async fn test_batch_fetch_caps_caller_concurrency() {
508        // Caller-provided concurrency above the cap should still work safely.
509        let requests = vec![
510            FetchRequest::new(""), // Will fail
511            FetchRequest::new(""), // Will fail
512        ];
513
514        let results = batch_fetch(requests, Some(usize::MAX)).await;
515        assert_eq!(results.len(), 2);
516        assert!(results[0].is_err());
517        assert!(results[1].is_err());
518    }
519
520    #[tokio::test]
521    async fn test_batch_fetch_empty_input() {
522        let results = batch_fetch(vec![], None).await;
523        assert!(results.is_empty());
524    }
525}