1#[cfg(feature = "bot-auth")]
7use crate::bot_auth::BotAuthConfig;
8use crate::content::ContentProcessor;
9use crate::dns::DnsPolicy;
10use crate::documents::DocumentConversion;
11use crate::error::FetchError;
12use crate::fetchers::FetcherRegistry;
13use crate::transport::{HttpTransport, ReqwestTransport};
14use crate::types::{FetchRequest, FetchResponse};
15use std::sync::Arc;
16use url::Url;
17
18#[derive(Clone, Default)]
20pub struct FetchOptions {
21 pub user_agent: Option<String>,
23 pub allow_prefixes: Vec<String>,
25 pub block_prefixes: Vec<String>,
27 pub enable_markdown: bool,
29 pub enable_text: bool,
31 pub dns_policy: DnsPolicy,
33 pub max_body_size: Option<usize>,
36 pub enable_save_to_file: bool,
38 pub respect_proxy_env: bool,
40 pub allowed_ports: Vec<u16>,
42 pub blocked_hosts: Vec<String>,
44 pub same_host_redirects_only: bool,
46 #[doc(hidden)]
48 pub redirect_origin: Option<Url>,
49 pub enable_render_rakers: bool,
51 #[cfg(feature = "bot-auth")]
54 pub bot_auth: Option<BotAuthConfig>,
55 pub transport: Option<Arc<dyn HttpTransport>>,
62 pub document_conversion: Option<DocumentConversion>,
65 pub content_processors: Vec<Arc<dyn ContentProcessor>>,
67}
68
69impl std::fmt::Debug for FetchOptions {
71 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
72 let mut d = f.debug_struct("FetchOptions");
73 d.field("user_agent", &self.user_agent)
74 .field("allow_prefixes", &self.allow_prefixes)
75 .field("block_prefixes", &self.block_prefixes)
76 .field("enable_markdown", &self.enable_markdown)
77 .field("enable_text", &self.enable_text)
78 .field("dns_policy", &self.dns_policy)
79 .field("max_body_size", &self.max_body_size)
80 .field("enable_save_to_file", &self.enable_save_to_file)
81 .field("respect_proxy_env", &self.respect_proxy_env)
82 .field("allowed_ports", &self.allowed_ports)
83 .field("blocked_hosts", &self.blocked_hosts)
84 .field("same_host_redirects_only", &self.same_host_redirects_only)
85 .field("redirect_origin", &self.redirect_origin)
86 .field("enable_render_rakers", &self.enable_render_rakers);
87 #[cfg(feature = "bot-auth")]
88 d.field("bot_auth", &self.bot_auth);
89 d.field(
90 "transport",
91 &self
92 .transport
93 .as_ref()
94 .map(|_| "<custom>")
95 .unwrap_or("None"),
96 )
97 .field("document_conversion", &self.document_conversion)
98 .field(
99 "content_processors",
100 &self
101 .content_processors
102 .iter()
103 .map(|processor| processor.name())
104 .collect::<Vec<_>>(),
105 );
106 d.finish()
107 }
108}
109
110impl FetchOptions {
111 pub(crate) fn transport(&self) -> Arc<dyn HttpTransport> {
114 match &self.transport {
115 Some(t) => Arc::clone(t),
116 None => Arc::new(ReqwestTransport::new()),
117 }
118 }
119
120 pub(crate) fn validate_url(&self, url: &Url) -> Result<(), FetchError> {
121 self.validate_host(url)?;
122 self.validate_port(url)?;
123 Ok(())
124 }
125
126 pub(crate) fn validate_redirect_target(
127 &self,
128 current_url: &Url,
129 next_url: &Url,
130 ) -> Result<(), FetchError> {
131 self.validate_url(next_url)?;
132
133 if self.same_host_redirects_only
134 && normalized_host(current_url) != normalized_host(next_url)
135 {
136 return Err(FetchError::BlockedUrl);
137 }
138
139 if let Some(origin) = &self.redirect_origin {
140 let same_origin = origin.scheme() == next_url.scheme()
141 && normalized_host(origin) == normalized_host(next_url)
142 && origin.port_or_known_default() == next_url.port_or_known_default();
143 if !same_origin {
144 return Err(FetchError::BlockedUrl);
145 }
146 }
147
148 Ok(())
149 }
150
151 fn validate_host(&self, url: &Url) -> Result<(), FetchError> {
152 let Some(host) = normalized_host(url) else {
153 return Ok(());
154 };
155
156 if self
157 .blocked_hosts
158 .iter()
159 .any(|rule| host_matches_rule(&host, rule))
160 {
161 return Err(FetchError::BlockedUrl);
162 }
163
164 Ok(())
165 }
166
167 fn validate_port(&self, url: &Url) -> Result<(), FetchError> {
168 if self.allowed_ports.is_empty() {
169 return Ok(());
170 }
171
172 let port = url.port_or_known_default().ok_or(FetchError::BlockedUrl)?;
173 if self.allowed_ports.contains(&port) {
174 Ok(())
175 } else {
176 Err(FetchError::BlockedUrl)
177 }
178 }
179}
180
181fn normalized_host(url: &Url) -> Option<String> {
182 url.host_str()
183 .map(|host| host.trim_end_matches('.').to_ascii_lowercase())
184}
185
186fn host_matches_rule(host: &str, rule: &str) -> bool {
187 let normalized_rule = rule.trim_end_matches('.').to_ascii_lowercase();
188
189 if let Some(suffix) = normalized_rule.strip_prefix('.') {
190 host == suffix || host.ends_with(&format!(".{suffix}"))
191 } else {
192 host == normalized_rule
193 }
194}
195
196pub async fn fetch(req: FetchRequest) -> Result<FetchResponse, FetchError> {
214 let options = FetchOptions {
215 enable_markdown: true,
216 enable_text: true,
217 ..Default::default()
218 };
219 fetch_with_options(req, options).await
220}
221
222pub async fn fetch_with_options(
227 mut req: FetchRequest,
228 options: FetchOptions,
229) -> Result<FetchResponse, FetchError> {
230 if req.url.is_empty() {
232 return Err(FetchError::MissingUrl);
233 }
234 req.normalize_url_for_fetch()?;
235
236 if req.wants_crawl() {
237 return crate::crawl::crawl_fetch_with_options(req, options).await;
238 }
239
240 let registry = FetcherRegistry::for_options(&options);
241 registry.fetch(req, options).await
242}
243
244const DEFAULT_BATCH_CONCURRENCY: usize = 5;
246const MAX_BATCH_CONCURRENCY: usize = 20;
248
249pub async fn batch_fetch(
276 requests: Vec<FetchRequest>,
277 concurrency: Option<usize>,
278) -> Vec<Result<FetchResponse, FetchError>> {
279 let options = FetchOptions {
280 enable_markdown: true,
281 enable_text: true,
282 ..Default::default()
283 };
284 batch_fetch_with_options(requests, options, concurrency).await
285}
286
287pub async fn batch_fetch_with_options(
292 requests: Vec<FetchRequest>,
293 options: FetchOptions,
294 concurrency: Option<usize>,
295) -> Vec<Result<FetchResponse, FetchError>> {
296 use futures::stream::{self, StreamExt};
297 use std::sync::Arc;
298
299 let concurrency = concurrency
300 .unwrap_or(DEFAULT_BATCH_CONCURRENCY)
301 .clamp(1, MAX_BATCH_CONCURRENCY);
302 let num_requests = requests.len();
303 let options = Arc::new(options);
304
305 let mut indexed_results: Vec<(usize, Result<FetchResponse, FetchError>)> =
306 stream::iter(requests.into_iter().enumerate())
307 .map(|(idx, req)| {
308 let options = Arc::clone(&options);
309 async move {
310 let registry = FetcherRegistry::for_options(&options);
311 let result = registry.fetch(req, (*options).clone()).await;
312 (idx, result)
313 }
314 })
315 .buffer_unordered(concurrency)
316 .collect()
317 .await;
318
319 indexed_results.sort_by_key(|(idx, _)| *idx);
321
322 let mut results: Vec<Result<FetchResponse, FetchError>> = (0..num_requests)
324 .map(|_| Err(FetchError::MissingUrl))
325 .collect();
326 for (idx, result) in indexed_results {
327 results[idx] = result;
328 }
329
330 results
331}
332
333#[cfg(test)]
334mod tests {
335 use super::*;
336
337 #[tokio::test]
338 async fn test_fetch_empty_url() {
339 let req = FetchRequest::new("");
340 let result = fetch(req).await;
341 assert!(matches!(result, Err(FetchError::MissingUrl)));
342 }
343
344 #[tokio::test]
345 async fn test_fetch_invalid_scheme() {
346 let req = FetchRequest::new("ftp://example.com");
347 let result = fetch(req).await;
348 assert!(matches!(result, Err(FetchError::InvalidUrlScheme)));
349 }
350
351 #[tokio::test]
352 async fn test_fetch_options_default() {
353 let options = FetchOptions::default();
354 assert!(options.user_agent.is_none());
355 assert!(options.allow_prefixes.is_empty());
356 assert!(options.block_prefixes.is_empty());
357 assert!(!options.enable_markdown);
358 assert!(!options.enable_text);
359 assert!(options.dns_policy.block_private);
361 assert!(options.max_body_size.is_none());
362 assert!(!options.enable_save_to_file);
363 assert!(!options.respect_proxy_env);
364 assert!(options.allowed_ports.is_empty());
365 assert!(options.blocked_hosts.is_empty());
366 assert!(!options.same_host_redirects_only);
367 }
368
369 #[test]
370 fn test_validate_url_blocks_configured_host_and_port() {
371 let options = FetchOptions {
372 allowed_ports: vec![443],
373 blocked_hosts: vec!["localhost".to_string(), ".internal".to_string()],
374 ..Default::default()
375 };
376
377 assert!(matches!(
378 options.validate_url(&Url::parse("https://api.internal").unwrap()),
379 Err(FetchError::BlockedUrl)
380 ));
381 assert!(matches!(
382 options.validate_url(&Url::parse("https://example.com:8443").unwrap()),
383 Err(FetchError::BlockedUrl)
384 ));
385 assert!(options
386 .validate_url(&Url::parse("https://example.com").unwrap())
387 .is_ok());
388 }
389
390 #[test]
391 fn test_validate_redirect_target_blocks_cross_host_when_enabled() {
392 let options = FetchOptions {
393 same_host_redirects_only: true,
394 ..Default::default()
395 };
396
397 let current = Url::parse("https://example.com/start").unwrap();
398 let next = Url::parse("https://www.example.com/end").unwrap();
399
400 assert!(matches!(
401 options.validate_redirect_target(¤t, &next),
402 Err(FetchError::BlockedUrl)
403 ));
404 }
405
406 #[tokio::test]
407 async fn test_batch_fetch_multiple_urls() {
408 use wiremock::matchers::{method, path};
409 use wiremock::{Mock, MockServer, ResponseTemplate};
410
411 let server = MockServer::start().await;
412 Mock::given(method("GET"))
413 .and(path("/page1"))
414 .respond_with(
415 ResponseTemplate::new(200)
416 .set_body_string("Page 1")
417 .insert_header("content-type", "text/plain"),
418 )
419 .mount(&server)
420 .await;
421 Mock::given(method("GET"))
422 .and(path("/page2"))
423 .respond_with(
424 ResponseTemplate::new(200)
425 .set_body_string("Page 2")
426 .insert_header("content-type", "text/plain"),
427 )
428 .mount(&server)
429 .await;
430
431 let requests = vec![
432 FetchRequest::new(format!("{}/page1", server.uri())),
433 FetchRequest::new(format!("{}/page2", server.uri())),
434 ];
435
436 let options = FetchOptions {
437 enable_markdown: true,
438 dns_policy: DnsPolicy::allow_all(),
439 ..Default::default()
440 };
441 let results = batch_fetch_with_options(requests, options, None).await;
442
443 assert_eq!(results.len(), 2);
444 assert!(results[0]
445 .as_ref()
446 .unwrap()
447 .content
448 .as_deref()
449 .unwrap()
450 .contains("Page 1"));
451 assert!(results[1]
452 .as_ref()
453 .unwrap()
454 .content
455 .as_deref()
456 .unwrap()
457 .contains("Page 2"));
458 }
459
460 #[tokio::test]
461 async fn test_batch_fetch_partial_failure() {
462 use wiremock::matchers::{method, path};
463 use wiremock::{Mock, MockServer, ResponseTemplate};
464
465 let server = MockServer::start().await;
466 Mock::given(method("GET"))
467 .and(path("/ok"))
468 .respond_with(
469 ResponseTemplate::new(200)
470 .set_body_string("OK")
471 .insert_header("content-type", "text/plain"),
472 )
473 .mount(&server)
474 .await;
475
476 let requests = vec![
477 FetchRequest::new(format!("{}/ok", server.uri())),
478 FetchRequest::new(""), ];
480
481 let options = FetchOptions {
482 dns_policy: DnsPolicy::allow_all(),
483 ..Default::default()
484 };
485 let results = batch_fetch_with_options(requests, options, None).await;
486
487 assert_eq!(results.len(), 2);
488 assert!(results[0].is_ok());
489 assert!(results[1].is_err());
490 }
491
492 #[tokio::test]
493 async fn test_batch_fetch_respects_concurrency_limit() {
494 let requests = vec![
496 FetchRequest::new(""), FetchRequest::new(""), ];
499
500 let results = batch_fetch(requests, Some(1)).await;
501 assert_eq!(results.len(), 2);
502 assert!(results[0].is_err());
503 assert!(results[1].is_err());
504 }
505
506 #[tokio::test]
507 async fn test_batch_fetch_caps_caller_concurrency() {
508 let requests = vec![
510 FetchRequest::new(""), FetchRequest::new(""), ];
513
514 let results = batch_fetch(requests, Some(usize::MAX)).await;
515 assert_eq!(results.len(), 2);
516 assert!(results[0].is_err());
517 assert!(results[1].is_err());
518 }
519
520 #[tokio::test]
521 async fn test_batch_fetch_empty_input() {
522 let results = batch_fetch(vec![], None).await;
523 assert!(results.is_empty());
524 }
525}