link_r/fetch/
mod.rs

1// Author: support@lituus.io
2// Copyright: lituus-io, all rights reserved
3//
4//! Fetching: retrieve one resource's bytes.
5//!
6//! [`Fetcher`] is a GAT async trait so the auth refresh future flows through
7//! unboxed. The trait itself is dependency-free (so the crawler can be tested with
8//! a mock); the concrete [`HttpFetcher`] (feature `http`) is
9//! gated. Filesystem reads live in the `fs` source, which reads files directly.
10
11#[cfg(feature = "http")]
12pub mod http;
13#[cfg(feature = "http")]
14#[cfg_attr(not(feature = "robots"), allow(unused_imports))]
15// consumed by the robots UA-parity check
16pub(crate) use http::DEFAULT_UA;
17
18#[cfg(feature = "http")]
19pub use http::HttpFetcher;
20
21use crate::error::Result;
22use crate::payload::DocPayload;
23use crate::resource::Resource;
24use crate::resource::ResourceKind;
25use compact_str::CompactString;
26use std::future::Future;
27use url::Url;
28
29/// Metadata about a fetched resource.
30#[derive(Clone, Debug)]
31pub struct FetchMeta {
32    /// The content kind, classified from the `Content-Type` (or path).
33    pub kind: ResourceKind,
34    /// A change token (`ETag`/SHA), if the source provided one.
35    pub etag: Option<CompactString>,
36    /// The transport status (e.g. HTTP status; 200 for non-HTTP).
37    pub status: u16,
38    /// The final URL after any redirects, when it differs from the requested one.
39    /// The crawler scope-checks and canonicalizes against this so a redirect can
40    /// neither escape the crawl scope nor split the canonical dedup key. `None`
41    /// means "no redirect happened" (or the source doesn't redirect).
42    pub final_url: Option<Url>,
43}
44
45/// A fetched resource: its metadata plus body payload.
46#[derive(Debug)]
47pub struct Fetched<'a> {
48    /// Metadata about the fetch.
49    pub meta: FetchMeta,
50    /// The body, in whichever zero-copy representation the fetcher produced.
51    pub payload: DocPayload<'a>,
52}
53
54/// Options controlling a fetch.
55#[derive(Clone, Copy, Debug, Default)]
56pub struct FetchOptions<'a> {
57    /// Conditional fetch: send `If-None-Match` and treat 304 as "unchanged".
58    pub if_none_match: Option<&'a str>,
59    /// Skip bodies larger than this many bytes.
60    pub max_bytes: Option<u64>,
61    /// Override the User-Agent for this fetch.
62    pub user_agent: Option<&'a str>,
63}
64
65/// Retrieves a single resource's bytes.
66pub trait Fetcher: Send + Sync {
67    /// The future returned by [`Fetcher::fetch`].
68    type FetchFuture<'a>: Future<Output = Result<Fetched<'a>>> + Send + 'a
69    where
70        Self: 'a;
71
72    /// Fetch `resource`.
73    fn fetch<'a>(&'a self, resource: &'a Resource, opts: FetchOptions<'a>)
74        -> Self::FetchFuture<'a>;
75}
link_r/fetch/mod.rs

link_r/fetch/
mod.rs
