Skip to main content

tea_coding_tools/web_fetch/
body.rs

1use std::borrow::Cow;
2use std::fmt;
3use std::io::Read as _;
4
5use dom_query::Document;
6use encoding_rs::{Encoding, UTF_8};
7use flate2::read::{DeflateDecoder, GzDecoder, ZlibDecoder};
8
9use super::{
10    FetchHttpHeaders, FetchProviderError, FetchProviderErrorCode, FetchTruncationReason,
11    MAX_FETCH_MAX_CHARS, MAX_FETCH_TITLE_BYTES,
12};
13
14/// Default maximum decoded response bytes accepted by the body pipeline.
15pub const DEFAULT_FETCH_DECODED_BYTES: usize = 4 * 1024 * 1024;
16/// Absolute maximum decoded response bytes accepted by body configuration.
17pub const MAX_FETCH_DECODED_BYTES: usize = 16 * 1024 * 1024;
18/// Default maximum HTML input bytes passed to the parser.
19pub const DEFAULT_FETCH_HTML_BYTES: usize = 1024 * 1024;
20/// Default maximum opening-tag markers accepted before HTML parsing.
21pub const DEFAULT_FETCH_HTML_ELEMENTS: usize = 50_000;
22
23const MIME_SNIFF_BYTES: usize = 512;
24
25/// Independent decoded-byte and HTML-parser limits.
26#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
27pub struct FetchBodyLimits {
28    decoded_bytes: usize,
29    html_bytes: usize,
30    html_elements: usize,
31}
32
33impl FetchBodyLimits {
34    /// Creates response-body limits.
35    ///
36    /// # Errors
37    ///
38    /// Rejects zero values, decoded bounds above 16 MiB, and HTML bounds above
39    /// the decoded-byte limit.
40    pub fn new(
41        max_decoded_bytes: usize,
42        max_html_bytes: usize,
43        max_html_elements: usize,
44    ) -> Result<Self, FetchProviderError> {
45        if !(1..=MAX_FETCH_DECODED_BYTES).contains(&max_decoded_bytes)
46            || !(1..=max_decoded_bytes).contains(&max_html_bytes)
47            || max_html_elements == 0
48        {
49            return Err(FetchProviderError::new(
50                FetchProviderErrorCode::InvalidConfiguration,
51            ));
52        }
53        Ok(Self {
54            decoded_bytes: max_decoded_bytes,
55            html_bytes: max_html_bytes,
56            html_elements: max_html_elements,
57        })
58    }
59
60    /// Returns the decoded-byte limit.
61    #[must_use]
62    pub const fn max_decoded_bytes(self) -> usize {
63        self.decoded_bytes
64    }
65
66    /// Returns the HTML input byte limit.
67    #[must_use]
68    pub const fn max_html_bytes(self) -> usize {
69        self.html_bytes
70    }
71
72    /// Returns the HTML element-marker limit.
73    #[must_use]
74    pub const fn max_html_elements(self) -> usize {
75        self.html_elements
76    }
77}
78
79impl Default for FetchBodyLimits {
80    fn default() -> Self {
81        Self {
82            decoded_bytes: DEFAULT_FETCH_DECODED_BYTES,
83            html_bytes: DEFAULT_FETCH_HTML_BYTES,
84            html_elements: DEFAULT_FETCH_HTML_ELEMENTS,
85        }
86    }
87}
88
89/// Canonical response content kind retained by the fetch result.
90#[derive(Debug, Clone, Copy, PartialEq, Eq)]
91pub enum FetchContentKind {
92    /// HTML or XHTML converted to bounded visible text.
93    Html,
94    /// Plain text.
95    PlainText,
96    /// Valid JSON text.
97    Json,
98}
99
100impl FetchContentKind {
101    /// Returns the canonical MIME type.
102    #[must_use]
103    pub const fn mime_type(self) -> &'static str {
104        match self {
105            Self::Html => "text/html; charset=utf-8",
106            Self::PlainText => "text/plain; charset=utf-8",
107            Self::Json => "application/json; charset=utf-8",
108        }
109    }
110}
111
112/// Normalized body ready for a provider-neutral [`super::FetchResult`].
113#[derive(Clone, PartialEq, Eq)]
114pub struct DecodedFetchBody {
115    kind: FetchContentKind,
116    title: Option<String>,
117    body: String,
118    truncation: Option<FetchTruncationReason>,
119    decoded_bytes: usize,
120}
121
122impl DecodedFetchBody {
123    /// Returns the canonical content kind.
124    #[must_use]
125    pub const fn kind(&self) -> FetchContentKind {
126        self.kind
127    }
128
129    /// Returns the canonical MIME type.
130    #[must_use]
131    pub const fn mime_type(&self) -> &'static str {
132        self.kind.mime_type()
133    }
134
135    /// Returns the optional bounded HTML title.
136    #[must_use]
137    pub fn title(&self) -> Option<&str> {
138        self.title.as_deref()
139    }
140
141    /// Returns normalized bounded body text.
142    #[must_use]
143    pub fn body(&self) -> &str {
144        &self.body
145    }
146
147    /// Returns the output truncation reason.
148    #[must_use]
149    pub const fn truncation(&self) -> Option<FetchTruncationReason> {
150        self.truncation
151    }
152
153    /// Returns bytes after content-encoding decompression.
154    #[must_use]
155    pub const fn decoded_bytes(&self) -> usize {
156        self.decoded_bytes
157    }
158}
159
160impl fmt::Debug for DecodedFetchBody {
161    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
162        formatter
163            .debug_struct("DecodedFetchBody")
164            .field("kind", &self.kind)
165            .field("has_title", &self.title.is_some())
166            .field("body_chars", &self.body.chars().count())
167            .field("truncation", &self.truncation)
168            .field("decoded_bytes", &self.decoded_bytes)
169            .finish_non_exhaustive()
170    }
171}
172
173/// Bounded decoder and extractor for raw HTTP response bytes.
174#[derive(Debug, Clone, Copy, PartialEq, Eq)]
175pub struct FetchBodyDecoder {
176    limits: FetchBodyLimits,
177}
178
179impl FetchBodyDecoder {
180    /// Creates a decoder with explicit independent limits.
181    #[must_use]
182    pub const fn new(limits: FetchBodyLimits) -> Self {
183        Self { limits }
184    }
185
186    /// Returns the immutable decoding and parser limits.
187    #[must_use]
188    pub const fn limits(self) -> FetchBodyLimits {
189        self.limits
190    }
191
192    /// Decodes, classifies, parses, and bounds one raw response body.
193    ///
194    /// # Errors
195    ///
196    /// Rejects unsupported content encodings/MIME types, invalid charset data,
197    /// decompression limit violations, malformed JSON, binary content, and HTML
198    /// parser inputs above the configured byte/element bounds.
199    pub fn decode(
200        &self,
201        headers: &FetchHttpHeaders,
202        raw: &[u8],
203        max_chars: usize,
204    ) -> Result<DecodedFetchBody, FetchProviderError> {
205        if !(1..=MAX_FETCH_MAX_CHARS).contains(&max_chars) {
206            return Err(FetchProviderError::new(
207                FetchProviderErrorCode::InvalidRequest,
208            ));
209        }
210        let decoded = decode_content_encoding(
211            headers.content_encoding(),
212            raw,
213            self.limits.max_decoded_bytes(),
214        )?;
215        let decoded_bytes = decoded.len();
216        let declared = declared_content(headers.content_type())?;
217        let sniffed = sniff_content(&decoded);
218        let kind = select_kind(declared.as_ref().map(|value| value.kind), sniffed)?;
219        let encoding = select_encoding(declared.as_ref(), kind, &decoded)?;
220        let text = decode_text(&decoded, encoding)?;
221        let (title, body) = match kind {
222            FetchContentKind::Html => self.extract_html(&text)?,
223            FetchContentKind::PlainText => (None, normalize_plain_text(&text)),
224            FetchContentKind::Json => {
225                serde_json::from_str::<serde_json::Value>(&text).map_err(|_| {
226                    FetchProviderError::new(FetchProviderErrorCode::MalformedResponse)
227                })?;
228                (None, normalize_plain_text(text.trim()))
229            }
230        };
231        let (body, truncation) = truncate_body(&body, max_chars);
232        Ok(DecodedFetchBody {
233            kind,
234            title,
235            body,
236            truncation,
237            decoded_bytes,
238        })
239    }
240
241    fn extract_html(self, html: &str) -> Result<(Option<String>, String), FetchProviderError> {
242        if html.len() > self.limits.max_html_bytes()
243            || html.bytes().filter(|byte| *byte == b'<').count() > self.limits.max_html_elements()
244        {
245            return Err(FetchProviderError::new(
246                FetchProviderErrorCode::ResponseTooLarge,
247            ));
248        }
249        let document = Document::from(html);
250        document
251            .select("script, style, noscript, template, svg")
252            .remove();
253        let title = bounded_title(&normalize_inline_text(&document.select("title").text()));
254        let body_selection = document.select("body");
255        let visible_text = if body_selection.exists() {
256            body_selection.formatted_text()
257        } else {
258            document.formatted_text()
259        };
260        let body = normalize_plain_text(visible_text.trim());
261        Ok((title, body))
262    }
263}
264
265impl Default for FetchBodyDecoder {
266    fn default() -> Self {
267        Self::new(FetchBodyLimits::default())
268    }
269}
270
271#[derive(Debug, Clone, Copy, PartialEq, Eq)]
272enum SniffedContent {
273    Html,
274    Json,
275    Text,
276    Binary,
277}
278
279#[derive(Debug, Clone, Copy)]
280struct DeclaredContent {
281    kind: FetchContentKind,
282    encoding: Option<&'static Encoding>,
283}
284
285fn declared_content(value: Option<&str>) -> Result<Option<DeclaredContent>, FetchProviderError> {
286    let Some(value) = value else {
287        return Ok(None);
288    };
289    let parsed = value
290        .parse::<mime::Mime>()
291        .map_err(|_| FetchProviderError::new(FetchProviderErrorCode::UnsupportedMime))?;
292    let kind = if parsed.type_() == mime::TEXT && parsed.subtype() == mime::HTML {
293        FetchContentKind::Html
294    } else if parsed.type_() == mime::TEXT && parsed.subtype() == mime::PLAIN {
295        FetchContentKind::PlainText
296    } else if parsed.type_() == mime::APPLICATION
297        && (parsed.subtype() == mime::JSON || parsed.suffix() == Some(mime::JSON))
298    {
299        FetchContentKind::Json
300    } else if parsed.type_() == mime::APPLICATION
301        && parsed.subtype().as_str() == "xhtml"
302        && parsed.suffix() == Some(mime::XML)
303    {
304        FetchContentKind::Html
305    } else {
306        return Err(FetchProviderError::new(
307            FetchProviderErrorCode::UnsupportedMime,
308        ));
309    };
310    let encoding = parsed
311        .get_param(mime::CHARSET)
312        .map(|label| {
313            Encoding::for_label(label.as_str().as_bytes())
314                .ok_or_else(|| FetchProviderError::new(FetchProviderErrorCode::UnsupportedMime))
315        })
316        .transpose()?;
317    Ok(Some(DeclaredContent { kind, encoding }))
318}
319
320fn sniff_content(bytes: &[u8]) -> SniffedContent {
321    if bytes.contains(&0) {
322        return SniffedContent::Binary;
323    }
324    let prefix = String::from_utf8_lossy(&bytes[..bytes.len().min(MIME_SNIFF_BYTES)]);
325    let normalized = prefix.trim_start().to_ascii_lowercase();
326    if normalized.starts_with("<!doctype html")
327        || normalized.starts_with("<html")
328        || normalized.starts_with("<head")
329        || normalized.starts_with("<body")
330    {
331        SniffedContent::Html
332    } else if normalized.starts_with('{') || normalized.starts_with('[') {
333        SniffedContent::Json
334    } else {
335        SniffedContent::Text
336    }
337}
338
339fn select_kind(
340    declared: Option<FetchContentKind>,
341    sniffed: SniffedContent,
342) -> Result<FetchContentKind, FetchProviderError> {
343    match (declared, sniffed) {
344        (_, SniffedContent::Binary) => Err(FetchProviderError::new(
345            FetchProviderErrorCode::UnsupportedMime,
346        )),
347        (Some(FetchContentKind::Json), _) | (None, SniffedContent::Json) => {
348            Ok(FetchContentKind::Json)
349        }
350        (Some(FetchContentKind::Html), _) | (_, SniffedContent::Html) => Ok(FetchContentKind::Html),
351        (Some(FetchContentKind::PlainText), _) | (None, SniffedContent::Text) => {
352            Ok(FetchContentKind::PlainText)
353        }
354    }
355}
356
357fn select_encoding(
358    declared: Option<&DeclaredContent>,
359    kind: FetchContentKind,
360    bytes: &[u8],
361) -> Result<(&'static Encoding, usize), FetchProviderError> {
362    if let Some((encoding, bom_bytes)) = Encoding::for_bom(bytes) {
363        if kind == FetchContentKind::Json && encoding != UTF_8 {
364            return Err(FetchProviderError::new(
365                FetchProviderErrorCode::MalformedResponse,
366            ));
367        }
368        return Ok((encoding, bom_bytes));
369    }
370    let encoding = declared.and_then(|value| value.encoding).unwrap_or(UTF_8);
371    if kind == FetchContentKind::Json && encoding != UTF_8 {
372        return Err(FetchProviderError::new(
373            FetchProviderErrorCode::MalformedResponse,
374        ));
375    }
376    Ok((encoding, 0))
377}
378
379fn decode_text<'a>(
380    bytes: &'a [u8],
381    (encoding, offset): (&'static Encoding, usize),
382) -> Result<Cow<'a, str>, FetchProviderError> {
383    let (text, _, had_errors) = encoding.decode(&bytes[offset..]);
384    if had_errors || text.contains('\0') {
385        Err(FetchProviderError::new(
386            FetchProviderErrorCode::MalformedResponse,
387        ))
388    } else {
389        Ok(text)
390    }
391}
392
393fn decode_content_encoding(
394    encoding: Option<&str>,
395    raw: &[u8],
396    max_bytes: usize,
397) -> Result<Vec<u8>, FetchProviderError> {
398    match encoding
399        .map(str::trim)
400        .map(str::to_ascii_lowercase)
401        .as_deref()
402    {
403        None | Some("" | "identity") => bounded_identity(raw, max_bytes),
404        Some("gzip" | "x-gzip") => read_bounded(GzDecoder::new(raw), max_bytes),
405        Some("deflate") => read_bounded(ZlibDecoder::new(raw), max_bytes).or_else(|error| {
406            if error.code() == FetchProviderErrorCode::MalformedResponse {
407                read_bounded(DeflateDecoder::new(raw), max_bytes)
408            } else {
409                Err(error)
410            }
411        }),
412        Some("br") => read_bounded(brotli::Decompressor::new(raw, 4096), max_bytes),
413        Some(_) => Err(FetchProviderError::new(
414            FetchProviderErrorCode::UnsupportedMime,
415        )),
416    }
417}
418
419fn bounded_identity(raw: &[u8], max_bytes: usize) -> Result<Vec<u8>, FetchProviderError> {
420    if raw.len() > max_bytes {
421        Err(FetchProviderError::new(
422            FetchProviderErrorCode::ResponseTooLarge,
423        ))
424    } else {
425        Ok(raw.to_vec())
426    }
427}
428
429fn read_bounded(
430    reader: impl std::io::Read,
431    max_bytes: usize,
432) -> Result<Vec<u8>, FetchProviderError> {
433    let mut reader = reader.take(max_bytes as u64 + 1);
434    let mut bytes = Vec::new();
435    reader
436        .read_to_end(&mut bytes)
437        .map_err(|_| FetchProviderError::new(FetchProviderErrorCode::MalformedResponse))?;
438    if bytes.len() > max_bytes {
439        Err(FetchProviderError::new(
440            FetchProviderErrorCode::ResponseTooLarge,
441        ))
442    } else {
443        Ok(bytes)
444    }
445}
446
447fn bounded_title(value: &str) -> Option<String> {
448    if value.is_empty() {
449        return None;
450    }
451    let mut end = value.len().min(MAX_FETCH_TITLE_BYTES);
452    while !value.is_char_boundary(end) {
453        end -= 1;
454    }
455    Some(value[..end].to_owned())
456}
457
458fn normalize_plain_text(value: &str) -> String {
459    value
460        .replace("\r\n", "\n")
461        .replace('\r', "\n")
462        .chars()
463        .filter(|character| !character.is_control() || matches!(character, '\n' | '\t'))
464        .collect::<String>()
465        .trim()
466        .to_owned()
467}
468
469fn normalize_inline_text(value: &str) -> String {
470    value.split_whitespace().collect::<Vec<_>>().join(" ")
471}
472
473fn truncate_body(value: &str, max_chars: usize) -> (String, Option<FetchTruncationReason>) {
474    let mut characters = value.chars();
475    let truncated = characters.by_ref().take(max_chars).collect::<String>();
476    if characters.next().is_some() {
477        (truncated, Some(FetchTruncationReason::BodyCharacters))
478    } else {
479        (truncated, None)
480    }
481}