1use std::borrow::Cow;
2use std::fmt;
3use std::io::Read as _;
4
5use dom_query::Document;
6use encoding_rs::{Encoding, UTF_8};
7use flate2::read::{DeflateDecoder, GzDecoder, ZlibDecoder};
8
9use super::{
10 FetchHttpHeaders, FetchProviderError, FetchProviderErrorCode, FetchTruncationReason,
11 MAX_FETCH_MAX_CHARS, MAX_FETCH_TITLE_BYTES,
12};
13
14pub const DEFAULT_FETCH_DECODED_BYTES: usize = 4 * 1024 * 1024;
16pub const MAX_FETCH_DECODED_BYTES: usize = 16 * 1024 * 1024;
18pub const DEFAULT_FETCH_HTML_BYTES: usize = 1024 * 1024;
20pub const DEFAULT_FETCH_HTML_ELEMENTS: usize = 50_000;
22
23const MIME_SNIFF_BYTES: usize = 512;
24
25#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
27pub struct FetchBodyLimits {
28 decoded_bytes: usize,
29 html_bytes: usize,
30 html_elements: usize,
31}
32
33impl FetchBodyLimits {
34 pub fn new(
41 max_decoded_bytes: usize,
42 max_html_bytes: usize,
43 max_html_elements: usize,
44 ) -> Result<Self, FetchProviderError> {
45 if !(1..=MAX_FETCH_DECODED_BYTES).contains(&max_decoded_bytes)
46 || !(1..=max_decoded_bytes).contains(&max_html_bytes)
47 || max_html_elements == 0
48 {
49 return Err(FetchProviderError::new(
50 FetchProviderErrorCode::InvalidConfiguration,
51 ));
52 }
53 Ok(Self {
54 decoded_bytes: max_decoded_bytes,
55 html_bytes: max_html_bytes,
56 html_elements: max_html_elements,
57 })
58 }
59
60 #[must_use]
62 pub const fn max_decoded_bytes(self) -> usize {
63 self.decoded_bytes
64 }
65
66 #[must_use]
68 pub const fn max_html_bytes(self) -> usize {
69 self.html_bytes
70 }
71
72 #[must_use]
74 pub const fn max_html_elements(self) -> usize {
75 self.html_elements
76 }
77}
78
79impl Default for FetchBodyLimits {
80 fn default() -> Self {
81 Self {
82 decoded_bytes: DEFAULT_FETCH_DECODED_BYTES,
83 html_bytes: DEFAULT_FETCH_HTML_BYTES,
84 html_elements: DEFAULT_FETCH_HTML_ELEMENTS,
85 }
86 }
87}
88
89#[derive(Debug, Clone, Copy, PartialEq, Eq)]
91pub enum FetchContentKind {
92 Html,
94 PlainText,
96 Json,
98}
99
100impl FetchContentKind {
101 #[must_use]
103 pub const fn mime_type(self) -> &'static str {
104 match self {
105 Self::Html => "text/html; charset=utf-8",
106 Self::PlainText => "text/plain; charset=utf-8",
107 Self::Json => "application/json; charset=utf-8",
108 }
109 }
110}
111
112#[derive(Clone, PartialEq, Eq)]
114pub struct DecodedFetchBody {
115 kind: FetchContentKind,
116 title: Option<String>,
117 body: String,
118 truncation: Option<FetchTruncationReason>,
119 decoded_bytes: usize,
120}
121
122impl DecodedFetchBody {
123 #[must_use]
125 pub const fn kind(&self) -> FetchContentKind {
126 self.kind
127 }
128
129 #[must_use]
131 pub const fn mime_type(&self) -> &'static str {
132 self.kind.mime_type()
133 }
134
135 #[must_use]
137 pub fn title(&self) -> Option<&str> {
138 self.title.as_deref()
139 }
140
141 #[must_use]
143 pub fn body(&self) -> &str {
144 &self.body
145 }
146
147 #[must_use]
149 pub const fn truncation(&self) -> Option<FetchTruncationReason> {
150 self.truncation
151 }
152
153 #[must_use]
155 pub const fn decoded_bytes(&self) -> usize {
156 self.decoded_bytes
157 }
158}
159
160impl fmt::Debug for DecodedFetchBody {
161 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
162 formatter
163 .debug_struct("DecodedFetchBody")
164 .field("kind", &self.kind)
165 .field("has_title", &self.title.is_some())
166 .field("body_chars", &self.body.chars().count())
167 .field("truncation", &self.truncation)
168 .field("decoded_bytes", &self.decoded_bytes)
169 .finish_non_exhaustive()
170 }
171}
172
173#[derive(Debug, Clone, Copy, PartialEq, Eq)]
175pub struct FetchBodyDecoder {
176 limits: FetchBodyLimits,
177}
178
179impl FetchBodyDecoder {
180 #[must_use]
182 pub const fn new(limits: FetchBodyLimits) -> Self {
183 Self { limits }
184 }
185
186 #[must_use]
188 pub const fn limits(self) -> FetchBodyLimits {
189 self.limits
190 }
191
192 pub fn decode(
200 &self,
201 headers: &FetchHttpHeaders,
202 raw: &[u8],
203 max_chars: usize,
204 ) -> Result<DecodedFetchBody, FetchProviderError> {
205 if !(1..=MAX_FETCH_MAX_CHARS).contains(&max_chars) {
206 return Err(FetchProviderError::new(
207 FetchProviderErrorCode::InvalidRequest,
208 ));
209 }
210 let decoded = decode_content_encoding(
211 headers.content_encoding(),
212 raw,
213 self.limits.max_decoded_bytes(),
214 )?;
215 let decoded_bytes = decoded.len();
216 let declared = declared_content(headers.content_type())?;
217 let sniffed = sniff_content(&decoded);
218 let kind = select_kind(declared.as_ref().map(|value| value.kind), sniffed)?;
219 let encoding = select_encoding(declared.as_ref(), kind, &decoded)?;
220 let text = decode_text(&decoded, encoding)?;
221 let (title, body) = match kind {
222 FetchContentKind::Html => self.extract_html(&text)?,
223 FetchContentKind::PlainText => (None, normalize_plain_text(&text)),
224 FetchContentKind::Json => {
225 serde_json::from_str::<serde_json::Value>(&text).map_err(|_| {
226 FetchProviderError::new(FetchProviderErrorCode::MalformedResponse)
227 })?;
228 (None, normalize_plain_text(text.trim()))
229 }
230 };
231 let (body, truncation) = truncate_body(&body, max_chars);
232 Ok(DecodedFetchBody {
233 kind,
234 title,
235 body,
236 truncation,
237 decoded_bytes,
238 })
239 }
240
241 fn extract_html(self, html: &str) -> Result<(Option<String>, String), FetchProviderError> {
242 if html.len() > self.limits.max_html_bytes()
243 || html.bytes().filter(|byte| *byte == b'<').count() > self.limits.max_html_elements()
244 {
245 return Err(FetchProviderError::new(
246 FetchProviderErrorCode::ResponseTooLarge,
247 ));
248 }
249 let document = Document::from(html);
250 document
251 .select("script, style, noscript, template, svg")
252 .remove();
253 let title = bounded_title(&normalize_inline_text(&document.select("title").text()));
254 let body_selection = document.select("body");
255 let visible_text = if body_selection.exists() {
256 body_selection.formatted_text()
257 } else {
258 document.formatted_text()
259 };
260 let body = normalize_plain_text(visible_text.trim());
261 Ok((title, body))
262 }
263}
264
265impl Default for FetchBodyDecoder {
266 fn default() -> Self {
267 Self::new(FetchBodyLimits::default())
268 }
269}
270
271#[derive(Debug, Clone, Copy, PartialEq, Eq)]
272enum SniffedContent {
273 Html,
274 Json,
275 Text,
276 Binary,
277}
278
279#[derive(Debug, Clone, Copy)]
280struct DeclaredContent {
281 kind: FetchContentKind,
282 encoding: Option<&'static Encoding>,
283}
284
285fn declared_content(value: Option<&str>) -> Result<Option<DeclaredContent>, FetchProviderError> {
286 let Some(value) = value else {
287 return Ok(None);
288 };
289 let parsed = value
290 .parse::<mime::Mime>()
291 .map_err(|_| FetchProviderError::new(FetchProviderErrorCode::UnsupportedMime))?;
292 let kind = if parsed.type_() == mime::TEXT && parsed.subtype() == mime::HTML {
293 FetchContentKind::Html
294 } else if parsed.type_() == mime::TEXT && parsed.subtype() == mime::PLAIN {
295 FetchContentKind::PlainText
296 } else if parsed.type_() == mime::APPLICATION
297 && (parsed.subtype() == mime::JSON || parsed.suffix() == Some(mime::JSON))
298 {
299 FetchContentKind::Json
300 } else if parsed.type_() == mime::APPLICATION
301 && parsed.subtype().as_str() == "xhtml"
302 && parsed.suffix() == Some(mime::XML)
303 {
304 FetchContentKind::Html
305 } else {
306 return Err(FetchProviderError::new(
307 FetchProviderErrorCode::UnsupportedMime,
308 ));
309 };
310 let encoding = parsed
311 .get_param(mime::CHARSET)
312 .map(|label| {
313 Encoding::for_label(label.as_str().as_bytes())
314 .ok_or_else(|| FetchProviderError::new(FetchProviderErrorCode::UnsupportedMime))
315 })
316 .transpose()?;
317 Ok(Some(DeclaredContent { kind, encoding }))
318}
319
320fn sniff_content(bytes: &[u8]) -> SniffedContent {
321 if bytes.contains(&0) {
322 return SniffedContent::Binary;
323 }
324 let prefix = String::from_utf8_lossy(&bytes[..bytes.len().min(MIME_SNIFF_BYTES)]);
325 let normalized = prefix.trim_start().to_ascii_lowercase();
326 if normalized.starts_with("<!doctype html")
327 || normalized.starts_with("<html")
328 || normalized.starts_with("<head")
329 || normalized.starts_with("<body")
330 {
331 SniffedContent::Html
332 } else if normalized.starts_with('{') || normalized.starts_with('[') {
333 SniffedContent::Json
334 } else {
335 SniffedContent::Text
336 }
337}
338
339fn select_kind(
340 declared: Option<FetchContentKind>,
341 sniffed: SniffedContent,
342) -> Result<FetchContentKind, FetchProviderError> {
343 match (declared, sniffed) {
344 (_, SniffedContent::Binary) => Err(FetchProviderError::new(
345 FetchProviderErrorCode::UnsupportedMime,
346 )),
347 (Some(FetchContentKind::Json), _) | (None, SniffedContent::Json) => {
348 Ok(FetchContentKind::Json)
349 }
350 (Some(FetchContentKind::Html), _) | (_, SniffedContent::Html) => Ok(FetchContentKind::Html),
351 (Some(FetchContentKind::PlainText), _) | (None, SniffedContent::Text) => {
352 Ok(FetchContentKind::PlainText)
353 }
354 }
355}
356
357fn select_encoding(
358 declared: Option<&DeclaredContent>,
359 kind: FetchContentKind,
360 bytes: &[u8],
361) -> Result<(&'static Encoding, usize), FetchProviderError> {
362 if let Some((encoding, bom_bytes)) = Encoding::for_bom(bytes) {
363 if kind == FetchContentKind::Json && encoding != UTF_8 {
364 return Err(FetchProviderError::new(
365 FetchProviderErrorCode::MalformedResponse,
366 ));
367 }
368 return Ok((encoding, bom_bytes));
369 }
370 let encoding = declared.and_then(|value| value.encoding).unwrap_or(UTF_8);
371 if kind == FetchContentKind::Json && encoding != UTF_8 {
372 return Err(FetchProviderError::new(
373 FetchProviderErrorCode::MalformedResponse,
374 ));
375 }
376 Ok((encoding, 0))
377}
378
379fn decode_text<'a>(
380 bytes: &'a [u8],
381 (encoding, offset): (&'static Encoding, usize),
382) -> Result<Cow<'a, str>, FetchProviderError> {
383 let (text, _, had_errors) = encoding.decode(&bytes[offset..]);
384 if had_errors || text.contains('\0') {
385 Err(FetchProviderError::new(
386 FetchProviderErrorCode::MalformedResponse,
387 ))
388 } else {
389 Ok(text)
390 }
391}
392
393fn decode_content_encoding(
394 encoding: Option<&str>,
395 raw: &[u8],
396 max_bytes: usize,
397) -> Result<Vec<u8>, FetchProviderError> {
398 match encoding
399 .map(str::trim)
400 .map(str::to_ascii_lowercase)
401 .as_deref()
402 {
403 None | Some("" | "identity") => bounded_identity(raw, max_bytes),
404 Some("gzip" | "x-gzip") => read_bounded(GzDecoder::new(raw), max_bytes),
405 Some("deflate") => read_bounded(ZlibDecoder::new(raw), max_bytes).or_else(|error| {
406 if error.code() == FetchProviderErrorCode::MalformedResponse {
407 read_bounded(DeflateDecoder::new(raw), max_bytes)
408 } else {
409 Err(error)
410 }
411 }),
412 Some("br") => read_bounded(brotli::Decompressor::new(raw, 4096), max_bytes),
413 Some(_) => Err(FetchProviderError::new(
414 FetchProviderErrorCode::UnsupportedMime,
415 )),
416 }
417}
418
419fn bounded_identity(raw: &[u8], max_bytes: usize) -> Result<Vec<u8>, FetchProviderError> {
420 if raw.len() > max_bytes {
421 Err(FetchProviderError::new(
422 FetchProviderErrorCode::ResponseTooLarge,
423 ))
424 } else {
425 Ok(raw.to_vec())
426 }
427}
428
429fn read_bounded(
430 reader: impl std::io::Read,
431 max_bytes: usize,
432) -> Result<Vec<u8>, FetchProviderError> {
433 let mut reader = reader.take(max_bytes as u64 + 1);
434 let mut bytes = Vec::new();
435 reader
436 .read_to_end(&mut bytes)
437 .map_err(|_| FetchProviderError::new(FetchProviderErrorCode::MalformedResponse))?;
438 if bytes.len() > max_bytes {
439 Err(FetchProviderError::new(
440 FetchProviderErrorCode::ResponseTooLarge,
441 ))
442 } else {
443 Ok(bytes)
444 }
445}
446
447fn bounded_title(value: &str) -> Option<String> {
448 if value.is_empty() {
449 return None;
450 }
451 let mut end = value.len().min(MAX_FETCH_TITLE_BYTES);
452 while !value.is_char_boundary(end) {
453 end -= 1;
454 }
455 Some(value[..end].to_owned())
456}
457
458fn normalize_plain_text(value: &str) -> String {
459 value
460 .replace("\r\n", "\n")
461 .replace('\r', "\n")
462 .chars()
463 .filter(|character| !character.is_control() || matches!(character, '\n' | '\t'))
464 .collect::<String>()
465 .trim()
466 .to_owned()
467}
468
469fn normalize_inline_text(value: &str) -> String {
470 value.split_whitespace().collect::<Vec<_>>().join(" ")
471}
472
473fn truncate_body(value: &str, max_chars: usize) -> (String, Option<FetchTruncationReason>) {
474 let mut characters = value.chars();
475 let truncated = characters.by_ref().take(max_chars).collect::<String>();
476 if characters.next().is_some() {
477 (truncated, Some(FetchTruncationReason::BodyCharacters))
478 } else {
479 (truncated, None)
480 }
481}