webfetch/convert/
mod.rs

1//! Output dispatcher: routes an HTML document to the requested format.
2
3pub mod markdown;
4pub mod structured;
5pub mod text;
6
7use scraper::Html;
8
9use crate::compress::compress_block;
10use crate::types::{ContentType, UrlReference};
11
12/// Elements whose contents never belong in extracted output (scripts,
13/// styling, embedded documents). Shared by every walker so the formats
14/// agree on what to drop.
15pub(crate) fn is_skippable(name: &str) -> bool {
16    matches!(
17        name,
18        "script" | "style" | "noscript" | "svg" | "head" | "template" | "iframe"
19    )
20}
21
22/// A converted document: the rendered `content` plus the references it cites.
23///
24/// `content` never carries the trailing `References:` block — assembling that
25/// is [`crate::refs::fit_to_budget`]'s job, because whether a reference survives
26/// depends on whether the (possibly truncated) body still cites it.
27pub struct Converted {
28    pub content: String,
29    pub references: Vec<UrlReference>,
30}
31
32/// Convert a parsed document to the requested content type.
33pub fn convert_parsed(doc: &Html, base_url: &str, content_type: ContentType) -> Converted {
34    match content_type {
35        ContentType::Text => {
36            let (body, references) = text::text_with_refs(doc, base_url);
37            Converted {
38                content: compress_block(&body),
39                references,
40            }
41        }
42        ContentType::Markdown => {
43            let (md, references) = markdown::markdown_with_refs(doc, base_url);
44            Converted {
45                content: compress_block(&md),
46                references,
47            }
48        }
49        ContentType::Structured => {
50            let parsed = structured::structured(doc, base_url);
51            Converted {
52                content: structured::to_json(&parsed),
53                references: parsed.references,
54            }
55        }
56    }
57}
58
59/// [`convert_parsed`] for callers holding raw HTML.
60///
61/// Prefer the parsed form: the full pipeline used to parse the same document
62/// twice — once for the title and metadata, once here — which was roughly a
63/// third of its total cost.
64pub fn convert(html: &str, base_url: &str, content_type: ContentType) -> Converted {
65    convert_parsed(&Html::parse_document(html), base_url, content_type)
66}
webfetch/convert/mod.rs

webfetch/convert/
mod.rs
