webfetch/convert/
structured.rs

1//! Structured conversion: emit the page as an ordered list of typed blocks,
2//! serialized to JSON. Links are preserved as reference indices (same scheme
3//! as the text path), so structured output is both machine-parseable and
4//! token-frugal inline.
5//!
6//! Blocks carry the kind they came from — heading (with its level), list item,
7//! code, quote, table row, paragraph — so a consumer can reconstruct document
8//! shape. Reading it back off flat text could not tell a heading from a
9//! sentence, which made "structured" no more structured than `text`.
10
11use ego_tree::NodeRef;
12use scraper::node::Node;
13use scraper::{ElementRef, Html};
14use serde::{Deserialize, Serialize};
15
16use super::text::RefCollector;
17use crate::compress::compress_text;
18use crate::extract;
19use crate::types::UrlReference;
20
21#[derive(Debug, Clone, Serialize, Deserialize)]
22pub struct StructuredDoc {
23    pub blocks: Vec<Block>,
24    pub references: Vec<UrlReference>,
25}
26
27#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
28pub struct Block {
29    pub kind: BlockKind,
30    /// Heading depth (1-6). Only present on [`BlockKind::Heading`].
31    #[serde(skip_serializing_if = "Option::is_none")]
32    pub level: Option<u8>,
33    pub text: String,
34}
35
36#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
37#[serde(rename_all = "lowercase")]
38pub enum BlockKind {
39    Heading,
40    Paragraph,
41    ListItem,
42    Code,
43    Quote,
44    TableRow,
45}
46
47/// Elements that open a block, and the kind they produce.
48fn block_kind(name: &str) -> Option<(BlockKind, Option<u8>)> {
49    let heading = |n: u8| Some((BlockKind::Heading, Some(n)));
50    match name {
51        "h1" => heading(1),
52        "h2" => heading(2),
53        "h3" => heading(3),
54        "h4" => heading(4),
55        "h5" => heading(5),
56        "h6" => heading(6),
57        "li" => Some((BlockKind::ListItem, None)),
58        "pre" => Some((BlockKind::Code, None)),
59        "blockquote" => Some((BlockKind::Quote, None)),
60        "tr" => Some((BlockKind::TableRow, None)),
61        "p" => Some((BlockKind::Paragraph, None)),
62        _ => None,
63    }
64}
65
66/// Walk the tree, emitting one block per block-level element.
67///
68/// Text that is not inside any block-level element still has to go somewhere;
69/// it accumulates in `loose` and is flushed as a paragraph when a block starts
70/// or the walk ends, so nothing is silently dropped.
71fn walk(node: NodeRef<Node>, blocks: &mut Vec<Block>, loose: &mut String, refs: &mut RefCollector) {
72    match node.value() {
73        Node::Text(t) => loose.push_str(&t[..]),
74        Node::Element(el) => {
75            let name = el.name();
76            if super::is_skippable(name) {
77                return;
78            }
79
80            if name == "a" {
81                let inner = ElementRef::wrap(node)
82                    .map(|e| e.text().collect::<String>())
83                    .unwrap_or_default();
84                let inner = compress_text(&inner);
85                loose.push_str(&inner);
86                if let Some(url) = el.attr("href").and_then(|h| refs.resolve(h)) {
87                    let idx = refs.index_for(url, &inner);
88                    loose.push_str(&format!(" [{idx}]"));
89                }
90                return;
91            }
92
93            match block_kind(name) {
94                Some((kind, level)) => {
95                    flush(blocks, loose);
96                    let mut inner = String::new();
97                    for child in node.children() {
98                        walk(child, blocks, &mut inner, refs);
99                    }
100                    let text = compress_text(&inner);
101                    if !text.is_empty() {
102                        blocks.push(Block { kind, level, text });
103                    }
104                }
105                None => {
106                    if matches!(name, "br" | "td" | "th") {
107                        loose.push(' ');
108                    }
109                    for child in node.children() {
110                        walk(child, blocks, loose, refs);
111                    }
112                }
113            }
114        }
115        _ => {}
116    }
117}
118
119/// Emit whatever loose text has accumulated as a paragraph block.
120fn flush(blocks: &mut Vec<Block>, loose: &mut String) {
121    let text = compress_text(loose);
122    loose.clear();
123    if !text.is_empty() {
124        blocks.push(Block {
125            kind: BlockKind::Paragraph,
126            level: None,
127            text,
128        });
129    }
130}
131
132/// Build a structured document from a parsed page.
133pub fn structured(doc: &Html, base_url: &str) -> StructuredDoc {
134    let root = match extract::content_root(doc) {
135        Some(el) => el,
136        None => {
137            return StructuredDoc {
138                blocks: Vec::new(),
139                references: Vec::new(),
140            }
141        }
142    };
143
144    let mut refs = RefCollector::new(base_url);
145    let mut blocks = Vec::new();
146    let mut loose = String::new();
147    for child in root.children() {
148        walk(child, &mut blocks, &mut loose, &mut refs);
149    }
150    flush(&mut blocks, &mut loose);
151
152    StructuredDoc {
153        blocks,
154        references: refs.references,
155    }
156}
157
158/// [`structured`] for callers holding raw HTML.
159pub fn html_to_structured(html: &str, base_url: &str) -> StructuredDoc {
160    structured(&Html::parse_document(html), base_url)
161}
162
163pub fn to_json(doc: &StructuredDoc) -> String {
164    serde_json::to_string_pretty(doc).unwrap_or_else(|_| "{}".to_string())
165}
166
167#[cfg(test)]
168mod tests {
169    use super::*;
170
171    #[test]
172    fn blocks_carry_their_kind() {
173        let html = "<article><h2>Setup</h2><p>Install it.</p>\
174                    <ul><li>first</li><li>second</li></ul>\
175                    <pre>cargo build</pre><blockquote>note</blockquote>\
176                    <table><tr><td>a</td><td>b</td></tr></table></article>";
177        let doc = html_to_structured(html, "https://x.test/");
178        let kinds: Vec<_> = doc.blocks.iter().map(|b| b.kind).collect();
179        assert_eq!(
180            kinds,
181            vec![
182                BlockKind::Heading,
183                BlockKind::Paragraph,
184                BlockKind::ListItem,
185                BlockKind::ListItem,
186                BlockKind::Code,
187                BlockKind::Quote,
188                BlockKind::TableRow,
189            ],
190            "blocks: {:?}",
191            doc.blocks
192        );
193        assert_eq!(doc.blocks[0].level, Some(2));
194        assert_eq!(doc.blocks[0].text, "Setup");
195        assert_eq!(doc.blocks[6].text, "a b");
196    }
197
198    #[test]
199    fn links_become_reference_markers() {
200        let html = r#"<article><p>See the <a href="/guide">guide</a>.</p></article>"#;
201        let doc = html_to_structured(html, "https://x.test/");
202        assert_eq!(doc.references.len(), 1);
203        assert_eq!(doc.references[0].url, "https://x.test/guide");
204        assert!(doc.blocks[0].text.contains("[1]"), "{:?}", doc.blocks);
205    }
206
207    #[test]
208    fn text_outside_any_block_is_still_captured() {
209        let html = "<article>bare text with no wrapper</article>";
210        let doc = html_to_structured(html, "https://x.test/");
211        assert_eq!(doc.blocks.len(), 1);
212        assert_eq!(doc.blocks[0].text, "bare text with no wrapper");
213    }
214}
webfetch/convert/structured.rs

webfetch/convert/
structured.rs
