docparser_core/plugins/
html.rs

1//! HTML 插件：手写标签扫描（零第三方依赖），对齐 Python `html.parser` 的块级语义。
2//!
3//! 规则（与 Python `_HTMLCollector` 一致）：
4//! * `h1..h6` → `Heading(level)`；
5//! * `p` / `div` / `br` → `Paragraph`；
6//! * `blockquote` → `Quote`；`li` → `List`（无序，不区分 ol/ul）；
7//! * `pre` → `Code`；`table/tr/td/th` → `Table`；
8//! * `script` / `style` / `noscript` / `svg` / `head` / `title` 整段跳过。
9//!
10//! 文本只做去首尾空白（不合并内部空白），与 Python 参考实现保持逐字可比。
11
12use crate::model::{Block, BlockKind, Document, Page, TableData};
13use crate::plugin::{FormatPlugin, RawDoc};
14
15const SKIP_TAGS: [&str; 6] = ["script", "style", "noscript", "svg", "head", "title"];
16
17fn heading_level(tag: &str) -> Option<u8> {
18    match tag {
19        "h1" => Some(1),
20        "h2" => Some(2),
21        "h3" => Some(3),
22        "h4" => Some(4),
23        "h5" => Some(5),
24        "h6" => Some(6),
25        _ => None,
26    }
27}
28
29fn named_entity(name: &str) -> Option<char> {
30    Some(match name {
31        "amp" => '&',
32        "lt" => '<',
33        "gt" => '>',
34        "quot" => '"',
35        "apos" => '\'',
36        "nbsp" => '\u{a0}',
37        "mdash" => '—',
38        "ndash" => '–',
39        "hellip" => '…',
40        "copy" => '©',
41        "reg" => '®',
42        "times" => '×',
43        "middot" => '·',
44        "ldquo" => '“',
45        "rdquo" => '”',
46        "lsquo" => '‘',
47        "rsquo" => '’',
48        _ => return None,
49    })
50}
51
52fn numeric_entity(body: &str) -> Option<char> {
53    let rest = body.strip_prefix('#')?;
54    let code = match rest.strip_prefix(['x', 'X']) {
55        Some(hex) => u32::from_str_radix(hex, 16).ok()?,
56        None => rest.parse::<u32>().ok()?,
57    };
58    char::from_u32(code)
59}
60
61/// 解实体：`&amp;` 等命名实体与 `&#NN;` / `&#xHH;`；未知实体原样保留。
62fn decode_entities(s: &str) -> String {
63    if !s.contains('&') {
64        return s.to_string();
65    }
66    let mut out = String::with_capacity(s.len());
67    let mut iter = s.char_indices().peekable();
68    while let Some((i, c)) = iter.next() {
69        if c != '&' {
70            out.push(c);
71            continue;
72        }
73        let rest = &s[i..];
74        let end = rest.find(';').filter(|e| *e <= 10);
75        let decoded = end.and_then(|e| {
76            let body = &rest[1..e];
77            named_entity(body).or_else(|| numeric_entity(body))
78        });
79        match (end, decoded) {
80            (Some(e), Some(ch)) => {
81                out.push(ch);
82                let skip_to = i + e + 1;
83                while let Some((j, _)) = iter.peek() {
84                    if *j < skip_to {
85                        iter.next();
86                    } else {
87                        break;
88                    }
89                }
90            }
91            _ => out.push('&'),
92        }
93    }
94    out
95}
96
97fn tag_name(raw: &str) -> String {
98    raw.split(|c: char| c.is_whitespace() || c == '/')
99        .next()
100        .unwrap_or("")
101        .to_ascii_lowercase()
102}
103
104#[derive(Default)]
105struct Collector {
106    blocks: Vec<Block>,
107    buf: String,
108    cell: String,
109    row: Option<Vec<String>>,
110    table: Option<Vec<Vec<String>>>,
111    heading: Option<u8>,
112    skip_depth: usize,
113}
114
115impl Collector {
116    fn flush(&mut self, kind: BlockKind) {
117        let text = self.buf.trim().to_string();
118        self.buf.clear();
119        if text.is_empty() {
120            return;
121        }
122        let mut block = Block {
123            kind,
124            content: text,
125            ..Default::default()
126        };
127        if kind == BlockKind::Heading {
128            block.level = self.heading.or(Some(1));
129        }
130        self.blocks.push(block);
131    }
132
133    fn start_tag(&mut self, name: &str) {
134        if SKIP_TAGS.contains(&name) {
135            self.skip_depth += 1;
136            return;
137        }
138        if self.skip_depth > 0 {
139            return;
140        }
141        if let Some(level) = heading_level(name) {
142            self.flush(BlockKind::Paragraph);
143            self.heading = Some(level);
144            return;
145        }
146        match name {
147            "p" | "div" | "br" | "li" | "blockquote" => self.flush(BlockKind::Paragraph),
148            "pre" => {
149                self.flush(BlockKind::Paragraph);
150                // pre 内空白按原文保留：这里不清 buf，只在 flush(Code) 时去首尾
151            }
152            "table" => {
153                self.flush(BlockKind::Paragraph);
154                self.table = Some(Vec::new());
155            }
156            "tr" => {
157                if self.table.is_some() {
158                    self.row = Some(Vec::new());
159                }
160            }
161            "td" | "th" => self.cell.clear(),
162            _ => {}
163        }
164    }
165
166    fn end_tag(&mut self, name: &str) {
167        if SKIP_TAGS.contains(&name) {
168            self.skip_depth = self.skip_depth.saturating_sub(1);
169            return;
170        }
171        if self.skip_depth > 0 {
172            return;
173        }
174        if heading_level(name).is_some() {
175            self.flush(BlockKind::Heading);
176            self.heading = None;
177            return;
178        }
179        match name {
180            "pre" => self.flush(BlockKind::Code),
181            "li" => self.flush(BlockKind::List),
182            "td" | "th" => {
183                let cell = self.cell.trim().to_string();
184                if let Some(row) = self.row.as_mut() {
185                    row.push(cell);
186                }
187                self.cell.clear();
188            }
189            "tr" => {
190                if let (Some(table), Some(row)) = (self.table.as_mut(), self.row.take()) {
191                    table.push(row);
192                }
193            }
194            "table" => {
195                if let Some(rows) = self.table.take() {
196                    if !rows.is_empty() {
197                        let content = rows[0].join(" | ");
198                        self.blocks.push(Block {
199                            kind: BlockKind::Table,
200                            content,
201                            table: Some(TableData { rows }),
202                            ..Default::default()
203                        });
204                    }
205                }
206            }
207            "p" | "div" => self.flush(BlockKind::Paragraph),
208            "blockquote" => self.flush(BlockKind::Quote),
209            _ => {}
210        }
211    }
212
213    fn on_text(&mut self, data: &str) {
214        if self.skip_depth > 0 || data.is_empty() {
215            return;
216        }
217        let decoded = decode_entities(data);
218        if self.row.is_some() {
219            self.cell.push_str(&decoded); // 表格内：文本只进单元格
220        } else {
221            self.buf.push_str(&decoded);
222        }
223    }
224
225    fn finish(mut self) -> Vec<Block> {
226        self.flush(BlockKind::Paragraph);
227        self.blocks
228    }
229}
230
231fn strip_comments(text: &str) -> String {
232    let mut out = String::with_capacity(text.len());
233    let mut rest = text;
234    while let Some(start) = rest.find("<!--") {
235        out.push_str(&rest[..start]);
236        match rest[start + 4..].find("-->") {
237            Some(end) => rest = &rest[start + 4 + end + 3..],
238            None => {
239                rest = "";
240                break;
241            }
242        }
243    }
244    out.push_str(rest);
245    out
246}
247
248/// 解析 HTML 为块序列（畸形输入尽量降级，不 panic）。
249pub fn parse_blocks(text: &str) -> Vec<Block> {
250    let mut collector = Collector::default();
251    let bytes = text.as_bytes();
252    let mut i = 0usize;
253    let mut text_start = 0usize;
254
255    while i < bytes.len() {
256        if bytes[i] != b'<' {
257            i += 1;
258            continue;
259        }
260        if text_start < i {
261            collector.on_text(&text[text_start..i]);
262        }
263        // 注释整段丢弃
264        if text[i..].starts_with("<!--") {
265            i = match text[i + 4..].find("-->") {
266                Some(rel) => i + 4 + rel + 3,
267                None => bytes.len(),
268            };
269            text_start = i;
270            continue;
271        }
272        match text[i..].find('>') {
273            Some(rel) => {
274                let raw = &text[i + 1..i + rel];
275                let trimmed = raw.trim();
276                if let Some(name) = trimmed.strip_prefix('/') {
277                    collector.end_tag(&tag_name(name));
278                } else if !trimmed.is_empty()
279                    && !trimmed.starts_with('!')
280                    && !trimmed.starts_with('?')
281                {
282                    collector.start_tag(&tag_name(trimmed));
283                }
284                i = i + rel + 1;
285            }
286            None => i = bytes.len(),
287        }
288        text_start = i;
289    }
290    if text_start < bytes.len() {
291        collector.on_text(&text[text_start..]);
292    }
293
294    let blocks = collector.finish();
295    if blocks.is_empty() {
296        // 无标签或全是被跳过的内容：退化为一段归一化纯文本（注释不算正文）
297        let plain = strip_comments(text)
298            .split_whitespace()
299            .collect::<Vec<_>>()
300            .join(" ");
301        if plain.is_empty() {
302            return vec![];
303        }
304        return vec![Block {
305            kind: BlockKind::Paragraph,
306            content: plain,
307            ..Default::default()
308        }];
309    }
310    blocks
311}
312
313/// html 插件：整页一组块。
314pub struct HtmlPlugin;
315
316impl FormatPlugin for HtmlPlugin {
317    fn id(&self) -> &str {
318        "html"
319    }
320    fn formats(&self) -> &[&str] {
321        &["html"]
322    }
323    fn default_parse_chain(&self) -> Vec<&'static str> {
324        vec!["html.parse"]
325    }
326    fn default_process_chain(&self) -> Vec<&'static str> {
327        vec![]
328    }
329    fn assemble(&self, raw: &RawDoc) -> Document {
330        let text = raw
331            .pages
332            .first()
333            .and_then(|p| p.items.first())
334            .map(|i| i.text.as_str())
335            .unwrap_or_default();
336        let blocks = parse_blocks(text);
337        let page_text = blocks
338            .iter()
339            .filter(|b| !b.content.is_empty())
340            .map(|b| b.content.as_str())
341            .collect::<Vec<_>>()
342            .join("\n\n");
343        Document {
344            pages: vec![Page {
345                blocks,
346                text: page_text,
347                ..Default::default()
348            }],
349            ..Default::default()
350        }
351    }
352}
353
354#[cfg(test)]
355mod tests {
356    use super::*;
357
358    #[test]
359    fn headings_paragraphs_and_lists() {
360        let html = "<html><head><title>t</title></head><body>
361            <h1>标题</h1><p>正文一</p><p>正文二</p>
362            <ul><li>甲</li><li>乙</li></ul>
363            <blockquote>引用</blockquote></body></html>";
364        let blocks = parse_blocks(html);
365        let kinds: Vec<BlockKind> = blocks.iter().map(|b| b.kind).collect();
366        assert_eq!(
367            kinds,
368            vec![
369                BlockKind::Heading,
370                BlockKind::Paragraph,
371                BlockKind::Paragraph,
372                BlockKind::List,
373                BlockKind::List,
374                BlockKind::Quote
375            ]
376        );
377        assert_eq!(blocks[0].content, "标题");
378        assert_eq!(blocks[0].level, Some(1));
379        assert_eq!(blocks[1].content, "正文一");
380        assert_eq!(blocks[3].content, "甲");
381    }
382
383    #[test]
384    fn table_rows_become_table_block() {
385        let html = "<table><tr><th>h1</th><th>h2</th></tr><tr><td>1</td><td>2</td></tr></table>";
386        let blocks = parse_blocks(html);
387        assert_eq!(blocks.len(), 1);
388        let rows = &blocks[0].table.as_ref().unwrap().rows;
389        assert_eq!(rows.len(), 2);
390        assert_eq!(rows[0], vec!["h1", "h2"]);
391        assert_eq!(rows[1], vec!["1", "2"]);
392        assert_eq!(blocks[0].content, "h1 | h2");
393    }
394
395    #[test]
396    fn script_style_and_entities() {
397        let html = "<style>p{color:red}</style><script>var a=1;</script><p>a &amp; b &lt;3 &nbsp;c &#65;</p>";
398        let blocks = parse_blocks(html);
399        assert_eq!(blocks.len(), 1);
400        assert!(blocks[0].content.contains("a & b <3"));
401        assert!(blocks[0].content.ends_with("c A"));
402    }
403
404    #[test]
405    fn pre_keeps_inner_newlines() {
406        let html = "<pre>line1\nline2</pre>";
407        let blocks = parse_blocks(html);
408        assert_eq!(blocks[0].kind, BlockKind::Code);
409        assert_eq!(blocks[0].content, "line1\nline2");
410    }
411
412    #[test]
413    fn plain_text_fallback_and_comments() {
414        assert_eq!(parse_blocks("no tags here").len(), 1);
415        assert!(parse_blocks("<!-- only comment -->").is_empty());
416        let html = "<body><!-- x --><p>visible</p></body>";
417        assert_eq!(parse_blocks(html)[0].content, "visible");
418    }
419
420    #[test]
421    fn malformed_html_does_not_panic() {
422        let _ = parse_blocks("<p>unclosed <b>bold");
423        let _ = parse_blocks("<");
424        let _ = parse_blocks("<<>>");
425    }
426}
docparser_core/plugins/html.rs

docparser_core/plugins/
html.rs
