docparser_core/plugins/
html.rs

1//! HTML 插件：手写标签扫描（零第三方依赖），对齐 Python `html.parser` 的块级语义。
2//!
3//! 规则（与 Python `_HTMLCollector` 一致）：
4//! * `h1..h6` → `Heading(level)`；
5//! * `p` / `div` / `br` → `Paragraph`；
6//! * `blockquote` → `Quote`；`li` → `List`（无序，不区分 ol/ul）；
7//! * `pre` → `Code`；`table/tr/td/th` → `Table`；
8//! * `script` / `style` / `noscript` / `svg` / `head` / `title` 整段跳过。
9//!
10//! 文本只做去首尾空白（不合并内部空白），与 Python 参考实现保持逐字可比。
11
12use crate::model::{Block, BlockKind, Document, Page, TableData};
13use crate::plugin::{FormatPlugin, RawDoc};
14
15const SKIP_TAGS: [&str; 6] = ["script", "style", "noscript", "svg", "head", "title"];
16
17fn heading_level(tag: &str) -> Option<u8> {
18    match tag {
19        "h1" => Some(1),
20        "h2" => Some(2),
21        "h3" => Some(3),
22        "h4" => Some(4),
23        "h5" => Some(5),
24        "h6" => Some(6),
25        _ => None,
26    }
27}
28
29fn named_entity(name: &str) -> Option<char> {
30    Some(match name {
31        "amp" => '&',
32        "lt" => '<',
33        "gt" => '>',
34        "quot" => '"',
35        "apos" => '\'',
36        "nbsp" => '\u{a0}',
37        "mdash" => '—',
38        "ndash" => '–',
39        "hellip" => '…',
40        "copy" => '©',
41        "reg" => '®',
42        "times" => '×',
43        "middot" => '·',
44        "ldquo" => '“',
45        "rdquo" => '”',
46        "lsquo" => '‘',
47        "rsquo" => '’',
48        _ => return None,
49    })
50}
51
52fn numeric_entity(body: &str) -> Option<char> {
53    let rest = body.strip_prefix('#')?;
54    let code = match rest.strip_prefix(['x', 'X']) {
55        Some(hex) => u32::from_str_radix(hex, 16).ok()?,
56        None => rest.parse::<u32>().ok()?,
57    };
58    char::from_u32(code)
59}
60
61/// 解实体：`&amp;` 等命名实体与 `&#NN;` / `&#xHH;`；未知实体原样保留。
62fn decode_entities(s: &str) -> String {
63    if !s.contains('&') {
64        return s.to_string();
65    }
66    let mut out = String::with_capacity(s.len());
67    let mut iter = s.char_indices().peekable();
68    while let Some((i, c)) = iter.next() {
69        if c != '&' {
70            out.push(c);
71            continue;
72        }
73        let rest = &s[i..];
74        let end = rest.find(';').filter(|e| *e <= 10);
75        let decoded = end.and_then(|e| {
76            let body = &rest[1..e];
77            named_entity(body).or_else(|| numeric_entity(body))
78        });
79        match (end, decoded) {
80            (Some(e), Some(ch)) => {
81                out.push(ch);
82                let skip_to = i + e + 1;
83                while let Some((j, _)) = iter.peek() {
84                    if *j < skip_to {
85                        iter.next();
86                    } else {
87                        break;
88                    }
89                }
90            }
91            _ => out.push('&'),
92        }
93    }
94    out
95}
96
97fn tag_name(raw: &str) -> String {
98    raw.split(|c: char| c.is_whitespace() || c == '/')
99        .next()
100        .unwrap_or("")
101        .to_ascii_lowercase()
102}
103
104#[derive(Default)]
105struct Collector {
106    blocks: Vec<Block>,
107    buf: String,
108    cell: String,
109    row: Option<Vec<String>>,
110    table: Option<Vec<Vec<String>>>,
111    heading: Option<u8>,
112    skip_depth: usize,
113}
114
115impl Collector {
116    fn flush(&mut self, kind: BlockKind) {
117        let text = self.buf.trim().to_string();
118        self.buf.clear();
119        if text.is_empty() {
120            return;
121        }
122        let mut block = Block {
123            kind,
124            content: text,
125            ..Default::default()
126        };
127        if kind == BlockKind::Heading {
128            block.level = self.heading.or(Some(1));
129        }
130        // 与 Python `_flush_text(BlockKind.LIST, ordered=False)` 一致：
131        // `li` 不区分 ol/ul，一律按无序列表标记
132        if kind == BlockKind::List {
133            block.ordered = Some(false);
134        }
135        self.blocks.push(block);
136    }
137
138    fn start_tag(&mut self, name: &str) {
139        if SKIP_TAGS.contains(&name) {
140            self.skip_depth += 1;
141            return;
142        }
143        if self.skip_depth > 0 {
144            return;
145        }
146        if let Some(level) = heading_level(name) {
147            self.flush(BlockKind::Paragraph);
148            self.heading = Some(level);
149            return;
150        }
151        match name {
152            "p" | "div" | "br" | "li" | "blockquote" => self.flush(BlockKind::Paragraph),
153            "pre" => {
154                self.flush(BlockKind::Paragraph);
155                // pre 内空白按原文保留：这里不清 buf，只在 flush(Code) 时去首尾
156            }
157            "table" => {
158                self.flush(BlockKind::Paragraph);
159                self.table = Some(Vec::new());
160            }
161            "tr" => {
162                if self.table.is_some() {
163                    self.row = Some(Vec::new());
164                }
165            }
166            "td" | "th" => self.cell.clear(),
167            _ => {}
168        }
169    }
170
171    fn end_tag(&mut self, name: &str) {
172        if SKIP_TAGS.contains(&name) {
173            self.skip_depth = self.skip_depth.saturating_sub(1);
174            return;
175        }
176        if self.skip_depth > 0 {
177            return;
178        }
179        if heading_level(name).is_some() {
180            self.flush(BlockKind::Heading);
181            self.heading = None;
182            return;
183        }
184        match name {
185            "pre" => self.flush(BlockKind::Code),
186            "li" => self.flush(BlockKind::List),
187            "td" | "th" => {
188                let cell = self.cell.trim().to_string();
189                if let Some(row) = self.row.as_mut() {
190                    row.push(cell);
191                }
192                self.cell.clear();
193            }
194            "tr" => {
195                if let (Some(table), Some(row)) = (self.table.as_mut(), self.row.take()) {
196                    table.push(row);
197                }
198            }
199            "table" => {
200                if let Some(rows) = self.table.take() {
201                    if !rows.is_empty() {
202                        let content = rows[0].join(" | ");
203                        self.blocks.push(Block {
204                            kind: BlockKind::Table,
205                            content,
206                            table: Some(TableData {
207                                rows,
208                                ..Default::default()
209                            }),
210                            ..Default::default()
211                        });
212                    }
213                }
214            }
215            "p" | "div" => self.flush(BlockKind::Paragraph),
216            "blockquote" => self.flush(BlockKind::Quote),
217            _ => {}
218        }
219    }
220
221    fn on_text(&mut self, data: &str) {
222        if self.skip_depth > 0 || data.is_empty() {
223            return;
224        }
225        let decoded = decode_entities(data);
226        if self.row.is_some() {
227            self.cell.push_str(&decoded); // 表格内：文本只进单元格
228        } else {
229            self.buf.push_str(&decoded);
230        }
231    }
232
233    fn finish(mut self) -> Vec<Block> {
234        self.flush(BlockKind::Paragraph);
235        self.blocks
236    }
237}
238
239fn strip_comments(text: &str) -> String {
240    let mut out = String::with_capacity(text.len());
241    let mut rest = text;
242    while let Some(start) = rest.find("<!--") {
243        out.push_str(&rest[..start]);
244        match rest[start + 4..].find("-->") {
245            Some(end) => rest = &rest[start + 4 + end + 3..],
246            None => {
247                rest = "";
248                break;
249            }
250        }
251    }
252    out.push_str(rest);
253    out
254}
255
256/// 解析 HTML 为块序列（畸形输入尽量降级，不 panic）。
257pub fn parse_blocks(text: &str) -> Vec<Block> {
258    let mut collector = Collector::default();
259    let bytes = text.as_bytes();
260    let mut i = 0usize;
261    let mut text_start = 0usize;
262
263    while i < bytes.len() {
264        if bytes[i] != b'<' {
265            i += 1;
266            continue;
267        }
268        if text_start < i {
269            collector.on_text(&text[text_start..i]);
270        }
271        // 注释整段丢弃
272        if text[i..].starts_with("<!--") {
273            i = match text[i + 4..].find("-->") {
274                Some(rel) => i + 4 + rel + 3,
275                None => bytes.len(),
276            };
277            text_start = i;
278            continue;
279        }
280        match text[i..].find('>') {
281            Some(rel) => {
282                let raw = &text[i + 1..i + rel];
283                let trimmed = raw.trim();
284                if let Some(name) = trimmed.strip_prefix('/') {
285                    collector.end_tag(&tag_name(name));
286                } else if !trimmed.is_empty()
287                    && !trimmed.starts_with('!')
288                    && !trimmed.starts_with('?')
289                {
290                    collector.start_tag(&tag_name(trimmed));
291                }
292                i = i + rel + 1;
293            }
294            None => i = bytes.len(),
295        }
296        text_start = i;
297    }
298    if text_start < bytes.len() {
299        collector.on_text(&text[text_start..]);
300    }
301
302    let blocks = collector.finish();
303    if blocks.is_empty() {
304        // 无标签或全是被跳过的内容：退化为一段归一化纯文本（注释不算正文）
305        let plain = strip_comments(text)
306            .split_whitespace()
307            .collect::<Vec<_>>()
308            .join(" ");
309        if plain.is_empty() {
310            return vec![];
311        }
312        return vec![Block {
313            kind: BlockKind::Paragraph,
314            content: plain,
315            ..Default::default()
316        }];
317    }
318    blocks
319}
320
321/// html 插件：整页一组块。
322pub struct HtmlPlugin;
323
324impl FormatPlugin for HtmlPlugin {
325    fn id(&self) -> &str {
326        "html"
327    }
328    fn formats(&self) -> &[&str] {
329        &["html"]
330    }
331    fn default_parse_chain(&self) -> Vec<&'static str> {
332        vec!["html.parse"]
333    }
334    fn default_process_chain(&self) -> Vec<&'static str> {
335        vec![]
336    }
337    fn assemble(&self, raw: &RawDoc) -> Document {
338        let text = raw
339            .pages
340            .first()
341            .and_then(|p| p.items.first())
342            .map(|i| i.text.as_str())
343            .unwrap_or_default();
344        let blocks = parse_blocks(text);
345        let page_text = blocks
346            .iter()
347            .filter(|b| !b.content.is_empty())
348            .map(|b| b.content.as_str())
349            .collect::<Vec<_>>()
350            .join("\n\n");
351        Document {
352            pages: vec![Page {
353                blocks,
354                text: page_text,
355                ..Default::default()
356            }],
357            ..Default::default()
358        }
359    }
360}
361
362#[cfg(test)]
363mod tests {
364    use super::*;
365
366    #[test]
367    fn headings_paragraphs_and_lists() {
368        let html = "<html><head><title>t</title></head><body>
369            <h1>标题</h1><p>正文一</p><p>正文二</p>
370            <ul><li>甲</li><li>乙</li></ul>
371            <blockquote>引用</blockquote></body></html>";
372        let blocks = parse_blocks(html);
373        let kinds: Vec<BlockKind> = blocks.iter().map(|b| b.kind).collect();
374        assert_eq!(
375            kinds,
376            vec![
377                BlockKind::Heading,
378                BlockKind::Paragraph,
379                BlockKind::Paragraph,
380                BlockKind::List,
381                BlockKind::List,
382                BlockKind::Quote
383            ]
384        );
385        assert_eq!(blocks[0].content, "标题");
386        assert_eq!(blocks[0].level, Some(1));
387        assert_eq!(blocks[1].content, "正文一");
388        assert_eq!(blocks[3].content, "甲");
389    }
390
391    #[test]
392    fn table_rows_become_table_block() {
393        let html = "<table><tr><th>h1</th><th>h2</th></tr><tr><td>1</td><td>2</td></tr></table>";
394        let blocks = parse_blocks(html);
395        assert_eq!(blocks.len(), 1);
396        let rows = &blocks[0].table.as_ref().unwrap().rows;
397        assert_eq!(rows.len(), 2);
398        assert_eq!(rows[0], vec!["h1", "h2"]);
399        assert_eq!(rows[1], vec!["1", "2"]);
400        assert_eq!(blocks[0].content, "h1 | h2");
401    }
402
403    #[test]
404    fn script_style_and_entities() {
405        let html = "<style>p{color:red}</style><script>var a=1;</script><p>a &amp; b &lt;3 &nbsp;c &#65;</p>";
406        let blocks = parse_blocks(html);
407        assert_eq!(blocks.len(), 1);
408        assert!(blocks[0].content.contains("a & b <3"));
409        assert!(blocks[0].content.ends_with("c A"));
410    }
411
412    #[test]
413    fn pre_keeps_inner_newlines() {
414        let html = "<pre>line1\nline2</pre>";
415        let blocks = parse_blocks(html);
416        assert_eq!(blocks[0].kind, BlockKind::Code);
417        assert_eq!(blocks[0].content, "line1\nline2");
418    }
419
420    #[test]
421    fn plain_text_fallback_and_comments() {
422        assert_eq!(parse_blocks("no tags here").len(), 1);
423        assert!(parse_blocks("<!-- only comment -->").is_empty());
424        let html = "<body><!-- x --><p>visible</p></body>";
425        assert_eq!(parse_blocks(html)[0].content, "visible");
426    }
427
428    #[test]
429    fn malformed_html_does_not_panic() {
430        let _ = parse_blocks("<p>unclosed <b>bold");
431        let _ = parse_blocks("<");
432        let _ = parse_blocks("<<>>");
433    }
434}
docparser_core/plugins/html.rs

docparser_core/plugins/
html.rs
