sim_codec_doc/
html.rs

1//! Bounded, inert HTML projection into the shared markup model.
2//! conformance: bounded HTML decoding produces the shared document model.
3
4use sim_kernel::Expr;
5
6use crate::{
7    BackendId, Inline, MarkupBackend, MarkupBlock, MarkupDecodeOptions, MarkupDoc,
8    MarkupEncodeOptions, MarkupError, MarkupFidelity, SourceDoc, Span, SpanState,
9};
10
11/// Resource limits and an optional authoritative HTTP charset.
12#[derive(Clone, Debug, PartialEq, Eq)]
13pub struct HtmlDecodeOptions {
14    /// Maximum accepted source bytes.
15    pub max_input_bytes: usize,
16    /// Maximum tags examined.
17    pub max_nodes: usize,
18    /// Maximum element nesting.
19    pub max_depth: usize,
20    /// Maximum normalized text bytes.
21    pub max_text_bytes: usize,
22    /// Charset supplied by HTTP, which takes precedence over a document declaration.
23    pub http_charset: Option<String>,
24}
25
26impl Default for HtmlDecodeOptions {
27    fn default() -> Self {
28        Self {
29            max_input_bytes: 2 * 1024 * 1024,
30            max_nodes: 100_000,
31            max_depth: 256,
32            max_text_bytes: 1024 * 1024,
33            http_charset: None,
34        }
35    }
36}
37
38/// Tolerant HTML backend. It never resolves URLs or executes active content.
39#[derive(Clone, Debug, Default)]
40pub struct HtmlBackend;
41
42impl MarkupBackend for HtmlBackend {
43    fn id(&self) -> BackendId {
44        BackendId::new("html")
45    }
46    fn decode(
47        &self,
48        input: &str,
49        opts: &MarkupDecodeOptions,
50    ) -> Result<(MarkupDoc, MarkupFidelity), MarkupError> {
51        decode_html_text(
52            input,
53            opts.preserve_source,
54            HtmlDecodeOptions::default(),
55            Vec::new(),
56        )
57    }
58    fn encode(
59        &self,
60        doc: &MarkupDoc,
61        _opts: &MarkupEncodeOptions,
62    ) -> Result<(String, MarkupFidelity), MarkupError> {
63        if let Some(source) = &doc.source
64            && source.backend.as_str() == "html"
65        {
66            return Ok((source.text.clone(), MarkupFidelity::exact(self.id())));
67        }
68        Err(MarkupError::Encode(
69            "HTML is an extraction backend; encoding requires preserved HTML source".into(),
70        ))
71    }
72}
73
74/// Decode HTML bytes with HTTP/document charset precedence and replacement warnings.
75pub fn decode_html_bytes(
76    input: &[u8],
77    opts: &HtmlDecodeOptions,
78) -> Result<(MarkupDoc, MarkupFidelity), MarkupError> {
79    if input.len() > opts.max_input_bytes {
80        return Err(MarkupError::Decode("HTML input byte limit exceeded".into()));
81    }
82    let declared = opts.http_charset.clone().or_else(|| sniff_charset(input));
83    let mut warnings = Vec::new();
84    let text = match declared.as_deref().map(|v| v.to_ascii_lowercase()) {
85        Some(label) if label == "iso-8859-1" || label == "windows-1252" => {
86            input.iter().map(|&b| char::from(b)).collect()
87        }
88        Some(label) if label != "utf-8" && label != "utf8" => {
89            warnings.push(format!("unsupported charset {label}; decoded as UTF-8"));
90            String::from_utf8_lossy(input).into_owned()
91        }
92        _ => String::from_utf8_lossy(input).into_owned(),
93    };
94    if std::str::from_utf8(input).is_err()
95        && !matches!(declared.as_deref(), Some("iso-8859-1" | "windows-1252"))
96    {
97        warnings.push("invalid UTF-8 replaced during decode".into());
98    }
99    decode_html_text(&text, true, opts.clone(), warnings)
100}
101
102fn sniff_charset(input: &[u8]) -> Option<String> {
103    let head = String::from_utf8_lossy(&input[..input.len().min(4096)]).to_ascii_lowercase();
104    let at = head.find("charset=")? + 8;
105    Some(
106        head[at..]
107            .trim_start_matches(['\'', '"'])
108            .split(|c: char| c == '\'' || c == '"' || c == ';' || c.is_whitespace() || c == '>')
109            .next()?
110            .to_owned(),
111    )
112}
113
114fn decode_html_text(
115    input: &str,
116    preserve_source: bool,
117    limits: HtmlDecodeOptions,
118    warnings: Vec<String>,
119) -> Result<(MarkupDoc, MarkupFidelity), MarkupError> {
120    if input.len() > limits.max_input_bytes {
121        return Err(MarkupError::Decode("HTML input byte limit exceeded".into()));
122    }
123    let mut p = Parser {
124        source: input,
125        pos: 0,
126        nodes: 0,
127        depth: 0,
128        text_bytes: 0,
129        limits,
130        blocks: Vec::new(),
131        stack: Vec::new(),
132        title: None,
133        attrs: Default::default(),
134        warnings,
135        suppressed: 0,
136    };
137    p.parse()?;
138    p.extract_structures();
139    let mut fidelity = MarkupFidelity::exact(BackendId::new("html"));
140    fidelity.warnings = p.warnings;
141    let doc = MarkupDoc {
142        title: p.title,
143        blocks: p.blocks,
144        attrs: p.attrs,
145        source: preserve_source.then(|| SourceDoc {
146            backend: BackendId::new("html"),
147            text: input.to_owned(),
148        }),
149    };
150    Ok((doc, fidelity))
151}
152
153struct Frame {
154    tag: String,
155    start: usize,
156    text: String,
157    href: Option<String>,
158    lang: Option<String>,
159}
160struct Parser<'a> {
161    source: &'a str,
162    pos: usize,
163    nodes: usize,
164    depth: usize,
165    text_bytes: usize,
166    limits: HtmlDecodeOptions,
167    blocks: Vec<MarkupBlock>,
168    stack: Vec<Frame>,
169    title: Option<String>,
170    attrs: std::collections::BTreeMap<String, Expr>,
171    warnings: Vec<String>,
172    suppressed: usize,
173}
174impl Parser<'_> {
175    fn extract_structures(&mut self) {
176        for (tag, ordered) in [("ul", false), ("ol", true)] {
177            for list in html_elements(self.source, tag) {
178                let items = html_elements(list, "li")
179                    .into_iter()
180                    .map(|item| {
181                        vec![MarkupBlock::Paragraph {
182                            content: vec![Inline::Text(normalize(&strip_tags(item)))],
183                            span: None,
184                        }]
185                    })
186                    .collect::<Vec<_>>();
187                if !items.is_empty() {
188                    self.blocks.push(MarkupBlock::List {
189                        ordered,
190                        items,
191                        span: None,
192                    });
193                }
194            }
195        }
196        for table in html_elements(self.source, "table") {
197            let mut rows = html_elements(table, "tr")
198                .into_iter()
199                .map(|row| {
200                    let mut cells = html_elements(row, "th");
201                    if cells.is_empty() {
202                        cells = html_elements(row, "td");
203                    }
204                    cells
205                        .into_iter()
206                        .map(|cell| vec![Inline::Text(normalize(&strip_tags(cell)))])
207                        .collect::<Vec<_>>()
208                })
209                .filter(|r| !r.is_empty())
210                .collect::<Vec<_>>();
211            if !rows.is_empty() {
212                let header = rows.remove(0);
213                self.blocks.push(MarkupBlock::Table {
214                    header,
215                    rows,
216                    span: None,
217                });
218            }
219        }
220        let readable = self
221            .blocks
222            .iter()
223            .filter_map(|b| match b {
224                MarkupBlock::Heading { text, .. }
225                | MarkupBlock::Paragraph { content: text, .. } => Some(
226                    text.iter()
227                        .filter_map(|i| {
228                            if let Inline::Text(v) = i {
229                                Some(v.as_str())
230                            } else {
231                                None
232                            }
233                        })
234                        .collect::<Vec<_>>()
235                        .join(" "),
236                ),
237                _ => None,
238            })
239            .collect::<Vec<_>>()
240            .join("\n");
241        self.attrs
242            .insert("readable-text".into(), Expr::String(readable));
243    }
244    fn parse(&mut self) -> Result<(), MarkupError> {
245        while self.pos < self.source.len() {
246            if self.source.as_bytes()[self.pos] == b'<' {
247                self.tag()?;
248            } else {
249                self.text()?;
250            }
251        }
252        while let Some(frame) = self.stack.pop() {
253            self.finish(frame, self.source.len());
254        }
255        Ok(())
256    }
257    fn tag(&mut self) -> Result<(), MarkupError> {
258        let start = self.pos;
259        let Some(rel) = self.source[start..].find('>') else {
260            self.pos = self.source.len();
261            return Ok(());
262        };
263        let end = start + rel + 1;
264        self.nodes += 1;
265        if self.nodes > self.limits.max_nodes {
266            return Err(MarkupError::Decode("HTML node limit exceeded".into()));
267        }
268        let raw = &self.source[start + 1..end - 1];
269        self.pos = end;
270        if raw.starts_with('!') || raw.starts_with('?') {
271            return Ok(());
272        }
273        let closing = raw.trim_start().starts_with('/');
274        let body = raw.trim().trim_start_matches('/').trim();
275        let name = body
276            .split_whitespace()
277            .next()
278            .unwrap_or("")
279            .trim_end_matches('/')
280            .to_ascii_lowercase();
281        if name.is_empty() {
282            return Ok(());
283        }
284        if closing {
285            if let Some(ix) = self.stack.iter().rposition(|f| f.tag == name) {
286                while self.stack.len() > ix {
287                    let f = self.stack.pop().unwrap();
288                    self.finish(f, end);
289                }
290            }
291            return Ok(());
292        }
293        if name == "meta"
294            && let Some(v) = attr(body, "name").zip(attr(body, "content"))
295        {
296            self.attrs.insert(
297                format!("meta:{}", v.0.to_ascii_lowercase()),
298                Expr::String(v.1),
299            );
300        }
301        if name == "link"
302            && attr(body, "rel").is_some_and(|v| v.eq_ignore_ascii_case("canonical"))
303            && let Some(v) = attr(body, "href")
304        {
305            self.attrs.insert("canonical-link".into(), Expr::String(v));
306        }
307        if name == "html"
308            && let Some(v) = attr(body, "lang")
309        {
310            self.attrs.insert("language".into(), Expr::String(v));
311        }
312        let active = matches!(
313            name.as_str(),
314            "script" | "style" | "form" | "object" | "embed" | "iframe"
315        );
316        if active {
317            self.suppressed += 1;
318            self.warnings
319                .push(format!("active or embedded <{name}> content omitted"));
320        }
321        if body.contains("on")
322            && body
323                .split_whitespace()
324                .any(|a| a.to_ascii_lowercase().starts_with("on") && a.contains('='))
325        {
326            self.warnings
327                .push(format!("event handler stripped from <{name}>"));
328        }
329        if !body.ends_with('/')
330            && !matches!(
331                name.as_str(),
332                "meta" | "link" | "img" | "br" | "hr" | "input" | "source"
333            )
334        {
335            self.depth += 1;
336            if self.depth > self.limits.max_depth {
337                return Err(MarkupError::Decode("HTML depth limit exceeded".into()));
338            }
339            self.stack.push(Frame {
340                tag: name,
341                start,
342                text: String::new(),
343                href: attr(body, "href"),
344                lang: attr(body, "class")
345                    .and_then(|v| v.strip_prefix("language-").map(str::to_owned)),
346            });
347        }
348        Ok(())
349    }
350    fn text(&mut self) -> Result<(), MarkupError> {
351        let end = self.source[self.pos..]
352            .find('<')
353            .map_or(self.source.len(), |v| self.pos + v);
354        let raw = &self.source[self.pos..end];
355        self.pos = end;
356        if self.suppressed == 0 {
357            let decoded = entities(raw);
358            self.text_bytes += decoded.len();
359            if self.text_bytes > self.limits.max_text_bytes {
360                return Err(MarkupError::Decode("HTML text limit exceeded".into()));
361            }
362            for f in &mut self.stack {
363                f.text.push_str(&decoded);
364            }
365        }
366        Ok(())
367    }
368    fn finish(&mut self, f: Frame, end: usize) {
369        self.depth = self.depth.saturating_sub(1);
370        if matches!(
371            f.tag.as_str(),
372            "script" | "style" | "form" | "object" | "embed" | "iframe"
373        ) {
374            self.suppressed = self.suppressed.saturating_sub(1);
375            return;
376        }
377        let text = normalize(&f.text);
378        if text.is_empty() {
379            return;
380        }
381        let span = Some(Span {
382            start: f.start,
383            end,
384            state: SpanState::Preserved,
385        });
386        let inline = || {
387            vec![if let Some(target) = &f.href {
388                Inline::Link {
389                    label: vec![Inline::Text(text.clone())],
390                    target: target.clone(),
391                }
392            } else {
393                Inline::Text(text.clone())
394            }]
395        };
396        match f.tag.as_str() {
397            "title" => self.title = Some(text),
398            "h1" | "h2" | "h3" | "h4" | "h5" | "h6" => self.blocks.push(MarkupBlock::Heading {
399                level: f.tag[1..].parse().unwrap_or(1),
400                text: inline(),
401                id: None,
402                span,
403            }),
404            "pre" => self.blocks.push(MarkupBlock::CodeBlock {
405                lang: f.lang,
406                code: text,
407                span,
408            }),
409            "blockquote" => self.blocks.push(MarkupBlock::Quote {
410                blocks: vec![MarkupBlock::Paragraph {
411                    content: inline(),
412                    span: span.clone(),
413                }],
414                span,
415            }),
416            "p" | "li" | "td" | "th" => self.blocks.push(MarkupBlock::Paragraph {
417                content: inline(),
418                span,
419            }),
420            _ => {}
421        }
422    }
423}
424fn attr(body: &str, wanted: &str) -> Option<String> {
425    for token in body.split_whitespace().skip(1) {
426        let (k, v) = token.split_once('=')?;
427        if k.eq_ignore_ascii_case(wanted) {
428            return Some(v.trim_matches(['\'', '"', '>']).to_owned());
429        }
430    }
431    None
432}
433fn normalize(s: &str) -> String {
434    s.split_whitespace().collect::<Vec<_>>().join(" ")
435}
436fn entities(s: &str) -> String {
437    s.replace("&amp;", "&")
438        .replace("&lt;", "<")
439        .replace("&gt;", ">")
440        .replace("&quot;", "\"")
441        .replace("&#39;", "'")
442        .replace("&nbsp;", " ")
443}
444fn html_elements<'a>(s: &'a str, tag: &str) -> Vec<&'a str> {
445    let mut out = Vec::new();
446    let open = format!("<{tag}");
447    let close = format!("</{tag}>");
448    let mut rest = s;
449    while let Some(a) = rest.to_ascii_lowercase().find(&open) {
450        let x = &rest[a..];
451        let Some(gt) = x.find('>') else { break };
452        let Some(b) = x[gt + 1..].to_ascii_lowercase().find(&close) else {
453            break;
454        };
455        out.push(&x[gt + 1..gt + 1 + b]);
456        rest = &x[gt + 1 + b + close.len()..];
457    }
458    out
459}
460fn strip_tags(s: &str) -> String {
461    let mut out = String::new();
462    let mut inside = false;
463    for c in s.chars() {
464        match c {
465            '<' => inside = true,
466            '>' => inside = false,
467            _ if !inside => out.push(c),
468            _ => {}
469        }
470    }
471    entities(&out)
472}
473
474#[cfg(test)]
475mod tests {
476    use super::*;
477    #[test]
478    fn inert_and_chunk_equivalent() {
479        let html=b"<html lang='en'><head><link rel='canonical' href='https://e/x'><script>panic()</script></head><body><h1>Hello &amp; hi</h1><p>Safe <a href='/x'>link</a></p></body></html>";
480        let (a, f) = decode_html_bytes(html, &Default::default()).unwrap();
481        let joined = [&html[..31], &html[31..]].concat();
482        let (b, _) = decode_html_bytes(&joined, &Default::default()).unwrap();
483        assert_eq!(a, b);
484        assert!(!format!("{:?}", a.blocks).contains("panic"));
485        assert!(f.warnings.iter().any(|w| w.contains("omitted")));
486    }
487}
488// conformance: bounded HTML decoding produces the shared document model.
sim_codec_doc/html.rs

sim_codec_doc/
html.rs
