html_entity_fix/
lib.rs

1//! # html-entity-fix
2//!
3//! Decode HTML entities (`&amp;`, `&lt;`, `&gt;`, `&quot;`, `&#39;`,
4//! `&#x27;`, etc.) inside text that was supposed to be plain.
5//!
6//! LLMs sometimes emit HTML-escaped text into JSON or chat output —
7//! usually because they over-corrected from a prior HTML context.
8//! This crate decodes the common cases without pulling in a full
9//! HTML parser.
10//!
11//! ## Example
12//!
13//! ```
14//! use html_entity_fix::fix;
15//! assert_eq!(fix("AT&amp;T"), "AT&T");
16//! assert_eq!(fix("&lt;tag&gt;"), "<tag>");
17//! assert_eq!(fix("&#39;hello&#39;"), "'hello'");
18//! ```
19
20#![deny(missing_docs)]
21
22/// Decode named and numeric HTML entities in `s`. Unknown entities are
23/// passed through unchanged.
24pub fn fix(s: &str) -> String {
25    let mut out = String::with_capacity(s.len());
26    let mut i = 0;
27    let bytes = s.as_bytes();
28    while i < bytes.len() {
29        if bytes[i] == b'&' {
30            if let Some((decoded, len)) = decode_entity(&s[i..]) {
31                out.push(decoded);
32                i += len;
33                continue;
34            }
35        }
36        // Safe to push a single byte directly only if it's ASCII; we
37        // fall back to taking one char in the general case so multi-byte
38        // UTF-8 sequences stay intact.
39        if bytes[i] < 0x80 {
40            out.push(bytes[i] as char);
41            i += 1;
42        } else {
43            let c = s[i..].chars().next().unwrap();
44            out.push(c);
45            i += c.len_utf8();
46        }
47    }
48    out
49}
50
51/// Returns `Some((decoded_char, bytes_consumed))` on a successful decode
52/// starting at `s` (which must begin with `&`).
53fn decode_entity(s: &str) -> Option<(char, usize)> {
54    let bytes = s.as_bytes();
55    if bytes.first() != Some(&b'&') {
56        return None;
57    }
58    let semi = s[1..].find(';')?;
59    let inside = &s[1..1 + semi];
60    let consumed = semi + 2; // & ... ;
61
62    // Numeric: &#NNN; or &#xHH;
63    if let Some(rest) = inside.strip_prefix('#') {
64        let code = if let Some(hex) = rest.strip_prefix('x').or(rest.strip_prefix('X')) {
65            u32::from_str_radix(hex, 16).ok()?
66        } else {
67            rest.parse::<u32>().ok()?
68        };
69        let c = char::from_u32(code)?;
70        return Some((c, consumed));
71    }
72
73    let c = match inside {
74        "amp" => '&',
75        "lt" => '<',
76        "gt" => '>',
77        "quot" => '"',
78        "apos" => '\'',
79        "nbsp" => '\u{00A0}',
80        "copy" => '©',
81        "reg" => '®',
82        "trade" => '™',
83        "hellip" => '…',
84        "mdash" => '—',
85        "ndash" => '–',
86        "lsquo" => '‘',
87        "rsquo" => '’',
88        "ldquo" => '“',
89        "rdquo" => '”',
90        _ => return None,
91    };
92    Some((c, consumed))
93}
html_entity_fix/lib.rs

html_entity_fix/
lib.rs
