mdream 1.6.0

Fastest HTML-to-Markdown converter. Zero dependencies, streaming support.
Documentation
use std::borrow::Cow;

#[path = "entities_generated.rs"]
mod generated;

use generated::{
  MAX_ENTITY_NAME_LENGTH, MAX_LEGACY_ENTITY_NAME_LENGTH, lookup_legacy_named_entity,
  lookup_named_entity,
};

#[inline]
pub(crate) fn decode_html_entities(text: &str) -> Cow<'_, str> {
  decode_html_entities_in_context(text, false, false)
}

#[inline]
pub(crate) fn decode_html_entities_for_markdown(text: &str) -> Cow<'_, str> {
  decode_html_entities_in_context(text, false, true)
}

#[inline]
pub(crate) fn decode_html_attribute_entities(text: &str) -> Cow<'_, str> {
  decode_html_entities_in_context(text, true, false)
}

#[inline]
fn decode_html_entities_in_context(
  text: &str,
  in_attribute: bool,
  protect_decoded_entity_references: bool,
) -> Cow<'_, str> {
  if !text.as_bytes().contains(&b'&') {
    return Cow::Borrowed(text);
  }
  Cow::Owned(decode_html_entities_alloc(
    text,
    in_attribute,
    protect_decoded_entity_references,
  ))
}

/// Resolve a numeric character reference value per the HTML standard.
fn decode_numeric_ref(code: u32) -> char {
  // Undefined Windows-1252 slots intentionally retain their C1 controls.
  const C1_REPLACEMENTS: [u32; 32] = [
    0x20AC, 0x0081, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0x0160, 0x2039,
    0x0152, 0x008D, 0x017D, 0x008F, 0x0090, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014,
    0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0x009D, 0x017E, 0x0178,
  ];

  let code = if (0x80..=0x9F).contains(&code) {
    C1_REPLACEMENTS[(code - 0x80) as usize]
  } else {
    code
  };
  if code == 0 || code > 0x10_FFFF || (0xD800..=0xDFFF).contains(&code) {
    return '\u{FFFD}';
  }
  char::from_u32(code).unwrap_or('\u{FFFD}')
}

#[inline]
fn is_ascii_alphanumeric(byte: u8) -> bool {
  byte.is_ascii_alphanumeric()
}

fn is_character_reference_tail(bytes: &[u8], start: usize) -> bool {
  let mut index = start;
  if bytes.get(index) == Some(&b'#') {
    index += 1;
    let hex = matches!(bytes.get(index), Some(b'x' | b'X'));
    if hex {
      index += 1;
    }
    let digit_start = index;
    while let Some(&byte) = bytes.get(index) {
      if byte.is_ascii_digit() || (hex && byte.is_ascii_hexdigit()) {
        index += 1;
      } else {
        break;
      }
    }
    return index > digit_start && bytes.get(index) == Some(&b';');
  }

  let name_start = index;
  while bytes.get(index).is_some_and(u8::is_ascii_alphanumeric) {
    index += 1;
  }
  index > name_start && bytes.get(index) == Some(&b';')
}

#[inline]
fn push_decoded_reference(
  result: &mut String,
  replacement: &str,
  bytes: &[u8],
  next: usize,
  protect_decoded_entity_references: bool,
) {
  if replacement == "&"
    && protect_decoded_entity_references
    && is_character_reference_tail(bytes, next)
  {
    result.push_str("\\&");
  } else {
    result.push_str(replacement);
  }
}

fn decode_html_entities_alloc(
  text: &str,
  in_attribute: bool,
  protect_decoded_entity_references: bool,
) -> String {
  let bytes = text.as_bytes();
  let len = bytes.len();
  let mut result = String::with_capacity(len);
  let mut i = 0;

  while i < len {
    if bytes[i] != b'&' {
      let plain_start = i;
      while i < len && bytes[i] != b'&' {
        i += 1;
      }
      result.push_str(&text[plain_start..i]);
      continue;
    }

    // Consume numeric references through the first non-digit. The semicolon
    // is optional; overflow saturates while all remaining digits are consumed.
    if i + 2 < len && bytes[i + 1] == b'#' {
      let mut end = i + 2;
      let is_hex = bytes[end] == b'x' || bytes[end] == b'X';
      if is_hex {
        end += 1;
      }
      let digit_start = end;
      let radix = if is_hex { 16 } else { 10 };
      let mut code_point = 0u32;

      while end < len {
        let digit = match bytes[end] {
          b'0'..=b'9' => Some(u32::from(bytes[end] - b'0')),
          b'A'..=b'F' if is_hex => Some(u32::from(bytes[end] - b'A' + 10)),
          b'a'..=b'f' if is_hex => Some(u32::from(bytes[end] - b'a' + 10)),
          _ => None,
        };
        let Some(digit) = digit else {
          break;
        };
        code_point = code_point
          .saturating_mul(radix)
          .saturating_add(digit)
          .min(0x11_0000);
        end += 1;
      }

      if end > digit_start {
        let replacement = decode_numeric_ref(code_point);
        let next = if end < len && bytes[end] == b';' {
          end + 1
        } else {
          end
        };
        if replacement == '&'
          && protect_decoded_entity_references
          && is_character_reference_tail(bytes, next)
        {
          result.push_str("\\&");
        } else {
          result.push(replacement);
        }
        if end < len && bytes[end] == b';' {
          end += 1;
        }
        i = end;
        continue;
      }
    }

    // A canonical name includes a semicolon. If that misses, find the longest
    // legacy name prefix; only those 106 names may omit their semicolon.
    let name_start = i + 1;
    let mut name_end = name_start;
    let scan_end = len.min(name_start + MAX_ENTITY_NAME_LENGTH);
    while name_end < scan_end && is_ascii_alphanumeric(bytes[name_end]) {
      name_end += 1;
    }

    if name_end > name_start
      && name_end < len
      && bytes[name_end] == b';'
      && let Some(replacement) = lookup_named_entity(&bytes[name_start..name_end])
    {
      push_decoded_reference(
        &mut result,
        replacement,
        bytes,
        name_end + 1,
        protect_decoded_entity_references,
      );
      i = name_end + 1;
      continue;
    }

    let mut legacy_end = name_end.min(name_start + MAX_LEGACY_ENTITY_NAME_LENGTH);
    let mut decoded_legacy = false;
    while legacy_end > name_start {
      let name = &bytes[name_start..legacy_end];
      if let Some(replacement) = lookup_legacy_named_entity(name) {
        let next = bytes.get(legacy_end).copied();
        let ambiguous_attribute =
          in_attribute && next.is_some_and(|byte| byte == b'=' || is_ascii_alphanumeric(byte));
        if !ambiguous_attribute {
          push_decoded_reference(
            &mut result,
            replacement,
            bytes,
            legacy_end,
            protect_decoded_entity_references,
          );
          i = legacy_end;
          decoded_legacy = true;
        }
        break;
      }
      legacy_end -= 1;
    }
    if decoded_legacy {
      continue;
    }

    result.push('&');
    i += 1;
  }

  result
}

#[cfg(test)]
mod tests {
  use super::*;

  #[test]
  fn no_ampersand_borrows() {
    assert!(matches!(
      decode_html_entities("plain text"),
      Cow::Borrowed(_)
    ));
  }

  #[test]
  fn decodes_canonical_names_using_the_longest_match() {
    assert_eq!(decode_html_entities("&copy; &notin; &thetasym;"), "© ∉ ϑ");
    assert_eq!(decode_html_entities("&notit;"), "¬it;");
  }

  #[test]
  fn decodes_semicolonless_numeric_references_without_swallowing() {
    assert_eq!(
      decode_html_entities("&#65 &#x41; &#65copy; &#x41zz;"),
      "A A Acopy; Azz;"
    );
  }

  #[test]
  fn numeric_references_follow_html_replacement_rules() {
    assert_eq!(
      decode_html_entities("&#x80; &#0; &#xD800; &#x110000; &#999999999999999999999;"),
      "€ � � � �"
    );
  }

  #[test]
  fn named_references_preserve_nbsp_and_legacy_uppercase_names() {
    assert_eq!(decode_html_entities("&nbsp;"), "\u{00A0}");
    assert_eq!(decode_html_entities("&COPY;"), "©");
  }

  #[test]
  fn attribute_ambiguity_only_applies_to_legacy_names() {
    assert_eq!(
      decode_html_attribute_entities("&copycat &copy=1 &copy! &copy;cat"),
      "&copycat &copy=1 ©! ©cat"
    );
    assert_eq!(decode_html_entities("&copycat"), "©cat");
  }

  #[test]
  fn unknown_references_do_not_scan_to_a_later_semicolon() {
    assert_eq!(decode_html_entities("&bogus &#65;"), "&bogus A");
    assert_eq!(decode_html_entities("&#nope; &#xnope;"), "&#nope; &#xnope;");
  }
}