rexact-core 0.2.0

//! Alphabet modes (FR-A07, docs/13 section 6).

use crate::charset::CharSet;
use rexact_syntax::{Dialect, Flags};

/// The universe of symbols a pattern is interpreted over. Reported with every
/// result.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum Alphabet {
    /// Unicode scalar values (surrogates excluded).
    Unicode,
    /// UTF-16 code units; a non-BMP character is two symbols.
    Utf16,
    /// Bytes.
    Bytes,
}

impl Alphabet {
    /// Name used in output and in the canonical form.
    pub const fn name(self) -> &'static str {
        match self {
            Alphabet::Unicode => "unicode",
            Alphabet::Utf16 => "utf16",
            Alphabet::Bytes => "bytes",
        }
    }

    /// Default mode for a dialect: ECMAScript without `u` works on UTF-16
    /// units, PCRE2 without `u` (UTF mode) on bytes, everything else on scalar
    /// values.
    pub fn default_for(dialect: Dialect, flags: Flags) -> Alphabet {
        match dialect {
            Dialect::Ecma if !flags.u => Alphabet::Utf16,
            // PCRE2 without UTF mode matches bytes.
            Dialect::Pcre2 if !flags.u => Alphabet::Bytes,
            _ => Alphabet::Unicode,
        }
    }

    /// Every symbol of the alphabet.
    pub fn universe(self) -> CharSet {
        match self {
            Alphabet::Unicode => CharSet::from_ranges([(0, 0xD7FF), (0xE000, 0x10FFFF)]),
            Alphabet::Utf16 => CharSet::range(0, 0xFFFF),
            Alphabet::Bytes => CharSet::range(0, 0xFF),
        }
    }

    /// Turn text into symbols (bytes mode uses the UTF-8 encoding).
    pub fn encode(self, s: &str) -> Vec<u32> {
        match self {
            Alphabet::Unicode => s.chars().map(u32::from).collect(),
            Alphabet::Utf16 => s.encode_utf16().map(u32::from).collect(),
            Alphabet::Bytes => s.bytes().map(u32::from).collect(),
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn encoding_of_a_non_bmp_character_depends_on_the_mode() {
        assert_eq!(Alphabet::Unicode.encode("😀"), vec![0x1F600]);
        assert_eq!(Alphabet::Utf16.encode("😀"), vec![0xD83D, 0xDE00]);
        assert_eq!(Alphabet::Bytes.encode("é"), vec![0xC3, 0xA9]);
    }

    #[test]
    fn unicode_universe_excludes_surrogates() {
        let u = Alphabet::Unicode.universe();
        assert!(
            u.contains(0xD7FF) && !u.contains(0xD800) && !u.contains(0xDFFF) && u.contains(0xE000)
        );
        assert_eq!(u.count(), 0x110000 - 0x800);
    }

    #[test]
    fn default_mode_follows_dialect_and_unicode_flag() {
        let u = Flags {
            u: true,
            ..Flags::none()
        };
        assert_eq!(
            Alphabet::default_for(Dialect::Ecma, Flags::none()),
            Alphabet::Utf16
        );
        assert_eq!(Alphabet::default_for(Dialect::Ecma, u), Alphabet::Unicode);
        assert_eq!(
            Alphabet::default_for(Dialect::Python, Flags::none()),
            Alphabet::Unicode
        );
    }
}