characterminator 0.1.2

find and eliminate characters outside an allowed set, per file type and per file, so text costs fewer tokens
Documentation
//! The builtin sets: one that is CODE, the rest DATA.
//!
//! Every preset but `ascii` ships as a data file written in the
//! `.ctrm-sets` grammar and compiled in with `include_str!` (V22), so the
//! contents a user reads and the contents the tool enforces are the same
//! bytes. `sets.ctrm-sets` holds the hand-written presets and
//! `hazard.ctrm-sets` the ones GENERATED from Unicode data (`src/lint/hazard:V34`),
//! and `locale/locales.ctrm-sets` the CLDR letter presets
//! (`src/charset/locale:V30`), generated by `cldr-letters.sh` beside it
//! and read lazily by `super::locale` (`src/charset/locale:V61`).
//!
//! `ascii` is the exception on purpose. `src/rules:V21` requires a run with
//! `--no-files --no-builtin-map --no-builtin-sets` to still resolve a
//! default set, so the one set that every path falls back to (`src/rules`
//! V1) cannot itself be a file that the flags remove.

use super::{
    CharRange, CharSet, ParseError, SetCatalog, SetDefinition, SetMember,
    parse_line,
};

/// The name of the intrinsic set, and the ONE spelling of `ascii` in code:
/// the set every unmatched path gets (`src/rules:V1`) and the root of the
/// fidelity tree (`src/fix:V27`) are this name, so they cannot drift apart.
pub(crate) const ASCII: &str = "ascii";

/// Tab and newline: the two control characters ASCII text is made of.
const WHITESPACE: CharRange = CharRange {
    start: '\u{0009}',
    end: '\u{000A}',
};

/// Space through tilde: every printable ASCII character.
const PRINTABLE: CharRange = CharRange {
    start: '\u{0020}',
    end: '\u{007E}',
};

/// `ascii` as a definition, so it composes like any other set (V25).
///
/// Stated in the same shape a data file would state it, so that a rule
/// naming `ascii` alongside a file-declared set takes one path through
/// resolution rather than two.
#[must_use]
pub(crate) fn ascii_definition() -> SetDefinition {
    SetDefinition {
        name: ASCII.to_owned(),
        members: vec![
            SetMember::Range(WHITESPACE),
            SetMember::Range(PRINTABLE),
        ],
    }
}

/// `ascii` resolved: printable ASCII plus tab and newline.
///
/// Carriage return is NOT here. It is the separate `cr` set, because a file
/// that grants it is making a decision about line endings rather than about
/// characters.
#[must_use]
pub(crate) fn ascii() -> CharSet {
    CharSet::new(ASCII.to_owned(), vec![WHITESPACE, PRINTABLE])
}

/// A catalog declaring the intrinsic set and nothing else.
///
/// The floor of the precedence chain in `src/rules:V19`: the builtin data
/// files, then discovered dotfiles, then flags are inserted over this, each
/// replacing a name the last one declared. Starting from `ascii` rather
/// than from nothing is what makes `--no-builtin-sets` survivable.
#[must_use]
pub(crate) fn intrinsic_catalog() -> SetCatalog {
    let mut catalog = SetCatalog::new();
    catalog.insert(ascii_definition());
    catalog
}

/// The preset data, in the `.ctrm-sets` grammar (V22): the hand-written
/// presets, then the generated hazard file last. The generated locale
/// letters (`src/charset/locale:V30`) are NOT here: they resolve lazily,
/// see `super::locale`.
///
/// Public as TEXT because that is what the precedence chain takes: the
/// builtin is the lowest source in `src/rules:V19` and arrives there the
/// same way a dotfile or a `--sets-file` does. One grammar, one parser,
/// and a preset a user overrides by declaring the name again.
///
/// TWO FILES, ONE TEXT. The chain holds a single builtin source, and the
/// hazard file is generated (`src/lint/hazard:V34`) so it cannot share a file
/// with presets written by hand. Joining them at compile time keeps the
/// chain's shape; the cost is that a builtin line number counts from the
/// top of the joined text, which only a defect in this crate would ever
/// show and which the parse test below keeps from shipping.
pub(crate) const SETS: &str = concat!(
    include_str!("sets.ctrm-sets"),
    include_str!("hazard.ctrm-sets")
);

/// The curated presets in the order `ctrm sets` lists them, each with its
/// one-line description (`src/cli/explain:V127`): the `#: <name> <text>`
/// lines of the preset file. Kept beside the sets they describe, so a
/// preset and the sentence about it are reviewed in one diff.
pub(crate) fn described() -> impl Iterator<Item = (&'static str, &'static str)>
{
    SETS.lines()
        .filter_map(|line| line.strip_prefix("#: ")?.split_once(' '))
}

/// The generated hazard file alone (`src/lint/hazard:V34`).
///
/// Kept apart from [`SETS`] because the lint node reads it DIRECTLY: a
/// hazard fires from these compiled-in bytes, never from the catalog a run
/// assembled, so neither a `.ctrm-sets` redeclaring a hazard name nor a
/// run without the builtin sets can empty the set that forbids.
pub(crate) const HAZARD: &str = include_str!("hazard.ctrm-sets");

/// The RGI emoji sequences, vendored from Unicode emoji data
/// (`src/fix/emoji:V62`): one `<kind> <sequence> [name]` line each. Not a set
/// file -- a set holds code points and a sequence is not one -- so it is
/// read through [`emoji_sequences`], by the lint node for its exemption
/// (`src/lint/hazard:V63`), compiled in so no configuration can widen it.
const EMOJI_SEQUENCES: &str = include_str!("emoji-sequences.txt");

/// What a vendored RGI sequence is, per the list's first word.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum SequenceKind {
    Zwj,
    Tag,
    Keycap,
    Presentation,
    Flag,
}

/// One vendored RGI emoji sequence, decoded.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct EmojiSequence {
    pub(crate) kind: SequenceKind,
    pub(crate) text: String,
}

/// Every sequence in the vendored list, in file order. A line this cannot
/// read is skipped rather than guessed at: `src/lint`'s test counts the
/// lines it needs, so a skipped one cannot ship.
pub(crate) fn emoji_sequences() -> impl Iterator<Item = EmojiSequence> {
    EMOJI_SEQUENCES.lines().filter_map(sequence_line)
}

/// One `<kind> <sequence> [name]` line; `None` for a comment.
fn sequence_line(line: &str) -> Option<EmojiSequence> {
    let mut words = line.split_whitespace();
    let kind = match words.next()? {
        "zwj" => SequenceKind::Zwj,
        "tag" => SequenceKind::Tag,
        "keycap" => SequenceKind::Keycap,
        "presentation" => SequenceKind::Presentation,
        "flag" => SequenceKind::Flag,
        _ => return None,
    };
    let text = super::code_points(words.next()?)?;
    Some(EmojiSequence { kind, text })
}

/// The hazard file's sets, and nothing else.
///
/// # Errors
///
/// As [`definitions`]: only a defect in the compiled-in file.
pub(crate) fn hazard_catalog() -> Result<SetCatalog, ParseError> {
    let mut catalog = SetCatalog::new();
    for line in HAZARD.lines() {
        if let Some(definition) = parse_line(line)? {
            catalog.insert(definition);
        }
    }
    Ok(catalog)
}

/// The presets the data file declares, unresolved.
///
/// Members stay unresolved for the reason [`SetDefinition`] states: a
/// preset here may name another, and `any` would otherwise expand in
/// every catalog that never uses it.
///
/// # Errors
///
/// Returns a [`ParseError`] if a line of the compiled-in file cannot be
/// read. That is a defect in THIS crate, not in anyone's configuration --
/// the test below is what keeps it from shipping -- but it is returned
/// rather than panicked, because a library that kills the process leaves
/// its caller no way to say which set was wrong.
pub(crate) fn definitions() -> Result<Vec<SetDefinition>, ParseError> {
    SETS.lines()
        .map(parse_line)
        .filter_map(Result::transpose)
        .collect()
}

/// The intrinsic set plus every preset the data file declares.
///
/// This is the floor of `src/rules:V19` with the builtin source present,
/// as [`intrinsic_catalog`] is the floor with it removed. The data file
/// is inserted OVER `ascii`, which costs nothing today and is the right
/// order the moment the file has anything to say about it.
///
/// # Errors
///
/// As [`definitions`].
pub(crate) fn catalog() -> Result<SetCatalog, ParseError> {
    let mut catalog = intrinsic_catalog();
    for definition in definitions()? {
        catalog.insert(definition);
    }
    Ok(catalog)
}

impl SetCatalog {
    /// Every set this build ships: `ascii`, the presets, the hazard classes
    /// and every CLDR locale (`src/charset/locale:V30`). What a library
    /// caller resolves names
    /// against; the binary assembles its own, layered with configuration.
    ///
    /// INFALLIBLE because the data is compiled in. A defect in it is this
    /// crate's, never the caller's, and `builtin_test` keeps it from
    /// shipping; were one to slip through, `ascii` alone still resolves.
    #[must_use]
    pub fn builtin() -> Self {
        let all = catalog()
            .and_then(|mut built| super::adopt_all(&mut built).map(|()| built));
        all.unwrap_or_else(|_| intrinsic_catalog())
    }
}

#[cfg(test)]
#[path = "builtin_test.rs"]
mod tests;

#[cfg(test)]
#[path = "growth_test.rs"]
mod growth_tests;