characterminator 0.1.2

find and eliminate characters outside an allowed set, per file type and per file, so text costs fewer tokens
Documentation
//! Positions over a `&str`: which character sits where.
//!
//! ALLOWED-NESS IS A PARAMETER. This node answers "what character is
//! where", never "is it allowed": the sets live in `src/charset` and the
//! precedence that picks one lives in `src/rules`, so the test arrives as
//! a closure rather than being looked up here (`src:V39`).

use super::{Hit, Position};

/// A walking cursor over one text.
///
/// It carries the line and column ONLY: the byte offset comes from the
/// `char_indices` walk itself, and a second copy of a number that is
/// already exact is a second chance to get it wrong.
struct Cursor {
    line: usize,
    column: usize,
}

impl Cursor {
    /// Both units are 1-BASED, because every editor and every compiler
    /// diagnostic a reader compares this against is (`V12`).
    const fn start() -> Self {
        Self { line: 1, column: 1 }
    }

    /// The column counts CHARACTERS, so it moves once per `char` whatever
    /// that char weighs in bytes.
    ///
    /// A line ends at the line feed, and nothing else. In a CRLF file the
    /// carriage return is an ordinary character on the line it closes:
    /// one terminator, rather than two spellings of one that then have to
    /// agree. A lone carriage return therefore does NOT start a line --
    /// the old Mac ending is deliberately not honoured, because guessing
    /// between the two would make the line number depend on a heuristic.
    fn advance(&mut self, character: char) {
        if character == '\n' {
            self.line = self.line.saturating_add(1);
            self.column = 1;
        } else {
            self.column = self.column.saturating_add(1);
        }
    }

    const fn position(&self, byte: usize) -> Position {
        Position {
            line: self.line,
            column: self.column,
            byte,
        }
    }
}

/// Every character of a text, each with its position, in byte order.
///
/// This is THE walk: `scan_str` filters it, and a caller whose question
/// is about a character's NEIGHBOURS rather than the character alone --
/// a carriage return before a line feed, whitespace before a line end
/// (`src/lint/pedantic:V55`) -- reads it whole. Exposing the walk rather than a
/// second cursor keeps one definition of where a line ends and what a
/// column counts, so a line-shaped finding and a character finding at
/// the same place always agree on its position.
pub(crate) fn located(text: &str) -> impl Iterator<Item = Hit> + '_ {
    text.char_indices()
        .scan(Cursor::start(), |cursor, (byte, character)| {
            let position = cursor.position(byte);
            cursor.advance(character);
            Some(Hit {
                position,
                character,
            })
        })
}

/// Locate every character the caller's test rejects.
///
/// `allowed` is the caller's rule and the only opinion in the call: a
/// `false` makes a hit, a `true` makes nothing. Hits come out in byte
/// order, which is the within-a-file half of V12's sort order; ordering
/// ACROSS files is `src/render`'s.
///
/// ```
/// use characterminator::scan_str;
///
/// let hits = scan_str("caf\u{e9}\n", |c| c.is_ascii());
/// let first = hits.first().map(|h| (h.character, h.position.column));
/// assert_eq!(first, Some(('\u{e9}', 4)));
/// ```
#[must_use]
pub fn scan_str(text: &str, allowed: impl Fn(char) -> bool) -> Vec<Hit> {
    located(text)
        .filter(|hit| !allowed(hit.character))
        .collect()
}

#[cfg(test)]
mod tests {
    use super::{located, scan_str};
    use crate::scan::{Hit, Position};

    /// Zero width space: invisible, two tokens wide, and the reason this
    /// tool exists.
    const ZWSP: char = '\u{200b}';

    /// A stand-in for the rule a real caller resolves, so the tests here
    /// exercise the WALK rather than any set.
    fn ascii_only(character: char) -> bool {
        character.is_ascii()
    }

    fn hit(line: usize, column: usize, byte: usize, found: char) -> Hit {
        Hit {
            position: Position { line, column, byte },
            character: found,
        }
    }

    #[test]
    fn column_counts_characters_not_bytes() {
        // "e with acute" weighs two bytes, so a byte column would call
        // the zero width space after it column 3.
        let hits = scan_str("\u{00e9}\u{200b}", ascii_only);

        let expected = vec![hit(1, 1, 0, '\u{00e9}'), hit(1, 2, 2, ZWSP)];

        assert_eq!(hits, expected);
    }

    #[test]
    fn carriage_return_line_feed_starts_one_new_line() {
        // The carriage return is column 2 of line 1; the hit lands at
        // column 1 of line 2, byte 3.
        let hits = scan_str("a\r\n\u{200b}b", ascii_only);

        assert_eq!(hits, vec![hit(2, 1, 3, ZWSP)]);
    }

    #[test]
    fn hits_arrive_in_byte_order_within_one_text() {
        let hits = scan_str("\u{200b}a\u{00a0}", ascii_only);

        assert_eq!(hits, vec![hit(1, 1, 0, ZWSP), hit(1, 3, 4, '\u{00a0}')]);
    }

    #[test]
    fn the_walk_yields_every_character_with_its_position() {
        let walked: Vec<Hit> = located("a\r\n\u{e9}").collect();
        let expected = vec![
            hit(1, 1, 0, 'a'),
            hit(1, 2, 1, '\r'),
            hit(1, 3, 2, '\n'),
            hit(2, 1, 3, '\u{e9}'),
        ];

        assert_eq!(walked, expected);
        assert_eq!(located("").count(), 0);
    }

    #[test]
    fn the_test_is_the_callers_and_nothing_is_reported_without_it() {
        let hits = scan_str("\u{200b}\u{00a0}", |_| true);

        assert_eq!(hits, Vec::new());
    }
}