rhaiper 0.2.0

Minifier for Rhai scripts, with Source Map v3 output
Documentation
//! Drives rhai's own tokenizer over a [`StrStream`], recording a token table.
//!
//! We use rhai's lexer rather than writing one so there is no second implementation of Rhai's
//! lexical rules to keep in sync. `include_comments` is enabled so comments arrive as tokens
//! instead of being silently skipped; that way consecutive tokens bound each other with nothing
//! but trivia in between, which is what makes byte spans recoverable.
//!
//! Token text always comes from slicing the source with those spans, never from the token
//! payload. Payloads are decoded rather than verbatim — `StringConstant` holds the unescaped
//! value, raw strings collapse into the same variant as quoted ones, and `Display for Token`
//! cannot reproduce either. Rhai's own `last_token` mechanism is deliberately *not* used: it is
//! also lossy, recording `1000000` for `1_000_000` (digit separators are stripped before it is
//! written) and only `#"` for a raw string, dropping the body.
//!
//! Spans are derived from the stream cursor rather than from rhai's `Position`, which is what lets
//! this work under `no_position` — there `Position` degrades to a unit struct whose `line()` and
//! `position()` always return `None`.
//!
//! Slicing is verified instead, by the tiling invariant in [`tokenize`]: spans must be ordered,
//! non-overlapping, and separated only by whitespace, covering the whole source.

use std::ops::Range;

use rhai::{Position, Token, TokenizeState};

use crate::stream::{LineIndex, StrStream};

/// A source location, in the units rhai reports: 1-based line and character column.
///
/// Carried instead of a rhai `Position` so diagnostics stay meaningful under `no_position`.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct At {
    pub line: u32,
    pub column: u32,
}

impl At {
    pub(crate) fn new(lines: &LineIndex, src: &str, offset: usize) -> Self {
        let (line, column) = lines.line_col_of(src, offset);
        Self {
            line: line + 1,
            column: column + 1,
        }
    }
}

impl std::fmt::Display for At {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(f, "line {}, column {}", self.line, self.column)
    }
}

/// One token, with its exact byte span in the source.
#[derive(Debug, Clone)]
pub struct Tok {
    pub token: Token,
    pub span: Range<usize>,
}

impl Tok {
    /// Verbatim source text of this token.
    pub fn text<'s>(&self, src: &'s str) -> &'s str {
        &src[self.span.clone()]
    }

    pub fn is_comment(&self) -> bool {
        matches!(self.token, Token::Comment(..))
    }
}

#[derive(Debug)]
pub enum LexFailure {
    /// The tokenizer produced a `LexError`.
    Lex { message: String, at: At },
    /// A token's span overlapped or preceded the previous token's.
    OverlappingSpan {
        at: At,
        span: Range<usize>,
        previous_end: usize,
    },
    /// Tokens did not tile the source: the gap before this token was not pure whitespace.
    NonWhitespaceGap { at: At, gap: String },
    /// Text after the last token was not pure whitespace.
    NonWhitespaceTail { tail: String },
    /// Reached end of input with an interpolated string still open.
    UnterminatedInterpolation,
}

impl std::fmt::Display for LexFailure {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self {
            Self::Lex { message, at } => write!(f, "{message} ({at})"),
            Self::OverlappingSpan {
                at,
                span,
                previous_end,
            } => write!(
                f,
                "span {span:?} at {at} overlaps the previous token, which ended at {previous_end}"
            ),
            Self::NonWhitespaceGap { at, gap } => {
                write!(f, "non-whitespace {gap:?} precedes the token at {at}")
            }
            Self::NonWhitespaceTail { tail } => {
                write!(f, "non-whitespace {tail:?} follows the last token")
            }
            Self::UnterminatedInterpolation => {
                f.write_str("input ended with an interpolated string still open")
            }
        }
    }
}

impl std::error::Error for LexFailure {}

/// Keeps the lexer's text/code mode in step across `` `...${...}...` `` strings.
///
/// This is the one place the tokenizer depends on parser cooperation: after an interpolation's
/// closing `}`, the lexer has to be told to resume reading string text rather than code. Rhai's
/// parser does it at `parser.rs:1456` by setting `is_within_text` on the shared control block,
/// which `TokenIterator::next` then turns into `is_within_text_terminated_by`. Driving
/// `get_next_token` ourselves, we set that field directly.
///
/// A `Token::InterpolatedString` is the text run ending at `${`, so each one opens a frame that
/// the following `{` deepens. Nested interpolations get their own frame, hence the stack.
fn track_interpolation(token: &Token, frames: &mut Vec<usize>, state: &mut TokenizeState) {
    match token {
        Token::InterpolatedString(..) => frames.push(0),
        Token::LeftBrace | Token::MapStart => {
            if let Some(depth) = frames.last_mut() {
                *depth += 1;
            }
        }
        Token::RightBrace => {
            if let Some(depth) = frames.last_mut() {
                *depth -= 1;
                if *depth == 0 {
                    frames.pop();
                    // Resume text mode; the run ends at the next back-tick or `${`.
                    state.is_within_text_terminated_by = Some("`".into());
                }
            }
        }
        _ => (),
    }
}

/// Skips the trivia between tokens, reporting any comment rhai consumed without returning.
///
/// Normally the only trivia is whitespace, because `include_comments` makes comments real tokens.
/// The exception is `metadata`: there a `//!` module comment is appended to `global_comments` and
/// **never returned** (`tokenizer.rs:2167-2175`), so the cursor advances over text we never saw.
/// Those runs are reported back so the caller can re-insert them as comment tokens, keeping the
/// tiling invariant exact and leaving their bytes accounted for in `stats`.
///
/// Only ASCII whitespace is skipped, matching what rhai's tokenizer skips. Anything else it would
/// have rejected as unexpected input, which surfaces as a lex error instead.
fn skip_trivia(src: &str, mut at: usize) -> (usize, Vec<Range<usize>>) {
    let mut swallowed = Vec::new();
    loop {
        at += src[at..]
            .bytes()
            .take_while(u8::is_ascii_whitespace)
            .count();

        if src[at..].starts_with("//!") {
            // Runs to the end of the line, newline included, matching what the scanner consumed.
            let end = src[at..]
                .find('\n')
                .map_or(src.len(), |offset| at + offset + 1);
            swallowed.push(at..end);
            at = end;
            continue;
        }
        return (at, swallowed);
    }
}

/// Records one token, checking the tiling invariant as it goes.
fn accept(
    src: &str,
    lines: &LineIndex,
    out: &mut Vec<Tok>,
    tiled: &mut usize,
    token: Token,
    span: Range<usize>,
) -> Result<(), LexFailure> {
    // Spans must advance, so no token can swallow another's text.
    if span.start < *tiled {
        return Err(LexFailure::OverlappingSpan {
            at: At::new(lines, src, span.start),
            span,
            previous_end: *tiled,
        });
    }
    // Everything between the previous token and this one is whitespace.
    let gap = &src[*tiled..span.start];
    if !gap.chars().all(char::is_whitespace) {
        return Err(LexFailure::NonWhitespaceGap {
            at: At::new(lines, src, span.start),
            gap: gap.to_string(),
        });
    }
    *tiled = span.end;
    out.push(Tok { token, span });
    Ok(())
}

/// Tokenizes `src`, returning every token including comments.
///
/// Verifies as it goes that the tokens exactly tile `src` with only whitespace in the gaps. That
/// check is what makes the rest of the pipeline safe: it catches any case where our byte cursor
/// and rhai's own idea of a token's extent disagree.
pub fn tokenize(src: &str) -> Result<Vec<Tok>, LexFailure> {
    let lines = LineIndex::new(src);
    let mut stream = StrStream::new(src);
    let mut state = TokenizeState {
        include_comments: true,
        ..Default::default()
    };
    let mut pos = Position::START;
    let mut out = Vec::new();
    let mut tiled = 0usize;
    // Brace depth of each enclosing `${...}`, innermost last. A frame at depth 0 is still waiting
    // for its opening `{`.
    let mut interpolations: Vec<usize> = Vec::new();

    loop {
        // Text mode resumes at the exact byte after an interpolation's `}`, so no trivia can
        // intervene — and the segment may legitimately *begin* with a space, as in `` `${x} tail` ``.
        // Skipping whitespace there would silently eat it out of the string.
        let resuming_text = state.is_within_text_terminated_by.is_some();
        let before = stream.byte_offset();

        // `get_next_token` records whether the *next* token may be a unary operator, and a comment
        // token clobbers it: rhai proper never emits comments, so after `= /*c*/` it would still
        // read `-1` as one negative literal, whereas an intervening comment token would split it
        // into `-` and `1`. Restoring the flag keeps our token stream identical to a real parse.
        let unary_before = state.next_token_cannot_be_unary;
        let (token, _) = rhai::get_next_token(&mut stream, &mut state, &mut pos);
        if let Token::Comment(..) = token {
            state.next_token_cannot_be_unary = unary_before;
        }
        let end = stream.byte_offset();

        let start = if resuming_text {
            before
        } else {
            let (start, swallowed) = skip_trivia(src, before);
            // Re-insert any comment rhai consumed without handing back, before the token that
            // followed it. Also matters at end of input, where a trailing `//!` would otherwise be
            // left over and trip the tail check.
            for span in swallowed {
                let text = src[span.clone()].to_string();
                accept(
                    src,
                    &lines,
                    &mut out,
                    &mut tiled,
                    Token::Comment(Box::new(text)),
                    span,
                )?;
            }
            start
        };

        if let Token::EOF = token {
            break;
        }
        if let Token::LexError(err) = &token {
            return Err(LexFailure::Lex {
                message: err.to_string(),
                at: At::new(&lines, src, start),
            });
        }

        track_interpolation(&token, &mut interpolations, &mut state);
        accept(src, &lines, &mut out, &mut tiled, token, start..end)?;
    }

    if !interpolations.is_empty() {
        return Err(LexFailure::UnterminatedInterpolation);
    }

    // The tokens plus whitespace must account for the entire source.
    let tail = &src[tiled..];
    if !tail.chars().all(char::is_whitespace) {
        return Err(LexFailure::NonWhitespaceTail {
            tail: tail.to_string(),
        });
    }

    Ok(out)
}