rhaiper 0.2.0

Minifier for Rhai scripts, with Source Map v3 output
Documentation
//! Turns a token table back into source text, as small as possible.

use rhai::{Position, Token, TokenizeState};

use crate::lex::Tok;
use crate::rename::Renames;
use crate::stream::{LineIndex, StrStream};

#[derive(Debug, Clone)]
pub struct Options {
    /// Maximum characters per output line.
    ///
    /// Not cosmetic. Rhai's `Position` stores its column as a `u16` and *saturates* on overflow,
    /// so every character past column 65535 of a line reports as column 65535 — which would make
    /// error positions, and therefore the source map, useless on a single-line output over 64K
    /// characters. Wrapping costs one byte per line.
    ///
    /// Exceeded only by a single token wider than the budget, such as a long string literal,
    /// since tokens cannot be split.
    pub max_line_len: usize,
    /// Shorten local variable names where it can be shown safe.
    ///
    /// Opt-in, and declined per script when the script does anything that reads names at runtime.
    /// `Minified::rename_declined` reports why when it is.
    pub rename: bool,
    /// Expression nesting limits, as `(global, function)`, for the engine used to validate the
    /// script.
    ///
    /// Needed because a host may raise its own limits: rhai's defaults are 32 and 16, and a script
    /// written for a more permissive engine would otherwise be rejected here as too complex.
    pub max_expr_depth: Option<(usize, usize)>,
}

impl Default for Options {
    fn default() -> Self {
        Self {
            max_line_len: 200,
            rename: false,
            max_expr_depth: None,
        }
    }
}

/// A generated-to-original position pair. Lines and columns are 0-based, columns in characters.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Mapping {
    pub gen_line: u32,
    pub gen_col: u32,
    pub src_line: u32,
    pub src_col: u32,
}

#[derive(Debug)]
pub struct Emitted {
    pub text: String,
    pub mappings: Vec<Mapping>,
}

/// Drops comments and semicolons the parser treats as optional.
///
/// Two eliminations are safe from the token stream alone:
///
/// * a `;` whose next significant token is `}` or end-of-input — both block and top-level
///   terminator loops accept those without a preceding semicolon;
/// * runs of `;`, which parse as empty statements and are discarded outright.
///
/// A third case — `;` directly after the `}` of an `if`/`while`/`for`/`switch`/`try`/`fn`/bare
/// block — is *not* handled here. It needs to know whether that `}` closed a statement block or
/// an expression, since `let x = #{a:1};` and `let x = { 1 };` both end in `};` and there the
/// semicolon is required. Token context cannot distinguish them; it would take the AST.
pub fn significant(toks: &[Tok]) -> Vec<&Tok> {
    let kept: Vec<&Tok> = toks.iter().filter(|t| !t.is_comment()).collect();
    let is_semi = |t: &Tok| matches!(t.token, Token::SemiColon);

    let mut out: Vec<&Tok> = Vec::with_capacity(kept.len());
    for (i, tok) in kept.iter().enumerate() {
        if is_semi(tok) {
            // Collapse runs of semicolons.
            if out.last().copied().is_some_and(is_semi) {
                continue;
            }
            // Keep one if dropping it would empty the enclosing block, so `{;}` stays `{;}`.
            let opens_block = out
                .last()
                .is_some_and(|p| matches!(p.token, Token::LeftBrace | Token::MapStart));
            if !opens_block {
                let next = kept[i + 1..].iter().find(|k| !is_semi(k));
                match next {
                    None => continue,
                    Some(k) if matches!(k.token, Token::RightBrace) => continue,
                    _ => (),
                }
            }
        }
        out.push(tok);
    }
    out
}

/// Byte length of the first token lexed from `probe`, or `None` if it does not lex.
fn first_token_len(probe: &str) -> Option<usize> {
    let mut stream = StrStream::new(probe);
    let mut state = TokenizeState::default();
    let mut pos = Position::START;
    match rhai::get_next_token(&mut stream, &mut state, &mut pos) {
        (Token::LexError(..), _) | (Token::EOF, _) => None,
        _ => Some(stream.byte_offset()),
    }
}

/// Whether `prev` and `next` need whitespace between them to keep lexing the same.
///
/// Decided by re-lexing rather than by a table of hazardous pairs: join the two and check that the
/// first token still ends exactly where `prev` did. If the boundary moved, the join created or
/// extended a token and a separator is required. This catches `-` + `-1` becoming the reserved
/// `--`, `let` + `x` becoming `letx`, `<` + `<` becoming `<<`, and so on, without enumerating them.
///
/// `lookahead` is the text following `next`, needed because the lexer's decisions depend on more
/// than two tokens: `1` + `.` alone would look like the float `1.`, but `1` + `.` + `abs` lexes as
/// three tokens and needs no separator.
///
/// Probing is only meaningful for tokens that lex the same standalone as they did in context, which
/// is why callers must first apply the adjacency rule in [`emit`]: an interpolated string's
/// segments only lex correctly in the lexer's text mode, so probing `" and $"` reads it as the
/// identifier `and` and wrongly demands a separator — which would break `${` apart.
fn needs_separator(prev: &str, next: &str, lookahead: &str) -> bool {
    if prev.is_empty() || next.is_empty() {
        return false;
    }
    // A token ending in whitespace (multi-line strings do) already separates itself.
    if prev.ends_with(|c: char| c.is_whitespace()) {
        return false;
    }
    let mut probe = String::with_capacity(prev.len() + next.len() + lookahead.len());
    probe.push_str(prev);
    probe.push_str(next);
    probe.push_str(lookahead);
    first_token_len(&probe) != Some(prev.len())
}

/// Emits the minified text and its mappings.
///
/// `renames` supplies replacement text for identifier tokens, keyed by starting byte offset; pass an
/// empty map to emit the source's own names.
pub fn emit(src: &str, toks: &[Tok], opts: &Options, renames: &Renames) -> Emitted {
    let lines = LineIndex::new(src);
    let kept = significant(toks);
    let texts: Vec<&str> = kept
        .iter()
        .map(|t| {
            renames
                .get(&t.span.start)
                .map_or_else(|| t.text(src), String::as_str)
        })
        .collect();

    let mut text = String::with_capacity(src.len());
    let mut mappings = Vec::with_capacity(kept.len());
    let (mut gen_line, mut gen_col) = (0u32, 0u32);
    let mut prev = "";
    let mut prev_end: Option<usize> = None;
    let mut prev_span_start: Option<usize> = None;

    for (i, tok) in kept.iter().enumerate() {
        // Enough following text for the lexer's lookahead, without paying to join the whole rest.
        let lookahead: String = texts[i + 1..]
            .iter()
            .flat_map(|s| s.chars())
            .take(4)
            .collect();

        // Width this token adds to the current line. A token spanning newlines (a multi-line raw
        // or back-tick string) resets the line itself, so only its first line counts.
        let width = texts[i].chars().take_while(|&c| c != '\n').count();
        // Tokens already touching in the source stay touching, and need no probe: the joined text
        // is a verbatim slice of the original, so it re-lexes to exactly these tokens. This is also
        // what keeps `${` and an interpolated string's resumed text intact, since those boundaries
        // are contiguous by construction and must never be separated.
        //
        // The reasoning only holds for text that is still the original's, so a renamed token on
        // either side falls through to the probe. String segments are never renamed, so the
        // interpolation boundaries keep the shortcut.
        let untouched = !renames.contains_key(&tok.span.start)
            && !prev_span_start.is_some_and(|s| renames.contains_key(&s));
        let contiguous = untouched && prev_end == Some(tok.span.start);

        let overflows = !contiguous && gen_col > 0 && gen_col as usize + width > opts.max_line_len;
        let separator = if overflows {
            // A newline is legal wherever a space is: whitespace is never significant between
            // tokens, and every multi-character construct is a single token.
            Some('\n')
        } else if !contiguous && needs_separator(prev, texts[i], &lookahead) {
            Some(' ')
        } else {
            None
        };
        if let Some(sep) = separator {
            text.push(sep);
            if sep == '\n' {
                gen_line += 1;
                gen_col = 0;
            } else {
                gen_col += 1;
            }
        }

        let (src_line, src_col) = lines.line_col_of(src, tok.span.start);
        mappings.push(Mapping {
            gen_line,
            gen_col,
            src_line,
            src_col,
        });

        text.push_str(texts[i]);
        for ch in texts[i].chars() {
            if ch == '\n' {
                gen_line += 1;
                gen_col = 0;
            } else {
                gen_col += 1;
            }
        }
        prev = texts[i];
        prev_end = Some(tok.span.end);
        prev_span_start = Some(tok.span.start);
    }

    Emitted { text, mappings }
}