rhaiper 0.2.0

Minifier for Rhai scripts, with Source Map v3 output
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
//! Minifier for Rhai scripts, with Source Map v3 output.
//!
//! Strips comments, collapses whitespace, and drops redundant semicolons, while recording where
//! every surviving token came from so a runtime error reported against the minified script can be
//! traced back to the original source.
//!
//! Correctness rests on re-lexing the output and requiring the token stream to be unchanged, so a
//! broken script is never emitted silently.

pub mod emit;
pub mod lex;
pub mod map;
pub mod rename;
pub mod stream;

pub use emit::{Mapping, Options};

use rhai::{Engine, Token};

#[derive(Debug)]
pub enum Error {
    /// The input script does not parse. Reported before any work is done, so callers get rhai's
    /// own diagnostics rather than a confusing failure later on.
    InvalidInput(Box<rhai::ParseError>),
    /// Tokenizing the input failed, or its tokens did not tile the source.
    Lex(lex::LexFailure),
    /// The output's token stream differs from the input's. Always a bug in this crate.
    TokenMismatch {
        index: usize,
        expected: String,
        found: String,
    },
    /// The output does not parse. Always a bug in this crate.
    OutputInvalid(Box<rhai::ParseError>),
}

impl std::fmt::Display for Error {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self {
            Self::InvalidInput(e) => write!(f, "input script does not parse: {e}"),
            Self::Lex(e) => write!(f, "{e}"),
            Self::TokenMismatch {
                index,
                expected,
                found,
            } => write!(
                f,
                "token {index} of the output is {found} but should be {expected} \
                 (this is a bug in rhaiper)"
            ),
            Self::OutputInvalid(e) => {
                write!(
                    f,
                    "minified output does not parse: {e} (this is a bug in rhaiper)"
                )
            }
        }
    }
}

impl std::error::Error for Error {}

impl From<lex::LexFailure> for Error {
    fn from(e: lex::LexFailure) -> Self {
        Self::Lex(e)
    }
}

#[derive(Debug)]
pub struct Minified {
    pub text: String,
    pub mappings: Vec<Mapping>,
    /// How many local bindings were shortened.
    pub renamed_bindings: usize,
    /// Why renaming was declined, when it was. Empty if renaming was not requested, or succeeded.
    pub rename_declined: Vec<String>,
}

impl Minified {
    /// Serializes a Source Map v3 document for this output.
    pub fn to_source_map(&self, src: &str, opts: &map::MapOptions) -> String {
        map::to_json(src, &self.text, &self.mappings, opts)
    }
}

/// Builds the engine used to parse and validate scripts.
///
/// `new_raw` because no registered functions are needed to parse, only to run. Expression-depth
/// limits are forwarded so a script written for a host with raised limits is not rejected here.
pub(crate) fn parsing_engine(opts: &Options) -> Engine {
    #[allow(unused_mut)]
    let mut engine = Engine::new_raw();

    // Renaming reads declarations off the AST, so the optimizer must not fold them away first.
    // Under `no_optimize` there is no optimizer and the AST is never optimized, so there is nothing
    // to turn off — and both the method and its argument type are gone.
    #[cfg(not(feature = "no_optimize"))]
    engine.set_optimization_level(rhai::OptimizationLevel::None);

    // Under `unchecked` rhai enforces no limits at all, so `max_expr_depth` has nothing to raise.
    #[cfg(not(feature = "unchecked"))]
    if let Some((global, _function)) = opts.max_expr_depth {
        // The function-depth parameter is itself `#[cfg(not(no_function))]` — with no functions
        // there is no function nesting to bound, so the method takes one argument instead of two.
        #[cfg(not(feature = "no_function"))]
        engine.set_max_expr_depths(global, _function);
        #[cfg(feature = "no_function")]
        engine.set_max_expr_depths(global);
    }
    let _ = opts;

    engine
}

/// Minifies `src`, verifying the result before returning it.
///
/// Uses a stock engine to parse. Scripts relying on host-registered custom syntax or operators need
/// [`minify_with_engine`] instead, since only the host can describe those.
///
/// ```
/// use rhaiper::{Options, map::MapOptions, minify};
///
/// let source = "\
/// // Convert a raw reading into millivolts.
/// fn to_millivolts(raw) {
///     let scale = 3300.0 / 4095.0;
///     raw * scale
/// }
/// to_millivolts(2048)
/// ";
///
/// let out = minify(source, &Options::default())?;
/// assert_eq!(
///     out.text,
///     "fn to_millivolts(raw){let scale=3300.0/4095.0;raw*scale}to_millivolts(2048)"
/// );
///
/// // A Source Map v3 document, so a reported position can be traced back.
/// let map = out.to_source_map(source, &MapOptions::default());
/// assert!(map.contains("\"version\":3"));
/// # Ok::<(), rhaiper::Error>(())
/// ```
///
/// With `rename` enabled, locals are shortened too — but never top-level bindings, function names,
/// or properties, and not at all if the script resolves names at runtime:
///
/// ```
/// use rhaiper::{Options, minify};
///
/// let opts = Options { rename: true, ..Default::default() };
/// let out = minify("fn f(reading) { let scaled = reading * 2; scaled } f(1)", &opts)?;
/// assert_eq!(out.text, "fn f(b){let a=b*2;a}f(1)");
/// assert_eq!(out.renamed_bindings, 2);
///
/// // `eval` reads names at runtime, so renaming is declined and the reason reported.
/// let out = minify("fn f() { let x = 1; eval(\"x\") } f()", &opts)?;
/// assert_eq!(out.renamed_bindings, 0);
/// assert!(out.rename_declined[0].contains("eval"));
/// # Ok::<(), rhaiper::Error>(())
/// ```
pub fn minify(src: &str, opts: &Options) -> Result<Minified, Error> {
    minify_with_engine(&parsing_engine(opts), src, opts)
}

/// Minifies `src` using a caller-supplied engine.
///
/// Pass the same engine the script will run under, so anything it registers — custom syntax, custom
/// operators, disabled symbols — is understood while parsing. Without that, a script using custom
/// syntax is rejected as unparseable.
///
/// Set [`OptimizationLevel::None`] on the engine when renaming. Otherwise the optimizer may fold away
/// declarations before the scope analysis sees them, and renaming will be declined rather than get it
/// wrong.
///
/// [`OptimizationLevel::None`]: rhai::OptimizationLevel
pub fn minify_with_engine(engine: &Engine, src: &str, opts: &Options) -> Result<Minified, Error> {
    engine
        .compile(src)
        .map_err(|e| Error::InvalidInput(Box::new(e)))?;

    let toks = lex::tokenize(src)?;

    let renaming = if opts.rename {
        rename::plan(engine, src, &toks)
    } else {
        rename::Outcome::default()
    };
    let emitted = emit::emit(src, &toks, opts, &renaming.renames);

    // The output must lex to exactly the token stream the input reduces to. This is what lets the
    // separator rule be merely conservative rather than provably complete.
    let expected: Vec<(&Token, &str)> = emit::significant(&toks)
        .iter()
        .map(|t| {
            let text = renaming
                .renames
                .get(&t.span.start)
                .map_or_else(|| t.text(src), String::as_str);
            (&t.token, text)
        })
        .collect();
    let round_trip = lex::tokenize(&emitted.text)?;
    let found: Vec<(&Token, &str)> = emit::significant(&round_trip)
        .iter()
        .map(|t| (&t.token, t.text(&emitted.text)))
        .collect();
    verify_tokens(&expected, &found)?;

    engine
        .compile(&emitted.text)
        .map_err(|e| Error::OutputInvalid(Box::new(e)))?;

    Ok(Minified {
        text: emitted.text,
        mappings: emitted.mappings,
        renamed_bindings: renaming.renamed_bindings,
        rename_declined: renaming.declined,
    })
}

/// Requires the output's tokens to match the input's, allowing only the planned renames.
///
/// Each entry pairs a token with the text it should have in the output — its own source text, or its
/// replacement if the rename plan covers it. Identifiers are therefore compared against that
/// expected text rather than against the input name, which is what makes shadowing work: two
/// distinct bindings that happen to share a name are supposed to end up with *different* new names,
/// so no mapping between old and new names could be one-to-one. Every other token must be identical.
///
/// What this pins down is that the emitter applied exactly the plan and altered nothing else.
/// Whether the plan itself picked the right tokens is established separately, by construction in
/// `rename::scope` and by the behavioural tests.
fn verify_tokens(expected: &[(&Token, &str)], found: &[(&Token, &str)]) -> Result<(), Error> {
    if expected.len() != found.len() {
        return Err(Error::TokenMismatch {
            index: expected.len().min(found.len()),
            expected: format!("{} tokens", expected.len()),
            found: format!("{} tokens", found.len()),
        });
    }

    for (index, ((want, want_text), (got, got_text))) in
        expected.iter().zip(found.iter()).enumerate()
    {
        let ok = match (want, got) {
            (Token::Identifier(..), Token::Identifier(..)) => want_text == got_text,
            _ => want == got,
        };
        if !ok {
            return Err(Error::TokenMismatch {
                index,
                expected: format!("{want_text:?} ({want:?})"),
                found: format!("{got_text:?} ({got:?})"),
            });
        }
    }
    Ok(())
}

#[cfg(test)]
mod tests {
    use super::lex::tokenize;
    use super::{Options, minify};

    /// The tiling invariant, asserted by `tokenize` itself: tokens must exactly cover the source
    /// with only whitespace in the gaps.
    fn tiles(src: &str) -> Vec<String> {
        match tokenize(src) {
            Ok(toks) => toks.iter().map(|t| t.text(src).to_string()).collect(),
            Err(e) => panic!("tokenize({src:?}) failed: {e}"),
        }
    }

    fn min(src: &str) -> String {
        match minify(src, &Options::default()) {
            Ok(m) => m.text,
            Err(e) => panic!("minify({src:?}) failed: {e}"),
        }
    }

    #[test]
    fn plain_statements() {
        assert_eq!(tiles("let x = 1;"), ["let", "x", "=", "1", ";"]);
    }

    #[test]
    fn comments_are_tokens() {
        // A line comment's span includes the newline that terminates it, so dropping the token
        // drops the newline with it.
        assert_eq!(
            tiles("let x = 1; // trailing\nx"),
            ["let", "x", "=", "1", ";", "// trailing\n", "x"]
        );
        assert_eq!(
            tiles("a /* one /* two */ */ b"),
            ["a", "/* one /* two */ */", "b"]
        );
    }

    #[test]
    fn string_literals_round_trip_verbatim() {
        // Payloads are decoded, so these only tile correctly via byte spans.
        assert_eq!(tiles(r#""a\nb""#), [r#""a\nb""#]);
        assert_eq!(
            tiles(r#""quote\" and \\ back""#),
            [r#""quote\" and \\ back""#]
        );
        assert_eq!(
            tiles(r##"#"raw \n not escaped"#"##),
            [r##"#"raw \n not escaped"#"##]
        );
        assert_eq!(
            tiles(r###"##"nested "# inside"##"###),
            [r###"##"nested "# inside"##"###]
        );
        assert_eq!(tiles(r"'\n'"), [r"'\n'"]);
        assert_eq!(tiles(r"'\x41'"), [r"'\x41'"]);
    }

    #[test]
    fn numeric_literals_keep_their_written_form() {
        assert_eq!(tiles("0xFF"), ["0xFF"]);
        assert_eq!(tiles("1_000_000"), ["1_000_000"]);
        assert_eq!(tiles("1.5e-3"), ["1.5e-3"]);
        assert_eq!(tiles("0b1010"), ["0b1010"]);
        assert_eq!(tiles("0o777"), ["0o777"]);
    }

    #[test]
    fn negative_literal_folds_the_sign_into_the_token() {
        // This is the shape that breaks rhai's own `compact_script`: the literal's verbatim text
        // is "-1", so joining it to a preceding `-` yields `--`, a reserved symbol.
        assert_eq!(tiles("a - -1"), ["a", "-", "-1"]);
    }

    #[test]
    fn non_ascii_positions_resolve() {
        assert_eq!(
            tiles(r#"let s = "héllo 🎉"; s"#),
            ["let", "s", "=", r#""héllo 🎉""#, ";", "s"]
        );
    }

    #[test]
    fn interpolated_strings() {
        assert_eq!(tiles("`a${x}b`"), ["`a$", "{", "x", "}", "b`"]);
        // The trailing segment here is empty, so rhai reports it at `Position::NONE`.
        assert_eq!(tiles("`${1+2}`"), ["`$", "{", "1", "+", "2", "}", "`"]);
        assert_eq!(
            tiles("`m ${ #{a:1}.a } z`"),
            ["`m $", "{", "#{", "a", ":", "1", "}", ".", "a", "}", " z`"]
        );
    }

    #[test]
    fn nested_interpolation() {
        assert_eq!(
            tiles("`outer ${ `inner ${y}` } tail`"),
            [
                "`outer $", "{", "`inner $", "{", "y", "}", "`", "}", " tail`"
            ]
        );
    }

    #[test]
    fn interpolation_containing_a_block() {
        // Braces inside the interpolation must not be mistaken for its terminator.
        assert_eq!(
            tiles("`v ${ if a { 1 } else { 2 } }`"),
            [
                "`v $", "{", "if", "a", "{", "1", "}", "else", "{", "2", "}", "}", "`"
            ]
        );
    }

    #[test]
    fn minifies_whitespace_and_comments() {
        assert_eq!(
            min("let count = 0; // running total\nfn step(x) {\n    count += x;\n}\n"),
            "let count=0;fn step(x){count+=x}"
        );
    }

    #[test]
    fn keeps_required_separators() {
        assert_eq!(min("let  x  =  1 ;  x"), "let x=1;x");
        assert_eq!(min("if a in b { 1 }"), "if a in b{1}");
        // `1 . abs()` may be joined: `1.abs` still lexes as three tokens.
        assert_eq!(min("let y = 1 . abs();"), "let y=1.abs()");
        // But `( )` is two tokens and joining them would make the single token `()`, so a space
        // stays. Costs a byte on an unusual input; `f()` in the source is already one token.
        assert_eq!(min("let y = 1 . abs ( ) ;"), "let y=1.abs( )");
    }

    #[test]
    fn separates_signs_that_would_form_reserved_symbols() {
        // The cases rhai's own `compact_script` gets wrong.
        assert_eq!(min("let x = a - -1;"), "let x=a- -1");
        assert_eq!(min("let x = a + +b;"), "let x=a+ +b");
        assert_eq!(min("let x = a - -b;"), "let x=a- -b");
    }

    #[test]
    fn comments_do_not_disturb_unary_lexing() {
        // Rhai never emits comment tokens itself, so `= /*c*/ -1` must still read as one negative
        // literal. If the comment token were allowed to clobber the unary flag, this would lex as
        // `-` then `1` and the output verification would reject a perfectly good script.
        assert_eq!(min("let x = /*c*/ -1;"), "let x=-1");
        // `+-` is not a reserved symbol (only `++` is), so these may be joined.
        assert_eq!(min("let x = 1 + /*c*/ -2;"), "let x=1+-2");
    }

    #[test]
    fn drops_redundant_semicolons() {
        assert_eq!(min("let x = 1;;;\nx;"), "let x=1;x");
        assert_eq!(min("fn f() { let a = 1; a; }"), "fn f(){let a=1;a}");
        // Not redundant: the semicolon terminates the `let`, it does not end a block.
        assert_eq!(min("let m = #{a:1};\nm"), "let m=#{a:1};m");
        assert_eq!(min("let b = { 1 };\nb"), "let b={1};b");
    }

    #[test]
    fn preserves_empty_block() {
        assert_eq!(min("fn f() { ; }"), "fn f(){;}");
    }

    #[test]
    fn wraps_long_output() {
        let src = (0..80)
            .map(|i| format!("let variable_number_{i} = {i};\n"))
            .collect::<String>();
        let out = minify(
            &src,
            &Options {
                max_line_len: 60,
                ..Default::default()
            },
        )
        .expect("minify");
        for line in out.text.lines() {
            assert!(
                line.chars().count() <= 60,
                "line of {} chars exceeds the budget: {line:?}",
                line.chars().count()
            );
        }
        assert!(
            out.text.lines().count() > 5,
            "expected several wrapped lines"
        );
    }
}