rubylang 0.1.18

Ruby as a fusevm frontend: lexer, parser, and compiler to fusevm::Chunk on a RubyHost heap — the first compiled standalone Ruby runtime
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
//! Ahead-of-time compilation (`ruby --build`).
//!
//! `--build` bundles the whole app: starting from the entrypoint it statically
//! resolves every literal `require`/`require_relative` (see `bundle.rs`), inlines
//! each required file, and lowers the combined program to fusevm bytecode, which
//! it warms into the on-disk cache (`cache.rs`). A subsequent `ruby FILE` then
//! runs the cached bundle directly — skipping lex/parse/lower AND needing none of
//! the required source files on disk. The report below is explicit user-requested
//! output.
//!
//! `--build --native` goes one step further and emits a **standalone native
//! executable** (`build_native`): no `ruby` interpreter, no `.rb` sources, no
//! cache. It uses the `fusevm` crate's `aot` feature — a Cranelift object emitter
//! — plus the rubylang runtime:
//!
//! 1. bundle → `Program` (main chunk + methods/classes/begins/procs), same as
//!    `build`.
//! 2. Every method/class-method/block chunk is tagged with a `native_id`
//!    (1, 2, …) and `fusevm::aot::compile_program_object(&prog.main, extras, app.o)`
//!    Cranelift-lowers *all* of them into one relocatable object: the main driver
//!    `fusevm_aot_entry`, one driver `fusevm_aot_fn_{native_id}` per method/block,
//!    the serialized main chunk (`fusevm_aot_chunk_blob`/`_len`), plus imports for
//!    the runtime shims and the frontend hook `fusevm_aot_register_builtins`.
//! 3. The full `Program` is serialized (`cache::program_to_blob`) and baked into
//!    a generated Rust frontend via `include_bytes!` — the `native_id` on each
//!    chunk rides along in the blob, so at runtime `host::run_chunk_on` reads it
//!    and dispatches the method/block to its native driver instead of interpreting.
//! 4. The frontend defines the blob symbols, declares the `fusevm_aot_fn_{i}`
//!    drivers and `fusevm_aot_entry`, registers their addresses (main's last)
//!    via `register_native_fns`, and a `main` that calls [`run_embedded`].
//!    `rustc` compiles it against the rubylang rlib (which statically links
//!    fusevm + the whole runtime) and links in `app.o`, producing the executable.
//!
//! [`run_embedded`] lives here, in the rubylang library, so it is baked into the
//! rlib the frontend links against. It seeds the host as `ruby FILE` does, loads
//! the embedded program, and runs the main chunk through `host::run_main`, so
//! method/block/const dispatch, `exit`, uncaught exceptions and `at_exit`
//! handlers behave identically to `ruby FILE`. fusevm's own
//! `fusevm_aot_run_embedded` and its [`fusevm_aot_register_builtins`] hook stay
//! linked but are not the entry point.

// ────────────────────────────────────────────────────────────────────────────
// Native method/block dispatch table (`--build --native` full lowering)
// ────────────────────────────────────────────────────────────────────────────

/// The C ABI of a chunk's AOT-lowered native driver, as fusevm emits it: it runs
/// the chunk's ops on `vm` and stores the result via `vm.take_aot_result()`.
type AotEntry = extern "C" fn(*mut fusevm::VM) -> i64;

/// The per-chunk native drivers a `--build --native` binary links in, indexed by
/// `Chunk::native_id - 1`. Populated once at startup by the generated frontend
/// (which alone can name the linked `fusevm_aot_fn_*` symbols) via
/// [`register_native_fns`]; empty in a normal interpreted run, where every chunk
/// has `native_id == 0` anyway.
static NATIVE_FNS: std::sync::OnceLock<Vec<AotEntry>> = std::sync::OnceLock::new();

/// Called once by the AOT frontend's `main` before the driver runs: `addrs` holds
/// the address of each linked `fusevm_aot_fn_{i}` (`fn as usize`), in native-id
/// order. Stored as typed entries so [`native_entry`] can hand one to the runtime.
///
/// # Safety
/// Each address must be a real `extern "C" fn(*mut VM) -> i64` linked into this
/// binary — which is exactly what the generated frontend passes.
pub fn register_native_fns(addrs: Vec<usize>) {
    let fns: Vec<AotEntry> = addrs
        .into_iter()
        .map(|a| unsafe { std::mem::transmute::<usize, AotEntry>(a) })
        .collect();
    let _ = NATIVE_FNS.set(fns);
}

/// The native driver for the chunk tagged `native_id`, if this binary AOT-lowered
/// it (`native_id != 0` and the table is registered). `None` means "interpret it"
/// — the default for a normal `ruby FILE` run.
pub fn native_entry(native_id: u32) -> Option<AotEntry> {
    if native_id == 0 {
        return None;
    }
    let f = NATIVE_FNS.get()?.get((native_id - 1) as usize).copied();
    if std::env::var_os("RUBYLANG_AOT_TRACE").is_some() {
        eprintln!(
            "[aot] dispatch native_id={native_id} -> {}",
            if f.is_some() { "NATIVE" } else { "miss" }
        );
    }
    f
}

/// Bundle `file` with everything it statically requires, compile the whole app to
/// bytecode, and store it in the cache with a dependency manifest. Returns a
/// one-line report of what was built.
pub fn build(file: &str) -> Result<String, String> {
    let (stmts, report) = crate::bundle::bundle(std::path::Path::new(file))?;
    let prog = crate::compiler::compile(&stmts, false)?;
    let (nmethods, nprocs, nops) = (prog.methods.len(), prog.procs.len(), prog.main.ops.len());

    // Dependency manifest: every inlined file's absolute path + content key, so a
    // later run detects (and recompiles past) a stale bundle after any bundled
    // file is edited. The cache is keyed on the canonical entrypoint path + its
    // source — what a plain `ruby FILE` run recomputes to find this bundle.
    let src = std::fs::read_to_string(file).map_err(|e| format!("cannot read {file}: {e}"))?;
    let abs = std::fs::canonicalize(file)
        .map_err(|e| format!("cannot read {file}: {e}"))?
        .to_string_lossy()
        .into_owned();
    let mut deps = Vec::with_capacity(report.files.len());
    for p in &report.files {
        let fsrc =
            std::fs::read_to_string(p).map_err(|e| format!("cannot read {}: {e}", p.display()))?;
        deps.push((
            p.to_string_lossy().into_owned(),
            crate::cache::key_for(&fsrc),
        ));
    }
    crate::cache::store_bundle(&abs, &src, &prog, deps)?;

    let nfiles = report.files.len();
    let dynamic = if report.dynamic > 0 {
        format!(", {} runtime require(s) left dynamic", report.dynamic)
    } else {
        String::new()
    };
    Ok(format!(
        "built {file}: {nfiles} file(s) bundled, {nops} top-level ops, {nmethods} methods, {nprocs} blocks{dynamic} -> ~/.rubylang/scripts.rkyv"
    ))
}

// ────────────────────────────────────────────────────────────────────────────
// Native standalone executable (`ruby --build --native`)
// ────────────────────────────────────────────────────────────────────────────

/// The standalone binary's run: what the generated `main` calls instead of
/// fusevm's `fusevm_aot_run_embedded`, whose return is the last expression's
/// value rather than an exit status and which never reaches the interpreter's
/// end of program.
///
/// It seeds the host as `ruby FILE` does for `script` (the entry file as given
/// to `--build`) — `ARGV` from the process arguments, `$0`, `__FILE__`,
/// `$LOAD_PATH` rooted at the cwd, `$VERBOSE`/`$DEBUG`, the prelude — then
/// runs the main chunk through `host::run_main` and `end_program`, so `exit`,
/// an uncaught exception's report and status, and `at_exit` handlers behave
/// exactly as they do interpreted. The main chunk still runs as machine code:
/// the frontend registers `fusevm_aot_entry` as the LAST native driver, and the
/// chunk is tagged with that slot, so `run_chunk_pooled` dispatches to it.
///
/// The embedded program is loaded before the prelude is compiled: its proc and
/// begin ids were fixed at build time from base 0 and its native drivers were
/// lowered against them, so it cannot be rebased; the prelude is rebased above
/// it instead. `synth_base` is the build's synthetic-name counter, so nothing
/// compiled here reuses a `__class_body__N` the program holds.
pub fn run_embedded(script: &str, synth_base: u64) -> i32 {
    let prog = embedded_program();
    let argv: Vec<String> = std::env::args().skip(1).collect();
    crate::host::reset_host();
    let r = crate::host::with_gvl(|| {
        seed_embedded_host();
        crate::host::push_file_path(script.to_string());
        crate::host::with_host(|h| h.set_program_args(&argv, script));
        crate::seed_verbosity(&crate::RunConfig::default());
        crate::compiler::advance_synth_ids(synth_base);
        let crate::compiler::Program {
            mut main,
            methods,
            classes,
            begins,
            procs,
        } = prog;
        crate::host::with_host(|h| h.load_program(methods, classes, begins, procs));
        crate::run_prelude()?;
        main.native_id = NATIVE_FNS.get().map_or(0, |f| f.len() as u32);
        crate::end_program(crate::host::run_main(main))
    });
    use std::io::Write;
    let _ = std::io::stdout().flush();
    match r {
        Ok(_) => 0,
        Err(e) => {
            eprintln!("{e}");
            1
        }
    }
}

/// The full program the generated frontend embeds, deserialized.
fn embedded_program() -> crate::compiler::Program {
    extern "C" {
        static rubylang_aot_program_blob: u8;
        static rubylang_aot_program_len: u64;
    }
    // SAFETY: the linked frontend defines both symbols; the blob is a fixed-size
    // array of exactly `rubylang_aot_program_len` bytes.
    let blob = unsafe {
        let len = rubylang_aot_program_len as usize;
        std::slice::from_raw_parts(&rubylang_aot_program_blob as *const u8, len)
    };
    crate::cache::program_from_blob(blob)
        .unwrap_or_else(|e| panic!("aot: embedded program blob: {e}"))
}

/// Root `$LOAD_PATH`/`$LOADED_FEATURES` and the require file-dir stack at the
/// cwd: a bundled binary needs no more, but a leftover dynamic `require` still
/// resolves against where the binary runs.
fn seed_embedded_host() {
    let cwd = std::env::current_dir().unwrap_or_default();
    crate::host::with_host(|h| h.init_load_path(&cwd.to_string_lossy()));
    crate::host::push_file_dir(cwd);
}

/// The AOT frontend hook fusevm's [`fusevm::aot::fusevm_aot_run_embedded`] calls
/// after it deserializes the embedded main chunk and builds the VM. Provided by
/// the frontend per fusevm's contract; here it lives in the rubylang library so
/// the generated frontend links it in via the rubylang rlib.
///
/// It mirrors `crate::host::run_chunk_on`'s VM setup (install every rubylang
/// builtin + the strict numeric hook) so the AOT main chunk dispatches Ruby ops
/// exactly as the interpreter does, then loads the embedded full `Program`
/// (methods/classes/begins/procs) into the thread-local host so a method call,
/// block yield, or namespaced-constant read from main resolves. Each method/block
/// chunk carries a `native_id`; when its body runs, `run_chunk_on` dispatches to
/// the matching `fusevm_aot_fn_{native_id}` native driver (registered by the
/// frontend) instead of interpreting — so loading the program here is what wires
/// the standalone binary's native method/block drivers to their chunk data.
///
/// The full program is carried in two extern symbols the generated frontend
/// defines (`rubylang_aot_program_blob` + `rubylang_aot_program_len`); the main
/// chunk fusevm embedded is ignored here (fusevm already runs it via the driver).
///
/// # Safety
/// `vm` is the valid `&mut VM` fusevm's runtime driver passes; the blob symbols
/// are defined by the linked frontend object.
#[no_mangle]
pub unsafe extern "C" fn fusevm_aot_register_builtins(vm: *mut fusevm::VM) {
    let vm = &mut *vm;
    crate::builtins::install(vm);
    vm.set_numeric_hook(std::sync::Arc::new(|op, a, b| {
        crate::builtins::numeric_hook(op, a, b)
    }));

    // Seed the host exactly as `eval_file` does before a normal run: fresh host,
    // `$LOAD_PATH`/`$LOADED_FEATURES` and the require file-dir stack rooted at the
    // cwd (a bundled binary needs no more, but a leftover dynamic `require` still
    // resolves against where the binary runs).
    crate::host::reset_host();
    seed_embedded_host();
    let prog = embedded_program();
    // Fresh host: proc/begin ids were compiled from base 0 and nothing is loaded
    // yet, so no rebasing is needed (see `compiler::rebase_program`).
    let crate::compiler::Program {
        main: _,
        methods,
        classes,
        begins,
        procs,
    } = prog;
    crate::host::with_host(|h| h.load_program(methods, classes, begins, procs));
}

/// Locate the rubylang runtime rlib and the deps dir holding fusevm + transitive
/// rlibs, relative to the running `ruby` binary (`.../target/<profile>/ruby` →
/// `librubylang.rlib` + `deps/` in the same dir). Returns an error a linked
/// standalone binary can't be produced without these (e.g. an installed `ruby`
/// stripped of the build tree).
fn locate_runtime() -> Result<(std::path::PathBuf, std::path::PathBuf), String> {
    let exe = std::env::current_exe().map_err(|e| format!("aot: current_exe: {e}"))?;
    let dir = exe
        .parent()
        .ok_or("aot: cannot find target dir next to the ruby binary")?;
    let rlib = dir.join("librubylang.rlib");
    let deps = dir.join("deps");
    if !rlib.exists() {
        return Err(format!(
            "aot: rubylang rlib not found at {} — --build --native needs the \
             build tree (run from a `cargo build` target dir)",
            rlib.display()
        ));
    }
    Ok((rlib, deps))
}

/// Bundle `file`, emit a native AOT object for its main chunk, bake the full
/// program into a generated frontend, and link a standalone executable next to
/// the source (`app.rb` → `app`). Returns a one-line report. The executable runs
/// the whole app with no `ruby` interpreter and no `.rb` files present.
pub fn build_native(file: &str) -> Result<String, String> {
    use std::path::Path;

    // 1. bundle → Program (identical front half to `build`).
    let (stmts, report) = crate::bundle::bundle(Path::new(file))?;
    let mut prog = crate::compiler::compile(&stmts, false)?;
    // Every synthetic name the program holds was minted below this; the binary
    // starts its own counter here (see `run_embedded`).
    let synth_base = crate::compiler::synth_ids_used();
    let (nmethods, nprocs, nops) = (prog.methods.len(), prog.procs.len(), prog.main.ops.len());

    // Output executable: strip the extension, place it beside the entrypoint.
    let src_abs =
        std::fs::canonicalize(file).map_err(|e| format!("aot: cannot read {file}: {e}"))?;
    let stem = src_abs
        .file_stem()
        .ok_or_else(|| format!("aot: no file stem for {file}"))?
        .to_string_lossy()
        .into_owned();
    let out_exe = src_abs
        .parent()
        .map(|p| p.join(&stem))
        .unwrap_or_else(|| Path::new(&stem).to_path_buf());

    // Scratch build dir (unique per process so concurrent builds don't collide).
    let work = std::env::temp_dir().join(format!("rubylang-aot-{}-{}", stem, std::process::id()));
    std::fs::create_dir_all(&work).map_err(|e| format!("aot: mkdir {}: {e}", work.display()))?;

    // 2. Assign a native slot to every method/block chunk (main is the AOT_ENTRY
    // driver fusevm's runtime calls directly). Then Cranelift-lower ALL of them —
    // main + each method/block body — into one object, each method/block under
    // `fusevm_aot_fn_{native_id}`. This is what makes method bodies run as native
    // machine code instead of the interpreter, not just the top-level chunk.
    // Carry each method's `slot_params` into the chunk's `aot_seeded_slots` so the
    // native driver treats the leading param slots as caller-seeded (definite-
    // assignment + integer entry guard). Procs bind their params host-side, so they
    // stay at 0.
    let mut next_id = 1u32;
    for (_, m) in &mut prog.methods {
        let c = std::sync::Arc::make_mut(&mut m.chunk);
        c.native_id = next_id;
        c.aot_seeded_slots = m.slot_params;
        next_id += 1;
    }
    for (_, c) in &mut prog.classes {
        for (_, m) in &mut c.methods {
            let c = std::sync::Arc::make_mut(&mut m.chunk);
            c.native_id = next_id;
            c.aot_seeded_slots = m.slot_params;
            next_id += 1;
        }
        for (_, m) in &mut c.class_methods {
            let c = std::sync::Arc::make_mut(&mut m.chunk);
            c.native_id = next_id;
            c.aot_seeded_slots = m.slot_params;
            next_id += 1;
        }
    }
    for p in &mut prog.procs {
        p.chunk.native_id = next_id;
        next_id += 1;
    }
    let n_native = next_id - 1;

    // Same walk order → `native_id` matches the symbol, so the frontend's table
    // slot `native_id - 1` resolves to this chunk's driver.
    let mut extras: Vec<(String, &fusevm::Chunk)> = Vec::new();
    for (_, m) in &prog.methods {
        extras.push((format!("fusevm_aot_fn_{}", m.chunk.native_id), &m.chunk));
    }
    for (_, c) in &prog.classes {
        for (_, m) in &c.methods {
            extras.push((format!("fusevm_aot_fn_{}", m.chunk.native_id), &m.chunk));
        }
        for (_, m) in &c.class_methods {
            extras.push((format!("fusevm_aot_fn_{}", m.chunk.native_id), &m.chunk));
        }
    }
    for p in &prog.procs {
        extras.push((format!("fusevm_aot_fn_{}", p.chunk.native_id), &p.chunk));
    }

    let obj = work.join("app.o");
    fusevm::aot::compile_program_object(&prog.main, &extras, &obj)?;

    // 3. serialize the full program and write the blob file the frontend embeds.
    let blob = crate::cache::program_to_blob(&prog)?;
    let blob_path = work.join("program.blob");
    std::fs::write(&blob_path, &blob)
        .map_err(|e| format!("aot: write {}: {e}", blob_path.display()))?;

    // 4. generate the frontend: it only uses `extern "C"` FFI + `include_bytes!`,
    // so it needs no `--extern fusevm`. `extern crate rubylang` pulls the whole
    // runtime (rubylang + statically-linked fusevm) into the link, which resolves
    // `fusevm_aot_run_embedded`, `fusevm_aot_register_builtins`, and every shim
    // the object imports. The blob is a fixed-size array so its symbol address is
    // the data itself (a `&[u8]` would export a fat pointer, not the bytes).
    // Declare the linked per-chunk drivers and register their addresses (in
    // native-id order) so the runtime can dispatch a method/block to its native
    // code. Only the frontend can name these symbols, so the table is built here.
    let mut fn_externs = String::new();
    let mut fn_addrs = String::new();
    for i in 1..=n_native {
        fn_externs.push_str(&format!("    fn fusevm_aot_fn_{i}(vm: *mut u8) -> i64;\n"));
        fn_addrs.push_str(&format!("fusevm_aot_fn_{i} as *const () as usize, "));
    }
    // The main chunk's driver goes last: `run_embedded` tags main with that slot.
    fn_addrs.push_str("fusevm_aot_entry as *const () as usize");

    let frontend = work.join("aot_main.rs");
    let frontend_src = format!(
        "//! Generated by `ruby --build --native` — do not edit.\n\
         #![allow(unused_extern_crates, non_upper_case_globals)]\n\
         extern crate rubylang;\n\n\
         #[no_mangle]\n\
         pub static rubylang_aot_program_blob: [u8; {len}] = *include_bytes!({blob:?});\n\
         #[no_mangle]\n\
         pub static rubylang_aot_program_len: u64 = {len};\n\n\
         extern \"C\" {{\n    fn fusevm_aot_entry(vm: *mut u8) -> i64;\n{fn_externs}}}\n\n\
         fn main() {{\n    \
         // Register every AOT-lowered driver (main's last) before the program runs.\n    \
         rubylang::aot::register_native_fns(vec![{fn_addrs}]);\n    \
         // Seed the host as `ruby FILE` does, run, and end through the\n    \
         // interpreter's own path: exit status, uncaught report, at_exit.\n    \
         std::process::exit(rubylang::aot::run_embedded({script:?}, {synth_base}));\n}}\n",
        len = blob.len(),
        blob = blob_path.to_string_lossy(),
        script = file,
    );
    std::fs::write(&frontend, &frontend_src)
        .map_err(|e| format!("aot: write {}: {e}", frontend.display()))?;

    // 5. link: rustc compiles the frontend against the rubylang rlib (+ deps dir
    // for fusevm & transitive rlibs) and folds in `app.o`.
    let (rlib, deps) = locate_runtime()?;
    let status = std::process::Command::new("rustc")
        .arg("--edition")
        .arg("2021")
        .arg(&frontend)
        .arg("--extern")
        .arg(format!("rubylang={}", rlib.display()))
        .arg("-L")
        .arg(format!("dependency={}", deps.display()))
        .arg("-C")
        .arg(format!("link-arg={}", obj.display()))
        // cranelift-object doesn't stamp a Mach-O platform load command; the
        // linker assumes macOS and links fine. Silence that benign warning so the
        // build report stays clean (harmless unknown-lint no-op on older rustc).
        .arg("-A")
        .arg("linker_messages")
        .arg("-o")
        .arg(&out_exe)
        .status()
        .map_err(|e| format!("aot: invoke rustc: {e}"))?;
    if !status.success() {
        return Err(format!(
            "aot: rustc link failed (status {status}); frontend + object kept in {}",
            work.display()
        ));
    }

    // 6. report path + size; clean the scratch dir (kept on link failure above).
    let size = std::fs::metadata(&out_exe).map(|m| m.len()).unwrap_or(0);
    let _ = std::fs::remove_dir_all(&work);
    let nfiles = report.files.len();
    Ok(format!(
        "built {file}: {nfiles} file(s) bundled, {nops} main ops, {nmethods} methods, \
         {nprocs} blocks, {n_native} chunk(s) lowered to native -> {} ({} KiB standalone executable)",
        out_exe.display(),
        size / 1024
    ))
}