Skip to main content

runec/
lib.rs

1pub mod bytecode;
2pub mod compiler;
3pub mod diagnostic_messages;
4pub mod diagnostics;
5pub mod error;
6pub mod lexer;
7pub mod lints;
8#[cfg(feature = "memprof")]
9pub mod memprof;
10pub mod parser;
11pub mod pointer;
12pub mod pointer_checker;
13pub mod symbol;
14pub mod symloader;
15pub mod token;
16pub mod trigger_table;
17pub mod typechecker;
18pub mod types;
19pub mod writer;
20
21use crate::bytecode::CompiledScript;
22use crate::compiler::Compiler;
23use crate::diagnostics::DiagnosticsCollector;
24use crate::lexer::Lexer;
25use crate::parser::{Parser, ScriptFile};
26use crate::symbol::SymbolRegistry;
27use crate::typechecker::TypeChecker;
28use crate::writer::ScriptWriter;
29use std::collections::HashMap;
30use std::error::Error;
31use std::fs;
32use std::path::{Path, PathBuf};
33use std::sync::Arc;
34use tracing::info;
35
36/// How the diagnostic layer fetches a file's source text (by its cache-key path)
37/// when building rustc-style help/suggestions. The batch path keeps every source
38/// in memory; the streaming path keeps only the on-disk paths and re-reads the
39/// handful of files that actually produce a warning — so the ~4 MB of source
40/// text isn't resident through the compile's peak. Cheap to copy (just borrows).
41#[derive(Clone, Copy)]
42pub enum SourceProvider<'a> {
43    /// Sources already resident, keyed by cache key.
44    Eager(&'a HashMap<String, Arc<String>>),
45    /// Cache key → on-disk path; re-read on demand.
46    Lazy(&'a HashMap<String, PathBuf>),
47}
48
49impl SourceProvider<'_> {
50    /// Source text for `key`, if available. Returns an owned `Arc` — a cheap
51    /// refcount bump for `Eager`, a fresh disk read for `Lazy`. Called only while
52    /// emitting diagnostic help, which is rare, so the re-read cost is immaterial.
53    pub fn get(&self, key: &str) -> Option<Arc<String>> {
54        match self {
55            SourceProvider::Eager(m) => m.get(key).cloned(),
56            SourceProvider::Lazy(m) => fs::read_to_string(m.get(key)?).ok().map(Arc::new),
57        }
58    }
59}
60
61/// Log live heap + global peak at a pipeline phase boundary. No-op unless built
62/// with `--features memprof`. Lets us attribute the footprint to each phase.
63#[cfg(feature = "memprof")]
64fn mem_mark(label: &str) {
65    info!(
66        "[mem] {:<22} live {:>7.1} MB | peak {:>7.1} MB | allocs {}",
67        label,
68        memprof::current_bytes() as f64 / 1_048_576.0,
69        memprof::peak_bytes() as f64 / 1_048_576.0,
70        memprof::alloc_count(),
71    );
72}
73#[cfg(not(feature = "memprof"))]
74#[inline]
75fn mem_mark(_label: &str) {}
76
77/// Per-map breakdown of the symbol registry's heap, to see the irreducible
78/// floor for a streaming rebuild. Built only under `--features memprof`.
79/// Accounts three components per map: the hashbrown table (inline `(K,V)`
80/// buckets + 1 control byte each), the key strings, and value-side heap.
81#[cfg(feature = "memprof")]
82fn mem_stats_registry(reg: &crate::symbol::SymbolRegistry) {
83    use crate::symbol::{Symbol, SymbolKind};
84    use crate::types::Type;
85    use std::collections::{HashMap, HashSet};
86    let t = std::mem::size_of::<Type>();
87    let mb = |b: usize| b as f64 / 1_048_576.0;
88
89    // Allocated bucket count: hashbrown keeps `capacity()` ≈ 7/8 of a power-of-two
90    // bucket array; an empty map allocates nothing.
91    fn tbl<K, V>(m: &HashMap<K, V>) -> usize {
92        if m.capacity() == 0 {
93            return 0;
94        }
95        let buckets = (m.capacity() * 8 / 7).next_power_of_two();
96        buckets * (std::mem::size_of::<(K, V)>() + 1)
97    }
98    fn keys_cap<V>(m: &HashMap<String, V>) -> usize {
99        m.keys().map(|k| k.capacity()).sum()
100    }
101    let sym_heap = |s: &Symbol| -> usize {
102        s.name.capacity()
103            + match &s.kind {
104                SymbolKind::Script {
105                    trigger,
106                    param_types,
107                    return_types,
108                    ..
109                } => trigger.capacity() + (param_types.capacity() + return_types.capacity()) * t,
110                SymbolKind::Command {
111                    param_types,
112                    return_types,
113                    ..
114                } => (param_types.capacity() + return_types.capacity()) * t,
115                SymbolKind::GameVar { category, .. } => category.capacity(),
116                SymbolKind::Constant { string_value, .. } => {
117                    string_value.as_ref().map_or(0, |s| s.capacity())
118                }
119                _ => 0,
120            }
121    };
122    let sym_map = |m: &HashMap<String, Symbol>| -> usize {
123        tbl(m) + keys_cap(m) + m.values().map(&sym_heap).sum::<usize>()
124    };
125    let i32_map = |m: &HashMap<String, i32>| -> usize { tbl(m) + keys_cap(m) };
126
127    let mut rows: Vec<(&str, usize, usize)> = vec![
128        (
129            "scripts",
130            reg.scripts.len(),
131            tbl(&reg.scripts) + keys_cap(&reg.scripts),
132        ),
133        ("commands", reg.commands.len(), sym_map(&reg.commands)),
134        ("game_vars", reg.game_vars.len(), sym_map(&reg.game_vars)),
135        ("constants", reg.constants.len(), sym_map(&reg.constants)),
136        (
137            "entity_ids",
138            reg.entity_ids.len(),
139            tbl(&reg.entity_ids)
140                + keys_cap(&reg.entity_ids)
141                + reg
142                    .entity_ids
143                    .values()
144                    .map(|e| e.variants.capacity() * std::mem::size_of::<(Type, i32)>())
145                    .sum::<usize>(),
146        ),
147        (
148            "scripts_by_trigger",
149            reg.scripts_by_trigger.len(),
150            tbl(&reg.scripts_by_trigger)
151                + keys_cap(&reg.scripts_by_trigger)
152                + reg
153                    .scripts_by_trigger
154                    .values()
155                    .map(|s| 16 + sym_heap(s.as_ref()))
156                    .sum::<usize>(),
157        ),
158        (
159            "command_param_types",
160            reg.command_param_types.len(),
161            tbl(&reg.command_param_types)
162                + keys_cap(&reg.command_param_types)
163                + reg
164                    .command_param_types
165                    .values()
166                    .map(|v| v.capacity() * t)
167                    .sum::<usize>(),
168        ),
169        ("script_ids", reg.script_ids.len(), i32_map(&reg.script_ids)),
170        (
171            "proc_script_ids",
172            reg.proc_script_ids.len(),
173            i32_map(&reg.proc_script_ids),
174        ),
175        (
176            "label_script_ids",
177            reg.label_script_ids.len(),
178            i32_map(&reg.label_script_ids),
179        ),
180        (
181            "trigger_script_ids",
182            reg.trigger_script_ids.len(),
183            i32_map(&reg.trigger_script_ids),
184        ),
185        (
186            "preloaded_script_ids",
187            reg.preloaded_script_ids.len(),
188            i32_map(&reg.preloaded_script_ids),
189        ),
190        ("components", reg.components.len(), i32_map(&reg.components)),
191        (
192            "interface_ids",
193            reg.interface_ids.len(),
194            i32_map(&reg.interface_ids),
195        ),
196        ("type_chars", reg.type_chars.len(), i32_map(&reg.type_chars)),
197        (
198            "dbcolumn_types",
199            reg.dbcolumn_types.len(),
200            tbl(&reg.dbcolumn_types) + keys_cap(&reg.dbcolumn_types),
201        ),
202    ];
203    let ts_tbl = {
204        let s: &HashSet<String> = &reg.test_scripts;
205        if s.capacity() == 0 {
206            0
207        } else {
208            (s.capacity() * 8 / 7).next_power_of_two() * (std::mem::size_of::<String>() + 1)
209                + s.iter().map(|k| k.capacity()).sum::<usize>()
210        }
211    };
212    rows.push(("test_scripts", reg.test_scripts.len(), ts_tbl));
213
214    rows.sort_by(|a, b| b.2.cmp(&a.2));
215    let total: usize = rows.iter().map(|r| r.2).sum();
216    info!(
217        "[mem] registry ~{:.2} MB (sizeof Symbol={}B, SymbolKind={}B, Type={}B):",
218        mb(total),
219        std::mem::size_of::<Symbol>(),
220        std::mem::size_of::<SymbolKind>(),
221        t,
222    );
223    for (name, n, bytes) in &rows {
224        info!(
225            "[mem]   {:<20} {:>6} entries  {:>7.3} MB",
226            name,
227            n,
228            mb(*bytes)
229        );
230    }
231}
232
233/// One-off breakdown of where the compiled-bytecode heap goes, to target
234/// memory work. Built only under `--features memprof`.
235#[cfg(feature = "memprof")]
236fn mem_stats_compiled(scripts: &[CompiledScript]) {
237    use crate::bytecode::{Instruction, Opcode, Operand};
238    let (mut n_instr, mut n_str, mut str_bytes) = (0usize, 0usize, 0usize);
239    let (mut n_switch, mut switch_entries) = (0usize, 0usize);
240    let (mut source_path_bytes, mut name_bytes) = (0usize, 0usize);
241    for s in scripts {
242        n_instr += s.instructions.len();
243        source_path_bytes += s.source_path.capacity();
244        name_bytes += s.name.capacity() + s.trigger.capacity();
245        for st in &s.strings {
246            n_str += 1;
247            str_bytes += st.len();
248        }
249        for sw in &s.switch_tables {
250            n_switch += 1;
251            switch_entries += sw.len();
252        }
253    }
254    let sz = std::mem::size_of::<Instruction>();
255    let mb = |b: usize| b as f64 / 1_048_576.0;
256    info!(
257        "[mem] sizeof Instruction={}B Operand={}B Opcode={}B CompiledScript={}B",
258        sz,
259        std::mem::size_of::<Operand>(),
260        std::mem::size_of::<Opcode>(),
261        std::mem::size_of::<CompiledScript>(),
262    );
263    info!(
264        "[mem] compiled: {} scripts, {} instrs => {:.2} MB instr structs",
265        scripts.len(),
266        n_instr,
267        mb(n_instr * sz)
268    );
269    info!(
270        "[mem]   Str operands: {} ({:.2} MB content + {:.2} MB Box<str> headers)",
271        n_str,
272        mb(str_bytes),
273        mb(n_str * 16)
274    );
275    info!(
276        "[mem]   SwitchTable operands: {} ({} entries => {:.2} MB)",
277        n_switch,
278        switch_entries,
279        mb(switch_entries * 16)
280    );
281    info!(
282        "[mem]   source_path {:.2} MB, names+triggers {:.2} MB",
283        mb(source_path_bytes),
284        mb(name_bytes)
285    );
286}
287
288struct ParsedFile {
289    path: PathBuf,
290    file: ScriptFile,
291    testscript_line: Option<usize>,
292    /// Canonicalized, normalized source path. Computed once during parsing and
293    /// reused as each compiled script's `source_path` (it is byte-identical to
294    /// the `source_cache` key), avoiding a second `canonicalize` syscall per
295    /// file in codegen.
296    source_path: String,
297}
298
299/// Map `f` over `items` across worker threads, returning results in the SAME
300/// order as the input. Dependency-free (uses `std::thread::scope`); each item
301/// is processed independently, so callers must keep the work pure (no shared
302/// mutable state). Falls back to a serial map for tiny inputs or single-core.
303pub(crate) fn parallel_map<T, R, F>(items: &[T], f: F) -> Vec<R>
304where
305    T: Sync,
306    R: Send,
307    F: Fn(&T) -> R + Sync,
308{
309    let threads = std::thread::available_parallelism()
310        .map(|n| n.get())
311        .unwrap_or(1);
312    if threads <= 1 || items.len() <= 1 {
313        return items.iter().map(&f).collect();
314    }
315    let threads = threads.min(items.len());
316    let chunk_size = items.len().div_ceil(threads);
317    let mut out: Vec<R> = Vec::with_capacity(items.len());
318    std::thread::scope(|scope| {
319        let handles: Vec<_> = items
320            .chunks(chunk_size)
321            .map(|chunk| {
322                let f = &f;
323                scope.spawn(move || chunk.iter().map(f).collect::<Vec<R>>())
324            })
325            .collect();
326        // Joining in spawn order concatenates chunks in input order, so the
327        // returned Vec matches a serial map element-for-element.
328        for h in handles {
329            out.extend(h.join().expect("worker thread panicked"));
330        }
331    });
332    out
333}
334
335/// Like `parallel_map`, but hands each worker a whole contiguous CHUNK of items
336/// so per-chunk setup (e.g. a checker's caches) can be reused across the chunk.
337/// Returns one result per chunk, in input order.
338pub(crate) fn parallel_chunks<T, R, F>(items: &[T], f: F) -> Vec<R>
339where
340    T: Sync,
341    R: Send,
342    F: Fn(&[T]) -> R + Sync,
343{
344    parallel_chunks_capped(items, usize::MAX, f)
345}
346
347/// `parallel_chunks` with an upper bound on the number of concurrent chunks.
348/// Fewer chunks → less peak memory when each chunk holds large per-chunk scratch
349/// (e.g. the pointer checker's CFG caches), at the cost of less parallelism.
350/// Result is independent of the chunk count, so output stays byte-identical.
351fn parallel_chunks_capped<T, R, F>(items: &[T], max_chunks: usize, f: F) -> Vec<R>
352where
353    T: Sync,
354    R: Send,
355    F: Fn(&[T]) -> R + Sync,
356{
357    let threads = std::thread::available_parallelism()
358        .map(|n| n.get())
359        .unwrap_or(1)
360        .min(max_chunks.max(1));
361    if threads <= 1 || items.len() <= 1 {
362        return vec![f(items)];
363    }
364    let threads = threads.min(items.len());
365    let chunk_size = items.len().div_ceil(threads);
366    let mut out: Vec<R> = Vec::new();
367    std::thread::scope(|scope| {
368        let handles: Vec<_> = items
369            .chunks(chunk_size)
370            .map(|chunk| {
371                let f = &f;
372                scope.spawn(move || f(chunk))
373            })
374            .collect();
375        for h in handles {
376            out.push(h.join().expect("worker thread panicked"));
377        }
378    });
379    out
380}
381
382/// Outcome of parsing one file off-thread. The owned source text and cache key
383/// travel back so the (single-threaded) merge can build the `Rc`-based caches
384/// without sending `Rc` across threads.
385struct FileParseResult {
386    path: PathBuf,
387    read: Result<ParsedSource, std::io::Error>,
388}
389
390struct ParsedSource {
391    cache_key: String,
392    raw_source: String,
393    outcome: ParseOutcome,
394}
395
396enum ParseOutcome {
397    Parsed {
398        file: ScriptFile,
399        testscript_line: Option<usize>,
400    },
401    Error {
402        line: usize,
403        position: usize,
404        message: String,
405        phase: diagnostics::Phase,
406    },
407}
408
409/// Read, lex and parse a single file off-thread. Pure: no shared state, so it
410/// is safe to run concurrently. Mirrors the original sequential body exactly —
411/// same source truncation, same cache-key normalization, same lex-then-parse
412/// error reporting — only restructured to return owned data for the merge.
413fn parse_one_file(path: &Path, include_tests: bool) -> FileParseResult {
414    let raw_source = match fs::read_to_string(path) {
415        Ok(s) => s,
416        Err(e) => {
417            return FileParseResult {
418                path: path.to_path_buf(),
419                read: Err(e),
420            };
421        }
422    };
423    let testscript_line = find_testscript_line(&raw_source);
424
425    let source_code = if !include_tests {
426        if let Some(line_no) = testscript_line {
427            truncate_at_line(&raw_source, line_no)
428        } else {
429            raw_source.clone()
430        }
431    } else {
432        strip_testscript_annotation(&raw_source)
433    };
434
435    let canonical = fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf());
436    let mut cache_key = canonical.to_string_lossy().into_owned();
437    if cache_key.starts_with("\\\\?\\") {
438        cache_key = cache_key[4..].to_string();
439    }
440    cache_key = cache_key.replace("\\Content\\", "\\content\\");
441
442    // Lexer/Parser borrow a `&PathBuf` for diagnostics; keep one alive locally.
443    let path_buf = path.to_path_buf();
444    let outcome = match Lexer::new(&source_code, &path_buf).tokenize() {
445        Err(e) => ParseOutcome::Error {
446            line: e.line,
447            position: e.position,
448            message: e.message,
449            phase: diagnostics::Phase::Lexing,
450        },
451        Ok(tokens) => match Parser::new(tokens, &path_buf).parse() {
452            Ok(file) => ParseOutcome::Parsed {
453                file,
454                testscript_line: if include_tests { testscript_line } else { None },
455            },
456            Err(e) => ParseOutcome::Error {
457                line: e.line,
458                position: e.position,
459                message: e.message,
460                phase: diagnostics::Phase::Parsing,
461            },
462        },
463    };
464
465    FileParseResult {
466        path: path_buf,
467        read: Ok(ParsedSource {
468            cache_key,
469            raw_source,
470            outcome,
471        }),
472    }
473}
474
475/// Compile scripts and write output. When `lint` is true, also run lint passes
476/// (unused locals, unreachable code) after code generation.
477pub fn compile_memory(
478    scripts_dir: &Path,
479    pack_dir: Option<&Path>,
480    lint: bool,
481) -> Result<(Vec<u8>, Vec<u8>), Box<dyn Error>> {
482    compile_memory_with_options(scripts_dir, pack_dir, lint, false, false)
483}
484
485/// Like `compile_memory` but includes `#testscript` sections when
486/// `include_tests` is true.
487pub fn compile_memory_with_tests(
488    scripts_dir: &Path,
489    pack_dir: Option<&Path>,
490    lint: bool,
491    include_tests: bool,
492) -> Result<(Vec<u8>, Vec<u8>), Box<dyn Error>> {
493    compile_memory_with_options(scripts_dir, pack_dir, lint, include_tests, false)
494}
495
496/// In-memory compile — returns the `(script.dat, script.idx)` bytes instead of
497/// writing them — choosing the compilation strategy.
498///
499/// With `low_mem`, the recompile pipeline is used: scripts are re-parsed and
500/// re-compiled on demand so the full compiled set is never resident (~16 MB heap
501/// vs ~93), at roughly 6× the time; the returned bytes are byte-identical to the
502/// default path. `include_tests` keeps `#testscript` sections. The
503/// `RUNEC_RECOMPILE` environment variable forces low-memory mode on as well.
504pub fn compile_memory_with_options(
505    scripts_dir: &Path,
506    pack_dir: Option<&Path>,
507    lint: bool,
508    include_tests: bool,
509    low_mem: bool,
510) -> Result<(Vec<u8>, Vec<u8>), Box<dyn Error>> {
511    if low_mem || std::env::var("RUNEC_RECOMPILE").is_ok_and(|v| !v.is_empty()) {
512        info!("[recompile] low-memory recompile pipeline enabled");
513        return compile_recompile_memory(scripts_dir, pack_dir, lint, include_tests);
514    }
515
516    let (compiled_scripts, diagnostics) = run_pipeline(scripts_dir, pack_dir, lint, include_tests)?;
517
518    info!("Writing compiled output to memory...");
519    let writer = ScriptWriter::new("".into());
520
521    diagnostics.print_all();
522    info!("Compilation complete!");
523
524    Ok(writer.build_all(&compiled_scripts)?)
525}
526
527/// Compile scripts and write output. When `lint` is true, also run lint passes
528/// (unused locals, unreachable code) after code generation.
529///
530/// Convenience wrapper over [`compile_with_options`] using the default (fast,
531/// fully-resident) pipeline. Kept signature-compatible for existing callers.
532pub fn compile(
533    scripts_dir: &Path,
534    pack_dir: Option<&Path>,
535    output_dir: &Path,
536    lint: bool,
537) -> Result<(), Box<dyn Error>> {
538    compile_with_options(scripts_dir, pack_dir, output_dir, lint, false)
539}
540
541/// Compile scripts and write output, choosing the compilation strategy.
542///
543/// With `low_mem`, the **recompile** pipeline is used: scripts are re-parsed and
544/// re-compiled on demand for the pointer check and written file-by-file, so the
545/// full set of compiled bytecode is never resident at once. That cuts peak usage
546/// to roughly 16 MB heap / 32 MB working set (from ~93 / ~111 MB) at about 6× the
547/// compile time. Output is byte-identical to the default path. The
548/// `RUNEC_RECOMPILE` environment variable forces this mode on as well.
549pub fn compile_with_options(
550    scripts_dir: &Path,
551    pack_dir: Option<&Path>,
552    output_dir: &Path,
553    lint: bool,
554    low_mem: bool,
555) -> Result<(), Box<dyn Error>> {
556    // The recompile pipeline never holds all compiled scripts (it re-compiles on
557    // demand for the pointer check + writes file-by-file). It does its own
558    // output, so it returns here. The default path below is unchanged.
559    if low_mem || std::env::var("RUNEC_RECOMPILE").is_ok_and(|v| !v.is_empty()) {
560        info!("[recompile] low-memory recompile pipeline enabled");
561        return compile_recompile(scripts_dir, pack_dir, output_dir, lint);
562    }
563
564    let (compiled_scripts, diagnostics) = run_pipeline(scripts_dir, pack_dir, lint, false)?;
565
566    let output = output_dir.display().to_string();
567    // Normalize separators for a clean, consistent log message (input paths may
568    // mix `/` and `\`, e.g. `data/pack\server`); the writer keeps the original.
569    info!("Writing output to {}...", output.replace('\\', "/"));
570    let writer = ScriptWriter::new(output);
571    writer.write_all(&compiled_scripts)?;
572
573    diagnostics.print_all();
574    info!("Compilation complete!");
575
576    Ok(())
577}
578
579/// Run all analysis passes (parse, type-check, codegen, pointer-check, lints)
580/// without writing output. Useful for editor tooling and CI lint gates.
581pub fn lint(scripts_dir: &Path, pack_dir: Option<&Path>) -> Result<(), Box<dyn Error>> {
582    let (_, diagnostics) = run_pipeline(scripts_dir, pack_dir, true, false)?;
583
584    diagnostics.print_all();
585    info!("Compilation complete!");
586
587    Ok(())
588}
589
590/// Runs the full compilation pipeline (parse, register, type-check, codegen,
591/// pointer-check, and optional lint passes), returning the compiled scripts
592/// and accumulated diagnostics for the caller to handle output.
593fn run_pipeline(
594    scripts_dir: &Path,
595    pack_dir: Option<&Path>,
596    lint: bool,
597    include_tests: bool,
598) -> Result<(Vec<CompiledScript>, DiagnosticsCollector), Box<dyn Error>> {
599    if !scripts_dir.exists() || !scripts_dir.is_dir() {
600        return Err(Box::new(error::CompilerError::FileNotFound(
601            scripts_dir.display().to_string(),
602        )));
603    }
604
605    // Prototype: opt into the streaming pipeline (never holds all ASTs at once)
606    // with RUNEC_STREAM=1. The default batch path below is unchanged.
607    if std::env::var("RUNEC_STREAM").is_ok_and(|v| !v.is_empty()) {
608        info!("[stream] streaming pipeline enabled (RUNEC_STREAM)");
609        return run_pipeline_streaming(scripts_dir, pack_dir, lint, include_tests);
610    }
611
612    let mut rs2_files: Vec<PathBuf> = get_rs2_files(scripts_dir, "rs2")?;
613
614    rs2_files.sort();
615    info!("Found {} script files", rs2_files.len());
616    mem_mark("startup");
617
618    let mut diagnostics = DiagnosticsCollector::new();
619
620    // Phase 1: Parse all files. Reading, lexing and parsing are independent
621    // per file, so fan them out across threads; the results are merged back in
622    // input order so registration/ID assignment stays deterministic.
623    info!("Parsing script files into syntax trees...");
624    let parse_results = parallel_map(&rs2_files, |path| parse_one_file(path, include_tests));
625
626    let mut all_files: Vec<ParsedFile> = Vec::new();
627    let mut source_cache: HashMap<String, Arc<String>> = HashMap::new();
628    // Raw source text keyed by on-disk path, reused by `generate_script_pack`
629    // so it doesn't re-read every file from disk a second time.
630    let mut raw_by_path: HashMap<PathBuf, Arc<String>> = HashMap::new();
631    for result in parse_results {
632        let parsed = match result.read {
633            Ok(p) => p,
634            // A read failure aborts the whole compile, exactly as the previous
635            // `fs::read_to_string(path)?` did.
636            Err(e) => return Err(Box::new(e)),
637        };
638        let rc = Arc::new(parsed.raw_source);
639        // The cache key is the canonicalized source path; reuse it as each
640        // compiled script's `source_path` so codegen needn't canonicalize again.
641        let source_path = parsed.cache_key.clone();
642        source_cache.insert(parsed.cache_key, rc.clone());
643        raw_by_path.insert(result.path.clone(), rc);
644        match parsed.outcome {
645            ParseOutcome::Parsed {
646                file,
647                testscript_line,
648            } => all_files.push(ParsedFile {
649                path: result.path,
650                file,
651                testscript_line,
652                source_path,
653            }),
654            ParseOutcome::Error {
655                line,
656                position,
657                message,
658                phase,
659            } => diagnostics.error(result.path, line, position, message, phase),
660        }
661    }
662
663    if diagnostics.has_errors() {
664        diagnostics.print_all();
665        return Err(Box::new(error::CompilerError::TypeError(
666            "Parsing failed".to_string(),
667        )));
668    }
669
670    mem_mark("after parse");
671
672    // Phase 2: Register all scripts. Build the pack/config registry, then
673    // regenerate script.pack, load pre-assigned IDs, and register every script.
674    info!("Registering scripts and building the symbol registry...");
675    let resolved_pack_dir = resolve_pack_dir(scripts_dir, pack_dir);
676    let mut registry = build_symbol_registry(scripts_dir, resolved_pack_dir.as_deref());
677    register_scripts(
678        &mut registry,
679        scripts_dir,
680        resolved_pack_dir.as_deref(),
681        &mut diagnostics,
682        &all_files,
683        &raw_by_path,
684    );
685    info!("  Registered {} scripts", registry.scripts.len());
686    // `raw_by_path` was only needed to (re)generate script.pack inside
687    // `register_scripts`; the source text itself stays reachable via
688    // `source_cache` (shared `Arc`s). Drop the redundant index now.
689    drop(raw_by_path);
690    // Pre-assigned script ids were consumed during registration; free them
691    // before codegen (the compile's peak-memory phase) rather than at scope end.
692    registry.drop_registration_scratch();
693
694    for pf in &all_files {
695        for script in &pf.file.scripts {
696            if let Some(msg) =
697                Compiler::validate_trigger_subject(&script.trigger, &script.name, &registry)
698            {
699                diagnostics.warning(
700                    pf.path.clone(),
701                    script.line,
702                    0,
703                    msg,
704                    diagnostics::Phase::SymbolRegistration,
705                );
706            }
707        }
708    }
709
710    mem_mark("after register");
711    #[cfg(feature = "memprof")]
712    mem_stats_registry(&registry);
713
714    // Phase 3: Type checking
715    info!("Type checking {} files...", all_files.len());
716    run_type_checker(&mut diagnostics, &all_files, &registry);
717
718    if diagnostics.has_errors() {
719        diagnostics.print_all();
720        return Err(Box::new(error::CompilerError::TypeError(
721            "Type checking failed".to_string(),
722        )));
723    }
724
725    mem_mark("after typecheck");
726
727    // Phase 4: Code generation
728    info!("Generating bytecode from type-checked scripts...");
729    let mut compiled_scripts = codegen(&all_files, &registry);
730    info!("  Generated {} compiled scripts", compiled_scripts.len());
731    compiled_scripts.sort_by_key(|s| s.id);
732    compiled_scripts.shrink_to_fit();
733    mem_mark("after codegen");
734    #[cfg(feature = "memprof")]
735    mem_stats_compiled(&compiled_scripts);
736
737    // The ASTs are consumed by codegen and never read again — pointer checking
738    // and lints work purely off `compiled_scripts` + the registry + the source
739    // cache. Free the syntax trees now so they don't coexist with the pointer
740    // checker's (per-thread) scratch, which is the compile's peak-memory moment.
741    drop(all_files);
742    mem_mark("after drop asts");
743
744    // Phase 4.5: Pointer checking & lints
745    info!("Checking pointers in generated code...");
746    check_pointers(
747        lint,
748        &mut diagnostics,
749        SourceProvider::Eager(&source_cache),
750        &registry,
751        &compiled_scripts,
752    );
753    mem_mark("after pointercheck");
754
755    Ok((compiled_scripts, diagnostics))
756}
757
758// ── Streaming pipeline (prototype, gated by RUNEC_STREAM) ────────────────────
759// Goal: never materialize all ASTs (~59 MB) at once. Pass 1 parses in parallel
760// but keeps only per-script signatures, dropping each AST in-worker; the registry
761// is built from those. Pass 2 re-parses each file in parallel, type-checks and
762// compiles it, then drops the AST immediately — so syntax trees never accumulate.
763// Output is byte-identical to the batch path (same registration, same per-file
764// codegen, same stable id-sort). It still holds all compiled scripts (for the
765// cross-script pointer check + writer); shrinking that is the next increment.
766
767/// One script's registration-relevant signature — everything the registry and
768/// trigger diagnostics need — so the full AST can be dropped right after parsing.
769struct ScriptSig {
770    name: String,
771    trigger: String,
772    param_types: Vec<types::Type>,
773    return_types: Vec<types::Type>,
774    line: usize,
775}
776
777struct FileSigs {
778    path: PathBuf,
779    testscript_line: Option<usize>,
780    sigs: Vec<ScriptSig>,
781}
782
783enum SigOutcome {
784    Parsed {
785        sigs: Vec<ScriptSig>,
786        testscript_line: Option<usize>,
787    },
788    Error {
789        line: usize,
790        position: usize,
791        message: String,
792        phase: diagnostics::Phase,
793    },
794}
795
796struct FileSigsResult {
797    path: PathBuf,
798    /// `(cache_key, raw_source, outcome)` on success.
799    read: Result<(String, String, SigOutcome), std::io::Error>,
800}
801
802/// Parse one file, extract signatures, and DROP the AST in-worker, so a parallel
803/// sweep never holds every syntax tree at once.
804fn parse_one_file_sigs(path: &Path, include_tests: bool) -> FileSigsResult {
805    let r = parse_one_file(path, include_tests);
806    let read = r.read.map(|ps| {
807        let outcome = match ps.outcome {
808            ParseOutcome::Parsed {
809                file,
810                testscript_line,
811            } => {
812                let sigs = file
813                    .scripts
814                    .iter()
815                    .map(|d| ScriptSig {
816                        name: d.name.clone(),
817                        trigger: d.trigger.clone(),
818                        param_types: d.params.iter().map(|p| p.param_type).collect(),
819                        return_types: d.return_types.clone(),
820                        line: d.line,
821                    })
822                    .collect();
823                SigOutcome::Parsed {
824                    sigs,
825                    testscript_line,
826                }
827                // `file` (the AST) is dropped here — only the sigs travel back.
828            }
829            ParseOutcome::Error {
830                line,
831                position,
832                message,
833                phase,
834            } => SigOutcome::Error {
835                line,
836                position,
837                message,
838                phase,
839            },
840        };
841        (ps.cache_key, ps.raw_source, outcome)
842    });
843    FileSigsResult { path: r.path, read }
844}
845
846fn run_pipeline_streaming(
847    scripts_dir: &Path,
848    pack_dir: Option<&Path>,
849    lint: bool,
850    include_tests: bool,
851) -> Result<(Vec<CompiledScript>, DiagnosticsCollector), Box<dyn Error>> {
852    let mut rs2_files: Vec<PathBuf> = get_rs2_files(scripts_dir, "rs2")?;
853    rs2_files.sort();
854    info!("Found {} script files", rs2_files.len());
855    mem_mark("startup");
856
857    let mut diagnostics = DiagnosticsCollector::new();
858
859    // Pass 1: parse for signatures only (each AST dropped in its worker).
860    info!("[stream] Pass 1: parsing for signatures...");
861    let sig_results = parallel_map(&rs2_files, |path| parse_one_file_sigs(path, include_tests));
862
863    // Streaming keeps only `cache_key -> on-disk path`; source text is re-read
864    // lazily for the rare diagnostic that needs it, so the ~4 MB of sources isn't
865    // resident through the peak. `raw_by_path` still holds the text transiently
866    // for `script.pack` regeneration, then is dropped right after registration.
867    let mut source_paths: HashMap<String, PathBuf> = HashMap::new();
868    let mut raw_by_path: HashMap<PathBuf, Arc<String>> = HashMap::new();
869    let mut file_sigs: Vec<FileSigs> = Vec::new();
870    for r in sig_results {
871        let (cache_key, raw_source, outcome) = match r.read {
872            Ok(t) => t,
873            Err(e) => return Err(Box::new(e)),
874        };
875        source_paths.insert(cache_key, r.path.clone());
876        raw_by_path.insert(r.path.clone(), Arc::new(raw_source));
877        match outcome {
878            SigOutcome::Parsed {
879                sigs,
880                testscript_line,
881            } => file_sigs.push(FileSigs {
882                path: r.path,
883                testscript_line,
884                sigs,
885            }),
886            SigOutcome::Error {
887                line,
888                position,
889                message,
890                phase,
891            } => diagnostics.error(r.path, line, position, message, phase),
892        }
893    }
894
895    if diagnostics.has_errors() {
896        diagnostics.print_all();
897        return Err(Box::new(error::CompilerError::TypeError(
898            "Parsing failed".to_string(),
899        )));
900    }
901    mem_mark("after parse");
902
903    // Registration (from signatures, not ASTs).
904    info!("[stream] Registering scripts and building the symbol registry...");
905    let resolved_pack_dir = resolve_pack_dir(scripts_dir, pack_dir);
906    let mut registry = build_symbol_registry(scripts_dir, resolved_pack_dir.as_deref());
907    register_scripts_from_sigs(
908        &mut registry,
909        scripts_dir,
910        resolved_pack_dir.as_deref(),
911        &mut diagnostics,
912        &file_sigs,
913        &raw_by_path,
914    );
915    info!("  Registered {} scripts", registry.scripts.len());
916    drop(raw_by_path);
917    registry.drop_registration_scratch();
918
919    for fs in &file_sigs {
920        for sig in &fs.sigs {
921            if let Some(msg) =
922                Compiler::validate_trigger_subject(&sig.trigger, &sig.name, &registry)
923            {
924                diagnostics.warning(
925                    fs.path.clone(),
926                    sig.line,
927                    0,
928                    msg,
929                    diagnostics::Phase::SymbolRegistration,
930                );
931            }
932        }
933    }
934    // Signatures are done; only the registry + source cache carry forward.
935    drop(file_sigs);
936    mem_mark("after register");
937    #[cfg(feature = "memprof")]
938    mem_stats_registry(&registry);
939
940    // Pass 2: re-parse + type-check + compile per file, streaming. Parse can't
941    // fail here (pass 1 validated), so a failed re-parse just yields nothing.
942    info!("[stream] Pass 2: type-check + codegen (streaming)...");
943    let per_file = parallel_map(&rs2_files, |path| {
944        let parsed = match parse_one_file(path, include_tests).read {
945            Ok(p) => p,
946            Err(_) => return (Vec::new(), DiagnosticsCollector::new()),
947        };
948        let (file, testscript_line) = match parsed.outcome {
949            ParseOutcome::Parsed {
950                file,
951                testscript_line,
952            } => (file, testscript_line),
953            ParseOutcome::Error { .. } => return (Vec::new(), DiagnosticsCollector::new()),
954        };
955        let mut type_checker = TypeChecker::new(&registry);
956        type_checker.check_file_with_test_boundary(&file, path, testscript_line);
957        let mut compiled = Vec::new();
958        // Compile only when this file type-checks clean (codegen assumes
959        // well-typed input). On any global error the whole compile aborts below,
960        // so output produced here would be discarded anyway.
961        if !type_checker.diagnostics.has_errors() {
962            let mut compiler = Compiler::new(&registry);
963            for script in &file.scripts {
964                if script.trigger == "command" {
965                    continue;
966                }
967                let mut c = compiler.compile_script(script);
968                c.source_path = parsed.cache_key.clone();
969                c.shrink_to_fit();
970                compiled.push(c);
971            }
972        }
973        (compiled, type_checker.diagnostics)
974        // `file` (the AST) is dropped here.
975    });
976
977    let mut compiled_scripts: Vec<CompiledScript> = Vec::new();
978    for (compiled, diags) in per_file {
979        compiled_scripts.extend(compiled);
980        diagnostics.merge(diags);
981    }
982    if diagnostics.has_errors() {
983        diagnostics.print_all();
984        return Err(Box::new(error::CompilerError::TypeError(
985            "Type checking failed".to_string(),
986        )));
987    }
988    // Stable sort by id matches the batch path (file order preserved among
989    // equal ids → identical writer last-write-wins behaviour).
990    compiled_scripts.sort_by_key(|s| s.id);
991    compiled_scripts.shrink_to_fit();
992    info!("  Generated {} compiled scripts", compiled_scripts.len());
993    mem_mark("after codegen");
994    #[cfg(feature = "memprof")]
995    mem_stats_compiled(&compiled_scripts);
996
997    check_pointers(
998        lint,
999        &mut diagnostics,
1000        SourceProvider::Lazy(&source_paths),
1001        &registry,
1002        &compiled_scripts,
1003    );
1004    mem_mark("after pointercheck");
1005
1006    Ok((compiled_scripts, diagnostics))
1007}
1008
1009/// `register_scripts` for the streaming path: identical logic, driven by the
1010/// extracted signatures instead of the (already-dropped) ASTs.
1011fn register_scripts_from_sigs(
1012    registry: &mut SymbolRegistry,
1013    scripts_dir: &Path,
1014    resolved_pack_dir: Option<&Path>,
1015    diagnostics: &mut DiagnosticsCollector,
1016    file_sigs: &[FileSigs],
1017    raw_by_path: &HashMap<PathBuf, Arc<String>>,
1018) {
1019    if let Some(pd) = resolved_pack_dir {
1020        symloader::generate_script_pack(scripts_dir, pd, raw_by_path);
1021        symloader::load_script_ids(registry, pd);
1022    }
1023
1024    use crate::diagnostic_messages as msg;
1025    let mut registered_scripts: std::collections::HashSet<String> =
1026        std::collections::HashSet::new();
1027
1028    for fs in file_sigs.iter() {
1029        for sig in &fs.sigs {
1030            if sig.trigger == "command" {
1031                continue;
1032            }
1033
1034            if !trigger_table::is_valid_trigger(&sig.trigger) {
1035                diagnostics.warning(
1036                    fs.path.clone(),
1037                    sig.line,
1038                    0,
1039                    msg::fmt(msg::SCRIPT_TRIGGER_INVALID, &[&sig.trigger]),
1040                    crate::diagnostics::Phase::SymbolRegistration,
1041                );
1042            }
1043
1044            let key = format!("{}:{}", sig.trigger, sig.name);
1045            if !registered_scripts.insert(key) {
1046                diagnostics.warning(
1047                    fs.path.clone(),
1048                    sig.line,
1049                    0,
1050                    msg::fmt(msg::SCRIPT_REDECLARATION, &[&sig.trigger, &sig.name]),
1051                    crate::diagnostics::Phase::SymbolRegistration,
1052                );
1053            }
1054
1055            if !sig.return_types.is_empty() && !trigger_table::allows_returns(&sig.trigger) {
1056                diagnostics.warning(
1057                    fs.path.clone(),
1058                    sig.line,
1059                    0,
1060                    msg::fmt(msg::SCRIPT_TRIGGER_NO_RETURNS, &[&sig.trigger]),
1061                    diagnostics::Phase::SymbolRegistration,
1062                );
1063            }
1064
1065            registry.register_script(
1066                sig.name.clone(),
1067                sig.trigger.clone(),
1068                sig.param_types.clone(),
1069                sig.return_types.clone(),
1070            );
1071
1072            if let Some(ts_line) = fs.testscript_line
1073                && sig.line >= ts_line
1074            {
1075                registry.mark_test_script(&sig.trigger, &sig.name);
1076            }
1077        }
1078    }
1079}
1080
1081fn check_pointers(
1082    lint: bool,
1083    diagnostics: &mut DiagnosticsCollector,
1084    source: SourceProvider,
1085    registry: &SymbolRegistry,
1086    compiled_scripts: &[CompiledScript],
1087) {
1088    {
1089        use crate::pointer_checker::{PointerChecker, ScriptSource};
1090        // Validate scripts across threads: each chunk gets its own checker that
1091        // shares the read-only scripts/registry/source cache but keeps its own
1092        // (purely memoizing) caches. Diagnostics are merged in script order, so
1093        // the result is identical to a single-threaded run.
1094        //
1095        // The read-only command/script lookup tables are built ONCE here and
1096        // shared (`Arc`) across every chunk checker, instead of each thread
1097        // rebuilding the full ~700-command pointer table + per-script reverse
1098        // maps. `names` is just the script order, so derive it directly rather
1099        // than spinning up a throwaway checker for `script_names()`.
1100        let shared = PointerChecker::build_shared(compiled_scripts, registry);
1101        let names: Vec<String> = compiled_scripts.iter().map(|s| s.name.clone()).collect();
1102        // Each chunk's checker retains its CFG caches until the chunk finishes,
1103        // so concurrency multiplies peak scratch. RUNEC_PTR_CHUNKS caps the
1104        // concurrent chunk count to trade pointer-check speed for memory; the
1105        // result is chunk-count-independent, so diagnostics stay byte-identical.
1106        let max_chunks = std::env::var("RUNEC_PTR_CHUNKS")
1107            .ok()
1108            .and_then(|s| s.parse::<usize>().ok())
1109            .filter(|&n| n > 0)
1110            .unwrap_or(usize::MAX);
1111        let chunk_diags = parallel_chunks_capped(&names, max_chunks, |chunk| {
1112            let mut checker = PointerChecker::with_shared(
1113                ScriptSource::Resident(compiled_scripts),
1114                registry,
1115                shared.clone(),
1116            );
1117            checker.set_source_cache(source);
1118            checker.validate_names(chunk)
1119        });
1120        let mut pointer_diags = DiagnosticsCollector::new();
1121        for d in chunk_diags {
1122            pointer_diags.merge(d);
1123        }
1124        let ptr_warnings = pointer_diags.warning_count();
1125        if ptr_warnings > 0 {
1126            info!("  Pointer check: {} warning(s)", ptr_warnings);
1127        }
1128        diagnostics.merge(pointer_diags);
1129    }
1130    #[cfg(feature = "memprof")]
1131    crate::pointer_checker::cfg_trace_report();
1132
1133    // Phase 4.6: Lint passes (optional)
1134    if lint {
1135        info!("Running lint passes (unused locals, unreachable code)...");
1136        let lint_diags = lints::run_lints(compiled_scripts, Some(source));
1137        let lint_warnings = lint_diags.warning_count();
1138        if lint_warnings > 0 {
1139            info!("  Lints: {} warning(s)", lint_warnings);
1140        }
1141        diagnostics.merge(lint_diags);
1142    }
1143}
1144
1145fn codegen(all_files: &[ParsedFile], registry: &SymbolRegistry) -> Vec<CompiledScript> {
1146    // Each file compiles independently against the read-only registry, so fan
1147    // the files out across threads. Output order doesn't matter — the caller
1148    // sorts compiled scripts by id — so a simple flatten of the per-file results
1149    // is enough.
1150    parallel_map(all_files, |pf| compile_file(pf, registry))
1151        .into_iter()
1152        .flatten()
1153        .collect()
1154}
1155
1156/// Compile every (non-`command`) script in one file. Creating a `Compiler` per
1157/// file is cheap now that it only borrows the registry, and keeps the codegen
1158/// scratch state thread-local.
1159fn compile_file(pf: &ParsedFile, registry: &SymbolRegistry) -> Vec<CompiledScript> {
1160    let mut compiler = Compiler::new(registry);
1161
1162    let mut compiled_scripts = Vec::new();
1163    for script in &pf.file.scripts {
1164        if script.trigger == "command" {
1165            continue;
1166        }
1167        let mut compiled = compiler.compile_script(script);
1168        // `source_path` was canonicalized once during parsing; reuse it.
1169        compiled.source_path = pf.source_path.clone();
1170        compiled.shrink_to_fit();
1171        compiled_scripts.push(compiled);
1172    }
1173    compiled_scripts
1174}
1175
1176fn run_type_checker(
1177    diagnostics: &mut DiagnosticsCollector,
1178    all_files: &[ParsedFile],
1179    registry: &SymbolRegistry,
1180) {
1181    // Each file type-checks independently against the read-only registry, so
1182    // check chunks of files in parallel; per-chunk diagnostics are merged in
1183    // file order, matching a single-threaded run.
1184    let chunk_diags = parallel_chunks(all_files, |chunk| {
1185        let mut type_checker = TypeChecker::new(registry);
1186        for pf in chunk {
1187            type_checker.check_file_with_test_boundary(&pf.file, &pf.path, pf.testscript_line);
1188        }
1189        type_checker.diagnostics
1190    });
1191    for d in chunk_diags {
1192        diagnostics.merge(d);
1193    }
1194}
1195
1196/// Resolve the pack directory: explicit `--pack`, else a sibling `pack/`, else
1197/// `content/pack` up the tree.
1198fn resolve_pack_dir(scripts_dir: &Path, pack_dir: Option<&Path>) -> Option<PathBuf> {
1199    pack_dir
1200        .map(PathBuf::from)
1201        .filter(|p| p.exists())
1202        .or_else(|| {
1203            scripts_dir
1204                .parent()
1205                .map(|p| p.join("pack"))
1206                .filter(|p| p.exists())
1207        })
1208        .or_else(|| {
1209            scripts_dir
1210                .ancestors()
1211                .nth(3)
1212                .map(|p| p.join("content/pack"))
1213                .filter(|p| p.exists())
1214        })
1215}
1216
1217/// Build the part of the registry that does NOT depend on the parsed scripts:
1218/// pack files, `.constant`/`.var`/`.dbtable` configs, and engine command params.
1219/// This runs concurrently with parsing; `script.pack` (re)generation and script
1220/// registration — which need the parsed sources/ASTs — happen afterwards in
1221/// `register_scripts`.
1222fn build_symbol_registry(scripts_dir: &Path, resolved_pack_dir: Option<&Path>) -> SymbolRegistry {
1223    let mut registry = SymbolRegistry::new();
1224
1225    if let Some(pd) = resolved_pack_dir {
1226        info!("  Loading packs from {}", pd.display());
1227        symloader::load_packs(&mut registry, pd);
1228        info!(
1229            "  Loaded {} commands, {} game vars",
1230            registry.commands.len(),
1231            registry.game_vars.len(),
1232        );
1233    } else {
1234        info!("  No pack directory found - symbol resolution will be limited");
1235    }
1236
1237    // Walk the scripts tree once and let the constant/game-var/dbtable loaders
1238    // filter this shared list by extension, instead of each re-walking the tree
1239    // (previously 6 separate recursive traversals over the same directories).
1240    let mut config_files: Vec<PathBuf> = Vec::new();
1241    symloader::collect_all_files(scripts_dir, &mut config_files);
1242
1243    symloader::load_constant_files(&mut registry, &config_files);
1244
1245    info!(
1246        "  Loaded {} constants from .constant files",
1247        registry.constants.len()
1248    );
1249
1250    symloader::load_game_var_types(&mut registry, &config_files);
1251
1252    if let Some(pd) = resolved_pack_dir {
1253        let dbtable_ids = symloader::load_dbtable_pack(&pd.join("dbtable.pack"));
1254        if !dbtable_ids.is_empty() {
1255            symloader::load_dbtable_configs(&mut registry, &config_files, &dbtable_ids);
1256            info!(
1257                "  Registered {} dbcolumn compound IDs",
1258                registry.dbcolumn_types.len()
1259            );
1260        }
1261    }
1262
1263    let engine_rs2 = scripts_dir.join("engine.rs2");
1264
1265    if engine_rs2.exists() {
1266        symloader::load_engine_command_params(&mut registry, &engine_rs2);
1267    } else {
1268        if let Ok(entries) = std::fs::read_dir(scripts_dir) {
1269            for entry in entries.flatten() {
1270                let candidate = entry.path().join("engine.rs2");
1271                if candidate.exists() {
1272                    crate::symloader::load_engine_command_params(&mut registry, &candidate);
1273                    break;
1274                }
1275            }
1276        }
1277    }
1278
1279    symloader::patch_command_return_types(&mut registry);
1280
1281    registry
1282}
1283
1284/// Finalize the registry once parsing is done: regenerate `script.pack` from the
1285/// parsed sources, load pre-assigned script IDs, then register every parsed
1286/// script (emitting trigger/redeclaration/return diagnostics). Builds on the
1287/// registry produced concurrently by `build_symbol_registry`.
1288fn register_scripts(
1289    registry: &mut SymbolRegistry,
1290    scripts_dir: &Path,
1291    resolved_pack_dir: Option<&Path>,
1292    diagnostics: &mut DiagnosticsCollector,
1293    all_files: &[ParsedFile],
1294    raw_by_path: &HashMap<PathBuf, Arc<String>>,
1295) {
1296    if let Some(pd) = resolved_pack_dir {
1297        symloader::generate_script_pack(scripts_dir, pd, raw_by_path);
1298        symloader::load_script_ids(registry, pd);
1299    }
1300
1301    {
1302        use crate::diagnostic_messages as msg;
1303
1304        let mut registered_scripts: std::collections::HashSet<String> =
1305            std::collections::HashSet::new();
1306
1307        for pf in all_files.iter() {
1308            for script in &pf.file.scripts {
1309                if script.trigger == "command" {
1310                    continue;
1311                }
1312
1313                if !trigger_table::is_valid_trigger(&script.trigger) {
1314                    diagnostics.warning(
1315                        pf.path.clone(),
1316                        script.line,
1317                        0,
1318                        msg::fmt(msg::SCRIPT_TRIGGER_INVALID, &[&script.trigger]),
1319                        crate::diagnostics::Phase::SymbolRegistration,
1320                    );
1321                }
1322
1323                let key = format!("{}:{}", script.trigger, script.name);
1324                if !registered_scripts.insert(key.clone()) {
1325                    diagnostics.warning(
1326                        pf.path.clone(),
1327                        script.line,
1328                        0,
1329                        msg::fmt(msg::SCRIPT_REDECLARATION, &[&script.trigger, &script.name]),
1330                        crate::diagnostics::Phase::SymbolRegistration,
1331                    );
1332                }
1333
1334                if !script.return_types.is_empty()
1335                    && !trigger_table::allows_returns(&script.trigger)
1336                {
1337                    diagnostics.warning(
1338                        pf.path.clone(),
1339                        script.line,
1340                        0,
1341                        msg::fmt(msg::SCRIPT_TRIGGER_NO_RETURNS, &[&script.trigger]),
1342                        diagnostics::Phase::SymbolRegistration,
1343                    );
1344                }
1345
1346                let param_types: Vec<types::Type> =
1347                    script.params.iter().map(|p| p.param_type).collect();
1348                registry.register_script(
1349                    script.name.clone(),
1350                    script.trigger.clone(),
1351                    param_types,
1352                    script.return_types.clone(),
1353                );
1354
1355                if let Some(ts_line) = pf.testscript_line
1356                    && script.line >= ts_line
1357                {
1358                    registry.mark_test_script(&script.trigger, &script.name);
1359                }
1360            }
1361        }
1362    }
1363}
1364
1365// ── Streaming recompile pipeline (prototype, gated by RUNEC_RECOMPILE) ────────
1366// Never holds all compiled scripts. Pass 2 only type-checks; the pointer check
1367// re-compiles each script on demand (bounded cache — measured ~1.3x re-parse on
1368// rs-majula, the call graph has high file locality); output is produced by
1369// re-compiling file-by-file into a by-id buffer. Output + diagnostics are
1370// byte-identical to the batch path (deterministic re-compile, same id order).
1371
1372/// Re-compiles scripts on demand for the pointer checker, caching `Rc`s with a
1373/// bounded FIFO so the full compiled set is never resident. A cache miss
1374/// re-parses the script's file and compiles all its scripts at once (siblings
1375/// share the parse), keying each by its full `[trigger,name]` to its index.
1376struct RecompileStore<'a> {
1377    idx_to_path: Vec<PathBuf>,
1378    /// file path -> its script idxs in declaration order (command-skipped).
1379    /// Position-based so duplicate `[trigger,name]` declarations each map to the
1380    /// correct distinct idx.
1381    file_to_idxs: HashMap<PathBuf, Vec<usize>>,
1382    registry: &'a SymbolRegistry,
1383    include_tests: bool,
1384    cache: HashMap<usize, std::rc::Rc<CompiledScript>>,
1385    order: std::collections::VecDeque<usize>,
1386    cap: usize,
1387}
1388
1389impl RecompileStore<'_> {
1390    fn get(&mut self, idx: usize) -> std::rc::Rc<CompiledScript> {
1391        if let Some(rc) = self.cache.get(&idx) {
1392            return std::rc::Rc::clone(rc);
1393        }
1394        let path = self.idx_to_path[idx].clone();
1395        let parsed = parse_one_file(&path, self.include_tests);
1396        if let Ok(ps) = parsed.read
1397            && let ParseOutcome::Parsed { file, .. } = ps.outcome
1398        {
1399            // The j-th non-command declaration in this file maps to idxs[j],
1400            // matching how the metas were built (same parse, same skip rule).
1401            let idxs = self.file_to_idxs.get(&path).cloned().unwrap_or_default();
1402            let mut compiler = Compiler::new(self.registry);
1403            let mut j = 0usize;
1404            for script in &file.scripts {
1405                if script.trigger == "command" {
1406                    continue;
1407                }
1408                if let Some(&i) = idxs.get(j) {
1409                    let mut c = compiler.compile_script(script);
1410                    c.source_path = ps.cache_key.clone();
1411                    self.insert(i, std::rc::Rc::new(c));
1412                }
1413                j += 1;
1414            }
1415        }
1416        std::rc::Rc::clone(
1417            self.cache
1418                .get(&idx)
1419                .expect("recompile store: script not produced by its file"),
1420        )
1421    }
1422
1423    fn insert(&mut self, idx: usize, rc: std::rc::Rc<CompiledScript>) {
1424        if self.cache.insert(idx, rc).is_none() {
1425            self.order.push_back(idx);
1426            while self.order.len() > self.cap {
1427                if let Some(old) = self.order.pop_front()
1428                    && old != idx
1429                {
1430                    self.cache.remove(&old);
1431                }
1432            }
1433        }
1434    }
1435}
1436
1437/// Per-slot encoded output blobs (indexed by script id) plus accumulated
1438/// diagnostics — what the recompile core hands back before the bytes are written
1439/// to disk or assembled in memory.
1440type RecompiledOutput = (Vec<Vec<u8>>, DiagnosticsCollector);
1441
1442/// Core of the recompile (low-memory) pipeline: runs every phase re-compiling
1443/// scripts on demand — never holding all compiled bytecode at once — and returns
1444/// the per-slot encoded output blobs plus diagnostics, without writing or
1445/// printing. The disk and in-memory entry points (`compile_recompile`,
1446/// `compile_recompile_memory`) wrap this. On a phase error it prints diagnostics
1447/// and returns `Err`, matching the batch path.
1448fn recompile_to_encoded(
1449    scripts_dir: &Path,
1450    pack_dir: Option<&Path>,
1451    lint: bool,
1452    include_tests: bool,
1453) -> Result<RecompiledOutput, Box<dyn Error>> {
1454    use crate::pointer_checker::{PointerChecker, ScriptSource};
1455
1456    let mut rs2_files: Vec<PathBuf> = get_rs2_files(scripts_dir, "rs2")?;
1457    rs2_files.sort();
1458    info!("Found {} script files", rs2_files.len());
1459    mem_mark("startup");
1460    let mut diagnostics = DiagnosticsCollector::new();
1461
1462    // Pass 1: signatures only (ASTs dropped in-worker); keep file paths for lazy
1463    // source + raw text for script.pack regeneration.
1464    info!("[recompile] Pass 1: parsing for signatures...");
1465    let sig_results = parallel_map(&rs2_files, |path| parse_one_file_sigs(path, include_tests));
1466    let mut source_paths: HashMap<String, PathBuf> = HashMap::new();
1467    let mut raw_by_path: HashMap<PathBuf, Arc<String>> = HashMap::new();
1468    let mut file_sigs: Vec<FileSigs> = Vec::new();
1469    for r in sig_results {
1470        let (cache_key, raw_source, outcome) = match r.read {
1471            Ok(t) => t,
1472            Err(e) => return Err(Box::new(e)),
1473        };
1474        source_paths.insert(cache_key, r.path.clone());
1475        raw_by_path.insert(r.path.clone(), Arc::new(raw_source));
1476        match outcome {
1477            SigOutcome::Parsed {
1478                sigs,
1479                testscript_line,
1480            } => file_sigs.push(FileSigs {
1481                path: r.path,
1482                testscript_line,
1483                sigs,
1484            }),
1485            SigOutcome::Error {
1486                line,
1487                position,
1488                message,
1489                phase,
1490            } => diagnostics.error(r.path, line, position, message, phase),
1491        }
1492    }
1493    if diagnostics.has_errors() {
1494        diagnostics.print_all();
1495        return Err(Box::new(error::CompilerError::TypeError(
1496            "Parsing failed".to_string(),
1497        )));
1498    }
1499    mem_mark("after parse");
1500
1501    // Registration.
1502    info!("[recompile] Registering scripts and building the symbol registry...");
1503    let resolved_pack_dir = resolve_pack_dir(scripts_dir, pack_dir);
1504    let mut registry = build_symbol_registry(scripts_dir, resolved_pack_dir.as_deref());
1505    register_scripts_from_sigs(
1506        &mut registry,
1507        scripts_dir,
1508        resolved_pack_dir.as_deref(),
1509        &mut diagnostics,
1510        &file_sigs,
1511        &raw_by_path,
1512    );
1513    info!("  Registered {} scripts", registry.scripts.len());
1514    drop(raw_by_path);
1515    registry.drop_registration_scratch();
1516    for fs in &file_sigs {
1517        for sig in &fs.sigs {
1518            if let Some(msg) =
1519                Compiler::validate_trigger_subject(&sig.trigger, &sig.name, &registry)
1520            {
1521                diagnostics.warning(
1522                    fs.path.clone(),
1523                    sig.line,
1524                    0,
1525                    msg,
1526                    diagnostics::Phase::SymbolRegistration,
1527                );
1528            }
1529        }
1530    }
1531    mem_mark("after register");
1532
1533    // Pass 2: type-check only (re-parse, no compiled scripts retained).
1534    info!("[recompile] Pass 2: type-check (streaming, no bytecode held)...");
1535    let tc_diags = parallel_map(&rs2_files, |path| {
1536        let parsed = match parse_one_file(path, include_tests).read {
1537            Ok(p) => p,
1538            Err(_) => return DiagnosticsCollector::new(),
1539        };
1540        let (file, testscript_line) = match parsed.outcome {
1541            ParseOutcome::Parsed {
1542                file,
1543                testscript_line,
1544            } => (file, testscript_line),
1545            ParseOutcome::Error { .. } => return DiagnosticsCollector::new(),
1546        };
1547        let mut type_checker = TypeChecker::new(&registry);
1548        type_checker.check_file_with_test_boundary(&file, path, testscript_line);
1549        type_checker.diagnostics
1550    });
1551    for d in tc_diags {
1552        diagnostics.merge(d);
1553    }
1554    if diagnostics.has_errors() {
1555        diagnostics.print_all();
1556        return Err(Box::new(error::CompilerError::TypeError(
1557            "Type checking failed".to_string(),
1558        )));
1559    }
1560    mem_mark("after typecheck");
1561
1562    // Build script metas in the same idx order as the batch path: file order
1563    // (file_sigs / declaration order, command-skipped), then a stable sort by id.
1564    // ids + names come from the registry — no bytecode needed.
1565    let mut pre: Vec<(i32, String, PathBuf)> = Vec::new();
1566    for fs in &file_sigs {
1567        for sig in &fs.sigs {
1568            if sig.trigger == "command" {
1569                continue;
1570            }
1571            let id = registry
1572                .script_id_for_trigger(&sig.trigger, &sig.name)
1573                .or_else(|| registry.script_id(&sig.name))
1574                .unwrap_or(-1);
1575            pre.push((
1576                id,
1577                format!("[{},{}]", sig.trigger, sig.name),
1578                fs.path.clone(),
1579            ));
1580        }
1581    }
1582    let n = pre.len();
1583    // Stable sort indices by id → the batch's idx order (file/decl pre-order
1584    // preserved among equal ids). `inv` maps a pre-order position back to its idx.
1585    let mut order: Vec<usize> = (0..n).collect();
1586    order.sort_by_key(|&p| pre[p].0);
1587    let idents: Vec<(i32, String)> = order
1588        .iter()
1589        .map(|&p| (pre[p].0, pre[p].1.clone()))
1590        .collect();
1591    let idx_to_path: Vec<PathBuf> = order.iter().map(|&p| pre[p].2.clone()).collect();
1592    let mut inv = vec![0usize; n];
1593    for (idx, &p) in order.iter().enumerate() {
1594        inv[p] = idx;
1595    }
1596    let mut file_to_idxs: HashMap<PathBuf, Vec<usize>> = HashMap::new();
1597    for (p, m) in pre.iter().enumerate() {
1598        file_to_idxs.entry(m.2.clone()).or_default().push(inv[p]);
1599    }
1600    let max_id = pre.iter().map(|m| m.0).max().unwrap_or(-1);
1601    let slot_count = if max_id < 0 { 0 } else { (max_id + 1) as usize };
1602    let len = n;
1603    drop(pre);
1604
1605    // Pointer check via on-demand re-compile (single-threaded: the store getter
1606    // is `FnMut`/`!Sync`). Diagnostics come out in id order, as in the batch path.
1607    info!("[recompile] Pointer check (re-compile on demand)...");
1608    let shared = PointerChecker::build_shared_idents(&idents, &registry);
1609    {
1610        // Cache cap must exceed the largest file's script count, or loading that
1611        // file would evict its own just-compiled scripts. A small margin over
1612        // that keeps cross-file locality high (measured ~1.1-1.3x re-parse).
1613        let cap = file_to_idxs
1614            .values()
1615            .map(|v| v.len())
1616            .max()
1617            .unwrap_or(0)
1618            .max(128);
1619        let mut store = RecompileStore {
1620            idx_to_path,
1621            file_to_idxs,
1622            registry: &registry,
1623            include_tests,
1624            cache: HashMap::new(),
1625            order: std::collections::VecDeque::new(),
1626            cap,
1627        };
1628        let getter: Box<dyn FnMut(usize) -> std::rc::Rc<CompiledScript>> =
1629            Box::new(move |idx| store.get(idx));
1630        let scripts = ScriptSource::Recompile {
1631            len,
1632            get: std::cell::RefCell::new(getter),
1633        };
1634        let mut checker = PointerChecker::with_shared(scripts, &registry, shared);
1635        checker.set_source_cache(SourceProvider::Lazy(&source_paths));
1636        let names: Vec<String> = idents.iter().map(|(_, n)| n.clone()).collect();
1637        let ptr_diags = checker.validate_names(&names);
1638        let w = ptr_diags.warning_count();
1639        if w > 0 {
1640            info!("  Pointer check: {} warning(s)", w);
1641        }
1642        diagnostics.merge(ptr_diags);
1643    }
1644    mem_mark("after pointercheck");
1645
1646    // Output: re-compile each file once, encode each script into its id slot,
1647    // optionally lint — never holding all compiled scripts at once. The caller
1648    // writes the blobs to disk or assembles them in memory.
1649    info!("[recompile] Re-compiling + encoding output...");
1650    let mut encoded: Vec<Vec<u8>> = vec![Vec::new(); slot_count];
1651    for path in &rs2_files {
1652        let parsed = match parse_one_file(path, include_tests).read {
1653            Ok(p) => p,
1654            Err(_) => continue,
1655        };
1656        let file = match parsed.outcome {
1657            ParseOutcome::Parsed { file, .. } => file,
1658            ParseOutcome::Error { .. } => continue,
1659        };
1660        let mut compiler = Compiler::new(&registry);
1661        let mut file_scripts: Vec<CompiledScript> = Vec::new();
1662        for script in &file.scripts {
1663            if script.trigger == "command" {
1664                continue;
1665            }
1666            let mut c = compiler.compile_script(script);
1667            c.source_path = parsed.cache_key.clone();
1668            file_scripts.push(c);
1669        }
1670        for c in &file_scripts {
1671            if c.id >= 0 && (c.id as usize) < slot_count {
1672                encoded[c.id as usize] = crate::writer::encode_script(c);
1673            }
1674        }
1675        if lint {
1676            diagnostics.merge(lints::run_lints(
1677                &file_scripts,
1678                Some(SourceProvider::Lazy(&source_paths)),
1679            ));
1680        }
1681    }
1682    mem_mark("after codegen");
1683    Ok((encoded, diagnostics))
1684}
1685
1686/// Full compile via the recompile (low-memory) pipeline: writes `script.dat` /
1687/// `script.idx` to `output_dir`, then prints diagnostics.
1688fn compile_recompile(
1689    scripts_dir: &Path,
1690    pack_dir: Option<&Path>,
1691    output_dir: &Path,
1692    lint: bool,
1693) -> Result<(), Box<dyn Error>> {
1694    let (encoded, diagnostics) = recompile_to_encoded(scripts_dir, pack_dir, lint, false)?;
1695    info!(
1696        "Writing output to {}...",
1697        output_dir.display().to_string().replace('\\', "/")
1698    );
1699    ScriptWriter::new(output_dir.display().to_string()).write_encoded(&encoded)?;
1700    diagnostics.print_all();
1701    info!("Compilation complete!");
1702    Ok(())
1703}
1704
1705/// In-memory compile via the recompile (low-memory) pipeline: returns the
1706/// `(script.dat, script.idx)` bytes, then prints diagnostics. Byte-identical to
1707/// the in-memory batch path (`compile_memory*`).
1708fn compile_recompile_memory(
1709    scripts_dir: &Path,
1710    pack_dir: Option<&Path>,
1711    lint: bool,
1712    include_tests: bool,
1713) -> Result<(Vec<u8>, Vec<u8>), Box<dyn Error>> {
1714    let (encoded, diagnostics) = recompile_to_encoded(scripts_dir, pack_dir, lint, include_tests)?;
1715    info!("Writing compiled output to memory...");
1716    let out = ScriptWriter::new("".into()).build_encoded(&encoded);
1717    diagnostics.print_all();
1718    info!("Compilation complete!");
1719    Ok(out)
1720}
1721
1722fn find_testscript_line(source: &str) -> Option<usize> {
1723    for (i, line) in source.lines().enumerate() {
1724        if line.trim() == "#testscript" {
1725            return Some(i + 1);
1726        }
1727    }
1728    None
1729}
1730
1731fn truncate_at_line(source: &str, line_no: usize) -> String {
1732    source
1733        .lines()
1734        .take(line_no - 1)
1735        .collect::<Vec<_>>()
1736        .join("\n")
1737}
1738
1739fn strip_testscript_annotation(source: &str) -> String {
1740    source
1741        .lines()
1742        .map(|line| {
1743            if line.trim() == "#testscript" {
1744                ""
1745            } else {
1746                line
1747            }
1748        })
1749        .collect::<Vec<_>>()
1750        .join("\n")
1751}
1752
1753fn get_rs2_files(dir: &Path, ext: &str) -> Result<Vec<PathBuf>, Box<dyn Error>> {
1754    let mut rs2_files: Vec<PathBuf> = Vec::new();
1755    search_dir_recursively(dir, ext, &mut rs2_files)?;
1756    Ok(rs2_files)
1757}
1758
1759fn search_dir_recursively(
1760    dir: &Path,
1761    ext: &str,
1762    out: &mut Vec<PathBuf>,
1763) -> Result<(), Box<dyn Error>> {
1764    for entry in fs::read_dir(dir)? {
1765        let entry = entry?;
1766        let path = entry.path();
1767        if symloader::entry_is_dir(&entry, &path) {
1768            search_dir_recursively(&path, ext, out)?;
1769        } else if path.extension().and_then(|e| e.to_str()) == Some(ext) {
1770            out.push(path);
1771        }
1772    }
1773    Ok(())
1774}