Skip to main content

runec/
filesystem.rs

1use crate::bytecode::CompiledScript;
2use crate::clientscript::{self, format::Format};
3use crate::compiler::Compiler;
4use crate::diagnostics::DiagnosticsCollector;
5use crate::lexer::Lexer;
6#[cfg(feature = "memprof")]
7use crate::memprof;
8use crate::parallel::{
9    background, parallel_chunks, parallel_chunks_capped, parallel_map, parallel_map_balanced,
10};
11use crate::parser::{Parser, ScriptFile};
12use crate::project::{Options, Project, SourceText, Target};
13use crate::sources::Scope;
14use crate::symbol::SymbolRegistry;
15use crate::typechecker::TypeChecker;
16use crate::writer::ScriptWriter;
17use crate::{diagnostics, error, lints, sources, symloader, trigger_table, types};
18use std::collections::{BTreeMap, HashMap};
19use std::error::Error;
20use std::fmt::Write;
21use std::fs;
22use std::panic;
23use std::path::{Path, PathBuf};
24use std::sync::Arc;
25use tracing::{info, warn};
26
27pub type ScriptIds = BTreeMap<String, i32>;
28
29/// Build an editor project from every matching source in the scripts tree. No ID file
30/// is generated or modified; all further editing happens in memory.
31pub fn editor_project(
32    source: &Path,
33    pack: Option<&Path>,
34    client: bool,
35) -> Result<Project, Box<dyn Error>> {
36    editor_project_for_scope(
37        source,
38        pack,
39        if client { Scope::Client } else { Scope::Rune },
40    )
41}
42
43/// [`editor_project`] for a server, client or lobby target.
44pub fn editor_project_for_scope(
45    source: &Path,
46    pack: Option<&Path>,
47    scope: Scope,
48) -> Result<Project, Box<dyn Error>> {
49    if scope == Scope::Global {
50        return Err("GlobalScript belongs to the server and client targets".into());
51    }
52    let client = scope == Scope::Client;
53    let dir = sources::target_dir(source, scope);
54    let pack = resolve_pack_dir(&dir, pack);
55    let mut editor_roots = sources::roots(&dir);
56    let mut registry = if client {
57        clientscript::registry(
58            &dir,
59            pack.as_deref()
60                .ok_or("Client opcode definitions require --pack")?,
61        )?
62    } else {
63        build_symbol_registry(&dir, pack.as_deref(), scope)?
64    };
65    let target = if client {
66        registry.preloaded_script_ids.clear();
67        for (key, id) in clientscript::script_ids(pack.as_deref().unwrap())? {
68            let (trigger, name) = key
69                .trim_start_matches('[')
70                .trim_end_matches(']')
71                .split_once(',')
72                .ok_or("Invalid client ID")?;
73            registry.set_script_id(trigger, name, id);
74        }
75        Target::Client(Format::load(
76            &pack.as_ref().unwrap().join("cs2_command.pack"),
77        )?)
78    } else if scope == Scope::Lobby {
79        Target::Lobby
80    } else {
81        Target::Rune
82    };
83    let mut project = Project::new(
84        target,
85        registry,
86        Options {
87            include_tests: true,
88            skip_shared_lints: client
89                && sources::script_files(&dir, "rs2")?
90                    .iter()
91                    .any(|path| path.extension().is_some_and(|ext| ext == "rs2")),
92            ..Options::default()
93        },
94    );
95    let paths = sources::script_files(&dir, scope.extension())?;
96    let files = read_sources(&paths)?;
97    for id in project.open_all(files) {
98        if client {
99            let ids = project
100                .document(id)
101                .and_then(|doc| doc.syntax())
102                .into_iter()
103                .flat_map(|file| &file.scripts)
104                .filter_map(|script| {
105                    script
106                        .name
107                        .strip_prefix("clientscript_")
108                        .and_then(|id| id.parse::<i32>().ok())
109                        .map(|id| (script.trigger.clone(), script.name.clone(), id))
110                })
111                .collect::<Vec<_>>();
112            for (trigger, name, id) in ids {
113                project.registry.set_script_id(&trigger, &name, id);
114            }
115        }
116    }
117    if !client {
118        let paths = sources::script_files(&dir, "cs2")?
119            .into_iter()
120            .filter(|path| {
121                path.extension().is_some_and(|ext| ext == "cs2")
122                    && path.file_name().is_some_and(|name| name != "engine.cs2")
123            })
124            .collect::<Vec<_>>();
125        project.open_all(read_sources(&paths)?);
126    }
127    let mut files = sources::config_files(&dir);
128    if let Some(pack) = &pack {
129        editor_roots.push(pack.clone());
130        symloader::collect_all_files(pack, &mut files);
131        // Interfaces and defaults live beside the scripts they're for.
132        if let Some(content) = pack.parent() {
133            // Other cache assets are opened on demand, without preloading their
134            // large directories or accepting every revision beside a custom pack.
135            for folder in [
136                "animations",
137                "defaults",
138                "fontmetrics",
139                "interfaces",
140                "instruments",
141                "jingles",
142                "loading_backgrounds",
143                "loading_sprites",
144                "maps",
145                "materials",
146                "models",
147                "skeletons",
148                "songs",
149                "sprites",
150                "synth",
151                "textures",
152                "worldmap",
153            ] {
154                let root = content.join(folder);
155                if root.is_dir() {
156                    editor_roots.push(root);
157                }
158            }
159            for (folder, extension) in [("interfaces", "if3"), ("defaults", "defaults")] {
160                let Ok(entries) = fs::read_dir(content.join(folder)) else {
161                    continue;
162                };
163                files.extend(
164                    entries
165                        .flatten()
166                        .map(|entry| entry.path())
167                        .filter(|path| path.extension().is_some_and(|ext| ext == extension)),
168                );
169            }
170        }
171    }
172    files.sort();
173    files.dedup();
174    files.retain(|path| config_kind(path).is_some());
175    let configs = read_sources(&files)?;
176    for ((uri, config_scope, version, text), path) in configs.into_iter().zip(&files) {
177        project.open_config(uri, config_scope, version, text, config_kind(path).unwrap());
178    }
179    project.editor_roots = Some(
180        editor_roots
181            .iter()
182            .map(|root| file_uri(root))
183            .collect::<Result<_, _>>()?,
184    );
185    Ok(project)
186}
187
188/// Read files and name them by URI, in parallel, failing on the first error in
189/// input order.
190fn read_sources(paths: &[PathBuf]) -> Result<Vec<SourceText>, Box<dyn Error>> {
191    parallel_map_balanced(paths, |path| {
192        let text = fs::read_to_string(path).map_err(|error| error.to_string())?;
193        let uri = file_uri(path).map_err(|error| error.to_string())?;
194        let scope = Scope::for_file(path, &text).map_err(|error| error.to_string())?;
195        Ok((uri, scope, 0, text))
196    })
197    .into_iter()
198    .collect::<Result<Vec<_>, String>>()
199    .map_err(Into::into)
200}
201
202pub(crate) fn config_kind(path: &Path) -> Option<String> {
203    if path.file_name()?.to_str()? == "floorcol.txt" {
204        return Some("floorcol".into());
205    }
206    let extension = path.extension()?.to_str()?;
207    if matches!(extension, "pack" | "order" | "hashes") {
208        return Some(format!("pack:{}", path.file_stem()?.to_str()?));
209    }
210    (matches!(
211        extension,
212        "constant" | "if" | "if3" | "defaults" | "anim" | "material" | "screen" | "jm2"
213    ) || types::Type::from_name(extension).is_some()
214        || extension
215            .strip_prefix("config")
216            .is_some_and(|number| number.parse::<u32>().is_ok()))
217    .then(|| extension.to_string())
218}
219
220pub(crate) fn file_uri(path: &Path) -> Result<String, Box<dyn Error>> {
221    let absolute = fs::canonicalize(path)?
222        .to_string_lossy()
223        .trim_start_matches("\\\\?\\")
224        .replace('\\', "/");
225    let mut uri = String::from(if absolute.starts_with('/') {
226        "file://"
227    } else {
228        "file:///"
229    });
230    for byte in absolute.bytes() {
231        if byte.is_ascii_alphanumeric() || b"/-._~:".contains(&byte) {
232            uri.push(byte as char);
233        } else {
234            write!(uri, "%{byte:02X}")?;
235        }
236    }
237    Ok(uri)
238}
239
240/// How the diagnostic layer fetches a file's source text (by its cache-key path)
241/// when building rustc-style help/suggestions. The batch path keeps every source
242/// in memory; the streaming path keeps only the on-disk paths and re-reads the
243/// handful of files that actually produce a warning — so the ~4 MB of source
244/// text isn't resident through the compile's peak. Cheap to copy (just borrows).
245#[derive(Clone, Copy)]
246pub enum SourceProvider<'a> {
247    /// Sources already resident, keyed by cache key.
248    Eager(&'a HashMap<String, Arc<String>>),
249    /// Cache key → on-disk path; re-read on demand.
250    Lazy(&'a HashMap<String, PathBuf>),
251}
252
253impl SourceProvider<'_> {
254    /// Source text for `key`, if available. Returns an owned `Arc` — a cheap
255    /// refcount bump for `Eager`, a fresh disk read for `Lazy`. Called only while
256    /// emitting diagnostic help, which is rare, so the re-read cost is immaterial.
257    pub fn get(&self, key: &str) -> Option<Arc<String>> {
258        match self {
259            SourceProvider::Eager(m) => m.get(key).cloned(),
260            SourceProvider::Lazy(m) => fs::read_to_string(m.get(key)?).ok().map(Arc::new),
261        }
262    }
263}
264
265/// Log live heap + global peak at a pipeline phase boundary. No-op unless built
266/// with `--features memprof`. Lets us attribute the footprint to each phase.
267#[cfg(feature = "memprof")]
268fn mem_mark(label: &str) {
269    info!(
270        "[mem] {:<22} live {:>7.1} MB | peak {:>7.1} MB | allocs {}",
271        label,
272        memprof::current_bytes() as f64 / 1_048_576.0,
273        memprof::peak_bytes() as f64 / 1_048_576.0,
274        memprof::alloc_count(),
275    );
276}
277#[cfg(not(feature = "memprof"))]
278#[inline]
279fn mem_mark(_label: &str) {}
280
281/// Per-map breakdown of the symbol registry's heap, to see the irreducible
282/// floor for a streaming rebuild. Built only under `--features memprof`.
283/// Accounts three components per map: the hashbrown table (inline `(K,V)`
284/// buckets + 1 control byte each), the key strings, and value-side heap.
285#[cfg(feature = "memprof")]
286fn mem_stats_registry(reg: &crate::symbol::SymbolRegistry) {
287    use crate::symbol::{Symbol, SymbolKind};
288    use crate::types::Type;
289    use std::collections::{HashMap, HashSet};
290    let t = std::mem::size_of::<Type>();
291    let mb = |b: usize| b as f64 / 1_048_576.0;
292
293    // Allocated bucket count: hashbrown keeps `capacity()` ≈ 7/8 of a power-of-two
294    // bucket array; an empty map allocates nothing.
295    fn tbl<K, V>(m: &HashMap<K, V>) -> usize {
296        if m.capacity() == 0 {
297            return 0;
298        }
299        let buckets = (m.capacity() * 8 / 7).next_power_of_two();
300        buckets * (std::mem::size_of::<(K, V)>() + 1)
301    }
302    fn keys_cap<V>(m: &HashMap<String, V>) -> usize {
303        m.keys().map(|k| k.capacity()).sum()
304    }
305    let sym_heap = |s: &Symbol| -> usize {
306        s.name.capacity()
307            + match &s.kind {
308                SymbolKind::Script {
309                    trigger,
310                    param_types,
311                    return_types,
312                    ..
313                } => trigger.capacity() + (param_types.capacity() + return_types.capacity()) * t,
314                SymbolKind::Command {
315                    param_types,
316                    return_types,
317                    ..
318                } => (param_types.capacity() + return_types.capacity()) * t,
319                SymbolKind::GameVar { category, .. } => category.capacity(),
320                SymbolKind::Constant { string_value, .. } => {
321                    string_value.as_ref().map_or(0, |s| s.capacity())
322                }
323                _ => 0,
324            }
325    };
326    let sym_map = |m: &HashMap<String, Symbol>| -> usize {
327        tbl(m) + keys_cap(m) + m.values().map(&sym_heap).sum::<usize>()
328    };
329    let i32_map = |m: &HashMap<String, i32>| -> usize { tbl(m) + keys_cap(m) };
330
331    let mut rows: Vec<(&str, usize, usize)> = vec![
332        (
333            "scripts",
334            reg.scripts.len(),
335            tbl(&reg.scripts) + keys_cap(&reg.scripts),
336        ),
337        ("commands", reg.commands.len(), sym_map(&reg.commands)),
338        ("game_vars", reg.game_vars.len(), sym_map(&reg.game_vars)),
339        ("constants", reg.constants.len(), sym_map(&reg.constants)),
340        (
341            "entity_ids",
342            reg.entity_ids.len(),
343            tbl(&reg.entity_ids)
344                + keys_cap(&reg.entity_ids)
345                + reg
346                    .entity_ids
347                    .values()
348                    .map(|e| e.variants.capacity() * std::mem::size_of::<(Type, i32)>())
349                    .sum::<usize>(),
350        ),
351        (
352            "scripts_by_trigger",
353            reg.scripts_by_trigger.len(),
354            tbl(&reg.scripts_by_trigger)
355                + keys_cap(&reg.scripts_by_trigger)
356                + reg
357                    .scripts_by_trigger
358                    .values()
359                    .map(|s| 16 + sym_heap(s.as_ref()))
360                    .sum::<usize>(),
361        ),
362        (
363            "command_param_types",
364            reg.command_param_types.len(),
365            tbl(&reg.command_param_types)
366                + keys_cap(&reg.command_param_types)
367                + reg
368                    .command_param_types
369                    .values()
370                    .map(|v| v.capacity() * t)
371                    .sum::<usize>(),
372        ),
373        ("script_ids", reg.script_ids.len(), i32_map(&reg.script_ids)),
374        (
375            "proc_script_ids",
376            reg.proc_script_ids.len(),
377            i32_map(&reg.proc_script_ids),
378        ),
379        (
380            "label_script_ids",
381            reg.label_script_ids.len(),
382            i32_map(&reg.label_script_ids),
383        ),
384        (
385            "trigger_script_ids",
386            reg.trigger_script_ids.len(),
387            i32_map(&reg.trigger_script_ids),
388        ),
389        (
390            "preloaded_script_ids",
391            reg.preloaded_script_ids.len(),
392            i32_map(&reg.preloaded_script_ids),
393        ),
394        ("components", reg.components.len(), i32_map(&reg.components)),
395        (
396            "interface_ids",
397            reg.interface_ids.len(),
398            i32_map(&reg.interface_ids),
399        ),
400        ("type_chars", reg.type_chars.len(), i32_map(&reg.type_chars)),
401        (
402            "dbcolumn_types",
403            reg.dbcolumn_types.len(),
404            tbl(&reg.dbcolumn_types) + keys_cap(&reg.dbcolumn_types),
405        ),
406    ];
407    let ts_tbl = {
408        let s: &HashSet<String> = &reg.test_scripts;
409        if s.capacity() == 0 {
410            0
411        } else {
412            (s.capacity() * 8 / 7).next_power_of_two() * (std::mem::size_of::<String>() + 1)
413                + s.iter().map(|k| k.capacity()).sum::<usize>()
414        }
415    };
416    rows.push(("test_scripts", reg.test_scripts.len(), ts_tbl));
417
418    rows.sort_by_key(|row| std::cmp::Reverse(row.2));
419    let total: usize = rows.iter().map(|r| r.2).sum();
420    info!(
421        "[mem] registry ~{:.2} MB (sizeof Symbol={}B, SymbolKind={}B, Type={}B):",
422        mb(total),
423        std::mem::size_of::<Symbol>(),
424        std::mem::size_of::<SymbolKind>(),
425        t,
426    );
427    for (name, n, bytes) in &rows {
428        info!(
429            "[mem]   {:<20} {:>6} entries  {:>7.3} MB",
430            name,
431            n,
432            mb(*bytes)
433        );
434    }
435}
436
437/// One-off breakdown of where the compiled-bytecode heap goes, to target
438/// memory work. Built only under `--features memprof`.
439#[cfg(feature = "memprof")]
440fn mem_stats_compiled(scripts: &[CompiledScript]) {
441    use crate::bytecode::{Instruction, Opcode, Operand};
442    let (mut n_instr, mut n_str, mut str_bytes) = (0usize, 0usize, 0usize);
443    let (mut n_switch, mut switch_entries) = (0usize, 0usize);
444    let (mut source_path_bytes, mut name_bytes) = (0usize, 0usize);
445    for s in scripts {
446        n_instr += s.instructions.len();
447        source_path_bytes += s.source_path.capacity();
448        name_bytes += s.name.capacity() + s.trigger.capacity();
449        for st in &s.strings {
450            n_str += 1;
451            str_bytes += st.len();
452        }
453        for sw in &s.switch_tables {
454            n_switch += 1;
455            switch_entries += sw.len();
456        }
457    }
458    let sz = std::mem::size_of::<Instruction>();
459    let mb = |b: usize| b as f64 / 1_048_576.0;
460    info!(
461        "[mem] sizeof Instruction={}B Operand={}B Opcode={}B CompiledScript={}B",
462        sz,
463        std::mem::size_of::<Operand>(),
464        std::mem::size_of::<Opcode>(),
465        std::mem::size_of::<CompiledScript>(),
466    );
467    info!(
468        "[mem] compiled: {} scripts, {} instrs => {:.2} MB instr structs",
469        scripts.len(),
470        n_instr,
471        mb(n_instr * sz)
472    );
473    info!(
474        "[mem]   Str operands: {} ({:.2} MB content + {:.2} MB Box<str> headers)",
475        n_str,
476        mb(str_bytes),
477        mb(n_str * 16)
478    );
479    info!(
480        "[mem]   SwitchTable operands: {} ({} entries => {:.2} MB)",
481        n_switch,
482        switch_entries,
483        mb(switch_entries * 16)
484    );
485    info!(
486        "[mem]   source_path {:.2} MB, names+triggers {:.2} MB",
487        mb(source_path_bytes),
488        mb(name_bytes)
489    );
490}
491
492struct ParsedFile {
493    path: PathBuf,
494    file: ScriptFile,
495    testscript_line: Option<usize>,
496    /// Canonicalized, normalized source path. Computed once during parsing and
497    /// reused as each compiled script's `source_path` (it is byte-identical to
498    /// the `source_cache` key), avoiding a second `canonicalize` syscall per
499    /// file in codegen.
500    source_path: String,
501}
502
503/// Outcome of parsing one file off-thread. The owned source text and cache key
504/// travel back so the (single-threaded) merge can build the `Rc`-based caches
505/// without sending `Rc` across threads.
506struct FileParseResult {
507    path: PathBuf,
508    read: Result<ParsedSource, std::io::Error>,
509}
510
511struct ParsedSource {
512    cache_key: String,
513    raw_source: String,
514    outcome: ParseOutcome,
515}
516
517enum ParseOutcome {
518    Parsed {
519        file: ScriptFile,
520        testscript_line: Option<usize>,
521    },
522    Error {
523        line: usize,
524        position: usize,
525        message: String,
526        phase: diagnostics::Phase,
527    },
528}
529
530/// Read, lex and parse a single file off-thread. Pure: no shared state, so it
531/// is safe to run concurrently. Mirrors the original sequential body exactly —
532/// same source truncation, same cache-key normalization, same lex-then-parse
533/// error reporting — only restructured to return owned data for the merge.
534fn parse_one_file(path: &Path, include_tests: bool) -> FileParseResult {
535    let raw_source = match fs::read_to_string(path) {
536        Ok(s) => s,
537        Err(e) => {
538            return FileParseResult {
539                path: path.to_path_buf(),
540                read: Err(e),
541            };
542        }
543    };
544    let testscript_line = find_testscript_line(&raw_source);
545
546    let source_code = if !include_tests {
547        if let Some(line_no) = testscript_line {
548            truncate_at_line(&raw_source, line_no)
549        } else {
550            raw_source.clone()
551        }
552    } else {
553        strip_testscript_annotation(&raw_source)
554    };
555
556    let canonical = fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf());
557    let mut cache_key = canonical.to_string_lossy().into_owned();
558    if cache_key.starts_with("\\\\?\\") {
559        cache_key = cache_key[4..].to_string();
560    }
561    cache_key = cache_key.replace("\\Content\\", "\\content\\");
562
563    // Lexer/Parser borrow a `&PathBuf` for diagnostics; keep one alive locally.
564    let path_buf = path.to_path_buf();
565    let outcome = match Lexer::new(&source_code, &path_buf).tokenize() {
566        Err(e) => ParseOutcome::Error {
567            line: e.line,
568            position: e.position,
569            message: e.message,
570            phase: diagnostics::Phase::Lexing,
571        },
572        Ok(tokens) => match Parser::new(tokens, &path_buf).parse() {
573            Ok(file) => ParseOutcome::Parsed {
574                file,
575                testscript_line: if include_tests { testscript_line } else { None },
576            },
577            Err(e) => ParseOutcome::Error {
578                line: e.line,
579                position: e.position,
580                message: e.message,
581                phase: diagnostics::Phase::Parsing,
582            },
583        },
584    };
585
586    FileParseResult {
587        path: path_buf,
588        read: Ok(ParsedSource {
589            cache_key,
590            raw_source,
591            outcome,
592        }),
593    }
594}
595
596/// Compile scripts and write output. When `lint` is true, also run lint passes
597/// (unused locals, unreachable code) after code generation.
598pub fn compile_memory(
599    scripts_dir: &Path,
600    pack_dir: Option<&Path>,
601    lint: bool,
602) -> Result<(Vec<u8>, Vec<u8>), Box<dyn Error>> {
603    compile_memory_with_options(scripts_dir, pack_dir, lint, false, false)
604}
605
606/// Like `compile_memory` but includes `#testscript` sections when
607/// `include_tests` is true.
608pub fn compile_memory_with_tests(
609    scripts_dir: &Path,
610    pack_dir: Option<&Path>,
611    lint: bool,
612    include_tests: bool,
613) -> Result<(Vec<u8>, Vec<u8>), Box<dyn Error>> {
614    compile_memory_with_options(scripts_dir, pack_dir, lint, include_tests, false)
615}
616
617/// In-memory compile — returns the `(script.dat, script.idx)` bytes instead of
618/// writing them — choosing the compilation strategy.
619///
620/// With `low_mem`, the recompile pipeline is used: scripts are re-parsed and
621/// re-compiled on demand so the full compiled set is never resident (~16 MB heap
622/// vs ~93), at roughly 6× the time; the returned bytes are byte-identical to the
623/// default path. `include_tests` keeps `#testscript` sections. The
624/// `RUNEC_RECOMPILE` environment variable forces low-memory mode on as well.
625pub fn compile_memory_with_options(
626    scripts_dir: &Path,
627    pack_dir: Option<&Path>,
628    lint: bool,
629    include_tests: bool,
630    low_mem: bool,
631) -> Result<(Vec<u8>, Vec<u8>), Box<dyn Error>> {
632    compile_memory_inner(
633        Scope::Rune,
634        scripts_dir,
635        pack_dir,
636        lint,
637        include_tests,
638        low_mem,
639        &mut ScriptIds::new(),
640    )
641}
642
643/// Reuses a world's script IDs across reloads, reserving removed scripts' IDs.
644pub fn compile_memory_with_ids(
645    scripts_dir: &Path,
646    pack_dir: Option<&Path>,
647    ids: &mut ScriptIds,
648) -> Result<(Vec<u8>, Vec<u8>), Box<dyn Error>> {
649    compile_memory_inner(Scope::Rune, scripts_dir, pack_dir, false, false, false, ids)
650}
651
652/// [`compile_memory`] for lobbyscript: `.ls2` sources, `ls2_command.pack` and `engine.ls2`.
653pub fn compile_lobby_memory(
654    scripts_dir: &Path,
655    pack_dir: Option<&Path>,
656    lint: bool,
657) -> Result<(Vec<u8>, Vec<u8>), Box<dyn Error>> {
658    compile_lobby_memory_with_options(scripts_dir, pack_dir, lint, false, false)
659}
660
661/// [`compile_memory_with_options`] for lobbyscript.
662pub fn compile_lobby_memory_with_options(
663    scripts_dir: &Path,
664    pack_dir: Option<&Path>,
665    lint: bool,
666    include_tests: bool,
667    low_mem: bool,
668) -> Result<(Vec<u8>, Vec<u8>), Box<dyn Error>> {
669    compile_memory_inner(
670        Scope::Lobby,
671        scripts_dir,
672        pack_dir,
673        lint,
674        include_tests,
675        low_mem,
676        &mut ScriptIds::new(),
677    )
678}
679
680/// [`compile_memory_with_ids`] for lobbyscript. Lobby IDs are separate from the server's.
681pub fn compile_lobby_memory_with_ids(
682    scripts_dir: &Path,
683    pack_dir: Option<&Path>,
684    ids: &mut ScriptIds,
685) -> Result<(Vec<u8>, Vec<u8>), Box<dyn Error>> {
686    compile_memory_inner(
687        Scope::Lobby,
688        scripts_dir,
689        pack_dir,
690        false,
691        false,
692        false,
693        ids,
694    )
695}
696
697fn compile_memory_inner(
698    scope: Scope,
699    scripts_dir: &Path,
700    pack_dir: Option<&Path>,
701    lint: bool,
702    include_tests: bool,
703    low_mem: bool,
704    ids: &mut ScriptIds,
705) -> Result<(Vec<u8>, Vec<u8>), Box<dyn Error>> {
706    if low_mem || std::env::var("RUNEC_RECOMPILE").is_ok_and(|v| !v.is_empty()) {
707        info!("[recompile] low-memory recompile pipeline enabled");
708        return compile_recompile_memory(scope, scripts_dir, pack_dir, lint, include_tests, ids);
709    }
710
711    let (compiled_scripts, diagnostics) =
712        run_pipeline(scope, scripts_dir, pack_dir, lint, include_tests, ids)?;
713
714    info!("Writing compiled output to memory...");
715    let writer = ScriptWriter::new("".into());
716
717    diagnostics.print_all();
718    info!("Compilation complete!");
719
720    Ok(writer.build_all(&compiled_scripts)?)
721}
722
723/// Compile scripts and write output. When `lint` is true, also run lint passes
724/// (unused locals, unreachable code) after code generation.
725///
726/// Convenience wrapper over [`compile_with_options`] using the default (fast,
727/// fully-resident) pipeline. Kept signature-compatible for existing callers.
728pub fn compile(
729    scripts_dir: &Path,
730    pack_dir: Option<&Path>,
731    output_dir: &Path,
732    lint: bool,
733) -> Result<(), Box<dyn Error>> {
734    compile_with_options(scripts_dir, pack_dir, output_dir, lint, false)
735}
736
737/// Compile scripts and write output, choosing the compilation strategy.
738///
739/// With `low_mem`, the **recompile** pipeline is used: scripts are re-parsed and
740/// re-compiled on demand for the pointer check and written file-by-file, so the
741/// full set of compiled bytecode is never resident at once. That cuts peak usage
742/// to roughly 16 MB heap / 32 MB working set (from ~93 / ~111 MB) at about 6× the
743/// compile time. Output is byte-identical to the default path. The
744/// `RUNEC_RECOMPILE` environment variable forces this mode on as well.
745pub fn compile_with_options(
746    scripts_dir: &Path,
747    pack_dir: Option<&Path>,
748    output_dir: &Path,
749    lint: bool,
750    low_mem: bool,
751) -> Result<(), Box<dyn Error>> {
752    compile_output(
753        Scope::Rune,
754        scripts_dir,
755        pack_dir,
756        output_dir,
757        lint,
758        low_mem,
759    )
760}
761
762/// [`compile`] for lobbyscript: `.ls2` sources, `ls2_command.pack` and `engine.ls2`.
763pub fn compile_lobby(
764    scripts_dir: &Path,
765    pack_dir: Option<&Path>,
766    output_dir: &Path,
767    lint: bool,
768) -> Result<(), Box<dyn Error>> {
769    compile_lobby_with_options(scripts_dir, pack_dir, output_dir, lint, false)
770}
771
772/// [`compile_with_options`] for lobbyscript.
773pub fn compile_lobby_with_options(
774    scripts_dir: &Path,
775    pack_dir: Option<&Path>,
776    output_dir: &Path,
777    lint: bool,
778    low_mem: bool,
779) -> Result<(), Box<dyn Error>> {
780    compile_output(
781        Scope::Lobby,
782        scripts_dir,
783        pack_dir,
784        output_dir,
785        lint,
786        low_mem,
787    )
788}
789
790fn compile_output(
791    scope: Scope,
792    scripts_dir: &Path,
793    pack_dir: Option<&Path>,
794    output_dir: &Path,
795    lint: bool,
796    low_mem: bool,
797) -> Result<(), Box<dyn Error>> {
798    // The recompile pipeline never holds all compiled scripts (it re-compiles on
799    // demand for the pointer check + writes file-by-file). It does its own
800    // output, so it returns here. The default path below is unchanged.
801    if low_mem || std::env::var("RUNEC_RECOMPILE").is_ok_and(|v| !v.is_empty()) {
802        info!("[recompile] low-memory recompile pipeline enabled");
803        return compile_recompile(scope, scripts_dir, pack_dir, output_dir, lint);
804    }
805
806    let (compiled_scripts, diagnostics) = run_pipeline(
807        scope,
808        scripts_dir,
809        pack_dir,
810        lint,
811        false,
812        &mut ScriptIds::new(),
813    )?;
814
815    let output = output_dir.display().to_string();
816    // Normalize separators for a clean, consistent log message (input paths may
817    // mix `/` and `\`, e.g. `data/pack\server`); the writer keeps the original.
818    info!("Writing output to {}...", output.replace('\\', "/"));
819    let writer = ScriptWriter::new(output);
820    writer.write_all(&compiled_scripts)?;
821
822    diagnostics.print_all();
823    info!("Compilation complete!");
824
825    Ok(())
826}
827
828/// Run all analysis passes (parse, type-check, codegen, pointer-check, lints)
829/// without writing output. Useful for editor tooling and CI lint gates.
830pub fn lint(scripts_dir: &Path, pack_dir: Option<&Path>) -> Result<(), Box<dyn Error>> {
831    lint_scope(Scope::Rune, scripts_dir, pack_dir)
832}
833
834/// [`lint`] for lobbyscript.
835pub fn lint_lobby(scripts_dir: &Path, pack_dir: Option<&Path>) -> Result<(), Box<dyn Error>> {
836    lint_scope(Scope::Lobby, scripts_dir, pack_dir)
837}
838
839fn lint_scope(
840    scope: Scope,
841    scripts_dir: &Path,
842    pack_dir: Option<&Path>,
843) -> Result<(), Box<dyn Error>> {
844    let (_, diagnostics) = run_pipeline(
845        scope,
846        scripts_dir,
847        pack_dir,
848        true,
849        false,
850        &mut ScriptIds::new(),
851    )?;
852
853    diagnostics.print_all();
854    info!("Compilation complete!");
855
856    Ok(())
857}
858
859/// Runs the full compilation pipeline (parse, register, type-check, codegen,
860/// pointer-check, and optional lint passes), returning the compiled scripts
861/// and accumulated diagnostics for the caller to handle output.
862fn run_pipeline(
863    scope: Scope,
864    scripts_dir: &Path,
865    pack_dir: Option<&Path>,
866    lint: bool,
867    include_tests: bool,
868    ids: &mut ScriptIds,
869) -> Result<(Vec<CompiledScript>, DiagnosticsCollector), Box<dyn Error>> {
870    let source_dir = sources::target_dir(scripts_dir, scope);
871    let scripts_dir = source_dir.as_path();
872    if !scripts_dir.exists() || !scripts_dir.is_dir() {
873        return Err(Box::new(error::CompilerError::FileNotFound(
874            scripts_dir.display().to_string(),
875        )));
876    }
877
878    // Prototype: opt into the streaming pipeline (never holds all ASTs at once)
879    // with RUNEC_STREAM=1. The default batch path below is unchanged.
880    if std::env::var("RUNEC_STREAM").is_ok_and(|v| !v.is_empty()) {
881        info!("[stream] streaming pipeline enabled (RUNEC_STREAM)");
882        return run_pipeline_streaming(scope, scripts_dir, pack_dir, lint, include_tests, ids);
883    }
884
885    let mut rs2_files: Vec<PathBuf> = get_rs2_files(scripts_dir, scope.extension())?;
886
887    rs2_files.sort();
888    info!("Found {} script files", rs2_files.len());
889    mem_mark("startup");
890
891    let mut diagnostics = DiagnosticsCollector::new();
892
893    // Phase 1: Parse all files. Reading, lexing and parsing are independent
894    // per file, so fan them out across threads; the results are merged back in
895    // input order so registration/ID assignment stays deterministic.
896    info!("Parsing script files into syntax trees...");
897    let parse_results = parallel_map(&rs2_files, |path| parse_one_file(path, include_tests));
898
899    let mut all_files: Vec<ParsedFile> = Vec::new();
900    let mut source_cache: HashMap<String, Arc<String>> = HashMap::new();
901    for result in parse_results {
902        let parsed = match result.read {
903            Ok(p) => p,
904            // A read failure aborts the whole compile, exactly as the previous
905            // `fs::read_to_string(path)?` did.
906            Err(e) => return Err(Box::new(e)),
907        };
908        let rc = Arc::new(parsed.raw_source);
909        // The cache key is the canonicalized source path; reuse it as each
910        // compiled script's `source_path` so codegen needn't canonicalize again.
911        let source_path = parsed.cache_key.clone();
912        source_cache.insert(parsed.cache_key, rc.clone());
913        match parsed.outcome {
914            ParseOutcome::Parsed {
915                file,
916                testscript_line,
917            } => all_files.push(ParsedFile {
918                path: result.path,
919                file,
920                testscript_line,
921                source_path,
922            }),
923            ParseOutcome::Error {
924                line,
925                position,
926                message,
927                phase,
928            } => diagnostics.error(result.path, line, position, message, phase),
929        }
930    }
931
932    if diagnostics.has_errors() {
933        diagnostics.print_all();
934        return Err(Box::new(error::CompilerError::TypeError(
935            "Parsing failed".to_string(),
936        )));
937    }
938
939    mem_mark("after parse");
940
941    // Phase 2: assign IDs and register every parsed script.
942    info!("Registering scripts and building the symbol registry...");
943    let resolved_pack_dir = resolve_pack_dir(scripts_dir, pack_dir);
944    let mut registry = build_symbol_registry(scripts_dir, resolved_pack_dir.as_deref(), scope)?;
945    register_scripts(scope, &mut registry, ids, &mut diagnostics, &all_files);
946    info!("  Registered {} scripts", registry.scripts.len());
947    // Pre-assigned script ids were consumed during registration; free them
948    // before codegen (the compile's peak-memory phase) rather than at scope end.
949    registry.drop_registration_scratch();
950
951    for pf in &all_files {
952        for script in &pf.file.scripts {
953            if let Some(msg) =
954                Compiler::validate_trigger_subject(&script.trigger, &script.name, &registry)
955            {
956                diagnostics.warning(
957                    pf.path.clone(),
958                    script.line,
959                    0,
960                    msg,
961                    diagnostics::Phase::SymbolRegistration,
962                );
963            }
964        }
965    }
966
967    mem_mark("after register");
968    #[cfg(feature = "memprof")]
969    mem_stats_registry(&registry);
970
971    // Phase 3: Type checking
972    info!("Type checking {} files...", all_files.len());
973    run_type_checker(&mut diagnostics, &all_files, &registry);
974
975    if diagnostics.has_errors() {
976        diagnostics.print_all();
977        return Err(Box::new(error::CompilerError::TypeError(
978            "Type checking failed".to_string(),
979        )));
980    }
981
982    mem_mark("after typecheck");
983
984    // Phase 4: Code generation
985    info!("Generating bytecode from type-checked scripts...");
986    let mut compiled_scripts = codegen(&all_files, &registry, &mut diagnostics);
987    if diagnostics.has_errors() {
988        diagnostics.print_all();
989        return Err(Box::new(error::CompilerError::CodeGenError(
990            "Code generation failed".into(),
991        )));
992    }
993    info!("  Generated {} compiled scripts", compiled_scripts.len());
994    compiled_scripts.sort_by_key(|s| s.id);
995    compiled_scripts.shrink_to_fit();
996    mem_mark("after codegen");
997    #[cfg(feature = "memprof")]
998    mem_stats_compiled(&compiled_scripts);
999
1000    // The ASTs are consumed by codegen and never read again — pointer checking
1001    // and lints work purely off `compiled_scripts` + the registry + the source
1002    // cache. Free the syntax trees now so they don't coexist with the pointer
1003    // checker's (per-thread) scratch, which is the compile's peak-memory moment.
1004    drop(all_files);
1005    mem_mark("after drop asts");
1006
1007    // Phase 4.5: Pointer checking & lints
1008    info!("Checking pointers in generated code...");
1009    check_pointers(
1010        lint,
1011        &mut diagnostics,
1012        SourceProvider::Eager(&source_cache),
1013        &registry,
1014        &compiled_scripts,
1015    );
1016    mem_mark("after pointercheck");
1017
1018    Ok((compiled_scripts, diagnostics))
1019}
1020
1021// ── Streaming pipeline (prototype, gated by RUNEC_STREAM) ────────────────────
1022// Goal: never materialize all ASTs (~59 MB) at once. Pass 1 parses in parallel
1023// but keeps only per-script signatures, dropping each AST in-worker; the registry
1024// is built from those. Pass 2 re-parses each file in parallel, type-checks and
1025// compiles it, then drops the AST immediately — so syntax trees never accumulate.
1026// Output is byte-identical to the batch path (same registration, same per-file
1027// codegen, same stable id-sort). It still holds all compiled scripts (for the
1028// cross-script pointer check + writer); shrinking that is the next increment.
1029
1030/// One script's registration-relevant signature — everything the registry and
1031/// trigger diagnostics need — so the full AST can be dropped right after parsing.
1032struct ScriptSig {
1033    name: String,
1034    trigger: String,
1035    param_types: Vec<types::Type>,
1036    return_types: Vec<types::Type>,
1037    line: usize,
1038}
1039
1040struct FileSigs {
1041    path: PathBuf,
1042    testscript_line: Option<usize>,
1043    sigs: Vec<ScriptSig>,
1044}
1045
1046enum SigOutcome {
1047    Parsed {
1048        sigs: Vec<ScriptSig>,
1049        testscript_line: Option<usize>,
1050    },
1051    Error {
1052        line: usize,
1053        position: usize,
1054        message: String,
1055        phase: diagnostics::Phase,
1056    },
1057}
1058
1059struct FileSigsResult {
1060    path: PathBuf,
1061    read: Result<(String, SigOutcome), std::io::Error>,
1062}
1063
1064/// Parse one file, extract signatures, and DROP the AST in-worker, so a parallel
1065/// sweep never holds every syntax tree at once.
1066fn parse_one_file_sigs(path: &Path, include_tests: bool) -> FileSigsResult {
1067    let r = parse_one_file(path, include_tests);
1068    let read = r.read.map(|ps| {
1069        let outcome = match ps.outcome {
1070            ParseOutcome::Parsed {
1071                file,
1072                testscript_line,
1073            } => {
1074                let sigs = file
1075                    .scripts
1076                    .iter()
1077                    .map(|d| ScriptSig {
1078                        name: d.name.clone(),
1079                        trigger: d.trigger.clone(),
1080                        param_types: d.params.iter().map(|p| p.param_type).collect(),
1081                        return_types: d.return_types.clone(),
1082                        line: d.line,
1083                    })
1084                    .collect();
1085                SigOutcome::Parsed {
1086                    sigs,
1087                    testscript_line,
1088                }
1089                // `file` (the AST) is dropped here — only the sigs travel back.
1090            }
1091            ParseOutcome::Error {
1092                line,
1093                position,
1094                message,
1095                phase,
1096            } => SigOutcome::Error {
1097                line,
1098                position,
1099                message,
1100                phase,
1101            },
1102        };
1103        (ps.cache_key, outcome)
1104    });
1105    FileSigsResult { path: r.path, read }
1106}
1107
1108fn run_pipeline_streaming(
1109    scope: Scope,
1110    scripts_dir: &Path,
1111    pack_dir: Option<&Path>,
1112    lint: bool,
1113    include_tests: bool,
1114    ids: &mut ScriptIds,
1115) -> Result<(Vec<CompiledScript>, DiagnosticsCollector), Box<dyn Error>> {
1116    let mut rs2_files: Vec<PathBuf> = get_rs2_files(scripts_dir, scope.extension())?;
1117    rs2_files.sort();
1118    info!("Found {} script files", rs2_files.len());
1119    mem_mark("startup");
1120
1121    let mut diagnostics = DiagnosticsCollector::new();
1122
1123    // Pass 1: parse for signatures only (each AST dropped in its worker).
1124    info!("[stream] Pass 1: parsing for signatures...");
1125    let sig_results = parallel_map(&rs2_files, |path| parse_one_file_sigs(path, include_tests));
1126
1127    // Streaming keeps only `cache_key -> on-disk path`; source text is re-read
1128    // lazily for the rare diagnostic that needs it, so the ~4 MB of sources isn't
1129    // resident through the peak.
1130    let mut source_paths: HashMap<String, PathBuf> = HashMap::new();
1131    let mut file_sigs: Vec<FileSigs> = Vec::new();
1132    for r in sig_results {
1133        let (cache_key, outcome) = match r.read {
1134            Ok(t) => t,
1135            Err(e) => return Err(Box::new(e)),
1136        };
1137        source_paths.insert(cache_key, r.path.clone());
1138        match outcome {
1139            SigOutcome::Parsed {
1140                sigs,
1141                testscript_line,
1142            } => file_sigs.push(FileSigs {
1143                path: r.path,
1144                testscript_line,
1145                sigs,
1146            }),
1147            SigOutcome::Error {
1148                line,
1149                position,
1150                message,
1151                phase,
1152            } => diagnostics.error(r.path, line, position, message, phase),
1153        }
1154    }
1155
1156    if diagnostics.has_errors() {
1157        diagnostics.print_all();
1158        return Err(Box::new(error::CompilerError::TypeError(
1159            "Parsing failed".to_string(),
1160        )));
1161    }
1162    mem_mark("after parse");
1163
1164    // Registration (from signatures, not ASTs).
1165    info!("[stream] Registering scripts and building the symbol registry...");
1166    let resolved_pack_dir = resolve_pack_dir(scripts_dir, pack_dir);
1167    let mut registry = build_symbol_registry(scripts_dir, resolved_pack_dir.as_deref(), scope)?;
1168    register_scripts_from_sigs(scope, &mut registry, ids, &mut diagnostics, &file_sigs);
1169    info!("  Registered {} scripts", registry.scripts.len());
1170    registry.drop_registration_scratch();
1171
1172    for fs in &file_sigs {
1173        for sig in &fs.sigs {
1174            if let Some(msg) =
1175                Compiler::validate_trigger_subject(&sig.trigger, &sig.name, &registry)
1176            {
1177                diagnostics.warning(
1178                    fs.path.clone(),
1179                    sig.line,
1180                    0,
1181                    msg,
1182                    diagnostics::Phase::SymbolRegistration,
1183                );
1184            }
1185        }
1186    }
1187    // Signatures are done; only the registry + source cache carry forward.
1188    drop(file_sigs);
1189    mem_mark("after register");
1190    #[cfg(feature = "memprof")]
1191    mem_stats_registry(&registry);
1192
1193    // Pass 2: re-parse + type-check + compile per file, streaming. Parse can't
1194    // fail here (pass 1 validated), so a failed re-parse just yields nothing.
1195    info!("[stream] Pass 2: type-check + codegen (streaming)...");
1196    let per_file = parallel_map(&rs2_files, |path| {
1197        let parsed = match parse_one_file(path, include_tests).read {
1198            Ok(p) => p,
1199            Err(_) => return (Vec::new(), DiagnosticsCollector::new()),
1200        };
1201        let (file, testscript_line) = match parsed.outcome {
1202            ParseOutcome::Parsed {
1203                file,
1204                testscript_line,
1205            } => (file, testscript_line),
1206            ParseOutcome::Error { .. } => return (Vec::new(), DiagnosticsCollector::new()),
1207        };
1208        let mut type_checker = TypeChecker::new(&registry);
1209        type_checker.check_file_with_test_boundary(&file, path, testscript_line);
1210        let mut compiled = Vec::new();
1211        // Compile only when this file type-checks clean (codegen assumes
1212        // well-typed input). On any global error the whole compile aborts below,
1213        // so output produced here would be discarded anyway.
1214        if !type_checker.diagnostics.has_errors() {
1215            let mut compiler = Compiler::new(&registry);
1216            for script in &file.scripts {
1217                if script.trigger == "command" {
1218                    continue;
1219                }
1220                let Some(mut c) =
1221                    compiler.compile_checked(script, path, &mut type_checker.diagnostics)
1222                else {
1223                    continue;
1224                };
1225                c.source_path = parsed.cache_key.clone();
1226                c.shrink_to_fit();
1227                compiled.push(c);
1228            }
1229        }
1230        (compiled, type_checker.diagnostics)
1231        // `file` (the AST) is dropped here.
1232    });
1233
1234    let mut compiled_scripts: Vec<CompiledScript> = Vec::new();
1235    for (compiled, diags) in per_file {
1236        compiled_scripts.extend(compiled);
1237        diagnostics.merge(diags);
1238    }
1239    if diagnostics.has_errors() {
1240        diagnostics.print_all();
1241        return Err(Box::new(error::CompilerError::TypeError(
1242            "Type checking failed".to_string(),
1243        )));
1244    }
1245    // Stable sort by id matches the batch path (file order preserved among
1246    // equal ids → identical writer last-write-wins behaviour).
1247    compiled_scripts.sort_by_key(|s| s.id);
1248    compiled_scripts.shrink_to_fit();
1249    info!("  Generated {} compiled scripts", compiled_scripts.len());
1250    mem_mark("after codegen");
1251    #[cfg(feature = "memprof")]
1252    mem_stats_compiled(&compiled_scripts);
1253
1254    check_pointers(
1255        lint,
1256        &mut diagnostics,
1257        SourceProvider::Lazy(&source_paths),
1258        &registry,
1259        &compiled_scripts,
1260    );
1261    mem_mark("after pointercheck");
1262
1263    Ok((compiled_scripts, diagnostics))
1264}
1265
1266fn assign_script_ids<'a>(
1267    registry: &mut SymbolRegistry,
1268    ids: &mut ScriptIds,
1269    scripts: impl Iterator<Item = (&'a str, &'a str)>,
1270) {
1271    let mut next = ids.values().max().map_or(0, |id| id + 1);
1272    let mut scripts: Vec<_> = scripts
1273        .filter(|(trigger, _)| *trigger != "command")
1274        .collect();
1275    scripts.sort_unstable();
1276    scripts.dedup();
1277    for (trigger, name) in scripts {
1278        let id = *ids.entry(format!("{trigger}:{name}")).or_insert_with(|| {
1279            let id = next;
1280            next += 1;
1281            id
1282        });
1283        registry.set_script_id(trigger, name, id);
1284    }
1285}
1286
1287/// `register_scripts` for the streaming path: identical logic, driven by the
1288/// extracted signatures instead of the (already-dropped) ASTs.
1289fn register_scripts_from_sigs(
1290    scope: Scope,
1291    registry: &mut SymbolRegistry,
1292    ids: &mut ScriptIds,
1293    diagnostics: &mut DiagnosticsCollector,
1294    file_sigs: &[FileSigs],
1295) {
1296    assign_script_ids(
1297        registry,
1298        ids,
1299        file_sigs.iter().flat_map(|file| {
1300            file.sigs
1301                .iter()
1302                .map(|sig| (sig.trigger.as_str(), sig.name.as_str()))
1303        }),
1304    );
1305
1306    use crate::diagnostic_messages as msg;
1307    let mut registered_scripts: std::collections::HashSet<String> =
1308        std::collections::HashSet::new();
1309
1310    for fs in file_sigs.iter() {
1311        for sig in &fs.sigs {
1312            Arc::make_mut(&mut registry.visibility).declare_script(
1313                &fs.path,
1314                &sig.trigger,
1315                &sig.name,
1316            );
1317            if scope == Scope::Lobby && !trigger_table::is_lobby_trigger(&sig.trigger, &fs.path) {
1318                diagnostics.error(
1319                    fs.path.clone(),
1320                    sig.line,
1321                    0,
1322                    msg::fmt(msg::SCRIPT_TRIGGER_NOT_LOBBY, &[&sig.trigger]),
1323                    diagnostics::Phase::SymbolRegistration,
1324                );
1325            }
1326            if sig.trigger == "command" {
1327                continue;
1328            }
1329
1330            if scope != Scope::Lobby && !trigger_table::is_valid_trigger(&sig.trigger) {
1331                diagnostics.warning(
1332                    fs.path.clone(),
1333                    sig.line,
1334                    0,
1335                    msg::fmt(msg::SCRIPT_TRIGGER_INVALID, &[&sig.trigger]),
1336                    crate::diagnostics::Phase::SymbolRegistration,
1337                );
1338            }
1339
1340            let key = format!("{}:{}", sig.trigger, sig.name);
1341            if !registered_scripts.insert(key) {
1342                diagnostics.warning(
1343                    fs.path.clone(),
1344                    sig.line,
1345                    0,
1346                    msg::fmt(msg::SCRIPT_REDECLARATION, &[&sig.trigger, &sig.name]),
1347                    crate::diagnostics::Phase::SymbolRegistration,
1348                );
1349            }
1350
1351            if !sig.return_types.is_empty() && !trigger_table::allows_returns(&sig.trigger) {
1352                diagnostics.warning(
1353                    fs.path.clone(),
1354                    sig.line,
1355                    0,
1356                    msg::fmt(msg::SCRIPT_TRIGGER_NO_RETURNS, &[&sig.trigger]),
1357                    diagnostics::Phase::SymbolRegistration,
1358                );
1359            }
1360
1361            registry.register_script(
1362                sig.name.clone(),
1363                sig.trigger.clone(),
1364                sig.param_types.clone(),
1365                sig.return_types.clone(),
1366            );
1367
1368            if let Some(ts_line) = fs.testscript_line
1369                && sig.line >= ts_line
1370            {
1371                registry.mark_test_script(&sig.trigger, &sig.name);
1372            }
1373        }
1374    }
1375}
1376
1377fn check_pointers(
1378    lint: bool,
1379    diagnostics: &mut DiagnosticsCollector,
1380    source: SourceProvider,
1381    registry: &SymbolRegistry,
1382    compiled_scripts: &[CompiledScript],
1383) {
1384    {
1385        use crate::pointer_checker::{PointerChecker, ScriptSource};
1386        // Validate scripts across threads: each chunk gets its own checker that
1387        // shares the read-only scripts/registry/source cache but keeps its own
1388        // (purely memoizing) caches. Diagnostics are merged in script order, so
1389        // the result is identical to a single-threaded run.
1390        //
1391        // The read-only command/script lookup tables are built ONCE here and
1392        // shared (`Arc`) across every chunk checker, instead of each thread
1393        // rebuilding the full ~700-command pointer table + per-script reverse
1394        // maps. `names` is just the script order, so derive it directly rather
1395        // than spinning up a throwaway checker for `script_names()`.
1396        let shared = PointerChecker::build_shared(compiled_scripts, registry, false);
1397        let names: Vec<String> = compiled_scripts.iter().map(|s| s.name.clone()).collect();
1398        // Each chunk's checker retains its CFG caches until the chunk finishes,
1399        // so concurrency multiplies peak scratch. RUNEC_PTR_CHUNKS caps the
1400        // concurrent chunk count to trade pointer-check speed for memory; the
1401        // result is chunk-count-independent, so diagnostics stay byte-identical.
1402        let max_chunks = std::env::var("RUNEC_PTR_CHUNKS")
1403            .ok()
1404            .and_then(|s| s.parse::<usize>().ok())
1405            .filter(|&n| n > 0)
1406            .unwrap_or(usize::MAX);
1407        let chunk_diags = parallel_chunks_capped(&names, max_chunks, |chunk| {
1408            let mut checker = PointerChecker::with_shared(
1409                ScriptSource::Resident(compiled_scripts),
1410                registry,
1411                shared.clone(),
1412            );
1413            checker.set_source_cache(source);
1414            checker.validate_names(chunk)
1415        });
1416        let mut pointer_diags = DiagnosticsCollector::new();
1417        for d in chunk_diags {
1418            pointer_diags.merge(d);
1419        }
1420        let ptr_warnings = pointer_diags.warning_count();
1421        if ptr_warnings > 0 {
1422            info!("  Pointer check: {} warning(s)", ptr_warnings);
1423        }
1424        diagnostics.merge(pointer_diags);
1425    }
1426    #[cfg(feature = "memprof")]
1427    crate::pointer_checker::cfg_trace_report();
1428
1429    // Phase 4.6: Lint passes (optional)
1430    if lint {
1431        info!("Running lint passes (unused locals, unreachable code)...");
1432        let lint_diags = lints::run_lints(compiled_scripts, Some(source));
1433        let lint_warnings = lint_diags.warning_count();
1434        if lint_warnings > 0 {
1435            info!("  Lints: {} warning(s)", lint_warnings);
1436        }
1437        diagnostics.merge(lint_diags);
1438    }
1439}
1440
1441fn codegen(
1442    all_files: &[ParsedFile],
1443    registry: &SymbolRegistry,
1444    diagnostics: &mut DiagnosticsCollector,
1445) -> Vec<CompiledScript> {
1446    // Each file compiles independently against the read-only registry, so fan
1447    // the files out across threads. Output order doesn't matter — the caller
1448    // sorts compiled scripts by id — so a simple flatten of the per-file results
1449    // is enough.
1450    let mut compiled = Vec::new();
1451    for (scripts, errors) in parallel_map(all_files, |pf| compile_file(pf, registry)) {
1452        compiled.extend(scripts);
1453        diagnostics.merge(errors);
1454    }
1455    compiled
1456}
1457
1458/// Compile every (non-`command`) script in one file. Creating a `Compiler` per
1459/// file is cheap now that it only borrows the registry, and keeps the codegen
1460/// scratch state thread-local.
1461fn compile_file(
1462    pf: &ParsedFile,
1463    registry: &SymbolRegistry,
1464) -> (Vec<CompiledScript>, DiagnosticsCollector) {
1465    let mut diagnostics = DiagnosticsCollector::new();
1466    let mut scripts =
1467        crate::compiler::compile_file(&pf.file, &pf.path, registry, false, &mut diagnostics);
1468    for script in &mut scripts {
1469        script.source_path = pf.source_path.clone();
1470        script.shrink_to_fit();
1471    }
1472    (scripts, diagnostics)
1473}
1474
1475fn run_type_checker(
1476    diagnostics: &mut DiagnosticsCollector,
1477    all_files: &[ParsedFile],
1478    registry: &SymbolRegistry,
1479) {
1480    // Each file type-checks independently against the read-only registry, so
1481    // check chunks of files in parallel; per-chunk diagnostics are merged in
1482    // file order, matching a single-threaded run.
1483    let chunk_diags = parallel_chunks(all_files, |chunk| {
1484        let mut type_checker = TypeChecker::new(registry);
1485        for pf in chunk {
1486            type_checker.check_file_with_test_boundary(&pf.file, &pf.path, pf.testscript_line);
1487        }
1488        type_checker.diagnostics
1489    });
1490    for d in chunk_diags {
1491        diagnostics.merge(d);
1492    }
1493}
1494
1495/// Resolve the pack directory: explicit `--pack`, else the nearest ancestor's `pack/`, else
1496/// `content/pack` up the tree.
1497fn resolve_pack_dir(scripts_dir: &Path, pack_dir: Option<&Path>) -> Option<PathBuf> {
1498    pack_dir
1499        .map(PathBuf::from)
1500        .filter(|p| p.exists())
1501        .or_else(|| {
1502            scripts_dir.ancestors().find_map(|dir| {
1503                let pack = dir.join("pack");
1504                pack.is_dir().then_some(pack)
1505            })
1506        })
1507        .or_else(|| {
1508            scripts_dir
1509                .ancestors()
1510                .nth(3)
1511                .map(|p| p.join("content/pack"))
1512                .filter(|p| p.exists())
1513        })
1514}
1515
1516/// Build the source-independent registry before registering parsed scripts.
1517fn build_symbol_registry(
1518    scripts_dir: &Path,
1519    resolved_pack_dir: Option<&Path>,
1520    scope: Scope,
1521) -> Result<SymbolRegistry, Box<dyn Error>> {
1522    let config_files = sources::target_config_files(scripts_dir, scope)?;
1523    let engine_rs2 = sources::engine_file(scripts_dir, scope)?;
1524    let commands = if scope == Scope::Lobby {
1525        "ls2_command.pack"
1526    } else {
1527        "command.pack"
1528    };
1529    // Visibility reads every source and config, and only its indexing needs
1530    // the registry built meanwhile.
1531    let visibility = {
1532        let dir = scripts_dir.to_path_buf();
1533        background(move || sources::Visibility::scan(&dir))
1534    };
1535    let mut registry = SymbolRegistry::new();
1536
1537    if let Some(pd) = resolved_pack_dir {
1538        info!("  Loading packs from {}", pd.display());
1539        symloader::load_packs_with_commands(&mut registry, pd, commands)?;
1540        info!(
1541            "  Loaded {} commands, {} game vars",
1542            registry.commands.len(),
1543            registry.game_vars.len(),
1544        );
1545    } else {
1546        info!("  No pack directory found - symbol resolution will be limited");
1547    }
1548
1549    // Walk the scripts tree once and let the constant/game-var/dbtable loaders
1550    // filter this shared list by extension, instead of each re-walking the tree
1551    // (previously 6 separate recursive traversals over the same directories).
1552    symloader::load_constant_files(&mut registry, &config_files);
1553
1554    info!(
1555        "  Loaded {} constants from .constant files",
1556        registry.constants.len()
1557    );
1558
1559    symloader::load_game_var_types(&mut registry, &config_files);
1560
1561    if let Some(pd) = resolved_pack_dir
1562        && let Err(error) = clientscript::load_clan_settings(&mut registry, &config_files, pd)
1563    {
1564        warn!("  Clan settings not loaded: {error}");
1565    }
1566
1567    if let Some(pd) = resolved_pack_dir {
1568        let dbtable_ids = symloader::load_dbtable_pack(&pd.join("dbtable.pack"));
1569        if !dbtable_ids.is_empty() {
1570            symloader::load_dbtable_configs(&mut registry, &config_files, &dbtable_ids);
1571            info!(
1572                "  Registered {} dbcolumn compound IDs",
1573                registry.dbcolumn_types.len()
1574            );
1575        }
1576    }
1577
1578    if let Some(engine_rs2) = engine_rs2 {
1579        symloader::load_engine_command_params(&mut registry, &engine_rs2);
1580    }
1581    symloader::load_clientscript_signatures(&mut registry, scripts_dir);
1582
1583    symloader::patch_command_return_types(&mut registry);
1584
1585    let mut visibility = visibility
1586        .join()
1587        .unwrap_or_else(|panic| panic::resume_unwind(panic));
1588    visibility.index_registry(&registry);
1589    registry.visibility = Arc::new(visibility);
1590
1591    Ok(registry)
1592}
1593
1594/// Assign IDs and register the parsed scripts, reporting declaration errors.
1595fn register_scripts(
1596    scope: Scope,
1597    registry: &mut SymbolRegistry,
1598    ids: &mut ScriptIds,
1599    diagnostics: &mut DiagnosticsCollector,
1600    all_files: &[ParsedFile],
1601) {
1602    assign_script_ids(
1603        registry,
1604        ids,
1605        all_files.iter().flat_map(|file| {
1606            file.file
1607                .scripts
1608                .iter()
1609                .map(|script| (script.trigger.as_str(), script.name.as_str()))
1610        }),
1611    );
1612
1613    {
1614        use crate::diagnostic_messages as msg;
1615
1616        let mut registered_scripts: std::collections::HashSet<String> =
1617            std::collections::HashSet::new();
1618
1619        for pf in all_files.iter() {
1620            Arc::make_mut(&mut registry.visibility).declare_source(&pf.path, &pf.file);
1621            for script in &pf.file.scripts {
1622                if scope == Scope::Lobby
1623                    && !trigger_table::is_lobby_trigger(&script.trigger, &pf.path)
1624                {
1625                    diagnostics.error(
1626                        pf.path.clone(),
1627                        script.line,
1628                        0,
1629                        msg::fmt(msg::SCRIPT_TRIGGER_NOT_LOBBY, &[&script.trigger]),
1630                        diagnostics::Phase::SymbolRegistration,
1631                    );
1632                }
1633                if script.trigger == "command" {
1634                    continue;
1635                }
1636
1637                if scope != Scope::Lobby && !trigger_table::is_valid_trigger(&script.trigger) {
1638                    diagnostics.warning(
1639                        pf.path.clone(),
1640                        script.line,
1641                        0,
1642                        msg::fmt(msg::SCRIPT_TRIGGER_INVALID, &[&script.trigger]),
1643                        crate::diagnostics::Phase::SymbolRegistration,
1644                    );
1645                }
1646
1647                let key = format!("{}:{}", script.trigger, script.name);
1648                if !registered_scripts.insert(key.clone()) {
1649                    diagnostics.warning(
1650                        pf.path.clone(),
1651                        script.line,
1652                        0,
1653                        msg::fmt(msg::SCRIPT_REDECLARATION, &[&script.trigger, &script.name]),
1654                        crate::diagnostics::Phase::SymbolRegistration,
1655                    );
1656                }
1657
1658                if !script.return_types.is_empty()
1659                    && !trigger_table::allows_returns(&script.trigger)
1660                {
1661                    diagnostics.warning(
1662                        pf.path.clone(),
1663                        script.line,
1664                        0,
1665                        msg::fmt(msg::SCRIPT_TRIGGER_NO_RETURNS, &[&script.trigger]),
1666                        diagnostics::Phase::SymbolRegistration,
1667                    );
1668                }
1669
1670                let param_types: Vec<types::Type> =
1671                    script.params.iter().map(|p| p.param_type).collect();
1672                registry.register_script(
1673                    script.name.clone(),
1674                    script.trigger.clone(),
1675                    param_types,
1676                    script.return_types.clone(),
1677                );
1678
1679                if let Some(ts_line) = pf.testscript_line
1680                    && script.line >= ts_line
1681                {
1682                    registry.mark_test_script(&script.trigger, &script.name);
1683                }
1684            }
1685        }
1686    }
1687}
1688
1689// ── Streaming recompile pipeline (prototype, gated by RUNEC_RECOMPILE) ────────
1690// Never holds all compiled scripts. Pass 2 only type-checks; the pointer check
1691// re-compiles each script on demand (bounded cache — measured ~1.3x re-parse on
1692// rs-majula, the call graph has high file locality); output is produced by
1693// re-compiling file-by-file into a by-id buffer. Output + diagnostics are
1694// byte-identical to the batch path (deterministic re-compile, same id order).
1695
1696/// Re-compiles scripts on demand for the pointer checker, caching `Rc`s with a
1697/// bounded FIFO so the full compiled set is never resident. A cache miss
1698/// re-parses the script's file and compiles all its scripts at once (siblings
1699/// share the parse), keying each by its full `[trigger,name]` to its index.
1700struct RecompileStore<'a> {
1701    idx_to_path: Vec<PathBuf>,
1702    /// file path -> its script idxs in declaration order (command-skipped).
1703    /// Position-based so duplicate `[trigger,name]` declarations each map to the
1704    /// correct distinct idx.
1705    file_to_idxs: HashMap<PathBuf, Vec<usize>>,
1706    registry: &'a SymbolRegistry,
1707    include_tests: bool,
1708    cache: HashMap<usize, std::rc::Rc<CompiledScript>>,
1709    order: std::collections::VecDeque<usize>,
1710    cap: usize,
1711}
1712
1713impl RecompileStore<'_> {
1714    fn get(&mut self, idx: usize) -> std::rc::Rc<CompiledScript> {
1715        if let Some(rc) = self.cache.get(&idx) {
1716            return std::rc::Rc::clone(rc);
1717        }
1718        let path = self.idx_to_path[idx].clone();
1719        let parsed = parse_one_file(&path, self.include_tests);
1720        if let Ok(ps) = parsed.read
1721            && let ParseOutcome::Parsed { file, .. } = ps.outcome
1722        {
1723            // The j-th non-command declaration in this file maps to idxs[j],
1724            // matching how the metas were built (same parse, same skip rule).
1725            let idxs = self.file_to_idxs.get(&path).cloned().unwrap_or_default();
1726            let mut compiler = Compiler::new(self.registry);
1727            let mut j = 0usize;
1728            for script in &file.scripts {
1729                if script.trigger == "command" {
1730                    continue;
1731                }
1732                if let Some(&i) = idxs.get(j) {
1733                    let mut c = compiler.compile_script(script);
1734                    c.source_path = ps.cache_key.clone();
1735                    self.insert(i, std::rc::Rc::new(c));
1736                }
1737                j += 1;
1738            }
1739        }
1740        std::rc::Rc::clone(
1741            self.cache
1742                .get(&idx)
1743                .expect("recompile store: script not produced by its file"),
1744        )
1745    }
1746
1747    fn insert(&mut self, idx: usize, rc: std::rc::Rc<CompiledScript>) {
1748        if self.cache.insert(idx, rc).is_none() {
1749            self.order.push_back(idx);
1750            while self.order.len() > self.cap {
1751                if let Some(old) = self.order.pop_front()
1752                    && old != idx
1753                {
1754                    self.cache.remove(&old);
1755                }
1756            }
1757        }
1758    }
1759}
1760
1761/// Per-slot encoded output blobs (indexed by script id) plus accumulated
1762/// diagnostics — what the recompile core hands back before the bytes are written
1763/// to disk or assembled in memory.
1764type RecompiledOutput = (Vec<Vec<u8>>, DiagnosticsCollector);
1765
1766/// Core of the recompile (low-memory) pipeline: runs every phase re-compiling
1767/// scripts on demand — never holding all compiled bytecode at once — and returns
1768/// the per-slot encoded output blobs plus diagnostics, without writing or
1769/// printing. The disk and in-memory entry points (`compile_recompile`,
1770/// `compile_recompile_memory`) wrap this. On a phase error it prints diagnostics
1771/// and returns `Err`, matching the batch path.
1772fn recompile_to_encoded(
1773    scope: Scope,
1774    scripts_dir: &Path,
1775    pack_dir: Option<&Path>,
1776    lint: bool,
1777    include_tests: bool,
1778    ids: &mut ScriptIds,
1779) -> Result<RecompiledOutput, Box<dyn Error>> {
1780    let source_dir = sources::target_dir(scripts_dir, scope);
1781    let scripts_dir = source_dir.as_path();
1782    use crate::pointer_checker::{PointerChecker, ScriptSource};
1783
1784    let mut rs2_files: Vec<PathBuf> = get_rs2_files(scripts_dir, scope.extension())?;
1785    rs2_files.sort();
1786    info!("Found {} script files", rs2_files.len());
1787    mem_mark("startup");
1788    let mut diagnostics = DiagnosticsCollector::new();
1789
1790    // Pass 1: signatures only (ASTs dropped in-worker); keep paths for lazy source reads.
1791    info!("[recompile] Pass 1: parsing for signatures...");
1792    let sig_results = parallel_map(&rs2_files, |path| parse_one_file_sigs(path, include_tests));
1793    let mut source_paths: HashMap<String, PathBuf> = HashMap::new();
1794    let mut file_sigs: Vec<FileSigs> = Vec::new();
1795    for r in sig_results {
1796        let (cache_key, outcome) = match r.read {
1797            Ok(t) => t,
1798            Err(e) => return Err(Box::new(e)),
1799        };
1800        source_paths.insert(cache_key, r.path.clone());
1801        match outcome {
1802            SigOutcome::Parsed {
1803                sigs,
1804                testscript_line,
1805            } => file_sigs.push(FileSigs {
1806                path: r.path,
1807                testscript_line,
1808                sigs,
1809            }),
1810            SigOutcome::Error {
1811                line,
1812                position,
1813                message,
1814                phase,
1815            } => diagnostics.error(r.path, line, position, message, phase),
1816        }
1817    }
1818    if diagnostics.has_errors() {
1819        diagnostics.print_all();
1820        return Err(Box::new(error::CompilerError::TypeError(
1821            "Parsing failed".to_string(),
1822        )));
1823    }
1824    mem_mark("after parse");
1825
1826    // Registration.
1827    info!("[recompile] Registering scripts and building the symbol registry...");
1828    let resolved_pack_dir = resolve_pack_dir(scripts_dir, pack_dir);
1829    let mut registry = build_symbol_registry(scripts_dir, resolved_pack_dir.as_deref(), scope)?;
1830    register_scripts_from_sigs(scope, &mut registry, ids, &mut diagnostics, &file_sigs);
1831    info!("  Registered {} scripts", registry.scripts.len());
1832    registry.drop_registration_scratch();
1833    for fs in &file_sigs {
1834        for sig in &fs.sigs {
1835            if let Some(msg) =
1836                Compiler::validate_trigger_subject(&sig.trigger, &sig.name, &registry)
1837            {
1838                diagnostics.warning(
1839                    fs.path.clone(),
1840                    sig.line,
1841                    0,
1842                    msg,
1843                    diagnostics::Phase::SymbolRegistration,
1844                );
1845            }
1846        }
1847    }
1848    mem_mark("after register");
1849
1850    // Pass 2: type-check only (re-parse, no compiled scripts retained).
1851    info!("[recompile] Pass 2: type-check (streaming, no bytecode held)...");
1852    let tc_diags = parallel_map(&rs2_files, |path| {
1853        let parsed = match parse_one_file(path, include_tests).read {
1854            Ok(p) => p,
1855            Err(_) => return DiagnosticsCollector::new(),
1856        };
1857        let (file, testscript_line) = match parsed.outcome {
1858            ParseOutcome::Parsed {
1859                file,
1860                testscript_line,
1861            } => (file, testscript_line),
1862            ParseOutcome::Error { .. } => return DiagnosticsCollector::new(),
1863        };
1864        let mut type_checker = TypeChecker::new(&registry);
1865        type_checker.check_file_with_test_boundary(&file, path, testscript_line);
1866        if !type_checker.diagnostics.has_errors() {
1867            let mut compiler = Compiler::new(&registry);
1868            for script in &file.scripts {
1869                if script.trigger != "command" {
1870                    compiler.compile_checked(script, path, &mut type_checker.diagnostics);
1871                }
1872            }
1873        }
1874        type_checker.diagnostics
1875    });
1876    for d in tc_diags {
1877        diagnostics.merge(d);
1878    }
1879    if diagnostics.has_errors() {
1880        diagnostics.print_all();
1881        return Err(Box::new(error::CompilerError::TypeError(
1882            "Type checking failed".to_string(),
1883        )));
1884    }
1885    mem_mark("after typecheck");
1886
1887    // Build script metas in the same idx order as the batch path: file order
1888    // (file_sigs / declaration order, command-skipped), then a stable sort by id.
1889    // ids + names come from the registry — no bytecode needed.
1890    let mut pre: Vec<(i32, String, PathBuf)> = Vec::new();
1891    for fs in &file_sigs {
1892        for sig in &fs.sigs {
1893            if sig.trigger == "command" {
1894                continue;
1895            }
1896            let id = registry
1897                .script_id_for_trigger(&sig.trigger, &sig.name)
1898                .or_else(|| registry.script_id(&sig.name))
1899                .unwrap_or(-1);
1900            pre.push((
1901                id,
1902                format!("[{},{}]", sig.trigger, sig.name),
1903                fs.path.clone(),
1904            ));
1905        }
1906    }
1907    let n = pre.len();
1908    // Stable sort indices by id → the batch's idx order (file/decl pre-order
1909    // preserved among equal ids). `inv` maps a pre-order position back to its idx.
1910    let mut order: Vec<usize> = (0..n).collect();
1911    order.sort_by_key(|&p| pre[p].0);
1912    let idents: Vec<(i32, String)> = order
1913        .iter()
1914        .map(|&p| (pre[p].0, pre[p].1.clone()))
1915        .collect();
1916    let idx_to_path: Vec<PathBuf> = order.iter().map(|&p| pre[p].2.clone()).collect();
1917    let mut inv = vec![0usize; n];
1918    for (idx, &p) in order.iter().enumerate() {
1919        inv[p] = idx;
1920    }
1921    let mut file_to_idxs: HashMap<PathBuf, Vec<usize>> = HashMap::new();
1922    for (p, m) in pre.iter().enumerate() {
1923        file_to_idxs.entry(m.2.clone()).or_default().push(inv[p]);
1924    }
1925    let max_id = pre.iter().map(|m| m.0).max().unwrap_or(-1);
1926    let slot_count = if max_id < 0 { 0 } else { (max_id + 1) as usize };
1927    let len = n;
1928    drop(pre);
1929
1930    // Pointer check via on-demand re-compile (single-threaded: the store getter
1931    // is `FnMut`/`!Sync`). Diagnostics come out in id order, as in the batch path.
1932    info!("[recompile] Pointer check (re-compile on demand)...");
1933    let shared = PointerChecker::build_shared_idents(&idents, &registry);
1934    {
1935        // Cache cap must exceed the largest file's script count, or loading that
1936        // file would evict its own just-compiled scripts. A small margin over
1937        // that keeps cross-file locality high (measured ~1.1-1.3x re-parse).
1938        let cap = file_to_idxs
1939            .values()
1940            .map(|v| v.len())
1941            .max()
1942            .unwrap_or(0)
1943            .max(128);
1944        let mut store = RecompileStore {
1945            idx_to_path,
1946            file_to_idxs,
1947            registry: &registry,
1948            include_tests,
1949            cache: HashMap::new(),
1950            order: std::collections::VecDeque::new(),
1951            cap,
1952        };
1953        let getter: Box<dyn FnMut(usize) -> std::rc::Rc<CompiledScript>> =
1954            Box::new(move |idx| store.get(idx));
1955        let scripts = ScriptSource::Recompile {
1956            len,
1957            get: std::cell::RefCell::new(getter),
1958        };
1959        let mut checker = PointerChecker::with_shared(scripts, &registry, shared);
1960        checker.set_source_cache(SourceProvider::Lazy(&source_paths));
1961        let names: Vec<String> = idents.iter().map(|(_, n)| n.clone()).collect();
1962        let ptr_diags = checker.validate_names(&names);
1963        let w = ptr_diags.warning_count();
1964        if w > 0 {
1965            info!("  Pointer check: {} warning(s)", w);
1966        }
1967        diagnostics.merge(ptr_diags);
1968    }
1969    mem_mark("after pointercheck");
1970
1971    // Output: re-compile each file once, encode each script into its id slot,
1972    // optionally lint — never holding all compiled scripts at once. The caller
1973    // writes the blobs to disk or assembles them in memory.
1974    info!("[recompile] Re-compiling + encoding output...");
1975    let mut encoded: Vec<Vec<u8>> = vec![Vec::new(); slot_count];
1976    for path in &rs2_files {
1977        let parsed = match parse_one_file(path, include_tests).read {
1978            Ok(p) => p,
1979            Err(_) => continue,
1980        };
1981        let file = match parsed.outcome {
1982            ParseOutcome::Parsed { file, .. } => file,
1983            ParseOutcome::Error { .. } => continue,
1984        };
1985        let mut compiler = Compiler::new(&registry);
1986        let mut file_scripts: Vec<CompiledScript> = Vec::new();
1987        for script in &file.scripts {
1988            if script.trigger == "command" {
1989                continue;
1990            }
1991            let Some(mut c) = compiler.compile_checked(script, path, &mut diagnostics) else {
1992                continue;
1993            };
1994            c.source_path = parsed.cache_key.clone();
1995            file_scripts.push(c);
1996        }
1997        for c in &file_scripts {
1998            if c.id >= 0 && (c.id as usize) < slot_count {
1999                encoded[c.id as usize] = crate::writer::encode_script(c);
2000            }
2001        }
2002        if lint {
2003            diagnostics.merge(lints::run_lints(
2004                &file_scripts,
2005                Some(SourceProvider::Lazy(&source_paths)),
2006            ));
2007        }
2008    }
2009    mem_mark("after codegen");
2010    Ok((encoded, diagnostics))
2011}
2012
2013/// Full compile via the recompile (low-memory) pipeline: writes `script.dat` /
2014/// `script.idx` to `output_dir`, then prints diagnostics.
2015fn compile_recompile(
2016    scope: Scope,
2017    scripts_dir: &Path,
2018    pack_dir: Option<&Path>,
2019    output_dir: &Path,
2020    lint: bool,
2021) -> Result<(), Box<dyn Error>> {
2022    let (encoded, diagnostics) = recompile_to_encoded(
2023        scope,
2024        scripts_dir,
2025        pack_dir,
2026        lint,
2027        false,
2028        &mut ScriptIds::new(),
2029    )?;
2030    info!(
2031        "Writing output to {}...",
2032        output_dir.display().to_string().replace('\\', "/")
2033    );
2034    ScriptWriter::new(output_dir.display().to_string()).write_encoded(&encoded)?;
2035    diagnostics.print_all();
2036    info!("Compilation complete!");
2037    Ok(())
2038}
2039
2040/// In-memory compile via the recompile (low-memory) pipeline: returns the
2041/// `(script.dat, script.idx)` bytes, then prints diagnostics. Byte-identical to
2042/// the in-memory batch path (`compile_memory*`).
2043fn compile_recompile_memory(
2044    scope: Scope,
2045    scripts_dir: &Path,
2046    pack_dir: Option<&Path>,
2047    lint: bool,
2048    include_tests: bool,
2049    ids: &mut ScriptIds,
2050) -> Result<(Vec<u8>, Vec<u8>), Box<dyn Error>> {
2051    let (encoded, diagnostics) =
2052        recompile_to_encoded(scope, scripts_dir, pack_dir, lint, include_tests, ids)?;
2053    info!("Writing compiled output to memory...");
2054    let out = ScriptWriter::new("".into()).build_encoded(&encoded);
2055    diagnostics.print_all();
2056    info!("Compilation complete!");
2057    Ok(out)
2058}
2059
2060pub(crate) fn find_testscript_line(source: &str) -> Option<usize> {
2061    for (i, line) in source.lines().enumerate() {
2062        if line.trim() == "#testscript" {
2063            return Some(i + 1);
2064        }
2065    }
2066    None
2067}
2068
2069fn truncate_at_line(source: &str, line_no: usize) -> String {
2070    source
2071        .lines()
2072        .take(line_no - 1)
2073        .collect::<Vec<_>>()
2074        .join("\n")
2075}
2076
2077fn strip_testscript_annotation(source: &str) -> String {
2078    source
2079        .lines()
2080        .map(|line| {
2081            if line.trim() == "#testscript" {
2082                ""
2083            } else {
2084                line
2085            }
2086        })
2087        .collect::<Vec<_>>()
2088        .join("\n")
2089}
2090
2091fn get_rs2_files(dir: &Path, ext: &str) -> Result<Vec<PathBuf>, Box<dyn Error>> {
2092    Ok(sources::script_files(dir, ext)?)
2093}