Skip to main content

runec/
filesystem.rs

1use crate::bytecode::CompiledScript;
2use crate::clientscript::{self, format::Format};
3use crate::compiler::Compiler;
4use crate::diagnostics::DiagnosticsCollector;
5use crate::lexer::Lexer;
6#[cfg(feature = "memprof")]
7use crate::memprof;
8use crate::parallel::{
9    background, parallel_chunks, parallel_chunks_capped, parallel_map, parallel_map_balanced,
10};
11use crate::parser::{Parser, ScriptFile};
12use crate::project::{Options, Project, SourceText, Target};
13use crate::sources::Scope;
14use crate::symbol::SymbolRegistry;
15use crate::typechecker::TypeChecker;
16use crate::writer::ScriptWriter;
17use crate::{diagnostics, error, lints, sources, symloader, trigger_table, types};
18use std::collections::{BTreeMap, HashMap};
19use std::error::Error;
20use std::fmt::Write;
21use std::fs;
22use std::panic;
23use std::path::{Path, PathBuf};
24use std::sync::Arc;
25use tracing::{info, warn};
26
27pub type ScriptIds = BTreeMap<String, i32>;
28
29/// Build an editor project from every matching source in the scripts tree. No ID file
30/// is generated or modified; all further editing happens in memory.
31pub fn editor_project(
32    source: &Path,
33    pack: Option<&Path>,
34    client: bool,
35) -> Result<Project, Box<dyn Error>> {
36    let scope = if client { Scope::Client } else { Scope::Rune };
37    let dir = sources::target_dir(source, scope);
38    let pack = resolve_pack_dir(&dir, pack);
39    let mut editor_roots = sources::roots(&dir);
40    let mut registry = if client {
41        clientscript::registry(
42            &dir,
43            pack.as_deref()
44                .ok_or("Client opcode definitions require --pack")?,
45        )?
46    } else {
47        build_symbol_registry(&dir, pack.as_deref())?
48    };
49    let target = if client {
50        registry.preloaded_script_ids.clear();
51        for (key, id) in clientscript::script_ids(pack.as_deref().unwrap())? {
52            let (trigger, name) = key
53                .trim_start_matches('[')
54                .trim_end_matches(']')
55                .split_once(',')
56                .ok_or("Invalid client ID")?;
57            registry.set_script_id(trigger, name, id);
58        }
59        Target::Client(Format::load(
60            &pack.as_ref().unwrap().join("cs2_command.pack"),
61        )?)
62    } else {
63        Target::Rune
64    };
65    let mut project = Project::new(
66        target,
67        registry,
68        Options {
69            include_tests: true,
70            skip_shared_lints: client
71                && sources::script_files(&dir, "rs2")?
72                    .iter()
73                    .any(|path| path.extension().is_some_and(|ext| ext == "rs2")),
74            ..Options::default()
75        },
76    );
77    let paths = sources::script_files(&dir, if client { "cs2" } else { "rs2" })?;
78    let files = read_sources(&paths)?;
79    for id in project.open_all(files) {
80        if client {
81            let ids = project
82                .document(id)
83                .and_then(|doc| doc.syntax())
84                .into_iter()
85                .flat_map(|file| &file.scripts)
86                .filter_map(|script| {
87                    script
88                        .name
89                        .strip_prefix("clientscript_")
90                        .and_then(|id| id.parse::<i32>().ok())
91                        .map(|id| (script.trigger.clone(), script.name.clone(), id))
92                })
93                .collect::<Vec<_>>();
94            for (trigger, name, id) in ids {
95                project.registry.set_script_id(&trigger, &name, id);
96            }
97        }
98    }
99    if !client {
100        let paths = sources::script_files(&dir, "cs2")?
101            .into_iter()
102            .filter(|path| {
103                path.extension().is_some_and(|ext| ext == "cs2")
104                    && path.file_name().is_some_and(|name| name != "engine.cs2")
105            })
106            .collect::<Vec<_>>();
107        project.open_all(read_sources(&paths)?);
108    }
109    let mut files = sources::config_files(&dir);
110    if let Some(pack) = &pack {
111        editor_roots.push(pack.clone());
112        symloader::collect_all_files(pack, &mut files);
113        // Interfaces and defaults live beside the scripts they're for.
114        if let Some(content) = pack.parent() {
115            // Other cache assets are opened on demand, without preloading their
116            // large directories or accepting every revision beside a custom pack.
117            for folder in [
118                "animations",
119                "defaults",
120                "fontmetrics",
121                "interfaces",
122                "instruments",
123                "jingles",
124                "loading_backgrounds",
125                "loading_sprites",
126                "maps",
127                "materials",
128                "models",
129                "skeletons",
130                "songs",
131                "sprites",
132                "synth",
133                "textures",
134                "worldmap",
135            ] {
136                let root = content.join(folder);
137                if root.is_dir() {
138                    editor_roots.push(root);
139                }
140            }
141            for (folder, extension) in [("interfaces", "if3"), ("defaults", "defaults")] {
142                let Ok(entries) = fs::read_dir(content.join(folder)) else {
143                    continue;
144                };
145                files.extend(
146                    entries
147                        .flatten()
148                        .map(|entry| entry.path())
149                        .filter(|path| path.extension().is_some_and(|ext| ext == extension)),
150                );
151            }
152        }
153    }
154    files.sort();
155    files.dedup();
156    files.retain(|path| config_kind(path).is_some());
157    let configs = read_sources(&files)?;
158    for ((uri, config_scope, version, text), path) in configs.into_iter().zip(&files) {
159        project.open_config(uri, config_scope, version, text, config_kind(path).unwrap());
160    }
161    project.editor_roots = Some(
162        editor_roots
163            .iter()
164            .map(|root| file_uri(root))
165            .collect::<Result<_, _>>()?,
166    );
167    Ok(project)
168}
169
170/// Read files and name them by URI, in parallel, failing on the first error in
171/// input order.
172fn read_sources(paths: &[PathBuf]) -> Result<Vec<SourceText>, Box<dyn Error>> {
173    parallel_map_balanced(paths, |path| {
174        let text = fs::read_to_string(path).map_err(|error| error.to_string())?;
175        let uri = file_uri(path).map_err(|error| error.to_string())?;
176        let scope = Scope::for_file(path, &text).map_err(|error| error.to_string())?;
177        Ok((uri, scope, 0, text))
178    })
179    .into_iter()
180    .collect::<Result<Vec<_>, String>>()
181    .map_err(Into::into)
182}
183
184pub(crate) fn config_kind(path: &Path) -> Option<String> {
185    if path.file_name()?.to_str()? == "floorcol.txt" {
186        return Some("floorcol".into());
187    }
188    let extension = path.extension()?.to_str()?;
189    if matches!(extension, "pack" | "order" | "hashes") {
190        return Some(format!("pack:{}", path.file_stem()?.to_str()?));
191    }
192    (matches!(
193        extension,
194        "constant" | "if" | "if3" | "defaults" | "anim" | "material" | "screen" | "jm2"
195    ) || types::Type::from_name(extension).is_some()
196        || extension
197            .strip_prefix("config")
198            .is_some_and(|number| number.parse::<u32>().is_ok()))
199    .then(|| extension.to_string())
200}
201
202pub(crate) fn file_uri(path: &Path) -> Result<String, Box<dyn Error>> {
203    let absolute = fs::canonicalize(path)?
204        .to_string_lossy()
205        .trim_start_matches("\\\\?\\")
206        .replace('\\', "/");
207    let mut uri = String::from(if absolute.starts_with('/') {
208        "file://"
209    } else {
210        "file:///"
211    });
212    for byte in absolute.bytes() {
213        if byte.is_ascii_alphanumeric() || b"/-._~:".contains(&byte) {
214            uri.push(byte as char);
215        } else {
216            write!(uri, "%{byte:02X}")?;
217        }
218    }
219    Ok(uri)
220}
221
222/// How the diagnostic layer fetches a file's source text (by its cache-key path)
223/// when building rustc-style help/suggestions. The batch path keeps every source
224/// in memory; the streaming path keeps only the on-disk paths and re-reads the
225/// handful of files that actually produce a warning — so the ~4 MB of source
226/// text isn't resident through the compile's peak. Cheap to copy (just borrows).
227#[derive(Clone, Copy)]
228pub enum SourceProvider<'a> {
229    /// Sources already resident, keyed by cache key.
230    Eager(&'a HashMap<String, Arc<String>>),
231    /// Cache key → on-disk path; re-read on demand.
232    Lazy(&'a HashMap<String, PathBuf>),
233}
234
235impl SourceProvider<'_> {
236    /// Source text for `key`, if available. Returns an owned `Arc` — a cheap
237    /// refcount bump for `Eager`, a fresh disk read for `Lazy`. Called only while
238    /// emitting diagnostic help, which is rare, so the re-read cost is immaterial.
239    pub fn get(&self, key: &str) -> Option<Arc<String>> {
240        match self {
241            SourceProvider::Eager(m) => m.get(key).cloned(),
242            SourceProvider::Lazy(m) => fs::read_to_string(m.get(key)?).ok().map(Arc::new),
243        }
244    }
245}
246
247/// Log live heap + global peak at a pipeline phase boundary. No-op unless built
248/// with `--features memprof`. Lets us attribute the footprint to each phase.
249#[cfg(feature = "memprof")]
250fn mem_mark(label: &str) {
251    info!(
252        "[mem] {:<22} live {:>7.1} MB | peak {:>7.1} MB | allocs {}",
253        label,
254        memprof::current_bytes() as f64 / 1_048_576.0,
255        memprof::peak_bytes() as f64 / 1_048_576.0,
256        memprof::alloc_count(),
257    );
258}
259#[cfg(not(feature = "memprof"))]
260#[inline]
261fn mem_mark(_label: &str) {}
262
263/// Per-map breakdown of the symbol registry's heap, to see the irreducible
264/// floor for a streaming rebuild. Built only under `--features memprof`.
265/// Accounts three components per map: the hashbrown table (inline `(K,V)`
266/// buckets + 1 control byte each), the key strings, and value-side heap.
267#[cfg(feature = "memprof")]
268fn mem_stats_registry(reg: &crate::symbol::SymbolRegistry) {
269    use crate::symbol::{Symbol, SymbolKind};
270    use crate::types::Type;
271    use std::collections::{HashMap, HashSet};
272    let t = std::mem::size_of::<Type>();
273    let mb = |b: usize| b as f64 / 1_048_576.0;
274
275    // Allocated bucket count: hashbrown keeps `capacity()` ≈ 7/8 of a power-of-two
276    // bucket array; an empty map allocates nothing.
277    fn tbl<K, V>(m: &HashMap<K, V>) -> usize {
278        if m.capacity() == 0 {
279            return 0;
280        }
281        let buckets = (m.capacity() * 8 / 7).next_power_of_two();
282        buckets * (std::mem::size_of::<(K, V)>() + 1)
283    }
284    fn keys_cap<V>(m: &HashMap<String, V>) -> usize {
285        m.keys().map(|k| k.capacity()).sum()
286    }
287    let sym_heap = |s: &Symbol| -> usize {
288        s.name.capacity()
289            + match &s.kind {
290                SymbolKind::Script {
291                    trigger,
292                    param_types,
293                    return_types,
294                    ..
295                } => trigger.capacity() + (param_types.capacity() + return_types.capacity()) * t,
296                SymbolKind::Command {
297                    param_types,
298                    return_types,
299                    ..
300                } => (param_types.capacity() + return_types.capacity()) * t,
301                SymbolKind::GameVar { category, .. } => category.capacity(),
302                SymbolKind::Constant { string_value, .. } => {
303                    string_value.as_ref().map_or(0, |s| s.capacity())
304                }
305                _ => 0,
306            }
307    };
308    let sym_map = |m: &HashMap<String, Symbol>| -> usize {
309        tbl(m) + keys_cap(m) + m.values().map(&sym_heap).sum::<usize>()
310    };
311    let i32_map = |m: &HashMap<String, i32>| -> usize { tbl(m) + keys_cap(m) };
312
313    let mut rows: Vec<(&str, usize, usize)> = vec![
314        (
315            "scripts",
316            reg.scripts.len(),
317            tbl(&reg.scripts) + keys_cap(&reg.scripts),
318        ),
319        ("commands", reg.commands.len(), sym_map(&reg.commands)),
320        ("game_vars", reg.game_vars.len(), sym_map(&reg.game_vars)),
321        ("constants", reg.constants.len(), sym_map(&reg.constants)),
322        (
323            "entity_ids",
324            reg.entity_ids.len(),
325            tbl(&reg.entity_ids)
326                + keys_cap(&reg.entity_ids)
327                + reg
328                    .entity_ids
329                    .values()
330                    .map(|e| e.variants.capacity() * std::mem::size_of::<(Type, i32)>())
331                    .sum::<usize>(),
332        ),
333        (
334            "scripts_by_trigger",
335            reg.scripts_by_trigger.len(),
336            tbl(&reg.scripts_by_trigger)
337                + keys_cap(&reg.scripts_by_trigger)
338                + reg
339                    .scripts_by_trigger
340                    .values()
341                    .map(|s| 16 + sym_heap(s.as_ref()))
342                    .sum::<usize>(),
343        ),
344        (
345            "command_param_types",
346            reg.command_param_types.len(),
347            tbl(&reg.command_param_types)
348                + keys_cap(&reg.command_param_types)
349                + reg
350                    .command_param_types
351                    .values()
352                    .map(|v| v.capacity() * t)
353                    .sum::<usize>(),
354        ),
355        ("script_ids", reg.script_ids.len(), i32_map(&reg.script_ids)),
356        (
357            "proc_script_ids",
358            reg.proc_script_ids.len(),
359            i32_map(&reg.proc_script_ids),
360        ),
361        (
362            "label_script_ids",
363            reg.label_script_ids.len(),
364            i32_map(&reg.label_script_ids),
365        ),
366        (
367            "trigger_script_ids",
368            reg.trigger_script_ids.len(),
369            i32_map(&reg.trigger_script_ids),
370        ),
371        (
372            "preloaded_script_ids",
373            reg.preloaded_script_ids.len(),
374            i32_map(&reg.preloaded_script_ids),
375        ),
376        ("components", reg.components.len(), i32_map(&reg.components)),
377        (
378            "interface_ids",
379            reg.interface_ids.len(),
380            i32_map(&reg.interface_ids),
381        ),
382        ("type_chars", reg.type_chars.len(), i32_map(&reg.type_chars)),
383        (
384            "dbcolumn_types",
385            reg.dbcolumn_types.len(),
386            tbl(&reg.dbcolumn_types) + keys_cap(&reg.dbcolumn_types),
387        ),
388    ];
389    let ts_tbl = {
390        let s: &HashSet<String> = &reg.test_scripts;
391        if s.capacity() == 0 {
392            0
393        } else {
394            (s.capacity() * 8 / 7).next_power_of_two() * (std::mem::size_of::<String>() + 1)
395                + s.iter().map(|k| k.capacity()).sum::<usize>()
396        }
397    };
398    rows.push(("test_scripts", reg.test_scripts.len(), ts_tbl));
399
400    rows.sort_by_key(|row| std::cmp::Reverse(row.2));
401    let total: usize = rows.iter().map(|r| r.2).sum();
402    info!(
403        "[mem] registry ~{:.2} MB (sizeof Symbol={}B, SymbolKind={}B, Type={}B):",
404        mb(total),
405        std::mem::size_of::<Symbol>(),
406        std::mem::size_of::<SymbolKind>(),
407        t,
408    );
409    for (name, n, bytes) in &rows {
410        info!(
411            "[mem]   {:<20} {:>6} entries  {:>7.3} MB",
412            name,
413            n,
414            mb(*bytes)
415        );
416    }
417}
418
419/// One-off breakdown of where the compiled-bytecode heap goes, to target
420/// memory work. Built only under `--features memprof`.
421#[cfg(feature = "memprof")]
422fn mem_stats_compiled(scripts: &[CompiledScript]) {
423    use crate::bytecode::{Instruction, Opcode, Operand};
424    let (mut n_instr, mut n_str, mut str_bytes) = (0usize, 0usize, 0usize);
425    let (mut n_switch, mut switch_entries) = (0usize, 0usize);
426    let (mut source_path_bytes, mut name_bytes) = (0usize, 0usize);
427    for s in scripts {
428        n_instr += s.instructions.len();
429        source_path_bytes += s.source_path.capacity();
430        name_bytes += s.name.capacity() + s.trigger.capacity();
431        for st in &s.strings {
432            n_str += 1;
433            str_bytes += st.len();
434        }
435        for sw in &s.switch_tables {
436            n_switch += 1;
437            switch_entries += sw.len();
438        }
439    }
440    let sz = std::mem::size_of::<Instruction>();
441    let mb = |b: usize| b as f64 / 1_048_576.0;
442    info!(
443        "[mem] sizeof Instruction={}B Operand={}B Opcode={}B CompiledScript={}B",
444        sz,
445        std::mem::size_of::<Operand>(),
446        std::mem::size_of::<Opcode>(),
447        std::mem::size_of::<CompiledScript>(),
448    );
449    info!(
450        "[mem] compiled: {} scripts, {} instrs => {:.2} MB instr structs",
451        scripts.len(),
452        n_instr,
453        mb(n_instr * sz)
454    );
455    info!(
456        "[mem]   Str operands: {} ({:.2} MB content + {:.2} MB Box<str> headers)",
457        n_str,
458        mb(str_bytes),
459        mb(n_str * 16)
460    );
461    info!(
462        "[mem]   SwitchTable operands: {} ({} entries => {:.2} MB)",
463        n_switch,
464        switch_entries,
465        mb(switch_entries * 16)
466    );
467    info!(
468        "[mem]   source_path {:.2} MB, names+triggers {:.2} MB",
469        mb(source_path_bytes),
470        mb(name_bytes)
471    );
472}
473
474struct ParsedFile {
475    path: PathBuf,
476    file: ScriptFile,
477    testscript_line: Option<usize>,
478    /// Canonicalized, normalized source path. Computed once during parsing and
479    /// reused as each compiled script's `source_path` (it is byte-identical to
480    /// the `source_cache` key), avoiding a second `canonicalize` syscall per
481    /// file in codegen.
482    source_path: String,
483}
484
485/// Outcome of parsing one file off-thread. The owned source text and cache key
486/// travel back so the (single-threaded) merge can build the `Rc`-based caches
487/// without sending `Rc` across threads.
488struct FileParseResult {
489    path: PathBuf,
490    read: Result<ParsedSource, std::io::Error>,
491}
492
493struct ParsedSource {
494    cache_key: String,
495    raw_source: String,
496    outcome: ParseOutcome,
497}
498
499enum ParseOutcome {
500    Parsed {
501        file: ScriptFile,
502        testscript_line: Option<usize>,
503    },
504    Error {
505        line: usize,
506        position: usize,
507        message: String,
508        phase: diagnostics::Phase,
509    },
510}
511
512/// Read, lex and parse a single file off-thread. Pure: no shared state, so it
513/// is safe to run concurrently. Mirrors the original sequential body exactly —
514/// same source truncation, same cache-key normalization, same lex-then-parse
515/// error reporting — only restructured to return owned data for the merge.
516fn parse_one_file(path: &Path, include_tests: bool) -> FileParseResult {
517    let raw_source = match fs::read_to_string(path) {
518        Ok(s) => s,
519        Err(e) => {
520            return FileParseResult {
521                path: path.to_path_buf(),
522                read: Err(e),
523            };
524        }
525    };
526    let testscript_line = find_testscript_line(&raw_source);
527
528    let source_code = if !include_tests {
529        if let Some(line_no) = testscript_line {
530            truncate_at_line(&raw_source, line_no)
531        } else {
532            raw_source.clone()
533        }
534    } else {
535        strip_testscript_annotation(&raw_source)
536    };
537
538    let canonical = fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf());
539    let mut cache_key = canonical.to_string_lossy().into_owned();
540    if cache_key.starts_with("\\\\?\\") {
541        cache_key = cache_key[4..].to_string();
542    }
543    cache_key = cache_key.replace("\\Content\\", "\\content\\");
544
545    // Lexer/Parser borrow a `&PathBuf` for diagnostics; keep one alive locally.
546    let path_buf = path.to_path_buf();
547    let outcome = match Lexer::new(&source_code, &path_buf).tokenize() {
548        Err(e) => ParseOutcome::Error {
549            line: e.line,
550            position: e.position,
551            message: e.message,
552            phase: diagnostics::Phase::Lexing,
553        },
554        Ok(tokens) => match Parser::new(tokens, &path_buf).parse() {
555            Ok(file) => ParseOutcome::Parsed {
556                file,
557                testscript_line: if include_tests { testscript_line } else { None },
558            },
559            Err(e) => ParseOutcome::Error {
560                line: e.line,
561                position: e.position,
562                message: e.message,
563                phase: diagnostics::Phase::Parsing,
564            },
565        },
566    };
567
568    FileParseResult {
569        path: path_buf,
570        read: Ok(ParsedSource {
571            cache_key,
572            raw_source,
573            outcome,
574        }),
575    }
576}
577
578/// Compile scripts and write output. When `lint` is true, also run lint passes
579/// (unused locals, unreachable code) after code generation.
580pub fn compile_memory(
581    scripts_dir: &Path,
582    pack_dir: Option<&Path>,
583    lint: bool,
584) -> Result<(Vec<u8>, Vec<u8>), Box<dyn Error>> {
585    compile_memory_with_options(scripts_dir, pack_dir, lint, false, false)
586}
587
588/// Like `compile_memory` but includes `#testscript` sections when
589/// `include_tests` is true.
590pub fn compile_memory_with_tests(
591    scripts_dir: &Path,
592    pack_dir: Option<&Path>,
593    lint: bool,
594    include_tests: bool,
595) -> Result<(Vec<u8>, Vec<u8>), Box<dyn Error>> {
596    compile_memory_with_options(scripts_dir, pack_dir, lint, include_tests, false)
597}
598
599/// In-memory compile — returns the `(script.dat, script.idx)` bytes instead of
600/// writing them — choosing the compilation strategy.
601///
602/// With `low_mem`, the recompile pipeline is used: scripts are re-parsed and
603/// re-compiled on demand so the full compiled set is never resident (~16 MB heap
604/// vs ~93), at roughly 6× the time; the returned bytes are byte-identical to the
605/// default path. `include_tests` keeps `#testscript` sections. The
606/// `RUNEC_RECOMPILE` environment variable forces low-memory mode on as well.
607pub fn compile_memory_with_options(
608    scripts_dir: &Path,
609    pack_dir: Option<&Path>,
610    lint: bool,
611    include_tests: bool,
612    low_mem: bool,
613) -> Result<(Vec<u8>, Vec<u8>), Box<dyn Error>> {
614    compile_memory_inner(
615        scripts_dir,
616        pack_dir,
617        lint,
618        include_tests,
619        low_mem,
620        &mut ScriptIds::new(),
621    )
622}
623
624/// Reuses a world's script IDs across reloads, reserving removed scripts' IDs.
625pub fn compile_memory_with_ids(
626    scripts_dir: &Path,
627    pack_dir: Option<&Path>,
628    ids: &mut ScriptIds,
629) -> Result<(Vec<u8>, Vec<u8>), Box<dyn Error>> {
630    compile_memory_inner(scripts_dir, pack_dir, false, false, false, ids)
631}
632
633fn compile_memory_inner(
634    scripts_dir: &Path,
635    pack_dir: Option<&Path>,
636    lint: bool,
637    include_tests: bool,
638    low_mem: bool,
639    ids: &mut ScriptIds,
640) -> Result<(Vec<u8>, Vec<u8>), Box<dyn Error>> {
641    if low_mem || std::env::var("RUNEC_RECOMPILE").is_ok_and(|v| !v.is_empty()) {
642        info!("[recompile] low-memory recompile pipeline enabled");
643        return compile_recompile_memory(scripts_dir, pack_dir, lint, include_tests, ids);
644    }
645
646    let (compiled_scripts, diagnostics) =
647        run_pipeline(scripts_dir, pack_dir, lint, include_tests, ids)?;
648
649    info!("Writing compiled output to memory...");
650    let writer = ScriptWriter::new("".into());
651
652    diagnostics.print_all();
653    info!("Compilation complete!");
654
655    Ok(writer.build_all(&compiled_scripts)?)
656}
657
658/// Compile scripts and write output. When `lint` is true, also run lint passes
659/// (unused locals, unreachable code) after code generation.
660///
661/// Convenience wrapper over [`compile_with_options`] using the default (fast,
662/// fully-resident) pipeline. Kept signature-compatible for existing callers.
663pub fn compile(
664    scripts_dir: &Path,
665    pack_dir: Option<&Path>,
666    output_dir: &Path,
667    lint: bool,
668) -> Result<(), Box<dyn Error>> {
669    compile_with_options(scripts_dir, pack_dir, output_dir, lint, false)
670}
671
672/// Compile scripts and write output, choosing the compilation strategy.
673///
674/// With `low_mem`, the **recompile** pipeline is used: scripts are re-parsed and
675/// re-compiled on demand for the pointer check and written file-by-file, so the
676/// full set of compiled bytecode is never resident at once. That cuts peak usage
677/// to roughly 16 MB heap / 32 MB working set (from ~93 / ~111 MB) at about 6× the
678/// compile time. Output is byte-identical to the default path. The
679/// `RUNEC_RECOMPILE` environment variable forces this mode on as well.
680pub fn compile_with_options(
681    scripts_dir: &Path,
682    pack_dir: Option<&Path>,
683    output_dir: &Path,
684    lint: bool,
685    low_mem: bool,
686) -> Result<(), Box<dyn Error>> {
687    // The recompile pipeline never holds all compiled scripts (it re-compiles on
688    // demand for the pointer check + writes file-by-file). It does its own
689    // output, so it returns here. The default path below is unchanged.
690    if low_mem || std::env::var("RUNEC_RECOMPILE").is_ok_and(|v| !v.is_empty()) {
691        info!("[recompile] low-memory recompile pipeline enabled");
692        return compile_recompile(scripts_dir, pack_dir, output_dir, lint);
693    }
694
695    let (compiled_scripts, diagnostics) =
696        run_pipeline(scripts_dir, pack_dir, lint, false, &mut ScriptIds::new())?;
697
698    let output = output_dir.display().to_string();
699    // Normalize separators for a clean, consistent log message (input paths may
700    // mix `/` and `\`, e.g. `data/pack\server`); the writer keeps the original.
701    info!("Writing output to {}...", output.replace('\\', "/"));
702    let writer = ScriptWriter::new(output);
703    writer.write_all(&compiled_scripts)?;
704
705    diagnostics.print_all();
706    info!("Compilation complete!");
707
708    Ok(())
709}
710
711/// Run all analysis passes (parse, type-check, codegen, pointer-check, lints)
712/// without writing output. Useful for editor tooling and CI lint gates.
713pub fn lint(scripts_dir: &Path, pack_dir: Option<&Path>) -> Result<(), Box<dyn Error>> {
714    let (_, diagnostics) = run_pipeline(scripts_dir, pack_dir, true, false, &mut ScriptIds::new())?;
715
716    diagnostics.print_all();
717    info!("Compilation complete!");
718
719    Ok(())
720}
721
722/// Runs the full compilation pipeline (parse, register, type-check, codegen,
723/// pointer-check, and optional lint passes), returning the compiled scripts
724/// and accumulated diagnostics for the caller to handle output.
725fn run_pipeline(
726    scripts_dir: &Path,
727    pack_dir: Option<&Path>,
728    lint: bool,
729    include_tests: bool,
730    ids: &mut ScriptIds,
731) -> Result<(Vec<CompiledScript>, DiagnosticsCollector), Box<dyn Error>> {
732    let source_dir = sources::target_dir(scripts_dir, sources::Scope::Rune);
733    let scripts_dir = source_dir.as_path();
734    if !scripts_dir.exists() || !scripts_dir.is_dir() {
735        return Err(Box::new(error::CompilerError::FileNotFound(
736            scripts_dir.display().to_string(),
737        )));
738    }
739
740    // Prototype: opt into the streaming pipeline (never holds all ASTs at once)
741    // with RUNEC_STREAM=1. The default batch path below is unchanged.
742    if std::env::var("RUNEC_STREAM").is_ok_and(|v| !v.is_empty()) {
743        info!("[stream] streaming pipeline enabled (RUNEC_STREAM)");
744        return run_pipeline_streaming(scripts_dir, pack_dir, lint, include_tests, ids);
745    }
746
747    let mut rs2_files: Vec<PathBuf> = get_rs2_files(scripts_dir, "rs2")?;
748
749    rs2_files.sort();
750    info!("Found {} script files", rs2_files.len());
751    mem_mark("startup");
752
753    let mut diagnostics = DiagnosticsCollector::new();
754
755    // Phase 1: Parse all files. Reading, lexing and parsing are independent
756    // per file, so fan them out across threads; the results are merged back in
757    // input order so registration/ID assignment stays deterministic.
758    info!("Parsing script files into syntax trees...");
759    let parse_results = parallel_map(&rs2_files, |path| parse_one_file(path, include_tests));
760
761    let mut all_files: Vec<ParsedFile> = Vec::new();
762    let mut source_cache: HashMap<String, Arc<String>> = HashMap::new();
763    for result in parse_results {
764        let parsed = match result.read {
765            Ok(p) => p,
766            // A read failure aborts the whole compile, exactly as the previous
767            // `fs::read_to_string(path)?` did.
768            Err(e) => return Err(Box::new(e)),
769        };
770        let rc = Arc::new(parsed.raw_source);
771        // The cache key is the canonicalized source path; reuse it as each
772        // compiled script's `source_path` so codegen needn't canonicalize again.
773        let source_path = parsed.cache_key.clone();
774        source_cache.insert(parsed.cache_key, rc.clone());
775        match parsed.outcome {
776            ParseOutcome::Parsed {
777                file,
778                testscript_line,
779            } => all_files.push(ParsedFile {
780                path: result.path,
781                file,
782                testscript_line,
783                source_path,
784            }),
785            ParseOutcome::Error {
786                line,
787                position,
788                message,
789                phase,
790            } => diagnostics.error(result.path, line, position, message, phase),
791        }
792    }
793
794    if diagnostics.has_errors() {
795        diagnostics.print_all();
796        return Err(Box::new(error::CompilerError::TypeError(
797            "Parsing failed".to_string(),
798        )));
799    }
800
801    mem_mark("after parse");
802
803    // Phase 2: assign IDs and register every parsed script.
804    info!("Registering scripts and building the symbol registry...");
805    let resolved_pack_dir = resolve_pack_dir(scripts_dir, pack_dir);
806    let mut registry = build_symbol_registry(scripts_dir, resolved_pack_dir.as_deref())?;
807    register_scripts(&mut registry, ids, &mut diagnostics, &all_files);
808    info!("  Registered {} scripts", registry.scripts.len());
809    // Pre-assigned script ids were consumed during registration; free them
810    // before codegen (the compile's peak-memory phase) rather than at scope end.
811    registry.drop_registration_scratch();
812
813    for pf in &all_files {
814        for script in &pf.file.scripts {
815            if let Some(msg) =
816                Compiler::validate_trigger_subject(&script.trigger, &script.name, &registry)
817            {
818                diagnostics.warning(
819                    pf.path.clone(),
820                    script.line,
821                    0,
822                    msg,
823                    diagnostics::Phase::SymbolRegistration,
824                );
825            }
826        }
827    }
828
829    mem_mark("after register");
830    #[cfg(feature = "memprof")]
831    mem_stats_registry(&registry);
832
833    // Phase 3: Type checking
834    info!("Type checking {} files...", all_files.len());
835    run_type_checker(&mut diagnostics, &all_files, &registry);
836
837    if diagnostics.has_errors() {
838        diagnostics.print_all();
839        return Err(Box::new(error::CompilerError::TypeError(
840            "Type checking failed".to_string(),
841        )));
842    }
843
844    mem_mark("after typecheck");
845
846    // Phase 4: Code generation
847    info!("Generating bytecode from type-checked scripts...");
848    let mut compiled_scripts = codegen(&all_files, &registry, &mut diagnostics);
849    if diagnostics.has_errors() {
850        diagnostics.print_all();
851        return Err(Box::new(error::CompilerError::CodeGenError(
852            "Code generation failed".into(),
853        )));
854    }
855    info!("  Generated {} compiled scripts", compiled_scripts.len());
856    compiled_scripts.sort_by_key(|s| s.id);
857    compiled_scripts.shrink_to_fit();
858    mem_mark("after codegen");
859    #[cfg(feature = "memprof")]
860    mem_stats_compiled(&compiled_scripts);
861
862    // The ASTs are consumed by codegen and never read again — pointer checking
863    // and lints work purely off `compiled_scripts` + the registry + the source
864    // cache. Free the syntax trees now so they don't coexist with the pointer
865    // checker's (per-thread) scratch, which is the compile's peak-memory moment.
866    drop(all_files);
867    mem_mark("after drop asts");
868
869    // Phase 4.5: Pointer checking & lints
870    info!("Checking pointers in generated code...");
871    check_pointers(
872        lint,
873        &mut diagnostics,
874        SourceProvider::Eager(&source_cache),
875        &registry,
876        &compiled_scripts,
877    );
878    mem_mark("after pointercheck");
879
880    Ok((compiled_scripts, diagnostics))
881}
882
883// ── Streaming pipeline (prototype, gated by RUNEC_STREAM) ────────────────────
884// Goal: never materialize all ASTs (~59 MB) at once. Pass 1 parses in parallel
885// but keeps only per-script signatures, dropping each AST in-worker; the registry
886// is built from those. Pass 2 re-parses each file in parallel, type-checks and
887// compiles it, then drops the AST immediately — so syntax trees never accumulate.
888// Output is byte-identical to the batch path (same registration, same per-file
889// codegen, same stable id-sort). It still holds all compiled scripts (for the
890// cross-script pointer check + writer); shrinking that is the next increment.
891
892/// One script's registration-relevant signature — everything the registry and
893/// trigger diagnostics need — so the full AST can be dropped right after parsing.
894struct ScriptSig {
895    name: String,
896    trigger: String,
897    param_types: Vec<types::Type>,
898    return_types: Vec<types::Type>,
899    line: usize,
900}
901
902struct FileSigs {
903    path: PathBuf,
904    testscript_line: Option<usize>,
905    sigs: Vec<ScriptSig>,
906}
907
908enum SigOutcome {
909    Parsed {
910        sigs: Vec<ScriptSig>,
911        testscript_line: Option<usize>,
912    },
913    Error {
914        line: usize,
915        position: usize,
916        message: String,
917        phase: diagnostics::Phase,
918    },
919}
920
921struct FileSigsResult {
922    path: PathBuf,
923    read: Result<(String, SigOutcome), std::io::Error>,
924}
925
926/// Parse one file, extract signatures, and DROP the AST in-worker, so a parallel
927/// sweep never holds every syntax tree at once.
928fn parse_one_file_sigs(path: &Path, include_tests: bool) -> FileSigsResult {
929    let r = parse_one_file(path, include_tests);
930    let read = r.read.map(|ps| {
931        let outcome = match ps.outcome {
932            ParseOutcome::Parsed {
933                file,
934                testscript_line,
935            } => {
936                let sigs = file
937                    .scripts
938                    .iter()
939                    .map(|d| ScriptSig {
940                        name: d.name.clone(),
941                        trigger: d.trigger.clone(),
942                        param_types: d.params.iter().map(|p| p.param_type).collect(),
943                        return_types: d.return_types.clone(),
944                        line: d.line,
945                    })
946                    .collect();
947                SigOutcome::Parsed {
948                    sigs,
949                    testscript_line,
950                }
951                // `file` (the AST) is dropped here — only the sigs travel back.
952            }
953            ParseOutcome::Error {
954                line,
955                position,
956                message,
957                phase,
958            } => SigOutcome::Error {
959                line,
960                position,
961                message,
962                phase,
963            },
964        };
965        (ps.cache_key, outcome)
966    });
967    FileSigsResult { path: r.path, read }
968}
969
970fn run_pipeline_streaming(
971    scripts_dir: &Path,
972    pack_dir: Option<&Path>,
973    lint: bool,
974    include_tests: bool,
975    ids: &mut ScriptIds,
976) -> Result<(Vec<CompiledScript>, DiagnosticsCollector), Box<dyn Error>> {
977    let mut rs2_files: Vec<PathBuf> = get_rs2_files(scripts_dir, "rs2")?;
978    rs2_files.sort();
979    info!("Found {} script files", rs2_files.len());
980    mem_mark("startup");
981
982    let mut diagnostics = DiagnosticsCollector::new();
983
984    // Pass 1: parse for signatures only (each AST dropped in its worker).
985    info!("[stream] Pass 1: parsing for signatures...");
986    let sig_results = parallel_map(&rs2_files, |path| parse_one_file_sigs(path, include_tests));
987
988    // Streaming keeps only `cache_key -> on-disk path`; source text is re-read
989    // lazily for the rare diagnostic that needs it, so the ~4 MB of sources isn't
990    // resident through the peak.
991    let mut source_paths: HashMap<String, PathBuf> = HashMap::new();
992    let mut file_sigs: Vec<FileSigs> = Vec::new();
993    for r in sig_results {
994        let (cache_key, outcome) = match r.read {
995            Ok(t) => t,
996            Err(e) => return Err(Box::new(e)),
997        };
998        source_paths.insert(cache_key, r.path.clone());
999        match outcome {
1000            SigOutcome::Parsed {
1001                sigs,
1002                testscript_line,
1003            } => file_sigs.push(FileSigs {
1004                path: r.path,
1005                testscript_line,
1006                sigs,
1007            }),
1008            SigOutcome::Error {
1009                line,
1010                position,
1011                message,
1012                phase,
1013            } => diagnostics.error(r.path, line, position, message, phase),
1014        }
1015    }
1016
1017    if diagnostics.has_errors() {
1018        diagnostics.print_all();
1019        return Err(Box::new(error::CompilerError::TypeError(
1020            "Parsing failed".to_string(),
1021        )));
1022    }
1023    mem_mark("after parse");
1024
1025    // Registration (from signatures, not ASTs).
1026    info!("[stream] Registering scripts and building the symbol registry...");
1027    let resolved_pack_dir = resolve_pack_dir(scripts_dir, pack_dir);
1028    let mut registry = build_symbol_registry(scripts_dir, resolved_pack_dir.as_deref())?;
1029    register_scripts_from_sigs(&mut registry, ids, &mut diagnostics, &file_sigs);
1030    info!("  Registered {} scripts", registry.scripts.len());
1031    registry.drop_registration_scratch();
1032
1033    for fs in &file_sigs {
1034        for sig in &fs.sigs {
1035            if let Some(msg) =
1036                Compiler::validate_trigger_subject(&sig.trigger, &sig.name, &registry)
1037            {
1038                diagnostics.warning(
1039                    fs.path.clone(),
1040                    sig.line,
1041                    0,
1042                    msg,
1043                    diagnostics::Phase::SymbolRegistration,
1044                );
1045            }
1046        }
1047    }
1048    // Signatures are done; only the registry + source cache carry forward.
1049    drop(file_sigs);
1050    mem_mark("after register");
1051    #[cfg(feature = "memprof")]
1052    mem_stats_registry(&registry);
1053
1054    // Pass 2: re-parse + type-check + compile per file, streaming. Parse can't
1055    // fail here (pass 1 validated), so a failed re-parse just yields nothing.
1056    info!("[stream] Pass 2: type-check + codegen (streaming)...");
1057    let per_file = parallel_map(&rs2_files, |path| {
1058        let parsed = match parse_one_file(path, include_tests).read {
1059            Ok(p) => p,
1060            Err(_) => return (Vec::new(), DiagnosticsCollector::new()),
1061        };
1062        let (file, testscript_line) = match parsed.outcome {
1063            ParseOutcome::Parsed {
1064                file,
1065                testscript_line,
1066            } => (file, testscript_line),
1067            ParseOutcome::Error { .. } => return (Vec::new(), DiagnosticsCollector::new()),
1068        };
1069        let mut type_checker = TypeChecker::new(&registry);
1070        type_checker.check_file_with_test_boundary(&file, path, testscript_line);
1071        let mut compiled = Vec::new();
1072        // Compile only when this file type-checks clean (codegen assumes
1073        // well-typed input). On any global error the whole compile aborts below,
1074        // so output produced here would be discarded anyway.
1075        if !type_checker.diagnostics.has_errors() {
1076            let mut compiler = Compiler::new(&registry);
1077            for script in &file.scripts {
1078                if script.trigger == "command" {
1079                    continue;
1080                }
1081                let Some(mut c) =
1082                    compiler.compile_checked(script, path, &mut type_checker.diagnostics)
1083                else {
1084                    continue;
1085                };
1086                c.source_path = parsed.cache_key.clone();
1087                c.shrink_to_fit();
1088                compiled.push(c);
1089            }
1090        }
1091        (compiled, type_checker.diagnostics)
1092        // `file` (the AST) is dropped here.
1093    });
1094
1095    let mut compiled_scripts: Vec<CompiledScript> = Vec::new();
1096    for (compiled, diags) in per_file {
1097        compiled_scripts.extend(compiled);
1098        diagnostics.merge(diags);
1099    }
1100    if diagnostics.has_errors() {
1101        diagnostics.print_all();
1102        return Err(Box::new(error::CompilerError::TypeError(
1103            "Type checking failed".to_string(),
1104        )));
1105    }
1106    // Stable sort by id matches the batch path (file order preserved among
1107    // equal ids → identical writer last-write-wins behaviour).
1108    compiled_scripts.sort_by_key(|s| s.id);
1109    compiled_scripts.shrink_to_fit();
1110    info!("  Generated {} compiled scripts", compiled_scripts.len());
1111    mem_mark("after codegen");
1112    #[cfg(feature = "memprof")]
1113    mem_stats_compiled(&compiled_scripts);
1114
1115    check_pointers(
1116        lint,
1117        &mut diagnostics,
1118        SourceProvider::Lazy(&source_paths),
1119        &registry,
1120        &compiled_scripts,
1121    );
1122    mem_mark("after pointercheck");
1123
1124    Ok((compiled_scripts, diagnostics))
1125}
1126
1127fn assign_script_ids<'a>(
1128    registry: &mut SymbolRegistry,
1129    ids: &mut ScriptIds,
1130    scripts: impl Iterator<Item = (&'a str, &'a str)>,
1131) {
1132    let mut next = ids.values().max().map_or(0, |id| id + 1);
1133    let mut scripts: Vec<_> = scripts
1134        .filter(|(trigger, _)| *trigger != "command")
1135        .collect();
1136    scripts.sort_unstable();
1137    scripts.dedup();
1138    for (trigger, name) in scripts {
1139        let id = *ids.entry(format!("{trigger}:{name}")).or_insert_with(|| {
1140            let id = next;
1141            next += 1;
1142            id
1143        });
1144        registry.set_script_id(trigger, name, id);
1145    }
1146}
1147
1148/// `register_scripts` for the streaming path: identical logic, driven by the
1149/// extracted signatures instead of the (already-dropped) ASTs.
1150fn register_scripts_from_sigs(
1151    registry: &mut SymbolRegistry,
1152    ids: &mut ScriptIds,
1153    diagnostics: &mut DiagnosticsCollector,
1154    file_sigs: &[FileSigs],
1155) {
1156    assign_script_ids(
1157        registry,
1158        ids,
1159        file_sigs.iter().flat_map(|file| {
1160            file.sigs
1161                .iter()
1162                .map(|sig| (sig.trigger.as_str(), sig.name.as_str()))
1163        }),
1164    );
1165
1166    use crate::diagnostic_messages as msg;
1167    let mut registered_scripts: std::collections::HashSet<String> =
1168        std::collections::HashSet::new();
1169
1170    for fs in file_sigs.iter() {
1171        for sig in &fs.sigs {
1172            Arc::make_mut(&mut registry.visibility).declare_script(
1173                &fs.path,
1174                &sig.trigger,
1175                &sig.name,
1176            );
1177            if sig.trigger == "command" {
1178                continue;
1179            }
1180
1181            if !trigger_table::is_valid_trigger(&sig.trigger) {
1182                diagnostics.warning(
1183                    fs.path.clone(),
1184                    sig.line,
1185                    0,
1186                    msg::fmt(msg::SCRIPT_TRIGGER_INVALID, &[&sig.trigger]),
1187                    crate::diagnostics::Phase::SymbolRegistration,
1188                );
1189            }
1190
1191            let key = format!("{}:{}", sig.trigger, sig.name);
1192            if !registered_scripts.insert(key) {
1193                diagnostics.warning(
1194                    fs.path.clone(),
1195                    sig.line,
1196                    0,
1197                    msg::fmt(msg::SCRIPT_REDECLARATION, &[&sig.trigger, &sig.name]),
1198                    crate::diagnostics::Phase::SymbolRegistration,
1199                );
1200            }
1201
1202            if !sig.return_types.is_empty() && !trigger_table::allows_returns(&sig.trigger) {
1203                diagnostics.warning(
1204                    fs.path.clone(),
1205                    sig.line,
1206                    0,
1207                    msg::fmt(msg::SCRIPT_TRIGGER_NO_RETURNS, &[&sig.trigger]),
1208                    diagnostics::Phase::SymbolRegistration,
1209                );
1210            }
1211
1212            registry.register_script(
1213                sig.name.clone(),
1214                sig.trigger.clone(),
1215                sig.param_types.clone(),
1216                sig.return_types.clone(),
1217            );
1218
1219            if let Some(ts_line) = fs.testscript_line
1220                && sig.line >= ts_line
1221            {
1222                registry.mark_test_script(&sig.trigger, &sig.name);
1223            }
1224        }
1225    }
1226}
1227
1228fn check_pointers(
1229    lint: bool,
1230    diagnostics: &mut DiagnosticsCollector,
1231    source: SourceProvider,
1232    registry: &SymbolRegistry,
1233    compiled_scripts: &[CompiledScript],
1234) {
1235    {
1236        use crate::pointer_checker::{PointerChecker, ScriptSource};
1237        // Validate scripts across threads: each chunk gets its own checker that
1238        // shares the read-only scripts/registry/source cache but keeps its own
1239        // (purely memoizing) caches. Diagnostics are merged in script order, so
1240        // the result is identical to a single-threaded run.
1241        //
1242        // The read-only command/script lookup tables are built ONCE here and
1243        // shared (`Arc`) across every chunk checker, instead of each thread
1244        // rebuilding the full ~700-command pointer table + per-script reverse
1245        // maps. `names` is just the script order, so derive it directly rather
1246        // than spinning up a throwaway checker for `script_names()`.
1247        let shared = PointerChecker::build_shared(compiled_scripts, registry, false);
1248        let names: Vec<String> = compiled_scripts.iter().map(|s| s.name.clone()).collect();
1249        // Each chunk's checker retains its CFG caches until the chunk finishes,
1250        // so concurrency multiplies peak scratch. RUNEC_PTR_CHUNKS caps the
1251        // concurrent chunk count to trade pointer-check speed for memory; the
1252        // result is chunk-count-independent, so diagnostics stay byte-identical.
1253        let max_chunks = std::env::var("RUNEC_PTR_CHUNKS")
1254            .ok()
1255            .and_then(|s| s.parse::<usize>().ok())
1256            .filter(|&n| n > 0)
1257            .unwrap_or(usize::MAX);
1258        let chunk_diags = parallel_chunks_capped(&names, max_chunks, |chunk| {
1259            let mut checker = PointerChecker::with_shared(
1260                ScriptSource::Resident(compiled_scripts),
1261                registry,
1262                shared.clone(),
1263            );
1264            checker.set_source_cache(source);
1265            checker.validate_names(chunk)
1266        });
1267        let mut pointer_diags = DiagnosticsCollector::new();
1268        for d in chunk_diags {
1269            pointer_diags.merge(d);
1270        }
1271        let ptr_warnings = pointer_diags.warning_count();
1272        if ptr_warnings > 0 {
1273            info!("  Pointer check: {} warning(s)", ptr_warnings);
1274        }
1275        diagnostics.merge(pointer_diags);
1276    }
1277    #[cfg(feature = "memprof")]
1278    crate::pointer_checker::cfg_trace_report();
1279
1280    // Phase 4.6: Lint passes (optional)
1281    if lint {
1282        info!("Running lint passes (unused locals, unreachable code)...");
1283        let lint_diags = lints::run_lints(compiled_scripts, Some(source));
1284        let lint_warnings = lint_diags.warning_count();
1285        if lint_warnings > 0 {
1286            info!("  Lints: {} warning(s)", lint_warnings);
1287        }
1288        diagnostics.merge(lint_diags);
1289    }
1290}
1291
1292fn codegen(
1293    all_files: &[ParsedFile],
1294    registry: &SymbolRegistry,
1295    diagnostics: &mut DiagnosticsCollector,
1296) -> Vec<CompiledScript> {
1297    // Each file compiles independently against the read-only registry, so fan
1298    // the files out across threads. Output order doesn't matter — the caller
1299    // sorts compiled scripts by id — so a simple flatten of the per-file results
1300    // is enough.
1301    let mut compiled = Vec::new();
1302    for (scripts, errors) in parallel_map(all_files, |pf| compile_file(pf, registry)) {
1303        compiled.extend(scripts);
1304        diagnostics.merge(errors);
1305    }
1306    compiled
1307}
1308
1309/// Compile every (non-`command`) script in one file. Creating a `Compiler` per
1310/// file is cheap now that it only borrows the registry, and keeps the codegen
1311/// scratch state thread-local.
1312fn compile_file(
1313    pf: &ParsedFile,
1314    registry: &SymbolRegistry,
1315) -> (Vec<CompiledScript>, DiagnosticsCollector) {
1316    let mut diagnostics = DiagnosticsCollector::new();
1317    let mut scripts =
1318        crate::compiler::compile_file(&pf.file, &pf.path, registry, false, &mut diagnostics);
1319    for script in &mut scripts {
1320        script.source_path = pf.source_path.clone();
1321        script.shrink_to_fit();
1322    }
1323    (scripts, diagnostics)
1324}
1325
1326fn run_type_checker(
1327    diagnostics: &mut DiagnosticsCollector,
1328    all_files: &[ParsedFile],
1329    registry: &SymbolRegistry,
1330) {
1331    // Each file type-checks independently against the read-only registry, so
1332    // check chunks of files in parallel; per-chunk diagnostics are merged in
1333    // file order, matching a single-threaded run.
1334    let chunk_diags = parallel_chunks(all_files, |chunk| {
1335        let mut type_checker = TypeChecker::new(registry);
1336        for pf in chunk {
1337            type_checker.check_file_with_test_boundary(&pf.file, &pf.path, pf.testscript_line);
1338        }
1339        type_checker.diagnostics
1340    });
1341    for d in chunk_diags {
1342        diagnostics.merge(d);
1343    }
1344}
1345
1346/// Resolve the pack directory: explicit `--pack`, else the nearest ancestor's `pack/`, else
1347/// `content/pack` up the tree.
1348fn resolve_pack_dir(scripts_dir: &Path, pack_dir: Option<&Path>) -> Option<PathBuf> {
1349    pack_dir
1350        .map(PathBuf::from)
1351        .filter(|p| p.exists())
1352        .or_else(|| {
1353            scripts_dir.ancestors().find_map(|dir| {
1354                let pack = dir.join("pack");
1355                pack.is_dir().then_some(pack)
1356            })
1357        })
1358        .or_else(|| {
1359            scripts_dir
1360                .ancestors()
1361                .nth(3)
1362                .map(|p| p.join("content/pack"))
1363                .filter(|p| p.exists())
1364        })
1365}
1366
1367/// Build the source-independent registry before registering parsed scripts.
1368fn build_symbol_registry(
1369    scripts_dir: &Path,
1370    resolved_pack_dir: Option<&Path>,
1371) -> Result<SymbolRegistry, Box<dyn Error>> {
1372    let config_files = sources::target_config_files(scripts_dir, Scope::Rune)?;
1373    let engine_rs2 = sources::engine_file(scripts_dir, Scope::Rune)?;
1374    // Visibility reads every source and config, and only its indexing needs
1375    // the registry built meanwhile.
1376    let visibility = {
1377        let dir = scripts_dir.to_path_buf();
1378        background(move || sources::Visibility::scan(&dir))
1379    };
1380    let mut registry = SymbolRegistry::new();
1381
1382    if let Some(pd) = resolved_pack_dir {
1383        info!("  Loading packs from {}", pd.display());
1384        symloader::load_packs(&mut registry, pd);
1385        info!(
1386            "  Loaded {} commands, {} game vars",
1387            registry.commands.len(),
1388            registry.game_vars.len(),
1389        );
1390    } else {
1391        info!("  No pack directory found - symbol resolution will be limited");
1392    }
1393
1394    // Walk the scripts tree once and let the constant/game-var/dbtable loaders
1395    // filter this shared list by extension, instead of each re-walking the tree
1396    // (previously 6 separate recursive traversals over the same directories).
1397    symloader::load_constant_files(&mut registry, &config_files);
1398
1399    info!(
1400        "  Loaded {} constants from .constant files",
1401        registry.constants.len()
1402    );
1403
1404    symloader::load_game_var_types(&mut registry, &config_files);
1405
1406    if let Some(pd) = resolved_pack_dir
1407        && let Err(error) = clientscript::load_clan_settings(&mut registry, &config_files, pd)
1408    {
1409        warn!("  Clan settings not loaded: {error}");
1410    }
1411
1412    if let Some(pd) = resolved_pack_dir {
1413        let dbtable_ids = symloader::load_dbtable_pack(&pd.join("dbtable.pack"));
1414        if !dbtable_ids.is_empty() {
1415            symloader::load_dbtable_configs(&mut registry, &config_files, &dbtable_ids);
1416            info!(
1417                "  Registered {} dbcolumn compound IDs",
1418                registry.dbcolumn_types.len()
1419            );
1420        }
1421    }
1422
1423    if let Some(engine_rs2) = engine_rs2 {
1424        symloader::load_engine_command_params(&mut registry, &engine_rs2);
1425    }
1426    symloader::load_clientscript_signatures(&mut registry, scripts_dir);
1427
1428    symloader::patch_command_return_types(&mut registry);
1429
1430    let mut visibility = visibility
1431        .join()
1432        .unwrap_or_else(|panic| panic::resume_unwind(panic));
1433    visibility.index_registry(&registry);
1434    registry.visibility = Arc::new(visibility);
1435
1436    Ok(registry)
1437}
1438
1439/// Assign IDs and register the parsed scripts, reporting declaration errors.
1440fn register_scripts(
1441    registry: &mut SymbolRegistry,
1442    ids: &mut ScriptIds,
1443    diagnostics: &mut DiagnosticsCollector,
1444    all_files: &[ParsedFile],
1445) {
1446    assign_script_ids(
1447        registry,
1448        ids,
1449        all_files.iter().flat_map(|file| {
1450            file.file
1451                .scripts
1452                .iter()
1453                .map(|script| (script.trigger.as_str(), script.name.as_str()))
1454        }),
1455    );
1456
1457    {
1458        use crate::diagnostic_messages as msg;
1459
1460        let mut registered_scripts: std::collections::HashSet<String> =
1461            std::collections::HashSet::new();
1462
1463        for pf in all_files.iter() {
1464            Arc::make_mut(&mut registry.visibility).declare_source(&pf.path, &pf.file);
1465            for script in &pf.file.scripts {
1466                if script.trigger == "command" {
1467                    continue;
1468                }
1469
1470                if !trigger_table::is_valid_trigger(&script.trigger) {
1471                    diagnostics.warning(
1472                        pf.path.clone(),
1473                        script.line,
1474                        0,
1475                        msg::fmt(msg::SCRIPT_TRIGGER_INVALID, &[&script.trigger]),
1476                        crate::diagnostics::Phase::SymbolRegistration,
1477                    );
1478                }
1479
1480                let key = format!("{}:{}", script.trigger, script.name);
1481                if !registered_scripts.insert(key.clone()) {
1482                    diagnostics.warning(
1483                        pf.path.clone(),
1484                        script.line,
1485                        0,
1486                        msg::fmt(msg::SCRIPT_REDECLARATION, &[&script.trigger, &script.name]),
1487                        crate::diagnostics::Phase::SymbolRegistration,
1488                    );
1489                }
1490
1491                if !script.return_types.is_empty()
1492                    && !trigger_table::allows_returns(&script.trigger)
1493                {
1494                    diagnostics.warning(
1495                        pf.path.clone(),
1496                        script.line,
1497                        0,
1498                        msg::fmt(msg::SCRIPT_TRIGGER_NO_RETURNS, &[&script.trigger]),
1499                        diagnostics::Phase::SymbolRegistration,
1500                    );
1501                }
1502
1503                let param_types: Vec<types::Type> =
1504                    script.params.iter().map(|p| p.param_type).collect();
1505                registry.register_script(
1506                    script.name.clone(),
1507                    script.trigger.clone(),
1508                    param_types,
1509                    script.return_types.clone(),
1510                );
1511
1512                if let Some(ts_line) = pf.testscript_line
1513                    && script.line >= ts_line
1514                {
1515                    registry.mark_test_script(&script.trigger, &script.name);
1516                }
1517            }
1518        }
1519    }
1520}
1521
1522// ── Streaming recompile pipeline (prototype, gated by RUNEC_RECOMPILE) ────────
1523// Never holds all compiled scripts. Pass 2 only type-checks; the pointer check
1524// re-compiles each script on demand (bounded cache — measured ~1.3x re-parse on
1525// rs-majula, the call graph has high file locality); output is produced by
1526// re-compiling file-by-file into a by-id buffer. Output + diagnostics are
1527// byte-identical to the batch path (deterministic re-compile, same id order).
1528
1529/// Re-compiles scripts on demand for the pointer checker, caching `Rc`s with a
1530/// bounded FIFO so the full compiled set is never resident. A cache miss
1531/// re-parses the script's file and compiles all its scripts at once (siblings
1532/// share the parse), keying each by its full `[trigger,name]` to its index.
1533struct RecompileStore<'a> {
1534    idx_to_path: Vec<PathBuf>,
1535    /// file path -> its script idxs in declaration order (command-skipped).
1536    /// Position-based so duplicate `[trigger,name]` declarations each map to the
1537    /// correct distinct idx.
1538    file_to_idxs: HashMap<PathBuf, Vec<usize>>,
1539    registry: &'a SymbolRegistry,
1540    include_tests: bool,
1541    cache: HashMap<usize, std::rc::Rc<CompiledScript>>,
1542    order: std::collections::VecDeque<usize>,
1543    cap: usize,
1544}
1545
1546impl RecompileStore<'_> {
1547    fn get(&mut self, idx: usize) -> std::rc::Rc<CompiledScript> {
1548        if let Some(rc) = self.cache.get(&idx) {
1549            return std::rc::Rc::clone(rc);
1550        }
1551        let path = self.idx_to_path[idx].clone();
1552        let parsed = parse_one_file(&path, self.include_tests);
1553        if let Ok(ps) = parsed.read
1554            && let ParseOutcome::Parsed { file, .. } = ps.outcome
1555        {
1556            // The j-th non-command declaration in this file maps to idxs[j],
1557            // matching how the metas were built (same parse, same skip rule).
1558            let idxs = self.file_to_idxs.get(&path).cloned().unwrap_or_default();
1559            let mut compiler = Compiler::new(self.registry);
1560            let mut j = 0usize;
1561            for script in &file.scripts {
1562                if script.trigger == "command" {
1563                    continue;
1564                }
1565                if let Some(&i) = idxs.get(j) {
1566                    let mut c = compiler.compile_script(script);
1567                    c.source_path = ps.cache_key.clone();
1568                    self.insert(i, std::rc::Rc::new(c));
1569                }
1570                j += 1;
1571            }
1572        }
1573        std::rc::Rc::clone(
1574            self.cache
1575                .get(&idx)
1576                .expect("recompile store: script not produced by its file"),
1577        )
1578    }
1579
1580    fn insert(&mut self, idx: usize, rc: std::rc::Rc<CompiledScript>) {
1581        if self.cache.insert(idx, rc).is_none() {
1582            self.order.push_back(idx);
1583            while self.order.len() > self.cap {
1584                if let Some(old) = self.order.pop_front()
1585                    && old != idx
1586                {
1587                    self.cache.remove(&old);
1588                }
1589            }
1590        }
1591    }
1592}
1593
1594/// Per-slot encoded output blobs (indexed by script id) plus accumulated
1595/// diagnostics — what the recompile core hands back before the bytes are written
1596/// to disk or assembled in memory.
1597type RecompiledOutput = (Vec<Vec<u8>>, DiagnosticsCollector);
1598
1599/// Core of the recompile (low-memory) pipeline: runs every phase re-compiling
1600/// scripts on demand — never holding all compiled bytecode at once — and returns
1601/// the per-slot encoded output blobs plus diagnostics, without writing or
1602/// printing. The disk and in-memory entry points (`compile_recompile`,
1603/// `compile_recompile_memory`) wrap this. On a phase error it prints diagnostics
1604/// and returns `Err`, matching the batch path.
1605fn recompile_to_encoded(
1606    scripts_dir: &Path,
1607    pack_dir: Option<&Path>,
1608    lint: bool,
1609    include_tests: bool,
1610    ids: &mut ScriptIds,
1611) -> Result<RecompiledOutput, Box<dyn Error>> {
1612    let source_dir = sources::target_dir(scripts_dir, sources::Scope::Rune);
1613    let scripts_dir = source_dir.as_path();
1614    use crate::pointer_checker::{PointerChecker, ScriptSource};
1615
1616    let mut rs2_files: Vec<PathBuf> = get_rs2_files(scripts_dir, "rs2")?;
1617    rs2_files.sort();
1618    info!("Found {} script files", rs2_files.len());
1619    mem_mark("startup");
1620    let mut diagnostics = DiagnosticsCollector::new();
1621
1622    // Pass 1: signatures only (ASTs dropped in-worker); keep paths for lazy source reads.
1623    info!("[recompile] Pass 1: parsing for signatures...");
1624    let sig_results = parallel_map(&rs2_files, |path| parse_one_file_sigs(path, include_tests));
1625    let mut source_paths: HashMap<String, PathBuf> = HashMap::new();
1626    let mut file_sigs: Vec<FileSigs> = Vec::new();
1627    for r in sig_results {
1628        let (cache_key, outcome) = match r.read {
1629            Ok(t) => t,
1630            Err(e) => return Err(Box::new(e)),
1631        };
1632        source_paths.insert(cache_key, r.path.clone());
1633        match outcome {
1634            SigOutcome::Parsed {
1635                sigs,
1636                testscript_line,
1637            } => file_sigs.push(FileSigs {
1638                path: r.path,
1639                testscript_line,
1640                sigs,
1641            }),
1642            SigOutcome::Error {
1643                line,
1644                position,
1645                message,
1646                phase,
1647            } => diagnostics.error(r.path, line, position, message, phase),
1648        }
1649    }
1650    if diagnostics.has_errors() {
1651        diagnostics.print_all();
1652        return Err(Box::new(error::CompilerError::TypeError(
1653            "Parsing failed".to_string(),
1654        )));
1655    }
1656    mem_mark("after parse");
1657
1658    // Registration.
1659    info!("[recompile] Registering scripts and building the symbol registry...");
1660    let resolved_pack_dir = resolve_pack_dir(scripts_dir, pack_dir);
1661    let mut registry = build_symbol_registry(scripts_dir, resolved_pack_dir.as_deref())?;
1662    register_scripts_from_sigs(&mut registry, ids, &mut diagnostics, &file_sigs);
1663    info!("  Registered {} scripts", registry.scripts.len());
1664    registry.drop_registration_scratch();
1665    for fs in &file_sigs {
1666        for sig in &fs.sigs {
1667            if let Some(msg) =
1668                Compiler::validate_trigger_subject(&sig.trigger, &sig.name, &registry)
1669            {
1670                diagnostics.warning(
1671                    fs.path.clone(),
1672                    sig.line,
1673                    0,
1674                    msg,
1675                    diagnostics::Phase::SymbolRegistration,
1676                );
1677            }
1678        }
1679    }
1680    mem_mark("after register");
1681
1682    // Pass 2: type-check only (re-parse, no compiled scripts retained).
1683    info!("[recompile] Pass 2: type-check (streaming, no bytecode held)...");
1684    let tc_diags = parallel_map(&rs2_files, |path| {
1685        let parsed = match parse_one_file(path, include_tests).read {
1686            Ok(p) => p,
1687            Err(_) => return DiagnosticsCollector::new(),
1688        };
1689        let (file, testscript_line) = match parsed.outcome {
1690            ParseOutcome::Parsed {
1691                file,
1692                testscript_line,
1693            } => (file, testscript_line),
1694            ParseOutcome::Error { .. } => return DiagnosticsCollector::new(),
1695        };
1696        let mut type_checker = TypeChecker::new(&registry);
1697        type_checker.check_file_with_test_boundary(&file, path, testscript_line);
1698        if !type_checker.diagnostics.has_errors() {
1699            let mut compiler = Compiler::new(&registry);
1700            for script in &file.scripts {
1701                if script.trigger != "command" {
1702                    compiler.compile_checked(script, path, &mut type_checker.diagnostics);
1703                }
1704            }
1705        }
1706        type_checker.diagnostics
1707    });
1708    for d in tc_diags {
1709        diagnostics.merge(d);
1710    }
1711    if diagnostics.has_errors() {
1712        diagnostics.print_all();
1713        return Err(Box::new(error::CompilerError::TypeError(
1714            "Type checking failed".to_string(),
1715        )));
1716    }
1717    mem_mark("after typecheck");
1718
1719    // Build script metas in the same idx order as the batch path: file order
1720    // (file_sigs / declaration order, command-skipped), then a stable sort by id.
1721    // ids + names come from the registry — no bytecode needed.
1722    let mut pre: Vec<(i32, String, PathBuf)> = Vec::new();
1723    for fs in &file_sigs {
1724        for sig in &fs.sigs {
1725            if sig.trigger == "command" {
1726                continue;
1727            }
1728            let id = registry
1729                .script_id_for_trigger(&sig.trigger, &sig.name)
1730                .or_else(|| registry.script_id(&sig.name))
1731                .unwrap_or(-1);
1732            pre.push((
1733                id,
1734                format!("[{},{}]", sig.trigger, sig.name),
1735                fs.path.clone(),
1736            ));
1737        }
1738    }
1739    let n = pre.len();
1740    // Stable sort indices by id → the batch's idx order (file/decl pre-order
1741    // preserved among equal ids). `inv` maps a pre-order position back to its idx.
1742    let mut order: Vec<usize> = (0..n).collect();
1743    order.sort_by_key(|&p| pre[p].0);
1744    let idents: Vec<(i32, String)> = order
1745        .iter()
1746        .map(|&p| (pre[p].0, pre[p].1.clone()))
1747        .collect();
1748    let idx_to_path: Vec<PathBuf> = order.iter().map(|&p| pre[p].2.clone()).collect();
1749    let mut inv = vec![0usize; n];
1750    for (idx, &p) in order.iter().enumerate() {
1751        inv[p] = idx;
1752    }
1753    let mut file_to_idxs: HashMap<PathBuf, Vec<usize>> = HashMap::new();
1754    for (p, m) in pre.iter().enumerate() {
1755        file_to_idxs.entry(m.2.clone()).or_default().push(inv[p]);
1756    }
1757    let max_id = pre.iter().map(|m| m.0).max().unwrap_or(-1);
1758    let slot_count = if max_id < 0 { 0 } else { (max_id + 1) as usize };
1759    let len = n;
1760    drop(pre);
1761
1762    // Pointer check via on-demand re-compile (single-threaded: the store getter
1763    // is `FnMut`/`!Sync`). Diagnostics come out in id order, as in the batch path.
1764    info!("[recompile] Pointer check (re-compile on demand)...");
1765    let shared = PointerChecker::build_shared_idents(&idents, &registry);
1766    {
1767        // Cache cap must exceed the largest file's script count, or loading that
1768        // file would evict its own just-compiled scripts. A small margin over
1769        // that keeps cross-file locality high (measured ~1.1-1.3x re-parse).
1770        let cap = file_to_idxs
1771            .values()
1772            .map(|v| v.len())
1773            .max()
1774            .unwrap_or(0)
1775            .max(128);
1776        let mut store = RecompileStore {
1777            idx_to_path,
1778            file_to_idxs,
1779            registry: &registry,
1780            include_tests,
1781            cache: HashMap::new(),
1782            order: std::collections::VecDeque::new(),
1783            cap,
1784        };
1785        let getter: Box<dyn FnMut(usize) -> std::rc::Rc<CompiledScript>> =
1786            Box::new(move |idx| store.get(idx));
1787        let scripts = ScriptSource::Recompile {
1788            len,
1789            get: std::cell::RefCell::new(getter),
1790        };
1791        let mut checker = PointerChecker::with_shared(scripts, &registry, shared);
1792        checker.set_source_cache(SourceProvider::Lazy(&source_paths));
1793        let names: Vec<String> = idents.iter().map(|(_, n)| n.clone()).collect();
1794        let ptr_diags = checker.validate_names(&names);
1795        let w = ptr_diags.warning_count();
1796        if w > 0 {
1797            info!("  Pointer check: {} warning(s)", w);
1798        }
1799        diagnostics.merge(ptr_diags);
1800    }
1801    mem_mark("after pointercheck");
1802
1803    // Output: re-compile each file once, encode each script into its id slot,
1804    // optionally lint — never holding all compiled scripts at once. The caller
1805    // writes the blobs to disk or assembles them in memory.
1806    info!("[recompile] Re-compiling + encoding output...");
1807    let mut encoded: Vec<Vec<u8>> = vec![Vec::new(); slot_count];
1808    for path in &rs2_files {
1809        let parsed = match parse_one_file(path, include_tests).read {
1810            Ok(p) => p,
1811            Err(_) => continue,
1812        };
1813        let file = match parsed.outcome {
1814            ParseOutcome::Parsed { file, .. } => file,
1815            ParseOutcome::Error { .. } => continue,
1816        };
1817        let mut compiler = Compiler::new(&registry);
1818        let mut file_scripts: Vec<CompiledScript> = Vec::new();
1819        for script in &file.scripts {
1820            if script.trigger == "command" {
1821                continue;
1822            }
1823            let Some(mut c) = compiler.compile_checked(script, path, &mut diagnostics) else {
1824                continue;
1825            };
1826            c.source_path = parsed.cache_key.clone();
1827            file_scripts.push(c);
1828        }
1829        for c in &file_scripts {
1830            if c.id >= 0 && (c.id as usize) < slot_count {
1831                encoded[c.id as usize] = crate::writer::encode_script(c);
1832            }
1833        }
1834        if lint {
1835            diagnostics.merge(lints::run_lints(
1836                &file_scripts,
1837                Some(SourceProvider::Lazy(&source_paths)),
1838            ));
1839        }
1840    }
1841    mem_mark("after codegen");
1842    Ok((encoded, diagnostics))
1843}
1844
1845/// Full compile via the recompile (low-memory) pipeline: writes `script.dat` /
1846/// `script.idx` to `output_dir`, then prints diagnostics.
1847fn compile_recompile(
1848    scripts_dir: &Path,
1849    pack_dir: Option<&Path>,
1850    output_dir: &Path,
1851    lint: bool,
1852) -> Result<(), Box<dyn Error>> {
1853    let (encoded, diagnostics) =
1854        recompile_to_encoded(scripts_dir, pack_dir, lint, false, &mut ScriptIds::new())?;
1855    info!(
1856        "Writing output to {}...",
1857        output_dir.display().to_string().replace('\\', "/")
1858    );
1859    ScriptWriter::new(output_dir.display().to_string()).write_encoded(&encoded)?;
1860    diagnostics.print_all();
1861    info!("Compilation complete!");
1862    Ok(())
1863}
1864
1865/// In-memory compile via the recompile (low-memory) pipeline: returns the
1866/// `(script.dat, script.idx)` bytes, then prints diagnostics. Byte-identical to
1867/// the in-memory batch path (`compile_memory*`).
1868fn compile_recompile_memory(
1869    scripts_dir: &Path,
1870    pack_dir: Option<&Path>,
1871    lint: bool,
1872    include_tests: bool,
1873    ids: &mut ScriptIds,
1874) -> Result<(Vec<u8>, Vec<u8>), Box<dyn Error>> {
1875    let (encoded, diagnostics) =
1876        recompile_to_encoded(scripts_dir, pack_dir, lint, include_tests, ids)?;
1877    info!("Writing compiled output to memory...");
1878    let out = ScriptWriter::new("".into()).build_encoded(&encoded);
1879    diagnostics.print_all();
1880    info!("Compilation complete!");
1881    Ok(out)
1882}
1883
1884pub(crate) fn find_testscript_line(source: &str) -> Option<usize> {
1885    for (i, line) in source.lines().enumerate() {
1886        if line.trim() == "#testscript" {
1887            return Some(i + 1);
1888        }
1889    }
1890    None
1891}
1892
1893fn truncate_at_line(source: &str, line_no: usize) -> String {
1894    source
1895        .lines()
1896        .take(line_no - 1)
1897        .collect::<Vec<_>>()
1898        .join("\n")
1899}
1900
1901fn strip_testscript_annotation(source: &str) -> String {
1902    source
1903        .lines()
1904        .map(|line| {
1905            if line.trim() == "#testscript" {
1906                ""
1907            } else {
1908                line
1909            }
1910        })
1911        .collect::<Vec<_>>()
1912        .join("\n")
1913}
1914
1915fn get_rs2_files(dir: &Path, ext: &str) -> Result<Vec<PathBuf>, Box<dyn Error>> {
1916    Ok(sources::script_files(dir, ext)?)
1917}