Skip to main content

runec/
filesystem.rs

1use crate::bytecode::CompiledScript;
2use crate::clientscript::{self, format::Format};
3use crate::compiler::Compiler;
4use crate::diagnostics::DiagnosticsCollector;
5use crate::lexer::Lexer;
6#[cfg(feature = "memprof")]
7use crate::memprof;
8use crate::parallel::{
9    parallel_chunks, parallel_chunks_capped, parallel_map, parallel_map_balanced,
10};
11use crate::parser::{Parser, ScriptFile};
12use crate::project::{Options, Project, SourceText, Target};
13use crate::sources::Scope;
14use crate::symbol::SymbolRegistry;
15use crate::typechecker::TypeChecker;
16use crate::writer::ScriptWriter;
17use crate::{diagnostics, error, lints, sources, symloader, trigger_table, types};
18use std::collections::{BTreeMap, HashMap};
19use std::error::Error;
20use std::fmt::Write;
21use std::fs;
22use std::panic;
23use std::path::{Path, PathBuf};
24use std::sync::Arc;
25use std::thread;
26use tracing::{info, warn};
27
28pub type ScriptIds = BTreeMap<String, i32>;
29
30/// Build an editor project using the legacy directory conventions. No ID file
31/// is generated or modified; all further editing happens in memory.
32pub fn editor_project(
33    source: &Path,
34    pack: Option<&Path>,
35    client: bool,
36) -> Result<Project, Box<dyn Error>> {
37    let scope = if client { Scope::Client } else { Scope::Rune };
38    let dir = sources::target_dir(source, scope);
39    let pack = resolve_pack_dir(&dir, pack);
40    let mut registry = if client {
41        clientscript::registry(
42            &dir,
43            pack.as_deref()
44                .ok_or("Client opcode definitions require --pack")?,
45        )?
46    } else {
47        build_symbol_registry(&dir, pack.as_deref())
48    };
49    let target = if client {
50        registry.preloaded_script_ids.clear();
51        for (key, id) in clientscript::script_ids(pack.as_deref().unwrap())? {
52            let (trigger, name) = key
53                .trim_start_matches('[')
54                .trim_end_matches(']')
55                .split_once(',')
56                .ok_or("Invalid client ID")?;
57            registry.set_script_id(trigger, name, id);
58        }
59        Target::Client(Format::load(
60            &pack.as_ref().unwrap().join("cs2_command.pack"),
61        )?)
62    } else {
63        Target::Rune
64    };
65    let mut project = Project::new(
66        target,
67        registry,
68        Options {
69            include_tests: true,
70            skip_shared_lints: client
71                && dir
72                    .parent()
73                    .is_some_and(|parent| parent.join("runescript").is_dir()),
74            ..Options::default()
75        },
76    );
77    let paths = sources::script_files(&dir, if client { "cs2" } else { "rs2" })?;
78    let files = read_sources(&paths, |path| {
79        if path.extension().is_some_and(|ext| ext == "gs2") {
80            Scope::Global
81        } else {
82            scope
83        }
84    })?;
85    for id in project.open_all(files) {
86        if client {
87            let ids = project
88                .document(id)
89                .and_then(|doc| doc.syntax())
90                .into_iter()
91                .flat_map(|file| &file.scripts)
92                .filter_map(|script| {
93                    script
94                        .name
95                        .strip_prefix("clientscript_")
96                        .and_then(|id| id.parse::<i32>().ok())
97                        .map(|id| (script.trigger.clone(), script.name.clone(), id))
98                })
99                .collect::<Vec<_>>();
100            for (trigger, name, id) in ids {
101                project.registry.set_script_id(&trigger, &name, id);
102            }
103        }
104    }
105    if !client {
106        let client_dir = if dir.file_name().is_some_and(|name| name == "runescript") {
107            dir.with_file_name("clientscript")
108        } else {
109            dir.join("clientscript")
110        };
111        if client_dir.is_dir() {
112            let paths = sources::script_files(&client_dir, "cs2")?
113                .into_iter()
114                .filter(|path| {
115                    path.extension().is_some_and(|ext| ext == "cs2")
116                        && path.file_name().is_some_and(|name| name != "engine.cs2")
117                })
118                .collect::<Vec<_>>();
119            project.open_all(read_sources(&paths, |_| Scope::Client)?);
120        }
121    }
122    let mut files = sources::config_files(&dir);
123    if let Some(pack) = &pack {
124        symloader::collect_all_files(pack, &mut files);
125        // Interfaces and defaults live beside the scripts they're for.
126        if let Some(content) = pack.parent() {
127            for (folder, extension) in [("interfaces", "if3"), ("defaults", "defaults")] {
128                let Ok(entries) = fs::read_dir(content.join(folder)) else {
129                    continue;
130                };
131                files.extend(
132                    entries
133                        .flatten()
134                        .map(|entry| entry.path())
135                        .filter(|path| path.extension().is_some_and(|ext| ext == extension)),
136                );
137            }
138        }
139    }
140    files.sort();
141    files.dedup();
142    files.retain(|path| config_kind(path).is_some());
143    let configs = read_sources(&files, |path| {
144        if path
145            .components()
146            .any(|part| part.as_os_str() == "globalscript")
147        {
148            Scope::Global
149        } else {
150            scope
151        }
152    })?;
153    for ((uri, config_scope, version, text), path) in configs.into_iter().zip(&files) {
154        project.open_config(uri, config_scope, version, text, config_kind(path).unwrap());
155    }
156    Ok(project)
157}
158
159/// Read files and name them by URI, in parallel, failing on the first error in
160/// input order.
161fn read_sources(
162    paths: &[PathBuf],
163    scope: impl Fn(&Path) -> Scope + Sync,
164) -> Result<Vec<SourceText>, Box<dyn Error>> {
165    parallel_map_balanced(paths, |path| {
166        let text = fs::read_to_string(path).map_err(|error| error.to_string())?;
167        let uri = file_uri(path).map_err(|error| error.to_string())?;
168        Ok((uri, scope(path), 0, text))
169    })
170    .into_iter()
171    .collect::<Result<Vec<_>, String>>()
172    .map_err(Into::into)
173}
174
175pub(crate) fn config_kind(path: &Path) -> Option<String> {
176    let extension = path.extension()?.to_str()?;
177    if matches!(extension, "pack" | "order" | "hashes") {
178        return Some(format!("pack:{}", path.file_stem()?.to_str()?));
179    }
180    (matches!(extension, "constant" | "if" | "if3" | "defaults" | "anim")
181        || types::Type::from_name(extension).is_some()
182        || extension
183            .strip_prefix("config")
184            .is_some_and(|number| number.parse::<u32>().is_ok()))
185    .then(|| extension.to_string())
186}
187
188pub(crate) fn file_uri(path: &Path) -> Result<String, Box<dyn Error>> {
189    let absolute = fs::canonicalize(path)?
190        .to_string_lossy()
191        .trim_start_matches("\\\\?\\")
192        .replace('\\', "/");
193    let mut uri = String::from(if absolute.starts_with('/') {
194        "file://"
195    } else {
196        "file:///"
197    });
198    for byte in absolute.bytes() {
199        if byte.is_ascii_alphanumeric() || b"/-._~:".contains(&byte) {
200            uri.push(byte as char);
201        } else {
202            write!(uri, "%{byte:02X}")?;
203        }
204    }
205    Ok(uri)
206}
207
208/// How the diagnostic layer fetches a file's source text (by its cache-key path)
209/// when building rustc-style help/suggestions. The batch path keeps every source
210/// in memory; the streaming path keeps only the on-disk paths and re-reads the
211/// handful of files that actually produce a warning — so the ~4 MB of source
212/// text isn't resident through the compile's peak. Cheap to copy (just borrows).
213#[derive(Clone, Copy)]
214pub enum SourceProvider<'a> {
215    /// Sources already resident, keyed by cache key.
216    Eager(&'a HashMap<String, Arc<String>>),
217    /// Cache key → on-disk path; re-read on demand.
218    Lazy(&'a HashMap<String, PathBuf>),
219}
220
221impl SourceProvider<'_> {
222    /// Source text for `key`, if available. Returns an owned `Arc` — a cheap
223    /// refcount bump for `Eager`, a fresh disk read for `Lazy`. Called only while
224    /// emitting diagnostic help, which is rare, so the re-read cost is immaterial.
225    pub fn get(&self, key: &str) -> Option<Arc<String>> {
226        match self {
227            SourceProvider::Eager(m) => m.get(key).cloned(),
228            SourceProvider::Lazy(m) => fs::read_to_string(m.get(key)?).ok().map(Arc::new),
229        }
230    }
231}
232
233/// Log live heap + global peak at a pipeline phase boundary. No-op unless built
234/// with `--features memprof`. Lets us attribute the footprint to each phase.
235#[cfg(feature = "memprof")]
236fn mem_mark(label: &str) {
237    info!(
238        "[mem] {:<22} live {:>7.1} MB | peak {:>7.1} MB | allocs {}",
239        label,
240        memprof::current_bytes() as f64 / 1_048_576.0,
241        memprof::peak_bytes() as f64 / 1_048_576.0,
242        memprof::alloc_count(),
243    );
244}
245#[cfg(not(feature = "memprof"))]
246#[inline]
247fn mem_mark(_label: &str) {}
248
249/// Per-map breakdown of the symbol registry's heap, to see the irreducible
250/// floor for a streaming rebuild. Built only under `--features memprof`.
251/// Accounts three components per map: the hashbrown table (inline `(K,V)`
252/// buckets + 1 control byte each), the key strings, and value-side heap.
253#[cfg(feature = "memprof")]
254fn mem_stats_registry(reg: &crate::symbol::SymbolRegistry) {
255    use crate::symbol::{Symbol, SymbolKind};
256    use crate::types::Type;
257    use std::collections::{HashMap, HashSet};
258    let t = std::mem::size_of::<Type>();
259    let mb = |b: usize| b as f64 / 1_048_576.0;
260
261    // Allocated bucket count: hashbrown keeps `capacity()` ≈ 7/8 of a power-of-two
262    // bucket array; an empty map allocates nothing.
263    fn tbl<K, V>(m: &HashMap<K, V>) -> usize {
264        if m.capacity() == 0 {
265            return 0;
266        }
267        let buckets = (m.capacity() * 8 / 7).next_power_of_two();
268        buckets * (std::mem::size_of::<(K, V)>() + 1)
269    }
270    fn keys_cap<V>(m: &HashMap<String, V>) -> usize {
271        m.keys().map(|k| k.capacity()).sum()
272    }
273    let sym_heap = |s: &Symbol| -> usize {
274        s.name.capacity()
275            + match &s.kind {
276                SymbolKind::Script {
277                    trigger,
278                    param_types,
279                    return_types,
280                    ..
281                } => trigger.capacity() + (param_types.capacity() + return_types.capacity()) * t,
282                SymbolKind::Command {
283                    param_types,
284                    return_types,
285                    ..
286                } => (param_types.capacity() + return_types.capacity()) * t,
287                SymbolKind::GameVar { category, .. } => category.capacity(),
288                SymbolKind::Constant { string_value, .. } => {
289                    string_value.as_ref().map_or(0, |s| s.capacity())
290                }
291                _ => 0,
292            }
293    };
294    let sym_map = |m: &HashMap<String, Symbol>| -> usize {
295        tbl(m) + keys_cap(m) + m.values().map(&sym_heap).sum::<usize>()
296    };
297    let i32_map = |m: &HashMap<String, i32>| -> usize { tbl(m) + keys_cap(m) };
298
299    let mut rows: Vec<(&str, usize, usize)> = vec![
300        (
301            "scripts",
302            reg.scripts.len(),
303            tbl(&reg.scripts) + keys_cap(&reg.scripts),
304        ),
305        ("commands", reg.commands.len(), sym_map(&reg.commands)),
306        ("game_vars", reg.game_vars.len(), sym_map(&reg.game_vars)),
307        ("constants", reg.constants.len(), sym_map(&reg.constants)),
308        (
309            "entity_ids",
310            reg.entity_ids.len(),
311            tbl(&reg.entity_ids)
312                + keys_cap(&reg.entity_ids)
313                + reg
314                    .entity_ids
315                    .values()
316                    .map(|e| e.variants.capacity() * std::mem::size_of::<(Type, i32)>())
317                    .sum::<usize>(),
318        ),
319        (
320            "scripts_by_trigger",
321            reg.scripts_by_trigger.len(),
322            tbl(&reg.scripts_by_trigger)
323                + keys_cap(&reg.scripts_by_trigger)
324                + reg
325                    .scripts_by_trigger
326                    .values()
327                    .map(|s| 16 + sym_heap(s.as_ref()))
328                    .sum::<usize>(),
329        ),
330        (
331            "command_param_types",
332            reg.command_param_types.len(),
333            tbl(&reg.command_param_types)
334                + keys_cap(&reg.command_param_types)
335                + reg
336                    .command_param_types
337                    .values()
338                    .map(|v| v.capacity() * t)
339                    .sum::<usize>(),
340        ),
341        ("script_ids", reg.script_ids.len(), i32_map(&reg.script_ids)),
342        (
343            "proc_script_ids",
344            reg.proc_script_ids.len(),
345            i32_map(&reg.proc_script_ids),
346        ),
347        (
348            "label_script_ids",
349            reg.label_script_ids.len(),
350            i32_map(&reg.label_script_ids),
351        ),
352        (
353            "trigger_script_ids",
354            reg.trigger_script_ids.len(),
355            i32_map(&reg.trigger_script_ids),
356        ),
357        (
358            "preloaded_script_ids",
359            reg.preloaded_script_ids.len(),
360            i32_map(&reg.preloaded_script_ids),
361        ),
362        ("components", reg.components.len(), i32_map(&reg.components)),
363        (
364            "interface_ids",
365            reg.interface_ids.len(),
366            i32_map(&reg.interface_ids),
367        ),
368        ("type_chars", reg.type_chars.len(), i32_map(&reg.type_chars)),
369        (
370            "dbcolumn_types",
371            reg.dbcolumn_types.len(),
372            tbl(&reg.dbcolumn_types) + keys_cap(&reg.dbcolumn_types),
373        ),
374    ];
375    let ts_tbl = {
376        let s: &HashSet<String> = &reg.test_scripts;
377        if s.capacity() == 0 {
378            0
379        } else {
380            (s.capacity() * 8 / 7).next_power_of_two() * (std::mem::size_of::<String>() + 1)
381                + s.iter().map(|k| k.capacity()).sum::<usize>()
382        }
383    };
384    rows.push(("test_scripts", reg.test_scripts.len(), ts_tbl));
385
386    rows.sort_by_key(|row| std::cmp::Reverse(row.2));
387    let total: usize = rows.iter().map(|r| r.2).sum();
388    info!(
389        "[mem] registry ~{:.2} MB (sizeof Symbol={}B, SymbolKind={}B, Type={}B):",
390        mb(total),
391        std::mem::size_of::<Symbol>(),
392        std::mem::size_of::<SymbolKind>(),
393        t,
394    );
395    for (name, n, bytes) in &rows {
396        info!(
397            "[mem]   {:<20} {:>6} entries  {:>7.3} MB",
398            name,
399            n,
400            mb(*bytes)
401        );
402    }
403}
404
405/// One-off breakdown of where the compiled-bytecode heap goes, to target
406/// memory work. Built only under `--features memprof`.
407#[cfg(feature = "memprof")]
408fn mem_stats_compiled(scripts: &[CompiledScript]) {
409    use crate::bytecode::{Instruction, Opcode, Operand};
410    let (mut n_instr, mut n_str, mut str_bytes) = (0usize, 0usize, 0usize);
411    let (mut n_switch, mut switch_entries) = (0usize, 0usize);
412    let (mut source_path_bytes, mut name_bytes) = (0usize, 0usize);
413    for s in scripts {
414        n_instr += s.instructions.len();
415        source_path_bytes += s.source_path.capacity();
416        name_bytes += s.name.capacity() + s.trigger.capacity();
417        for st in &s.strings {
418            n_str += 1;
419            str_bytes += st.len();
420        }
421        for sw in &s.switch_tables {
422            n_switch += 1;
423            switch_entries += sw.len();
424        }
425    }
426    let sz = std::mem::size_of::<Instruction>();
427    let mb = |b: usize| b as f64 / 1_048_576.0;
428    info!(
429        "[mem] sizeof Instruction={}B Operand={}B Opcode={}B CompiledScript={}B",
430        sz,
431        std::mem::size_of::<Operand>(),
432        std::mem::size_of::<Opcode>(),
433        std::mem::size_of::<CompiledScript>(),
434    );
435    info!(
436        "[mem] compiled: {} scripts, {} instrs => {:.2} MB instr structs",
437        scripts.len(),
438        n_instr,
439        mb(n_instr * sz)
440    );
441    info!(
442        "[mem]   Str operands: {} ({:.2} MB content + {:.2} MB Box<str> headers)",
443        n_str,
444        mb(str_bytes),
445        mb(n_str * 16)
446    );
447    info!(
448        "[mem]   SwitchTable operands: {} ({} entries => {:.2} MB)",
449        n_switch,
450        switch_entries,
451        mb(switch_entries * 16)
452    );
453    info!(
454        "[mem]   source_path {:.2} MB, names+triggers {:.2} MB",
455        mb(source_path_bytes),
456        mb(name_bytes)
457    );
458}
459
460struct ParsedFile {
461    path: PathBuf,
462    file: ScriptFile,
463    testscript_line: Option<usize>,
464    /// Canonicalized, normalized source path. Computed once during parsing and
465    /// reused as each compiled script's `source_path` (it is byte-identical to
466    /// the `source_cache` key), avoiding a second `canonicalize` syscall per
467    /// file in codegen.
468    source_path: String,
469}
470
471/// Outcome of parsing one file off-thread. The owned source text and cache key
472/// travel back so the (single-threaded) merge can build the `Rc`-based caches
473/// without sending `Rc` across threads.
474struct FileParseResult {
475    path: PathBuf,
476    read: Result<ParsedSource, std::io::Error>,
477}
478
479struct ParsedSource {
480    cache_key: String,
481    raw_source: String,
482    outcome: ParseOutcome,
483}
484
485enum ParseOutcome {
486    Parsed {
487        file: ScriptFile,
488        testscript_line: Option<usize>,
489    },
490    Error {
491        line: usize,
492        position: usize,
493        message: String,
494        phase: diagnostics::Phase,
495    },
496}
497
498/// Read, lex and parse a single file off-thread. Pure: no shared state, so it
499/// is safe to run concurrently. Mirrors the original sequential body exactly —
500/// same source truncation, same cache-key normalization, same lex-then-parse
501/// error reporting — only restructured to return owned data for the merge.
502fn parse_one_file(path: &Path, include_tests: bool) -> FileParseResult {
503    let raw_source = match fs::read_to_string(path) {
504        Ok(s) => s,
505        Err(e) => {
506            return FileParseResult {
507                path: path.to_path_buf(),
508                read: Err(e),
509            };
510        }
511    };
512    let testscript_line = find_testscript_line(&raw_source);
513
514    let source_code = if !include_tests {
515        if let Some(line_no) = testscript_line {
516            truncate_at_line(&raw_source, line_no)
517        } else {
518            raw_source.clone()
519        }
520    } else {
521        strip_testscript_annotation(&raw_source)
522    };
523
524    let canonical = fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf());
525    let mut cache_key = canonical.to_string_lossy().into_owned();
526    if cache_key.starts_with("\\\\?\\") {
527        cache_key = cache_key[4..].to_string();
528    }
529    cache_key = cache_key.replace("\\Content\\", "\\content\\");
530
531    // Lexer/Parser borrow a `&PathBuf` for diagnostics; keep one alive locally.
532    let path_buf = path.to_path_buf();
533    let outcome = match Lexer::new(&source_code, &path_buf).tokenize() {
534        Err(e) => ParseOutcome::Error {
535            line: e.line,
536            position: e.position,
537            message: e.message,
538            phase: diagnostics::Phase::Lexing,
539        },
540        Ok(tokens) => match Parser::new(tokens, &path_buf).parse() {
541            Ok(file) => ParseOutcome::Parsed {
542                file,
543                testscript_line: if include_tests { testscript_line } else { None },
544            },
545            Err(e) => ParseOutcome::Error {
546                line: e.line,
547                position: e.position,
548                message: e.message,
549                phase: diagnostics::Phase::Parsing,
550            },
551        },
552    };
553
554    FileParseResult {
555        path: path_buf,
556        read: Ok(ParsedSource {
557            cache_key,
558            raw_source,
559            outcome,
560        }),
561    }
562}
563
564/// Compile scripts and write output. When `lint` is true, also run lint passes
565/// (unused locals, unreachable code) after code generation.
566pub fn compile_memory(
567    scripts_dir: &Path,
568    pack_dir: Option<&Path>,
569    lint: bool,
570) -> Result<(Vec<u8>, Vec<u8>), Box<dyn Error>> {
571    compile_memory_with_options(scripts_dir, pack_dir, lint, false, false)
572}
573
574/// Like `compile_memory` but includes `#testscript` sections when
575/// `include_tests` is true.
576pub fn compile_memory_with_tests(
577    scripts_dir: &Path,
578    pack_dir: Option<&Path>,
579    lint: bool,
580    include_tests: bool,
581) -> Result<(Vec<u8>, Vec<u8>), Box<dyn Error>> {
582    compile_memory_with_options(scripts_dir, pack_dir, lint, include_tests, false)
583}
584
585/// In-memory compile — returns the `(script.dat, script.idx)` bytes instead of
586/// writing them — choosing the compilation strategy.
587///
588/// With `low_mem`, the recompile pipeline is used: scripts are re-parsed and
589/// re-compiled on demand so the full compiled set is never resident (~16 MB heap
590/// vs ~93), at roughly 6× the time; the returned bytes are byte-identical to the
591/// default path. `include_tests` keeps `#testscript` sections. The
592/// `RUNEC_RECOMPILE` environment variable forces low-memory mode on as well.
593pub fn compile_memory_with_options(
594    scripts_dir: &Path,
595    pack_dir: Option<&Path>,
596    lint: bool,
597    include_tests: bool,
598    low_mem: bool,
599) -> Result<(Vec<u8>, Vec<u8>), Box<dyn Error>> {
600    compile_memory_inner(
601        scripts_dir,
602        pack_dir,
603        lint,
604        include_tests,
605        low_mem,
606        &mut ScriptIds::new(),
607    )
608}
609
610/// Reuses a world's script IDs across reloads, reserving removed scripts' IDs.
611pub fn compile_memory_with_ids(
612    scripts_dir: &Path,
613    pack_dir: Option<&Path>,
614    ids: &mut ScriptIds,
615) -> Result<(Vec<u8>, Vec<u8>), Box<dyn Error>> {
616    compile_memory_inner(scripts_dir, pack_dir, false, false, false, ids)
617}
618
619fn compile_memory_inner(
620    scripts_dir: &Path,
621    pack_dir: Option<&Path>,
622    lint: bool,
623    include_tests: bool,
624    low_mem: bool,
625    ids: &mut ScriptIds,
626) -> Result<(Vec<u8>, Vec<u8>), Box<dyn Error>> {
627    if low_mem || std::env::var("RUNEC_RECOMPILE").is_ok_and(|v| !v.is_empty()) {
628        info!("[recompile] low-memory recompile pipeline enabled");
629        return compile_recompile_memory(scripts_dir, pack_dir, lint, include_tests, ids);
630    }
631
632    let (compiled_scripts, diagnostics) =
633        run_pipeline(scripts_dir, pack_dir, lint, include_tests, ids)?;
634
635    info!("Writing compiled output to memory...");
636    let writer = ScriptWriter::new("".into());
637
638    diagnostics.print_all();
639    info!("Compilation complete!");
640
641    Ok(writer.build_all(&compiled_scripts)?)
642}
643
644/// Compile scripts and write output. When `lint` is true, also run lint passes
645/// (unused locals, unreachable code) after code generation.
646///
647/// Convenience wrapper over [`compile_with_options`] using the default (fast,
648/// fully-resident) pipeline. Kept signature-compatible for existing callers.
649pub fn compile(
650    scripts_dir: &Path,
651    pack_dir: Option<&Path>,
652    output_dir: &Path,
653    lint: bool,
654) -> Result<(), Box<dyn Error>> {
655    compile_with_options(scripts_dir, pack_dir, output_dir, lint, false)
656}
657
658/// Compile scripts and write output, choosing the compilation strategy.
659///
660/// With `low_mem`, the **recompile** pipeline is used: scripts are re-parsed and
661/// re-compiled on demand for the pointer check and written file-by-file, so the
662/// full set of compiled bytecode is never resident at once. That cuts peak usage
663/// to roughly 16 MB heap / 32 MB working set (from ~93 / ~111 MB) at about 6× the
664/// compile time. Output is byte-identical to the default path. The
665/// `RUNEC_RECOMPILE` environment variable forces this mode on as well.
666pub fn compile_with_options(
667    scripts_dir: &Path,
668    pack_dir: Option<&Path>,
669    output_dir: &Path,
670    lint: bool,
671    low_mem: bool,
672) -> Result<(), Box<dyn Error>> {
673    // The recompile pipeline never holds all compiled scripts (it re-compiles on
674    // demand for the pointer check + writes file-by-file). It does its own
675    // output, so it returns here. The default path below is unchanged.
676    if low_mem || std::env::var("RUNEC_RECOMPILE").is_ok_and(|v| !v.is_empty()) {
677        info!("[recompile] low-memory recompile pipeline enabled");
678        return compile_recompile(scripts_dir, pack_dir, output_dir, lint);
679    }
680
681    let (compiled_scripts, diagnostics) =
682        run_pipeline(scripts_dir, pack_dir, lint, false, &mut ScriptIds::new())?;
683
684    let output = output_dir.display().to_string();
685    // Normalize separators for a clean, consistent log message (input paths may
686    // mix `/` and `\`, e.g. `data/pack\server`); the writer keeps the original.
687    info!("Writing output to {}...", output.replace('\\', "/"));
688    let writer = ScriptWriter::new(output);
689    writer.write_all(&compiled_scripts)?;
690
691    diagnostics.print_all();
692    info!("Compilation complete!");
693
694    Ok(())
695}
696
697/// Run all analysis passes (parse, type-check, codegen, pointer-check, lints)
698/// without writing output. Useful for editor tooling and CI lint gates.
699pub fn lint(scripts_dir: &Path, pack_dir: Option<&Path>) -> Result<(), Box<dyn Error>> {
700    let (_, diagnostics) = run_pipeline(scripts_dir, pack_dir, true, false, &mut ScriptIds::new())?;
701
702    diagnostics.print_all();
703    info!("Compilation complete!");
704
705    Ok(())
706}
707
708/// Runs the full compilation pipeline (parse, register, type-check, codegen,
709/// pointer-check, and optional lint passes), returning the compiled scripts
710/// and accumulated diagnostics for the caller to handle output.
711fn run_pipeline(
712    scripts_dir: &Path,
713    pack_dir: Option<&Path>,
714    lint: bool,
715    include_tests: bool,
716    ids: &mut ScriptIds,
717) -> Result<(Vec<CompiledScript>, DiagnosticsCollector), Box<dyn Error>> {
718    let source_dir = sources::target_dir(scripts_dir, sources::Scope::Rune);
719    let scripts_dir = source_dir.as_path();
720    if !scripts_dir.exists() || !scripts_dir.is_dir() {
721        return Err(Box::new(error::CompilerError::FileNotFound(
722            scripts_dir.display().to_string(),
723        )));
724    }
725
726    // Prototype: opt into the streaming pipeline (never holds all ASTs at once)
727    // with RUNEC_STREAM=1. The default batch path below is unchanged.
728    if std::env::var("RUNEC_STREAM").is_ok_and(|v| !v.is_empty()) {
729        info!("[stream] streaming pipeline enabled (RUNEC_STREAM)");
730        return run_pipeline_streaming(scripts_dir, pack_dir, lint, include_tests, ids);
731    }
732
733    let mut rs2_files: Vec<PathBuf> = get_rs2_files(scripts_dir, "rs2")?;
734
735    rs2_files.sort();
736    info!("Found {} script files", rs2_files.len());
737    mem_mark("startup");
738
739    let mut diagnostics = DiagnosticsCollector::new();
740
741    // Phase 1: Parse all files. Reading, lexing and parsing are independent
742    // per file, so fan them out across threads; the results are merged back in
743    // input order so registration/ID assignment stays deterministic.
744    info!("Parsing script files into syntax trees...");
745    let parse_results = parallel_map(&rs2_files, |path| parse_one_file(path, include_tests));
746
747    let mut all_files: Vec<ParsedFile> = Vec::new();
748    let mut source_cache: HashMap<String, Arc<String>> = HashMap::new();
749    for result in parse_results {
750        let parsed = match result.read {
751            Ok(p) => p,
752            // A read failure aborts the whole compile, exactly as the previous
753            // `fs::read_to_string(path)?` did.
754            Err(e) => return Err(Box::new(e)),
755        };
756        let rc = Arc::new(parsed.raw_source);
757        // The cache key is the canonicalized source path; reuse it as each
758        // compiled script's `source_path` so codegen needn't canonicalize again.
759        let source_path = parsed.cache_key.clone();
760        source_cache.insert(parsed.cache_key, rc.clone());
761        match parsed.outcome {
762            ParseOutcome::Parsed {
763                file,
764                testscript_line,
765            } => all_files.push(ParsedFile {
766                path: result.path,
767                file,
768                testscript_line,
769                source_path,
770            }),
771            ParseOutcome::Error {
772                line,
773                position,
774                message,
775                phase,
776            } => diagnostics.error(result.path, line, position, message, phase),
777        }
778    }
779
780    if diagnostics.has_errors() {
781        diagnostics.print_all();
782        return Err(Box::new(error::CompilerError::TypeError(
783            "Parsing failed".to_string(),
784        )));
785    }
786
787    mem_mark("after parse");
788
789    // Phase 2: assign IDs and register every parsed script.
790    info!("Registering scripts and building the symbol registry...");
791    let resolved_pack_dir = resolve_pack_dir(scripts_dir, pack_dir);
792    let mut registry = build_symbol_registry(scripts_dir, resolved_pack_dir.as_deref());
793    register_scripts(&mut registry, ids, &mut diagnostics, &all_files);
794    info!("  Registered {} scripts", registry.scripts.len());
795    // Pre-assigned script ids were consumed during registration; free them
796    // before codegen (the compile's peak-memory phase) rather than at scope end.
797    registry.drop_registration_scratch();
798
799    for pf in &all_files {
800        for script in &pf.file.scripts {
801            if let Some(msg) =
802                Compiler::validate_trigger_subject(&script.trigger, &script.name, &registry)
803            {
804                diagnostics.warning(
805                    pf.path.clone(),
806                    script.line,
807                    0,
808                    msg,
809                    diagnostics::Phase::SymbolRegistration,
810                );
811            }
812        }
813    }
814
815    mem_mark("after register");
816    #[cfg(feature = "memprof")]
817    mem_stats_registry(&registry);
818
819    // Phase 3: Type checking
820    info!("Type checking {} files...", all_files.len());
821    run_type_checker(&mut diagnostics, &all_files, &registry);
822
823    if diagnostics.has_errors() {
824        diagnostics.print_all();
825        return Err(Box::new(error::CompilerError::TypeError(
826            "Type checking failed".to_string(),
827        )));
828    }
829
830    mem_mark("after typecheck");
831
832    // Phase 4: Code generation
833    info!("Generating bytecode from type-checked scripts...");
834    let mut compiled_scripts = codegen(&all_files, &registry, &mut diagnostics);
835    if diagnostics.has_errors() {
836        diagnostics.print_all();
837        return Err(Box::new(error::CompilerError::CodeGenError(
838            "Code generation failed".into(),
839        )));
840    }
841    info!("  Generated {} compiled scripts", compiled_scripts.len());
842    compiled_scripts.sort_by_key(|s| s.id);
843    compiled_scripts.shrink_to_fit();
844    mem_mark("after codegen");
845    #[cfg(feature = "memprof")]
846    mem_stats_compiled(&compiled_scripts);
847
848    // The ASTs are consumed by codegen and never read again — pointer checking
849    // and lints work purely off `compiled_scripts` + the registry + the source
850    // cache. Free the syntax trees now so they don't coexist with the pointer
851    // checker's (per-thread) scratch, which is the compile's peak-memory moment.
852    drop(all_files);
853    mem_mark("after drop asts");
854
855    // Phase 4.5: Pointer checking & lints
856    info!("Checking pointers in generated code...");
857    check_pointers(
858        lint,
859        &mut diagnostics,
860        SourceProvider::Eager(&source_cache),
861        &registry,
862        &compiled_scripts,
863    );
864    mem_mark("after pointercheck");
865
866    Ok((compiled_scripts, diagnostics))
867}
868
869// ── Streaming pipeline (prototype, gated by RUNEC_STREAM) ────────────────────
870// Goal: never materialize all ASTs (~59 MB) at once. Pass 1 parses in parallel
871// but keeps only per-script signatures, dropping each AST in-worker; the registry
872// is built from those. Pass 2 re-parses each file in parallel, type-checks and
873// compiles it, then drops the AST immediately — so syntax trees never accumulate.
874// Output is byte-identical to the batch path (same registration, same per-file
875// codegen, same stable id-sort). It still holds all compiled scripts (for the
876// cross-script pointer check + writer); shrinking that is the next increment.
877
878/// One script's registration-relevant signature — everything the registry and
879/// trigger diagnostics need — so the full AST can be dropped right after parsing.
880struct ScriptSig {
881    name: String,
882    trigger: String,
883    param_types: Vec<types::Type>,
884    return_types: Vec<types::Type>,
885    line: usize,
886}
887
888struct FileSigs {
889    path: PathBuf,
890    testscript_line: Option<usize>,
891    sigs: Vec<ScriptSig>,
892}
893
894enum SigOutcome {
895    Parsed {
896        sigs: Vec<ScriptSig>,
897        testscript_line: Option<usize>,
898    },
899    Error {
900        line: usize,
901        position: usize,
902        message: String,
903        phase: diagnostics::Phase,
904    },
905}
906
907struct FileSigsResult {
908    path: PathBuf,
909    read: Result<(String, SigOutcome), std::io::Error>,
910}
911
912/// Parse one file, extract signatures, and DROP the AST in-worker, so a parallel
913/// sweep never holds every syntax tree at once.
914fn parse_one_file_sigs(path: &Path, include_tests: bool) -> FileSigsResult {
915    let r = parse_one_file(path, include_tests);
916    let read = r.read.map(|ps| {
917        let outcome = match ps.outcome {
918            ParseOutcome::Parsed {
919                file,
920                testscript_line,
921            } => {
922                let sigs = file
923                    .scripts
924                    .iter()
925                    .map(|d| ScriptSig {
926                        name: d.name.clone(),
927                        trigger: d.trigger.clone(),
928                        param_types: d.params.iter().map(|p| p.param_type).collect(),
929                        return_types: d.return_types.clone(),
930                        line: d.line,
931                    })
932                    .collect();
933                SigOutcome::Parsed {
934                    sigs,
935                    testscript_line,
936                }
937                // `file` (the AST) is dropped here — only the sigs travel back.
938            }
939            ParseOutcome::Error {
940                line,
941                position,
942                message,
943                phase,
944            } => SigOutcome::Error {
945                line,
946                position,
947                message,
948                phase,
949            },
950        };
951        (ps.cache_key, outcome)
952    });
953    FileSigsResult { path: r.path, read }
954}
955
956fn run_pipeline_streaming(
957    scripts_dir: &Path,
958    pack_dir: Option<&Path>,
959    lint: bool,
960    include_tests: bool,
961    ids: &mut ScriptIds,
962) -> Result<(Vec<CompiledScript>, DiagnosticsCollector), Box<dyn Error>> {
963    let mut rs2_files: Vec<PathBuf> = get_rs2_files(scripts_dir, "rs2")?;
964    rs2_files.sort();
965    info!("Found {} script files", rs2_files.len());
966    mem_mark("startup");
967
968    let mut diagnostics = DiagnosticsCollector::new();
969
970    // Pass 1: parse for signatures only (each AST dropped in its worker).
971    info!("[stream] Pass 1: parsing for signatures...");
972    let sig_results = parallel_map(&rs2_files, |path| parse_one_file_sigs(path, include_tests));
973
974    // Streaming keeps only `cache_key -> on-disk path`; source text is re-read
975    // lazily for the rare diagnostic that needs it, so the ~4 MB of sources isn't
976    // resident through the peak.
977    let mut source_paths: HashMap<String, PathBuf> = HashMap::new();
978    let mut file_sigs: Vec<FileSigs> = Vec::new();
979    for r in sig_results {
980        let (cache_key, outcome) = match r.read {
981            Ok(t) => t,
982            Err(e) => return Err(Box::new(e)),
983        };
984        source_paths.insert(cache_key, r.path.clone());
985        match outcome {
986            SigOutcome::Parsed {
987                sigs,
988                testscript_line,
989            } => file_sigs.push(FileSigs {
990                path: r.path,
991                testscript_line,
992                sigs,
993            }),
994            SigOutcome::Error {
995                line,
996                position,
997                message,
998                phase,
999            } => diagnostics.error(r.path, line, position, message, phase),
1000        }
1001    }
1002
1003    if diagnostics.has_errors() {
1004        diagnostics.print_all();
1005        return Err(Box::new(error::CompilerError::TypeError(
1006            "Parsing failed".to_string(),
1007        )));
1008    }
1009    mem_mark("after parse");
1010
1011    // Registration (from signatures, not ASTs).
1012    info!("[stream] Registering scripts and building the symbol registry...");
1013    let resolved_pack_dir = resolve_pack_dir(scripts_dir, pack_dir);
1014    let mut registry = build_symbol_registry(scripts_dir, resolved_pack_dir.as_deref());
1015    register_scripts_from_sigs(&mut registry, ids, &mut diagnostics, &file_sigs);
1016    info!("  Registered {} scripts", registry.scripts.len());
1017    registry.drop_registration_scratch();
1018
1019    for fs in &file_sigs {
1020        for sig in &fs.sigs {
1021            if let Some(msg) =
1022                Compiler::validate_trigger_subject(&sig.trigger, &sig.name, &registry)
1023            {
1024                diagnostics.warning(
1025                    fs.path.clone(),
1026                    sig.line,
1027                    0,
1028                    msg,
1029                    diagnostics::Phase::SymbolRegistration,
1030                );
1031            }
1032        }
1033    }
1034    // Signatures are done; only the registry + source cache carry forward.
1035    drop(file_sigs);
1036    mem_mark("after register");
1037    #[cfg(feature = "memprof")]
1038    mem_stats_registry(&registry);
1039
1040    // Pass 2: re-parse + type-check + compile per file, streaming. Parse can't
1041    // fail here (pass 1 validated), so a failed re-parse just yields nothing.
1042    info!("[stream] Pass 2: type-check + codegen (streaming)...");
1043    let per_file = parallel_map(&rs2_files, |path| {
1044        let parsed = match parse_one_file(path, include_tests).read {
1045            Ok(p) => p,
1046            Err(_) => return (Vec::new(), DiagnosticsCollector::new()),
1047        };
1048        let (file, testscript_line) = match parsed.outcome {
1049            ParseOutcome::Parsed {
1050                file,
1051                testscript_line,
1052            } => (file, testscript_line),
1053            ParseOutcome::Error { .. } => return (Vec::new(), DiagnosticsCollector::new()),
1054        };
1055        let mut type_checker = TypeChecker::new(&registry);
1056        type_checker.check_file_with_test_boundary(&file, path, testscript_line);
1057        let mut compiled = Vec::new();
1058        // Compile only when this file type-checks clean (codegen assumes
1059        // well-typed input). On any global error the whole compile aborts below,
1060        // so output produced here would be discarded anyway.
1061        if !type_checker.diagnostics.has_errors() {
1062            let mut compiler = Compiler::new(&registry);
1063            for script in &file.scripts {
1064                if script.trigger == "command" {
1065                    continue;
1066                }
1067                let Some(mut c) =
1068                    compiler.compile_checked(script, path, &mut type_checker.diagnostics)
1069                else {
1070                    continue;
1071                };
1072                c.source_path = parsed.cache_key.clone();
1073                c.shrink_to_fit();
1074                compiled.push(c);
1075            }
1076        }
1077        (compiled, type_checker.diagnostics)
1078        // `file` (the AST) is dropped here.
1079    });
1080
1081    let mut compiled_scripts: Vec<CompiledScript> = Vec::new();
1082    for (compiled, diags) in per_file {
1083        compiled_scripts.extend(compiled);
1084        diagnostics.merge(diags);
1085    }
1086    if diagnostics.has_errors() {
1087        diagnostics.print_all();
1088        return Err(Box::new(error::CompilerError::TypeError(
1089            "Type checking failed".to_string(),
1090        )));
1091    }
1092    // Stable sort by id matches the batch path (file order preserved among
1093    // equal ids → identical writer last-write-wins behaviour).
1094    compiled_scripts.sort_by_key(|s| s.id);
1095    compiled_scripts.shrink_to_fit();
1096    info!("  Generated {} compiled scripts", compiled_scripts.len());
1097    mem_mark("after codegen");
1098    #[cfg(feature = "memprof")]
1099    mem_stats_compiled(&compiled_scripts);
1100
1101    check_pointers(
1102        lint,
1103        &mut diagnostics,
1104        SourceProvider::Lazy(&source_paths),
1105        &registry,
1106        &compiled_scripts,
1107    );
1108    mem_mark("after pointercheck");
1109
1110    Ok((compiled_scripts, diagnostics))
1111}
1112
1113fn assign_script_ids<'a>(
1114    registry: &mut SymbolRegistry,
1115    ids: &mut ScriptIds,
1116    scripts: impl Iterator<Item = (&'a str, &'a str)>,
1117) {
1118    let mut next = ids.values().max().map_or(0, |id| id + 1);
1119    let mut scripts: Vec<_> = scripts
1120        .filter(|(trigger, _)| *trigger != "command")
1121        .collect();
1122    scripts.sort_unstable();
1123    scripts.dedup();
1124    for (trigger, name) in scripts {
1125        let id = *ids.entry(format!("{trigger}:{name}")).or_insert_with(|| {
1126            let id = next;
1127            next += 1;
1128            id
1129        });
1130        registry.set_script_id(trigger, name, id);
1131    }
1132}
1133
1134/// `register_scripts` for the streaming path: identical logic, driven by the
1135/// extracted signatures instead of the (already-dropped) ASTs.
1136fn register_scripts_from_sigs(
1137    registry: &mut SymbolRegistry,
1138    ids: &mut ScriptIds,
1139    diagnostics: &mut DiagnosticsCollector,
1140    file_sigs: &[FileSigs],
1141) {
1142    assign_script_ids(
1143        registry,
1144        ids,
1145        file_sigs.iter().flat_map(|file| {
1146            file.sigs
1147                .iter()
1148                .map(|sig| (sig.trigger.as_str(), sig.name.as_str()))
1149        }),
1150    );
1151
1152    use crate::diagnostic_messages as msg;
1153    let mut registered_scripts: std::collections::HashSet<String> =
1154        std::collections::HashSet::new();
1155
1156    for fs in file_sigs.iter() {
1157        for sig in &fs.sigs {
1158            Arc::make_mut(&mut registry.visibility).declare_script(
1159                &fs.path,
1160                &sig.trigger,
1161                &sig.name,
1162            );
1163            if sig.trigger == "command" {
1164                continue;
1165            }
1166
1167            if !trigger_table::is_valid_trigger(&sig.trigger) {
1168                diagnostics.warning(
1169                    fs.path.clone(),
1170                    sig.line,
1171                    0,
1172                    msg::fmt(msg::SCRIPT_TRIGGER_INVALID, &[&sig.trigger]),
1173                    crate::diagnostics::Phase::SymbolRegistration,
1174                );
1175            }
1176
1177            let key = format!("{}:{}", sig.trigger, sig.name);
1178            if !registered_scripts.insert(key) {
1179                diagnostics.warning(
1180                    fs.path.clone(),
1181                    sig.line,
1182                    0,
1183                    msg::fmt(msg::SCRIPT_REDECLARATION, &[&sig.trigger, &sig.name]),
1184                    crate::diagnostics::Phase::SymbolRegistration,
1185                );
1186            }
1187
1188            if !sig.return_types.is_empty() && !trigger_table::allows_returns(&sig.trigger) {
1189                diagnostics.warning(
1190                    fs.path.clone(),
1191                    sig.line,
1192                    0,
1193                    msg::fmt(msg::SCRIPT_TRIGGER_NO_RETURNS, &[&sig.trigger]),
1194                    diagnostics::Phase::SymbolRegistration,
1195                );
1196            }
1197
1198            registry.register_script(
1199                sig.name.clone(),
1200                sig.trigger.clone(),
1201                sig.param_types.clone(),
1202                sig.return_types.clone(),
1203            );
1204
1205            if let Some(ts_line) = fs.testscript_line
1206                && sig.line >= ts_line
1207            {
1208                registry.mark_test_script(&sig.trigger, &sig.name);
1209            }
1210        }
1211    }
1212}
1213
1214fn check_pointers(
1215    lint: bool,
1216    diagnostics: &mut DiagnosticsCollector,
1217    source: SourceProvider,
1218    registry: &SymbolRegistry,
1219    compiled_scripts: &[CompiledScript],
1220) {
1221    {
1222        use crate::pointer_checker::{PointerChecker, ScriptSource};
1223        // Validate scripts across threads: each chunk gets its own checker that
1224        // shares the read-only scripts/registry/source cache but keeps its own
1225        // (purely memoizing) caches. Diagnostics are merged in script order, so
1226        // the result is identical to a single-threaded run.
1227        //
1228        // The read-only command/script lookup tables are built ONCE here and
1229        // shared (`Arc`) across every chunk checker, instead of each thread
1230        // rebuilding the full ~700-command pointer table + per-script reverse
1231        // maps. `names` is just the script order, so derive it directly rather
1232        // than spinning up a throwaway checker for `script_names()`.
1233        let shared = PointerChecker::build_shared(compiled_scripts, registry, false);
1234        let names: Vec<String> = compiled_scripts.iter().map(|s| s.name.clone()).collect();
1235        // Each chunk's checker retains its CFG caches until the chunk finishes,
1236        // so concurrency multiplies peak scratch. RUNEC_PTR_CHUNKS caps the
1237        // concurrent chunk count to trade pointer-check speed for memory; the
1238        // result is chunk-count-independent, so diagnostics stay byte-identical.
1239        let max_chunks = std::env::var("RUNEC_PTR_CHUNKS")
1240            .ok()
1241            .and_then(|s| s.parse::<usize>().ok())
1242            .filter(|&n| n > 0)
1243            .unwrap_or(usize::MAX);
1244        let chunk_diags = parallel_chunks_capped(&names, max_chunks, |chunk| {
1245            let mut checker = PointerChecker::with_shared(
1246                ScriptSource::Resident(compiled_scripts),
1247                registry,
1248                shared.clone(),
1249            );
1250            checker.set_source_cache(source);
1251            checker.validate_names(chunk)
1252        });
1253        let mut pointer_diags = DiagnosticsCollector::new();
1254        for d in chunk_diags {
1255            pointer_diags.merge(d);
1256        }
1257        let ptr_warnings = pointer_diags.warning_count();
1258        if ptr_warnings > 0 {
1259            info!("  Pointer check: {} warning(s)", ptr_warnings);
1260        }
1261        diagnostics.merge(pointer_diags);
1262    }
1263    #[cfg(feature = "memprof")]
1264    crate::pointer_checker::cfg_trace_report();
1265
1266    // Phase 4.6: Lint passes (optional)
1267    if lint {
1268        info!("Running lint passes (unused locals, unreachable code)...");
1269        let lint_diags = lints::run_lints(compiled_scripts, Some(source));
1270        let lint_warnings = lint_diags.warning_count();
1271        if lint_warnings > 0 {
1272            info!("  Lints: {} warning(s)", lint_warnings);
1273        }
1274        diagnostics.merge(lint_diags);
1275    }
1276}
1277
1278fn codegen(
1279    all_files: &[ParsedFile],
1280    registry: &SymbolRegistry,
1281    diagnostics: &mut DiagnosticsCollector,
1282) -> Vec<CompiledScript> {
1283    // Each file compiles independently against the read-only registry, so fan
1284    // the files out across threads. Output order doesn't matter — the caller
1285    // sorts compiled scripts by id — so a simple flatten of the per-file results
1286    // is enough.
1287    let mut compiled = Vec::new();
1288    for (scripts, errors) in parallel_map(all_files, |pf| compile_file(pf, registry)) {
1289        compiled.extend(scripts);
1290        diagnostics.merge(errors);
1291    }
1292    compiled
1293}
1294
1295/// Compile every (non-`command`) script in one file. Creating a `Compiler` per
1296/// file is cheap now that it only borrows the registry, and keeps the codegen
1297/// scratch state thread-local.
1298fn compile_file(
1299    pf: &ParsedFile,
1300    registry: &SymbolRegistry,
1301) -> (Vec<CompiledScript>, DiagnosticsCollector) {
1302    let mut diagnostics = DiagnosticsCollector::new();
1303    let mut scripts =
1304        crate::compiler::compile_file(&pf.file, &pf.path, registry, false, &mut diagnostics);
1305    for script in &mut scripts {
1306        script.source_path = pf.source_path.clone();
1307        script.shrink_to_fit();
1308    }
1309    (scripts, diagnostics)
1310}
1311
1312fn run_type_checker(
1313    diagnostics: &mut DiagnosticsCollector,
1314    all_files: &[ParsedFile],
1315    registry: &SymbolRegistry,
1316) {
1317    // Each file type-checks independently against the read-only registry, so
1318    // check chunks of files in parallel; per-chunk diagnostics are merged in
1319    // file order, matching a single-threaded run.
1320    let chunk_diags = parallel_chunks(all_files, |chunk| {
1321        let mut type_checker = TypeChecker::new(registry);
1322        for pf in chunk {
1323            type_checker.check_file_with_test_boundary(&pf.file, &pf.path, pf.testscript_line);
1324        }
1325        type_checker.diagnostics
1326    });
1327    for d in chunk_diags {
1328        diagnostics.merge(d);
1329    }
1330}
1331
1332/// Resolve the pack directory: explicit `--pack`, else the nearest ancestor's `pack/`, else
1333/// `content/pack` up the tree.
1334fn resolve_pack_dir(scripts_dir: &Path, pack_dir: Option<&Path>) -> Option<PathBuf> {
1335    pack_dir
1336        .map(PathBuf::from)
1337        .filter(|p| p.exists())
1338        .or_else(|| {
1339            scripts_dir.ancestors().find_map(|dir| {
1340                let pack = dir.join("pack");
1341                pack.is_dir().then_some(pack)
1342            })
1343        })
1344        .or_else(|| {
1345            scripts_dir
1346                .ancestors()
1347                .nth(3)
1348                .map(|p| p.join("content/pack"))
1349                .filter(|p| p.exists())
1350        })
1351}
1352
1353/// Build the source-independent registry before registering parsed scripts.
1354fn build_symbol_registry(scripts_dir: &Path, resolved_pack_dir: Option<&Path>) -> SymbolRegistry {
1355    // Visibility reads every source and config, and only its indexing needs
1356    // the registry built meanwhile.
1357    let visibility = {
1358        let dir = scripts_dir.to_path_buf();
1359        thread::spawn(move || sources::Visibility::scan(&dir))
1360    };
1361    let mut registry = SymbolRegistry::new();
1362
1363    if let Some(pd) = resolved_pack_dir {
1364        info!("  Loading packs from {}", pd.display());
1365        symloader::load_packs(&mut registry, pd);
1366        info!(
1367            "  Loaded {} commands, {} game vars",
1368            registry.commands.len(),
1369            registry.game_vars.len(),
1370        );
1371    } else {
1372        info!("  No pack directory found - symbol resolution will be limited");
1373    }
1374
1375    // Walk the scripts tree once and let the constant/game-var/dbtable loaders
1376    // filter this shared list by extension, instead of each re-walking the tree
1377    // (previously 6 separate recursive traversals over the same directories).
1378    let mut config_files: Vec<PathBuf> = Vec::new();
1379    config_files.extend(sources::config_files(scripts_dir));
1380
1381    symloader::load_constant_files(&mut registry, &config_files);
1382
1383    info!(
1384        "  Loaded {} constants from .constant files",
1385        registry.constants.len()
1386    );
1387
1388    symloader::load_game_var_types(&mut registry, &config_files);
1389
1390    if let Some(pd) = resolved_pack_dir
1391        && let Err(error) = clientscript::load_clan_settings(&mut registry, &config_files, pd)
1392    {
1393        warn!("  Clan settings not loaded: {error}");
1394    }
1395
1396    if let Some(pd) = resolved_pack_dir {
1397        let dbtable_ids = symloader::load_dbtable_pack(&pd.join("dbtable.pack"));
1398        if !dbtable_ids.is_empty() {
1399            symloader::load_dbtable_configs(&mut registry, &config_files, &dbtable_ids);
1400            info!(
1401                "  Registered {} dbcolumn compound IDs",
1402                registry.dbcolumn_types.len()
1403            );
1404        }
1405    }
1406
1407    let engine_rs2 = scripts_dir.join("engine.rs2");
1408
1409    if engine_rs2.exists() {
1410        symloader::load_engine_command_params(&mut registry, &engine_rs2);
1411    } else {
1412        if let Ok(entries) = std::fs::read_dir(scripts_dir) {
1413            for entry in entries.flatten() {
1414                let candidate = entry.path().join("engine.rs2");
1415                if candidate.exists() {
1416                    crate::symloader::load_engine_command_params(&mut registry, &candidate);
1417                    break;
1418                }
1419            }
1420        }
1421    }
1422
1423    let client_dir = if scripts_dir
1424        .file_name()
1425        .is_some_and(|name| name == "runescript")
1426    {
1427        scripts_dir.with_file_name("clientscript")
1428    } else {
1429        scripts_dir.join("clientscript")
1430    };
1431    if client_dir.is_dir() {
1432        symloader::load_clientscript_signatures(&mut registry, &client_dir);
1433    }
1434
1435    symloader::patch_command_return_types(&mut registry);
1436
1437    let mut visibility = visibility
1438        .join()
1439        .unwrap_or_else(|panic| panic::resume_unwind(panic));
1440    visibility.index_registry(&registry);
1441    registry.visibility = Arc::new(visibility);
1442
1443    registry
1444}
1445
1446/// Assign IDs and register the parsed scripts, reporting declaration errors.
1447fn register_scripts(
1448    registry: &mut SymbolRegistry,
1449    ids: &mut ScriptIds,
1450    diagnostics: &mut DiagnosticsCollector,
1451    all_files: &[ParsedFile],
1452) {
1453    assign_script_ids(
1454        registry,
1455        ids,
1456        all_files.iter().flat_map(|file| {
1457            file.file
1458                .scripts
1459                .iter()
1460                .map(|script| (script.trigger.as_str(), script.name.as_str()))
1461        }),
1462    );
1463
1464    {
1465        use crate::diagnostic_messages as msg;
1466
1467        let mut registered_scripts: std::collections::HashSet<String> =
1468            std::collections::HashSet::new();
1469
1470        for pf in all_files.iter() {
1471            Arc::make_mut(&mut registry.visibility).declare_source(&pf.path, &pf.file);
1472            for script in &pf.file.scripts {
1473                if script.trigger == "command" {
1474                    continue;
1475                }
1476
1477                if !trigger_table::is_valid_trigger(&script.trigger) {
1478                    diagnostics.warning(
1479                        pf.path.clone(),
1480                        script.line,
1481                        0,
1482                        msg::fmt(msg::SCRIPT_TRIGGER_INVALID, &[&script.trigger]),
1483                        crate::diagnostics::Phase::SymbolRegistration,
1484                    );
1485                }
1486
1487                let key = format!("{}:{}", script.trigger, script.name);
1488                if !registered_scripts.insert(key.clone()) {
1489                    diagnostics.warning(
1490                        pf.path.clone(),
1491                        script.line,
1492                        0,
1493                        msg::fmt(msg::SCRIPT_REDECLARATION, &[&script.trigger, &script.name]),
1494                        crate::diagnostics::Phase::SymbolRegistration,
1495                    );
1496                }
1497
1498                if !script.return_types.is_empty()
1499                    && !trigger_table::allows_returns(&script.trigger)
1500                {
1501                    diagnostics.warning(
1502                        pf.path.clone(),
1503                        script.line,
1504                        0,
1505                        msg::fmt(msg::SCRIPT_TRIGGER_NO_RETURNS, &[&script.trigger]),
1506                        diagnostics::Phase::SymbolRegistration,
1507                    );
1508                }
1509
1510                let param_types: Vec<types::Type> =
1511                    script.params.iter().map(|p| p.param_type).collect();
1512                registry.register_script(
1513                    script.name.clone(),
1514                    script.trigger.clone(),
1515                    param_types,
1516                    script.return_types.clone(),
1517                );
1518
1519                if let Some(ts_line) = pf.testscript_line
1520                    && script.line >= ts_line
1521                {
1522                    registry.mark_test_script(&script.trigger, &script.name);
1523                }
1524            }
1525        }
1526    }
1527}
1528
1529// ── Streaming recompile pipeline (prototype, gated by RUNEC_RECOMPILE) ────────
1530// Never holds all compiled scripts. Pass 2 only type-checks; the pointer check
1531// re-compiles each script on demand (bounded cache — measured ~1.3x re-parse on
1532// rs-majula, the call graph has high file locality); output is produced by
1533// re-compiling file-by-file into a by-id buffer. Output + diagnostics are
1534// byte-identical to the batch path (deterministic re-compile, same id order).
1535
1536/// Re-compiles scripts on demand for the pointer checker, caching `Rc`s with a
1537/// bounded FIFO so the full compiled set is never resident. A cache miss
1538/// re-parses the script's file and compiles all its scripts at once (siblings
1539/// share the parse), keying each by its full `[trigger,name]` to its index.
1540struct RecompileStore<'a> {
1541    idx_to_path: Vec<PathBuf>,
1542    /// file path -> its script idxs in declaration order (command-skipped).
1543    /// Position-based so duplicate `[trigger,name]` declarations each map to the
1544    /// correct distinct idx.
1545    file_to_idxs: HashMap<PathBuf, Vec<usize>>,
1546    registry: &'a SymbolRegistry,
1547    include_tests: bool,
1548    cache: HashMap<usize, std::rc::Rc<CompiledScript>>,
1549    order: std::collections::VecDeque<usize>,
1550    cap: usize,
1551}
1552
1553impl RecompileStore<'_> {
1554    fn get(&mut self, idx: usize) -> std::rc::Rc<CompiledScript> {
1555        if let Some(rc) = self.cache.get(&idx) {
1556            return std::rc::Rc::clone(rc);
1557        }
1558        let path = self.idx_to_path[idx].clone();
1559        let parsed = parse_one_file(&path, self.include_tests);
1560        if let Ok(ps) = parsed.read
1561            && let ParseOutcome::Parsed { file, .. } = ps.outcome
1562        {
1563            // The j-th non-command declaration in this file maps to idxs[j],
1564            // matching how the metas were built (same parse, same skip rule).
1565            let idxs = self.file_to_idxs.get(&path).cloned().unwrap_or_default();
1566            let mut compiler = Compiler::new(self.registry);
1567            let mut j = 0usize;
1568            for script in &file.scripts {
1569                if script.trigger == "command" {
1570                    continue;
1571                }
1572                if let Some(&i) = idxs.get(j) {
1573                    let mut c = compiler.compile_script(script);
1574                    c.source_path = ps.cache_key.clone();
1575                    self.insert(i, std::rc::Rc::new(c));
1576                }
1577                j += 1;
1578            }
1579        }
1580        std::rc::Rc::clone(
1581            self.cache
1582                .get(&idx)
1583                .expect("recompile store: script not produced by its file"),
1584        )
1585    }
1586
1587    fn insert(&mut self, idx: usize, rc: std::rc::Rc<CompiledScript>) {
1588        if self.cache.insert(idx, rc).is_none() {
1589            self.order.push_back(idx);
1590            while self.order.len() > self.cap {
1591                if let Some(old) = self.order.pop_front()
1592                    && old != idx
1593                {
1594                    self.cache.remove(&old);
1595                }
1596            }
1597        }
1598    }
1599}
1600
1601/// Per-slot encoded output blobs (indexed by script id) plus accumulated
1602/// diagnostics — what the recompile core hands back before the bytes are written
1603/// to disk or assembled in memory.
1604type RecompiledOutput = (Vec<Vec<u8>>, DiagnosticsCollector);
1605
1606/// Core of the recompile (low-memory) pipeline: runs every phase re-compiling
1607/// scripts on demand — never holding all compiled bytecode at once — and returns
1608/// the per-slot encoded output blobs plus diagnostics, without writing or
1609/// printing. The disk and in-memory entry points (`compile_recompile`,
1610/// `compile_recompile_memory`) wrap this. On a phase error it prints diagnostics
1611/// and returns `Err`, matching the batch path.
1612fn recompile_to_encoded(
1613    scripts_dir: &Path,
1614    pack_dir: Option<&Path>,
1615    lint: bool,
1616    include_tests: bool,
1617    ids: &mut ScriptIds,
1618) -> Result<RecompiledOutput, Box<dyn Error>> {
1619    let source_dir = sources::target_dir(scripts_dir, sources::Scope::Rune);
1620    let scripts_dir = source_dir.as_path();
1621    use crate::pointer_checker::{PointerChecker, ScriptSource};
1622
1623    let mut rs2_files: Vec<PathBuf> = get_rs2_files(scripts_dir, "rs2")?;
1624    rs2_files.sort();
1625    info!("Found {} script files", rs2_files.len());
1626    mem_mark("startup");
1627    let mut diagnostics = DiagnosticsCollector::new();
1628
1629    // Pass 1: signatures only (ASTs dropped in-worker); keep paths for lazy source reads.
1630    info!("[recompile] Pass 1: parsing for signatures...");
1631    let sig_results = parallel_map(&rs2_files, |path| parse_one_file_sigs(path, include_tests));
1632    let mut source_paths: HashMap<String, PathBuf> = HashMap::new();
1633    let mut file_sigs: Vec<FileSigs> = Vec::new();
1634    for r in sig_results {
1635        let (cache_key, outcome) = match r.read {
1636            Ok(t) => t,
1637            Err(e) => return Err(Box::new(e)),
1638        };
1639        source_paths.insert(cache_key, r.path.clone());
1640        match outcome {
1641            SigOutcome::Parsed {
1642                sigs,
1643                testscript_line,
1644            } => file_sigs.push(FileSigs {
1645                path: r.path,
1646                testscript_line,
1647                sigs,
1648            }),
1649            SigOutcome::Error {
1650                line,
1651                position,
1652                message,
1653                phase,
1654            } => diagnostics.error(r.path, line, position, message, phase),
1655        }
1656    }
1657    if diagnostics.has_errors() {
1658        diagnostics.print_all();
1659        return Err(Box::new(error::CompilerError::TypeError(
1660            "Parsing failed".to_string(),
1661        )));
1662    }
1663    mem_mark("after parse");
1664
1665    // Registration.
1666    info!("[recompile] Registering scripts and building the symbol registry...");
1667    let resolved_pack_dir = resolve_pack_dir(scripts_dir, pack_dir);
1668    let mut registry = build_symbol_registry(scripts_dir, resolved_pack_dir.as_deref());
1669    register_scripts_from_sigs(&mut registry, ids, &mut diagnostics, &file_sigs);
1670    info!("  Registered {} scripts", registry.scripts.len());
1671    registry.drop_registration_scratch();
1672    for fs in &file_sigs {
1673        for sig in &fs.sigs {
1674            if let Some(msg) =
1675                Compiler::validate_trigger_subject(&sig.trigger, &sig.name, &registry)
1676            {
1677                diagnostics.warning(
1678                    fs.path.clone(),
1679                    sig.line,
1680                    0,
1681                    msg,
1682                    diagnostics::Phase::SymbolRegistration,
1683                );
1684            }
1685        }
1686    }
1687    mem_mark("after register");
1688
1689    // Pass 2: type-check only (re-parse, no compiled scripts retained).
1690    info!("[recompile] Pass 2: type-check (streaming, no bytecode held)...");
1691    let tc_diags = parallel_map(&rs2_files, |path| {
1692        let parsed = match parse_one_file(path, include_tests).read {
1693            Ok(p) => p,
1694            Err(_) => return DiagnosticsCollector::new(),
1695        };
1696        let (file, testscript_line) = match parsed.outcome {
1697            ParseOutcome::Parsed {
1698                file,
1699                testscript_line,
1700            } => (file, testscript_line),
1701            ParseOutcome::Error { .. } => return DiagnosticsCollector::new(),
1702        };
1703        let mut type_checker = TypeChecker::new(&registry);
1704        type_checker.check_file_with_test_boundary(&file, path, testscript_line);
1705        if !type_checker.diagnostics.has_errors() {
1706            let mut compiler = Compiler::new(&registry);
1707            for script in &file.scripts {
1708                if script.trigger != "command" {
1709                    compiler.compile_checked(script, path, &mut type_checker.diagnostics);
1710                }
1711            }
1712        }
1713        type_checker.diagnostics
1714    });
1715    for d in tc_diags {
1716        diagnostics.merge(d);
1717    }
1718    if diagnostics.has_errors() {
1719        diagnostics.print_all();
1720        return Err(Box::new(error::CompilerError::TypeError(
1721            "Type checking failed".to_string(),
1722        )));
1723    }
1724    mem_mark("after typecheck");
1725
1726    // Build script metas in the same idx order as the batch path: file order
1727    // (file_sigs / declaration order, command-skipped), then a stable sort by id.
1728    // ids + names come from the registry — no bytecode needed.
1729    let mut pre: Vec<(i32, String, PathBuf)> = Vec::new();
1730    for fs in &file_sigs {
1731        for sig in &fs.sigs {
1732            if sig.trigger == "command" {
1733                continue;
1734            }
1735            let id = registry
1736                .script_id_for_trigger(&sig.trigger, &sig.name)
1737                .or_else(|| registry.script_id(&sig.name))
1738                .unwrap_or(-1);
1739            pre.push((
1740                id,
1741                format!("[{},{}]", sig.trigger, sig.name),
1742                fs.path.clone(),
1743            ));
1744        }
1745    }
1746    let n = pre.len();
1747    // Stable sort indices by id → the batch's idx order (file/decl pre-order
1748    // preserved among equal ids). `inv` maps a pre-order position back to its idx.
1749    let mut order: Vec<usize> = (0..n).collect();
1750    order.sort_by_key(|&p| pre[p].0);
1751    let idents: Vec<(i32, String)> = order
1752        .iter()
1753        .map(|&p| (pre[p].0, pre[p].1.clone()))
1754        .collect();
1755    let idx_to_path: Vec<PathBuf> = order.iter().map(|&p| pre[p].2.clone()).collect();
1756    let mut inv = vec![0usize; n];
1757    for (idx, &p) in order.iter().enumerate() {
1758        inv[p] = idx;
1759    }
1760    let mut file_to_idxs: HashMap<PathBuf, Vec<usize>> = HashMap::new();
1761    for (p, m) in pre.iter().enumerate() {
1762        file_to_idxs.entry(m.2.clone()).or_default().push(inv[p]);
1763    }
1764    let max_id = pre.iter().map(|m| m.0).max().unwrap_or(-1);
1765    let slot_count = if max_id < 0 { 0 } else { (max_id + 1) as usize };
1766    let len = n;
1767    drop(pre);
1768
1769    // Pointer check via on-demand re-compile (single-threaded: the store getter
1770    // is `FnMut`/`!Sync`). Diagnostics come out in id order, as in the batch path.
1771    info!("[recompile] Pointer check (re-compile on demand)...");
1772    let shared = PointerChecker::build_shared_idents(&idents, &registry);
1773    {
1774        // Cache cap must exceed the largest file's script count, or loading that
1775        // file would evict its own just-compiled scripts. A small margin over
1776        // that keeps cross-file locality high (measured ~1.1-1.3x re-parse).
1777        let cap = file_to_idxs
1778            .values()
1779            .map(|v| v.len())
1780            .max()
1781            .unwrap_or(0)
1782            .max(128);
1783        let mut store = RecompileStore {
1784            idx_to_path,
1785            file_to_idxs,
1786            registry: &registry,
1787            include_tests,
1788            cache: HashMap::new(),
1789            order: std::collections::VecDeque::new(),
1790            cap,
1791        };
1792        let getter: Box<dyn FnMut(usize) -> std::rc::Rc<CompiledScript>> =
1793            Box::new(move |idx| store.get(idx));
1794        let scripts = ScriptSource::Recompile {
1795            len,
1796            get: std::cell::RefCell::new(getter),
1797        };
1798        let mut checker = PointerChecker::with_shared(scripts, &registry, shared);
1799        checker.set_source_cache(SourceProvider::Lazy(&source_paths));
1800        let names: Vec<String> = idents.iter().map(|(_, n)| n.clone()).collect();
1801        let ptr_diags = checker.validate_names(&names);
1802        let w = ptr_diags.warning_count();
1803        if w > 0 {
1804            info!("  Pointer check: {} warning(s)", w);
1805        }
1806        diagnostics.merge(ptr_diags);
1807    }
1808    mem_mark("after pointercheck");
1809
1810    // Output: re-compile each file once, encode each script into its id slot,
1811    // optionally lint — never holding all compiled scripts at once. The caller
1812    // writes the blobs to disk or assembles them in memory.
1813    info!("[recompile] Re-compiling + encoding output...");
1814    let mut encoded: Vec<Vec<u8>> = vec![Vec::new(); slot_count];
1815    for path in &rs2_files {
1816        let parsed = match parse_one_file(path, include_tests).read {
1817            Ok(p) => p,
1818            Err(_) => continue,
1819        };
1820        let file = match parsed.outcome {
1821            ParseOutcome::Parsed { file, .. } => file,
1822            ParseOutcome::Error { .. } => continue,
1823        };
1824        let mut compiler = Compiler::new(&registry);
1825        let mut file_scripts: Vec<CompiledScript> = Vec::new();
1826        for script in &file.scripts {
1827            if script.trigger == "command" {
1828                continue;
1829            }
1830            let Some(mut c) = compiler.compile_checked(script, path, &mut diagnostics) else {
1831                continue;
1832            };
1833            c.source_path = parsed.cache_key.clone();
1834            file_scripts.push(c);
1835        }
1836        for c in &file_scripts {
1837            if c.id >= 0 && (c.id as usize) < slot_count {
1838                encoded[c.id as usize] = crate::writer::encode_script(c);
1839            }
1840        }
1841        if lint {
1842            diagnostics.merge(lints::run_lints(
1843                &file_scripts,
1844                Some(SourceProvider::Lazy(&source_paths)),
1845            ));
1846        }
1847    }
1848    mem_mark("after codegen");
1849    Ok((encoded, diagnostics))
1850}
1851
1852/// Full compile via the recompile (low-memory) pipeline: writes `script.dat` /
1853/// `script.idx` to `output_dir`, then prints diagnostics.
1854fn compile_recompile(
1855    scripts_dir: &Path,
1856    pack_dir: Option<&Path>,
1857    output_dir: &Path,
1858    lint: bool,
1859) -> Result<(), Box<dyn Error>> {
1860    let (encoded, diagnostics) =
1861        recompile_to_encoded(scripts_dir, pack_dir, lint, false, &mut ScriptIds::new())?;
1862    info!(
1863        "Writing output to {}...",
1864        output_dir.display().to_string().replace('\\', "/")
1865    );
1866    ScriptWriter::new(output_dir.display().to_string()).write_encoded(&encoded)?;
1867    diagnostics.print_all();
1868    info!("Compilation complete!");
1869    Ok(())
1870}
1871
1872/// In-memory compile via the recompile (low-memory) pipeline: returns the
1873/// `(script.dat, script.idx)` bytes, then prints diagnostics. Byte-identical to
1874/// the in-memory batch path (`compile_memory*`).
1875fn compile_recompile_memory(
1876    scripts_dir: &Path,
1877    pack_dir: Option<&Path>,
1878    lint: bool,
1879    include_tests: bool,
1880    ids: &mut ScriptIds,
1881) -> Result<(Vec<u8>, Vec<u8>), Box<dyn Error>> {
1882    let (encoded, diagnostics) =
1883        recompile_to_encoded(scripts_dir, pack_dir, lint, include_tests, ids)?;
1884    info!("Writing compiled output to memory...");
1885    let out = ScriptWriter::new("".into()).build_encoded(&encoded);
1886    diagnostics.print_all();
1887    info!("Compilation complete!");
1888    Ok(out)
1889}
1890
1891pub(crate) fn find_testscript_line(source: &str) -> Option<usize> {
1892    for (i, line) in source.lines().enumerate() {
1893        if line.trim() == "#testscript" {
1894            return Some(i + 1);
1895        }
1896    }
1897    None
1898}
1899
1900fn truncate_at_line(source: &str, line_no: usize) -> String {
1901    source
1902        .lines()
1903        .take(line_no - 1)
1904        .collect::<Vec<_>>()
1905        .join("\n")
1906}
1907
1908fn strip_testscript_annotation(source: &str) -> String {
1909    source
1910        .lines()
1911        .map(|line| {
1912            if line.trim() == "#testscript" {
1913                ""
1914            } else {
1915                line
1916            }
1917        })
1918        .collect::<Vec<_>>()
1919        .join("\n")
1920}
1921
1922fn get_rs2_files(dir: &Path, ext: &str) -> Result<Vec<PathBuf>, Box<dyn Error>> {
1923    Ok(sources::script_files(dir, ext)?)
1924}