Skip to main content

runec/
filesystem.rs

1use crate::bytecode::CompiledScript;
2use crate::clientscript::{self, format::Format};
3use crate::compiler::Compiler;
4use crate::diagnostics::DiagnosticsCollector;
5use crate::lexer::Lexer;
6#[cfg(feature = "memprof")]
7use crate::memprof;
8use crate::parallel::{
9    parallel_chunks, parallel_chunks_capped, parallel_map, parallel_map_balanced,
10};
11use crate::parser::{Parser, ScriptFile};
12use crate::project::{Options, Project, SourceText, Target};
13use crate::sources::Scope;
14use crate::symbol::SymbolRegistry;
15use crate::typechecker::TypeChecker;
16use crate::writer::ScriptWriter;
17use crate::{diagnostics, error, lints, sources, symloader, trigger_table, types};
18use std::collections::HashMap;
19use std::error::Error;
20use std::fmt::Write;
21use std::fs;
22use std::panic;
23use std::path::{Path, PathBuf};
24use std::sync::Arc;
25use std::thread;
26use tracing::{info, warn};
27
28/// Build an editor project using the legacy directory conventions. No ID file
29/// is generated or modified; all further editing happens in memory.
30pub fn editor_project(
31    source: &Path,
32    pack: Option<&Path>,
33    client: bool,
34) -> Result<Project, Box<dyn Error>> {
35    let scope = if client { Scope::Client } else { Scope::Rune };
36    let dir = sources::target_dir(source, scope);
37    let pack = resolve_pack_dir(&dir, pack);
38    let mut registry = if client {
39        clientscript::registry(
40            &dir,
41            pack.as_deref()
42                .ok_or("Client opcode definitions require --pack")?,
43        )?
44    } else {
45        build_symbol_registry(&dir, pack.as_deref())
46    };
47    let target = if client {
48        registry.preloaded_script_ids.clear();
49        for (key, id) in clientscript::script_ids(pack.as_deref().unwrap())? {
50            let (trigger, name) = key
51                .trim_start_matches('[')
52                .trim_end_matches(']')
53                .split_once(',')
54                .ok_or("Invalid client ID")?;
55            registry.set_script_id(trigger, name, id);
56        }
57        Target::Client(Format::load(
58            &pack.as_ref().unwrap().join("cs2_command.pack"),
59        )?)
60    } else {
61        if let Some(pack) = &pack {
62            symloader::load_script_ids(&mut registry, pack);
63        }
64        Target::Rune
65    };
66    let mut project = Project::new(
67        target,
68        registry,
69        Options {
70            include_tests: true,
71            skip_shared_lints: client
72                && dir
73                    .parent()
74                    .is_some_and(|parent| parent.join("runescript").is_dir()),
75            ..Options::default()
76        },
77    );
78    let paths = sources::script_files(&dir, if client { "cs2" } else { "rs2" })?;
79    let files = read_sources(&paths, |path| {
80        if path.extension().is_some_and(|ext| ext == "gs2") {
81            Scope::Global
82        } else {
83            scope
84        }
85    })?;
86    for id in project.open_all(files) {
87        if client {
88            let ids = project
89                .document(id)
90                .and_then(|doc| doc.syntax())
91                .into_iter()
92                .flat_map(|file| &file.scripts)
93                .filter_map(|script| {
94                    script
95                        .name
96                        .strip_prefix("clientscript_")
97                        .and_then(|id| id.parse::<i32>().ok())
98                        .map(|id| (script.trigger.clone(), script.name.clone(), id))
99                })
100                .collect::<Vec<_>>();
101            for (trigger, name, id) in ids {
102                project.registry.set_script_id(&trigger, &name, id);
103            }
104        }
105    }
106    if !client {
107        let client_dir = if dir.file_name().is_some_and(|name| name == "runescript") {
108            dir.with_file_name("clientscript")
109        } else {
110            dir.join("clientscript")
111        };
112        if client_dir.is_dir() {
113            let paths = sources::script_files(&client_dir, "cs2")?
114                .into_iter()
115                .filter(|path| {
116                    path.extension().is_some_and(|ext| ext == "cs2")
117                        && path.file_name().is_some_and(|name| name != "engine.cs2")
118                })
119                .collect::<Vec<_>>();
120            project.open_all(read_sources(&paths, |_| Scope::Client)?);
121        }
122    }
123    let mut files = sources::config_files(&dir);
124    if let Some(pack) = &pack {
125        symloader::collect_all_files(pack, &mut files);
126        // Interfaces and defaults live beside the scripts they're for.
127        if let Some(content) = pack.parent() {
128            for (folder, extension) in [("interfaces", "if3"), ("defaults", "defaults")] {
129                let Ok(entries) = fs::read_dir(content.join(folder)) else {
130                    continue;
131                };
132                files.extend(
133                    entries
134                        .flatten()
135                        .map(|entry| entry.path())
136                        .filter(|path| path.extension().is_some_and(|ext| ext == extension)),
137                );
138            }
139        }
140    }
141    files.sort();
142    files.dedup();
143    files.retain(|path| config_kind(path).is_some());
144    let configs = read_sources(&files, |path| {
145        if path
146            .components()
147            .any(|part| part.as_os_str() == "globalscript")
148        {
149            Scope::Global
150        } else {
151            scope
152        }
153    })?;
154    for ((uri, config_scope, version, text), path) in configs.into_iter().zip(&files) {
155        project.open_config(uri, config_scope, version, text, config_kind(path).unwrap());
156    }
157    Ok(project)
158}
159
160/// Read files and name them by URI, in parallel, failing on the first error in
161/// input order.
162fn read_sources(
163    paths: &[PathBuf],
164    scope: impl Fn(&Path) -> Scope + Sync,
165) -> Result<Vec<SourceText>, Box<dyn Error>> {
166    parallel_map_balanced(paths, |path| {
167        let text = fs::read_to_string(path).map_err(|error| error.to_string())?;
168        let uri = file_uri(path).map_err(|error| error.to_string())?;
169        Ok((uri, scope(path), 0, text))
170    })
171    .into_iter()
172    .collect::<Result<Vec<_>, String>>()
173    .map_err(Into::into)
174}
175
176pub(crate) fn config_kind(path: &Path) -> Option<String> {
177    let extension = path.extension()?.to_str()?;
178    if matches!(extension, "pack" | "order" | "hashes") {
179        return Some(format!("pack:{}", path.file_stem()?.to_str()?));
180    }
181    (matches!(extension, "constant" | "if" | "if3" | "defaults" | "anim")
182        || types::Type::from_name(extension).is_some()
183        || extension
184            .strip_prefix("config")
185            .is_some_and(|number| number.parse::<u32>().is_ok()))
186    .then(|| extension.to_string())
187}
188
189pub(crate) fn file_uri(path: &Path) -> Result<String, Box<dyn Error>> {
190    let absolute = fs::canonicalize(path)?
191        .to_string_lossy()
192        .trim_start_matches("\\\\?\\")
193        .replace('\\', "/");
194    let mut uri = String::from(if absolute.starts_with('/') {
195        "file://"
196    } else {
197        "file:///"
198    });
199    for byte in absolute.bytes() {
200        if byte.is_ascii_alphanumeric() || b"/-._~:".contains(&byte) {
201            uri.push(byte as char);
202        } else {
203            write!(uri, "%{byte:02X}")?;
204        }
205    }
206    Ok(uri)
207}
208
209/// How the diagnostic layer fetches a file's source text (by its cache-key path)
210/// when building rustc-style help/suggestions. The batch path keeps every source
211/// in memory; the streaming path keeps only the on-disk paths and re-reads the
212/// handful of files that actually produce a warning — so the ~4 MB of source
213/// text isn't resident through the compile's peak. Cheap to copy (just borrows).
214#[derive(Clone, Copy)]
215pub enum SourceProvider<'a> {
216    /// Sources already resident, keyed by cache key.
217    Eager(&'a HashMap<String, Arc<String>>),
218    /// Cache key → on-disk path; re-read on demand.
219    Lazy(&'a HashMap<String, PathBuf>),
220}
221
222impl SourceProvider<'_> {
223    /// Source text for `key`, if available. Returns an owned `Arc` — a cheap
224    /// refcount bump for `Eager`, a fresh disk read for `Lazy`. Called only while
225    /// emitting diagnostic help, which is rare, so the re-read cost is immaterial.
226    pub fn get(&self, key: &str) -> Option<Arc<String>> {
227        match self {
228            SourceProvider::Eager(m) => m.get(key).cloned(),
229            SourceProvider::Lazy(m) => fs::read_to_string(m.get(key)?).ok().map(Arc::new),
230        }
231    }
232}
233
234/// Log live heap + global peak at a pipeline phase boundary. No-op unless built
235/// with `--features memprof`. Lets us attribute the footprint to each phase.
236#[cfg(feature = "memprof")]
237fn mem_mark(label: &str) {
238    info!(
239        "[mem] {:<22} live {:>7.1} MB | peak {:>7.1} MB | allocs {}",
240        label,
241        memprof::current_bytes() as f64 / 1_048_576.0,
242        memprof::peak_bytes() as f64 / 1_048_576.0,
243        memprof::alloc_count(),
244    );
245}
246#[cfg(not(feature = "memprof"))]
247#[inline]
248fn mem_mark(_label: &str) {}
249
250/// Per-map breakdown of the symbol registry's heap, to see the irreducible
251/// floor for a streaming rebuild. Built only under `--features memprof`.
252/// Accounts three components per map: the hashbrown table (inline `(K,V)`
253/// buckets + 1 control byte each), the key strings, and value-side heap.
254#[cfg(feature = "memprof")]
255fn mem_stats_registry(reg: &crate::symbol::SymbolRegistry) {
256    use crate::symbol::{Symbol, SymbolKind};
257    use crate::types::Type;
258    use std::collections::{HashMap, HashSet};
259    let t = std::mem::size_of::<Type>();
260    let mb = |b: usize| b as f64 / 1_048_576.0;
261
262    // Allocated bucket count: hashbrown keeps `capacity()` ≈ 7/8 of a power-of-two
263    // bucket array; an empty map allocates nothing.
264    fn tbl<K, V>(m: &HashMap<K, V>) -> usize {
265        if m.capacity() == 0 {
266            return 0;
267        }
268        let buckets = (m.capacity() * 8 / 7).next_power_of_two();
269        buckets * (std::mem::size_of::<(K, V)>() + 1)
270    }
271    fn keys_cap<V>(m: &HashMap<String, V>) -> usize {
272        m.keys().map(|k| k.capacity()).sum()
273    }
274    let sym_heap = |s: &Symbol| -> usize {
275        s.name.capacity()
276            + match &s.kind {
277                SymbolKind::Script {
278                    trigger,
279                    param_types,
280                    return_types,
281                    ..
282                } => trigger.capacity() + (param_types.capacity() + return_types.capacity()) * t,
283                SymbolKind::Command {
284                    param_types,
285                    return_types,
286                    ..
287                } => (param_types.capacity() + return_types.capacity()) * t,
288                SymbolKind::GameVar { category, .. } => category.capacity(),
289                SymbolKind::Constant { string_value, .. } => {
290                    string_value.as_ref().map_or(0, |s| s.capacity())
291                }
292                _ => 0,
293            }
294    };
295    let sym_map = |m: &HashMap<String, Symbol>| -> usize {
296        tbl(m) + keys_cap(m) + m.values().map(&sym_heap).sum::<usize>()
297    };
298    let i32_map = |m: &HashMap<String, i32>| -> usize { tbl(m) + keys_cap(m) };
299
300    let mut rows: Vec<(&str, usize, usize)> = vec![
301        (
302            "scripts",
303            reg.scripts.len(),
304            tbl(&reg.scripts) + keys_cap(&reg.scripts),
305        ),
306        ("commands", reg.commands.len(), sym_map(&reg.commands)),
307        ("game_vars", reg.game_vars.len(), sym_map(&reg.game_vars)),
308        ("constants", reg.constants.len(), sym_map(&reg.constants)),
309        (
310            "entity_ids",
311            reg.entity_ids.len(),
312            tbl(&reg.entity_ids)
313                + keys_cap(&reg.entity_ids)
314                + reg
315                    .entity_ids
316                    .values()
317                    .map(|e| e.variants.capacity() * std::mem::size_of::<(Type, i32)>())
318                    .sum::<usize>(),
319        ),
320        (
321            "scripts_by_trigger",
322            reg.scripts_by_trigger.len(),
323            tbl(&reg.scripts_by_trigger)
324                + keys_cap(&reg.scripts_by_trigger)
325                + reg
326                    .scripts_by_trigger
327                    .values()
328                    .map(|s| 16 + sym_heap(s.as_ref()))
329                    .sum::<usize>(),
330        ),
331        (
332            "command_param_types",
333            reg.command_param_types.len(),
334            tbl(&reg.command_param_types)
335                + keys_cap(&reg.command_param_types)
336                + reg
337                    .command_param_types
338                    .values()
339                    .map(|v| v.capacity() * t)
340                    .sum::<usize>(),
341        ),
342        ("script_ids", reg.script_ids.len(), i32_map(&reg.script_ids)),
343        (
344            "proc_script_ids",
345            reg.proc_script_ids.len(),
346            i32_map(&reg.proc_script_ids),
347        ),
348        (
349            "label_script_ids",
350            reg.label_script_ids.len(),
351            i32_map(&reg.label_script_ids),
352        ),
353        (
354            "trigger_script_ids",
355            reg.trigger_script_ids.len(),
356            i32_map(&reg.trigger_script_ids),
357        ),
358        (
359            "preloaded_script_ids",
360            reg.preloaded_script_ids.len(),
361            i32_map(&reg.preloaded_script_ids),
362        ),
363        ("components", reg.components.len(), i32_map(&reg.components)),
364        (
365            "interface_ids",
366            reg.interface_ids.len(),
367            i32_map(&reg.interface_ids),
368        ),
369        ("type_chars", reg.type_chars.len(), i32_map(&reg.type_chars)),
370        (
371            "dbcolumn_types",
372            reg.dbcolumn_types.len(),
373            tbl(&reg.dbcolumn_types) + keys_cap(&reg.dbcolumn_types),
374        ),
375    ];
376    let ts_tbl = {
377        let s: &HashSet<String> = &reg.test_scripts;
378        if s.capacity() == 0 {
379            0
380        } else {
381            (s.capacity() * 8 / 7).next_power_of_two() * (std::mem::size_of::<String>() + 1)
382                + s.iter().map(|k| k.capacity()).sum::<usize>()
383        }
384    };
385    rows.push(("test_scripts", reg.test_scripts.len(), ts_tbl));
386
387    rows.sort_by_key(|row| std::cmp::Reverse(row.2));
388    let total: usize = rows.iter().map(|r| r.2).sum();
389    info!(
390        "[mem] registry ~{:.2} MB (sizeof Symbol={}B, SymbolKind={}B, Type={}B):",
391        mb(total),
392        std::mem::size_of::<Symbol>(),
393        std::mem::size_of::<SymbolKind>(),
394        t,
395    );
396    for (name, n, bytes) in &rows {
397        info!(
398            "[mem]   {:<20} {:>6} entries  {:>7.3} MB",
399            name,
400            n,
401            mb(*bytes)
402        );
403    }
404}
405
406/// One-off breakdown of where the compiled-bytecode heap goes, to target
407/// memory work. Built only under `--features memprof`.
408#[cfg(feature = "memprof")]
409fn mem_stats_compiled(scripts: &[CompiledScript]) {
410    use crate::bytecode::{Instruction, Opcode, Operand};
411    let (mut n_instr, mut n_str, mut str_bytes) = (0usize, 0usize, 0usize);
412    let (mut n_switch, mut switch_entries) = (0usize, 0usize);
413    let (mut source_path_bytes, mut name_bytes) = (0usize, 0usize);
414    for s in scripts {
415        n_instr += s.instructions.len();
416        source_path_bytes += s.source_path.capacity();
417        name_bytes += s.name.capacity() + s.trigger.capacity();
418        for st in &s.strings {
419            n_str += 1;
420            str_bytes += st.len();
421        }
422        for sw in &s.switch_tables {
423            n_switch += 1;
424            switch_entries += sw.len();
425        }
426    }
427    let sz = std::mem::size_of::<Instruction>();
428    let mb = |b: usize| b as f64 / 1_048_576.0;
429    info!(
430        "[mem] sizeof Instruction={}B Operand={}B Opcode={}B CompiledScript={}B",
431        sz,
432        std::mem::size_of::<Operand>(),
433        std::mem::size_of::<Opcode>(),
434        std::mem::size_of::<CompiledScript>(),
435    );
436    info!(
437        "[mem] compiled: {} scripts, {} instrs => {:.2} MB instr structs",
438        scripts.len(),
439        n_instr,
440        mb(n_instr * sz)
441    );
442    info!(
443        "[mem]   Str operands: {} ({:.2} MB content + {:.2} MB Box<str> headers)",
444        n_str,
445        mb(str_bytes),
446        mb(n_str * 16)
447    );
448    info!(
449        "[mem]   SwitchTable operands: {} ({} entries => {:.2} MB)",
450        n_switch,
451        switch_entries,
452        mb(switch_entries * 16)
453    );
454    info!(
455        "[mem]   source_path {:.2} MB, names+triggers {:.2} MB",
456        mb(source_path_bytes),
457        mb(name_bytes)
458    );
459}
460
461struct ParsedFile {
462    path: PathBuf,
463    file: ScriptFile,
464    testscript_line: Option<usize>,
465    /// Canonicalized, normalized source path. Computed once during parsing and
466    /// reused as each compiled script's `source_path` (it is byte-identical to
467    /// the `source_cache` key), avoiding a second `canonicalize` syscall per
468    /// file in codegen.
469    source_path: String,
470}
471
472/// Outcome of parsing one file off-thread. The owned source text and cache key
473/// travel back so the (single-threaded) merge can build the `Rc`-based caches
474/// without sending `Rc` across threads.
475struct FileParseResult {
476    path: PathBuf,
477    read: Result<ParsedSource, std::io::Error>,
478}
479
480struct ParsedSource {
481    cache_key: String,
482    raw_source: String,
483    outcome: ParseOutcome,
484}
485
486enum ParseOutcome {
487    Parsed {
488        file: ScriptFile,
489        testscript_line: Option<usize>,
490    },
491    Error {
492        line: usize,
493        position: usize,
494        message: String,
495        phase: diagnostics::Phase,
496    },
497}
498
499/// Read, lex and parse a single file off-thread. Pure: no shared state, so it
500/// is safe to run concurrently. Mirrors the original sequential body exactly —
501/// same source truncation, same cache-key normalization, same lex-then-parse
502/// error reporting — only restructured to return owned data for the merge.
503fn parse_one_file(path: &Path, include_tests: bool) -> FileParseResult {
504    let raw_source = match fs::read_to_string(path) {
505        Ok(s) => s,
506        Err(e) => {
507            return FileParseResult {
508                path: path.to_path_buf(),
509                read: Err(e),
510            };
511        }
512    };
513    let testscript_line = find_testscript_line(&raw_source);
514
515    let source_code = if !include_tests {
516        if let Some(line_no) = testscript_line {
517            truncate_at_line(&raw_source, line_no)
518        } else {
519            raw_source.clone()
520        }
521    } else {
522        strip_testscript_annotation(&raw_source)
523    };
524
525    let canonical = fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf());
526    let mut cache_key = canonical.to_string_lossy().into_owned();
527    if cache_key.starts_with("\\\\?\\") {
528        cache_key = cache_key[4..].to_string();
529    }
530    cache_key = cache_key.replace("\\Content\\", "\\content\\");
531
532    // Lexer/Parser borrow a `&PathBuf` for diagnostics; keep one alive locally.
533    let path_buf = path.to_path_buf();
534    let outcome = match Lexer::new(&source_code, &path_buf).tokenize() {
535        Err(e) => ParseOutcome::Error {
536            line: e.line,
537            position: e.position,
538            message: e.message,
539            phase: diagnostics::Phase::Lexing,
540        },
541        Ok(tokens) => match Parser::new(tokens, &path_buf).parse() {
542            Ok(file) => ParseOutcome::Parsed {
543                file,
544                testscript_line: if include_tests { testscript_line } else { None },
545            },
546            Err(e) => ParseOutcome::Error {
547                line: e.line,
548                position: e.position,
549                message: e.message,
550                phase: diagnostics::Phase::Parsing,
551            },
552        },
553    };
554
555    FileParseResult {
556        path: path_buf,
557        read: Ok(ParsedSource {
558            cache_key,
559            raw_source,
560            outcome,
561        }),
562    }
563}
564
565/// Compile scripts and write output. When `lint` is true, also run lint passes
566/// (unused locals, unreachable code) after code generation.
567pub fn compile_memory(
568    scripts_dir: &Path,
569    pack_dir: Option<&Path>,
570    lint: bool,
571) -> Result<(Vec<u8>, Vec<u8>), Box<dyn Error>> {
572    compile_memory_with_options(scripts_dir, pack_dir, lint, false, false)
573}
574
575/// Like `compile_memory` but includes `#testscript` sections when
576/// `include_tests` is true.
577pub fn compile_memory_with_tests(
578    scripts_dir: &Path,
579    pack_dir: Option<&Path>,
580    lint: bool,
581    include_tests: bool,
582) -> Result<(Vec<u8>, Vec<u8>), Box<dyn Error>> {
583    compile_memory_with_options(scripts_dir, pack_dir, lint, include_tests, false)
584}
585
586/// In-memory compile — returns the `(script.dat, script.idx)` bytes instead of
587/// writing them — choosing the compilation strategy.
588///
589/// With `low_mem`, the recompile pipeline is used: scripts are re-parsed and
590/// re-compiled on demand so the full compiled set is never resident (~16 MB heap
591/// vs ~93), at roughly 6× the time; the returned bytes are byte-identical to the
592/// default path. `include_tests` keeps `#testscript` sections. The
593/// `RUNEC_RECOMPILE` environment variable forces low-memory mode on as well.
594pub fn compile_memory_with_options(
595    scripts_dir: &Path,
596    pack_dir: Option<&Path>,
597    lint: bool,
598    include_tests: bool,
599    low_mem: bool,
600) -> Result<(Vec<u8>, Vec<u8>), Box<dyn Error>> {
601    if low_mem || std::env::var("RUNEC_RECOMPILE").is_ok_and(|v| !v.is_empty()) {
602        info!("[recompile] low-memory recompile pipeline enabled");
603        return compile_recompile_memory(scripts_dir, pack_dir, lint, include_tests);
604    }
605
606    let (compiled_scripts, diagnostics) = run_pipeline(scripts_dir, pack_dir, lint, include_tests)?;
607
608    info!("Writing compiled output to memory...");
609    let writer = ScriptWriter::new("".into());
610
611    diagnostics.print_all();
612    info!("Compilation complete!");
613
614    Ok(writer.build_all(&compiled_scripts)?)
615}
616
617/// Compile scripts and write output. When `lint` is true, also run lint passes
618/// (unused locals, unreachable code) after code generation.
619///
620/// Convenience wrapper over [`compile_with_options`] using the default (fast,
621/// fully-resident) pipeline. Kept signature-compatible for existing callers.
622pub fn compile(
623    scripts_dir: &Path,
624    pack_dir: Option<&Path>,
625    output_dir: &Path,
626    lint: bool,
627) -> Result<(), Box<dyn Error>> {
628    compile_with_options(scripts_dir, pack_dir, output_dir, lint, false)
629}
630
631/// Compile scripts and write output, choosing the compilation strategy.
632///
633/// With `low_mem`, the **recompile** pipeline is used: scripts are re-parsed and
634/// re-compiled on demand for the pointer check and written file-by-file, so the
635/// full set of compiled bytecode is never resident at once. That cuts peak usage
636/// to roughly 16 MB heap / 32 MB working set (from ~93 / ~111 MB) at about 6× the
637/// compile time. Output is byte-identical to the default path. The
638/// `RUNEC_RECOMPILE` environment variable forces this mode on as well.
639pub fn compile_with_options(
640    scripts_dir: &Path,
641    pack_dir: Option<&Path>,
642    output_dir: &Path,
643    lint: bool,
644    low_mem: bool,
645) -> Result<(), Box<dyn Error>> {
646    // The recompile pipeline never holds all compiled scripts (it re-compiles on
647    // demand for the pointer check + writes file-by-file). It does its own
648    // output, so it returns here. The default path below is unchanged.
649    if low_mem || std::env::var("RUNEC_RECOMPILE").is_ok_and(|v| !v.is_empty()) {
650        info!("[recompile] low-memory recompile pipeline enabled");
651        return compile_recompile(scripts_dir, pack_dir, output_dir, lint);
652    }
653
654    let (compiled_scripts, diagnostics) = run_pipeline(scripts_dir, pack_dir, lint, false)?;
655
656    let output = output_dir.display().to_string();
657    // Normalize separators for a clean, consistent log message (input paths may
658    // mix `/` and `\`, e.g. `data/pack\server`); the writer keeps the original.
659    info!("Writing output to {}...", output.replace('\\', "/"));
660    let writer = ScriptWriter::new(output);
661    writer.write_all(&compiled_scripts)?;
662
663    diagnostics.print_all();
664    info!("Compilation complete!");
665
666    Ok(())
667}
668
669/// Run all analysis passes (parse, type-check, codegen, pointer-check, lints)
670/// without writing output. Useful for editor tooling and CI lint gates.
671pub fn lint(scripts_dir: &Path, pack_dir: Option<&Path>) -> Result<(), Box<dyn Error>> {
672    let (_, diagnostics) = run_pipeline(scripts_dir, pack_dir, true, false)?;
673
674    diagnostics.print_all();
675    info!("Compilation complete!");
676
677    Ok(())
678}
679
680/// Runs the full compilation pipeline (parse, register, type-check, codegen,
681/// pointer-check, and optional lint passes), returning the compiled scripts
682/// and accumulated diagnostics for the caller to handle output.
683fn run_pipeline(
684    scripts_dir: &Path,
685    pack_dir: Option<&Path>,
686    lint: bool,
687    include_tests: bool,
688) -> Result<(Vec<CompiledScript>, DiagnosticsCollector), Box<dyn Error>> {
689    let source_dir = sources::target_dir(scripts_dir, sources::Scope::Rune);
690    let scripts_dir = source_dir.as_path();
691    if !scripts_dir.exists() || !scripts_dir.is_dir() {
692        return Err(Box::new(error::CompilerError::FileNotFound(
693            scripts_dir.display().to_string(),
694        )));
695    }
696
697    // Prototype: opt into the streaming pipeline (never holds all ASTs at once)
698    // with RUNEC_STREAM=1. The default batch path below is unchanged.
699    if std::env::var("RUNEC_STREAM").is_ok_and(|v| !v.is_empty()) {
700        info!("[stream] streaming pipeline enabled (RUNEC_STREAM)");
701        return run_pipeline_streaming(scripts_dir, pack_dir, lint, include_tests);
702    }
703
704    let mut rs2_files: Vec<PathBuf> = get_rs2_files(scripts_dir, "rs2")?;
705
706    rs2_files.sort();
707    info!("Found {} script files", rs2_files.len());
708    mem_mark("startup");
709
710    let mut diagnostics = DiagnosticsCollector::new();
711
712    // Phase 1: Parse all files. Reading, lexing and parsing are independent
713    // per file, so fan them out across threads; the results are merged back in
714    // input order so registration/ID assignment stays deterministic.
715    info!("Parsing script files into syntax trees...");
716    let parse_results = parallel_map(&rs2_files, |path| parse_one_file(path, include_tests));
717
718    let mut all_files: Vec<ParsedFile> = Vec::new();
719    let mut source_cache: HashMap<String, Arc<String>> = HashMap::new();
720    // Raw source text keyed by on-disk path, reused by `generate_script_pack`
721    // so it doesn't re-read every file from disk a second time.
722    let mut raw_by_path: HashMap<PathBuf, Arc<String>> = HashMap::new();
723    for result in parse_results {
724        let parsed = match result.read {
725            Ok(p) => p,
726            // A read failure aborts the whole compile, exactly as the previous
727            // `fs::read_to_string(path)?` did.
728            Err(e) => return Err(Box::new(e)),
729        };
730        let rc = Arc::new(parsed.raw_source);
731        // The cache key is the canonicalized source path; reuse it as each
732        // compiled script's `source_path` so codegen needn't canonicalize again.
733        let source_path = parsed.cache_key.clone();
734        source_cache.insert(parsed.cache_key, rc.clone());
735        raw_by_path.insert(result.path.clone(), rc);
736        match parsed.outcome {
737            ParseOutcome::Parsed {
738                file,
739                testscript_line,
740            } => all_files.push(ParsedFile {
741                path: result.path,
742                file,
743                testscript_line,
744                source_path,
745            }),
746            ParseOutcome::Error {
747                line,
748                position,
749                message,
750                phase,
751            } => diagnostics.error(result.path, line, position, message, phase),
752        }
753    }
754
755    if diagnostics.has_errors() {
756        diagnostics.print_all();
757        return Err(Box::new(error::CompilerError::TypeError(
758            "Parsing failed".to_string(),
759        )));
760    }
761
762    mem_mark("after parse");
763
764    // Phase 2: Register all scripts. Build the pack/config registry, then
765    // regenerate script.pack, load pre-assigned IDs, and register every script.
766    info!("Registering scripts and building the symbol registry...");
767    let resolved_pack_dir = resolve_pack_dir(scripts_dir, pack_dir);
768    let mut registry = build_symbol_registry(scripts_dir, resolved_pack_dir.as_deref());
769    register_scripts(
770        &mut registry,
771        scripts_dir,
772        resolved_pack_dir.as_deref(),
773        &mut diagnostics,
774        &all_files,
775        &raw_by_path,
776    );
777    info!("  Registered {} scripts", registry.scripts.len());
778    // `raw_by_path` was only needed to (re)generate script.pack inside
779    // `register_scripts`; the source text itself stays reachable via
780    // `source_cache` (shared `Arc`s). Drop the redundant index now.
781    drop(raw_by_path);
782    // Pre-assigned script ids were consumed during registration; free them
783    // before codegen (the compile's peak-memory phase) rather than at scope end.
784    registry.drop_registration_scratch();
785
786    for pf in &all_files {
787        for script in &pf.file.scripts {
788            if let Some(msg) =
789                Compiler::validate_trigger_subject(&script.trigger, &script.name, &registry)
790            {
791                diagnostics.warning(
792                    pf.path.clone(),
793                    script.line,
794                    0,
795                    msg,
796                    diagnostics::Phase::SymbolRegistration,
797                );
798            }
799        }
800    }
801
802    mem_mark("after register");
803    #[cfg(feature = "memprof")]
804    mem_stats_registry(&registry);
805
806    // Phase 3: Type checking
807    info!("Type checking {} files...", all_files.len());
808    run_type_checker(&mut diagnostics, &all_files, &registry);
809
810    if diagnostics.has_errors() {
811        diagnostics.print_all();
812        return Err(Box::new(error::CompilerError::TypeError(
813            "Type checking failed".to_string(),
814        )));
815    }
816
817    mem_mark("after typecheck");
818
819    // Phase 4: Code generation
820    info!("Generating bytecode from type-checked scripts...");
821    let mut compiled_scripts = codegen(&all_files, &registry, &mut diagnostics);
822    if diagnostics.has_errors() {
823        diagnostics.print_all();
824        return Err(Box::new(error::CompilerError::CodeGenError(
825            "Code generation failed".into(),
826        )));
827    }
828    info!("  Generated {} compiled scripts", compiled_scripts.len());
829    compiled_scripts.sort_by_key(|s| s.id);
830    compiled_scripts.shrink_to_fit();
831    mem_mark("after codegen");
832    #[cfg(feature = "memprof")]
833    mem_stats_compiled(&compiled_scripts);
834
835    // The ASTs are consumed by codegen and never read again — pointer checking
836    // and lints work purely off `compiled_scripts` + the registry + the source
837    // cache. Free the syntax trees now so they don't coexist with the pointer
838    // checker's (per-thread) scratch, which is the compile's peak-memory moment.
839    drop(all_files);
840    mem_mark("after drop asts");
841
842    // Phase 4.5: Pointer checking & lints
843    info!("Checking pointers in generated code...");
844    check_pointers(
845        lint,
846        &mut diagnostics,
847        SourceProvider::Eager(&source_cache),
848        &registry,
849        &compiled_scripts,
850    );
851    mem_mark("after pointercheck");
852
853    Ok((compiled_scripts, diagnostics))
854}
855
856// ── Streaming pipeline (prototype, gated by RUNEC_STREAM) ────────────────────
857// Goal: never materialize all ASTs (~59 MB) at once. Pass 1 parses in parallel
858// but keeps only per-script signatures, dropping each AST in-worker; the registry
859// is built from those. Pass 2 re-parses each file in parallel, type-checks and
860// compiles it, then drops the AST immediately — so syntax trees never accumulate.
861// Output is byte-identical to the batch path (same registration, same per-file
862// codegen, same stable id-sort). It still holds all compiled scripts (for the
863// cross-script pointer check + writer); shrinking that is the next increment.
864
865/// One script's registration-relevant signature — everything the registry and
866/// trigger diagnostics need — so the full AST can be dropped right after parsing.
867struct ScriptSig {
868    name: String,
869    trigger: String,
870    param_types: Vec<types::Type>,
871    return_types: Vec<types::Type>,
872    line: usize,
873}
874
875struct FileSigs {
876    path: PathBuf,
877    testscript_line: Option<usize>,
878    sigs: Vec<ScriptSig>,
879}
880
881enum SigOutcome {
882    Parsed {
883        sigs: Vec<ScriptSig>,
884        testscript_line: Option<usize>,
885    },
886    Error {
887        line: usize,
888        position: usize,
889        message: String,
890        phase: diagnostics::Phase,
891    },
892}
893
894struct FileSigsResult {
895    path: PathBuf,
896    /// `(cache_key, raw_source, outcome)` on success.
897    read: Result<(String, String, SigOutcome), std::io::Error>,
898}
899
900/// Parse one file, extract signatures, and DROP the AST in-worker, so a parallel
901/// sweep never holds every syntax tree at once.
902fn parse_one_file_sigs(path: &Path, include_tests: bool) -> FileSigsResult {
903    let r = parse_one_file(path, include_tests);
904    let read = r.read.map(|ps| {
905        let outcome = match ps.outcome {
906            ParseOutcome::Parsed {
907                file,
908                testscript_line,
909            } => {
910                let sigs = file
911                    .scripts
912                    .iter()
913                    .map(|d| ScriptSig {
914                        name: d.name.clone(),
915                        trigger: d.trigger.clone(),
916                        param_types: d.params.iter().map(|p| p.param_type).collect(),
917                        return_types: d.return_types.clone(),
918                        line: d.line,
919                    })
920                    .collect();
921                SigOutcome::Parsed {
922                    sigs,
923                    testscript_line,
924                }
925                // `file` (the AST) is dropped here — only the sigs travel back.
926            }
927            ParseOutcome::Error {
928                line,
929                position,
930                message,
931                phase,
932            } => SigOutcome::Error {
933                line,
934                position,
935                message,
936                phase,
937            },
938        };
939        (ps.cache_key, ps.raw_source, outcome)
940    });
941    FileSigsResult { path: r.path, read }
942}
943
944fn run_pipeline_streaming(
945    scripts_dir: &Path,
946    pack_dir: Option<&Path>,
947    lint: bool,
948    include_tests: bool,
949) -> Result<(Vec<CompiledScript>, DiagnosticsCollector), Box<dyn Error>> {
950    let mut rs2_files: Vec<PathBuf> = get_rs2_files(scripts_dir, "rs2")?;
951    rs2_files.sort();
952    info!("Found {} script files", rs2_files.len());
953    mem_mark("startup");
954
955    let mut diagnostics = DiagnosticsCollector::new();
956
957    // Pass 1: parse for signatures only (each AST dropped in its worker).
958    info!("[stream] Pass 1: parsing for signatures...");
959    let sig_results = parallel_map(&rs2_files, |path| parse_one_file_sigs(path, include_tests));
960
961    // Streaming keeps only `cache_key -> on-disk path`; source text is re-read
962    // lazily for the rare diagnostic that needs it, so the ~4 MB of sources isn't
963    // resident through the peak. `raw_by_path` still holds the text transiently
964    // for `script.pack` regeneration, then is dropped right after registration.
965    let mut source_paths: HashMap<String, PathBuf> = HashMap::new();
966    let mut raw_by_path: HashMap<PathBuf, Arc<String>> = HashMap::new();
967    let mut file_sigs: Vec<FileSigs> = Vec::new();
968    for r in sig_results {
969        let (cache_key, raw_source, outcome) = match r.read {
970            Ok(t) => t,
971            Err(e) => return Err(Box::new(e)),
972        };
973        source_paths.insert(cache_key, r.path.clone());
974        raw_by_path.insert(r.path.clone(), Arc::new(raw_source));
975        match outcome {
976            SigOutcome::Parsed {
977                sigs,
978                testscript_line,
979            } => file_sigs.push(FileSigs {
980                path: r.path,
981                testscript_line,
982                sigs,
983            }),
984            SigOutcome::Error {
985                line,
986                position,
987                message,
988                phase,
989            } => diagnostics.error(r.path, line, position, message, phase),
990        }
991    }
992
993    if diagnostics.has_errors() {
994        diagnostics.print_all();
995        return Err(Box::new(error::CompilerError::TypeError(
996            "Parsing failed".to_string(),
997        )));
998    }
999    mem_mark("after parse");
1000
1001    // Registration (from signatures, not ASTs).
1002    info!("[stream] Registering scripts and building the symbol registry...");
1003    let resolved_pack_dir = resolve_pack_dir(scripts_dir, pack_dir);
1004    let mut registry = build_symbol_registry(scripts_dir, resolved_pack_dir.as_deref());
1005    register_scripts_from_sigs(
1006        &mut registry,
1007        scripts_dir,
1008        resolved_pack_dir.as_deref(),
1009        &mut diagnostics,
1010        &file_sigs,
1011        &raw_by_path,
1012    );
1013    info!("  Registered {} scripts", registry.scripts.len());
1014    drop(raw_by_path);
1015    registry.drop_registration_scratch();
1016
1017    for fs in &file_sigs {
1018        for sig in &fs.sigs {
1019            if let Some(msg) =
1020                Compiler::validate_trigger_subject(&sig.trigger, &sig.name, &registry)
1021            {
1022                diagnostics.warning(
1023                    fs.path.clone(),
1024                    sig.line,
1025                    0,
1026                    msg,
1027                    diagnostics::Phase::SymbolRegistration,
1028                );
1029            }
1030        }
1031    }
1032    // Signatures are done; only the registry + source cache carry forward.
1033    drop(file_sigs);
1034    mem_mark("after register");
1035    #[cfg(feature = "memprof")]
1036    mem_stats_registry(&registry);
1037
1038    // Pass 2: re-parse + type-check + compile per file, streaming. Parse can't
1039    // fail here (pass 1 validated), so a failed re-parse just yields nothing.
1040    info!("[stream] Pass 2: type-check + codegen (streaming)...");
1041    let per_file = parallel_map(&rs2_files, |path| {
1042        let parsed = match parse_one_file(path, include_tests).read {
1043            Ok(p) => p,
1044            Err(_) => return (Vec::new(), DiagnosticsCollector::new()),
1045        };
1046        let (file, testscript_line) = match parsed.outcome {
1047            ParseOutcome::Parsed {
1048                file,
1049                testscript_line,
1050            } => (file, testscript_line),
1051            ParseOutcome::Error { .. } => return (Vec::new(), DiagnosticsCollector::new()),
1052        };
1053        let mut type_checker = TypeChecker::new(&registry);
1054        type_checker.check_file_with_test_boundary(&file, path, testscript_line);
1055        let mut compiled = Vec::new();
1056        // Compile only when this file type-checks clean (codegen assumes
1057        // well-typed input). On any global error the whole compile aborts below,
1058        // so output produced here would be discarded anyway.
1059        if !type_checker.diagnostics.has_errors() {
1060            let mut compiler = Compiler::new(&registry);
1061            for script in &file.scripts {
1062                if script.trigger == "command" {
1063                    continue;
1064                }
1065                let Some(mut c) =
1066                    compiler.compile_checked(script, path, &mut type_checker.diagnostics)
1067                else {
1068                    continue;
1069                };
1070                c.source_path = parsed.cache_key.clone();
1071                c.shrink_to_fit();
1072                compiled.push(c);
1073            }
1074        }
1075        (compiled, type_checker.diagnostics)
1076        // `file` (the AST) is dropped here.
1077    });
1078
1079    let mut compiled_scripts: Vec<CompiledScript> = Vec::new();
1080    for (compiled, diags) in per_file {
1081        compiled_scripts.extend(compiled);
1082        diagnostics.merge(diags);
1083    }
1084    if diagnostics.has_errors() {
1085        diagnostics.print_all();
1086        return Err(Box::new(error::CompilerError::TypeError(
1087            "Type checking failed".to_string(),
1088        )));
1089    }
1090    // Stable sort by id matches the batch path (file order preserved among
1091    // equal ids → identical writer last-write-wins behaviour).
1092    compiled_scripts.sort_by_key(|s| s.id);
1093    compiled_scripts.shrink_to_fit();
1094    info!("  Generated {} compiled scripts", compiled_scripts.len());
1095    mem_mark("after codegen");
1096    #[cfg(feature = "memprof")]
1097    mem_stats_compiled(&compiled_scripts);
1098
1099    check_pointers(
1100        lint,
1101        &mut diagnostics,
1102        SourceProvider::Lazy(&source_paths),
1103        &registry,
1104        &compiled_scripts,
1105    );
1106    mem_mark("after pointercheck");
1107
1108    Ok((compiled_scripts, diagnostics))
1109}
1110
1111/// `register_scripts` for the streaming path: identical logic, driven by the
1112/// extracted signatures instead of the (already-dropped) ASTs.
1113fn register_scripts_from_sigs(
1114    registry: &mut SymbolRegistry,
1115    scripts_dir: &Path,
1116    resolved_pack_dir: Option<&Path>,
1117    diagnostics: &mut DiagnosticsCollector,
1118    file_sigs: &[FileSigs],
1119    raw_by_path: &HashMap<PathBuf, Arc<String>>,
1120) {
1121    if let Some(pd) = resolved_pack_dir {
1122        symloader::generate_script_pack(scripts_dir, pd, raw_by_path);
1123        symloader::load_script_ids(registry, pd);
1124    }
1125
1126    use crate::diagnostic_messages as msg;
1127    let mut registered_scripts: std::collections::HashSet<String> =
1128        std::collections::HashSet::new();
1129
1130    for fs in file_sigs.iter() {
1131        for sig in &fs.sigs {
1132            Arc::make_mut(&mut registry.visibility).declare_script(
1133                &fs.path,
1134                &sig.trigger,
1135                &sig.name,
1136            );
1137            if sig.trigger == "command" {
1138                continue;
1139            }
1140
1141            if !trigger_table::is_valid_trigger(&sig.trigger) {
1142                diagnostics.warning(
1143                    fs.path.clone(),
1144                    sig.line,
1145                    0,
1146                    msg::fmt(msg::SCRIPT_TRIGGER_INVALID, &[&sig.trigger]),
1147                    crate::diagnostics::Phase::SymbolRegistration,
1148                );
1149            }
1150
1151            let key = format!("{}:{}", sig.trigger, sig.name);
1152            if !registered_scripts.insert(key) {
1153                diagnostics.warning(
1154                    fs.path.clone(),
1155                    sig.line,
1156                    0,
1157                    msg::fmt(msg::SCRIPT_REDECLARATION, &[&sig.trigger, &sig.name]),
1158                    crate::diagnostics::Phase::SymbolRegistration,
1159                );
1160            }
1161
1162            if !sig.return_types.is_empty() && !trigger_table::allows_returns(&sig.trigger) {
1163                diagnostics.warning(
1164                    fs.path.clone(),
1165                    sig.line,
1166                    0,
1167                    msg::fmt(msg::SCRIPT_TRIGGER_NO_RETURNS, &[&sig.trigger]),
1168                    diagnostics::Phase::SymbolRegistration,
1169                );
1170            }
1171
1172            registry.register_script(
1173                sig.name.clone(),
1174                sig.trigger.clone(),
1175                sig.param_types.clone(),
1176                sig.return_types.clone(),
1177            );
1178
1179            if let Some(ts_line) = fs.testscript_line
1180                && sig.line >= ts_line
1181            {
1182                registry.mark_test_script(&sig.trigger, &sig.name);
1183            }
1184        }
1185    }
1186}
1187
1188fn check_pointers(
1189    lint: bool,
1190    diagnostics: &mut DiagnosticsCollector,
1191    source: SourceProvider,
1192    registry: &SymbolRegistry,
1193    compiled_scripts: &[CompiledScript],
1194) {
1195    {
1196        use crate::pointer_checker::{PointerChecker, ScriptSource};
1197        // Validate scripts across threads: each chunk gets its own checker that
1198        // shares the read-only scripts/registry/source cache but keeps its own
1199        // (purely memoizing) caches. Diagnostics are merged in script order, so
1200        // the result is identical to a single-threaded run.
1201        //
1202        // The read-only command/script lookup tables are built ONCE here and
1203        // shared (`Arc`) across every chunk checker, instead of each thread
1204        // rebuilding the full ~700-command pointer table + per-script reverse
1205        // maps. `names` is just the script order, so derive it directly rather
1206        // than spinning up a throwaway checker for `script_names()`.
1207        let shared = PointerChecker::build_shared(compiled_scripts, registry, false);
1208        let names: Vec<String> = compiled_scripts.iter().map(|s| s.name.clone()).collect();
1209        // Each chunk's checker retains its CFG caches until the chunk finishes,
1210        // so concurrency multiplies peak scratch. RUNEC_PTR_CHUNKS caps the
1211        // concurrent chunk count to trade pointer-check speed for memory; the
1212        // result is chunk-count-independent, so diagnostics stay byte-identical.
1213        let max_chunks = std::env::var("RUNEC_PTR_CHUNKS")
1214            .ok()
1215            .and_then(|s| s.parse::<usize>().ok())
1216            .filter(|&n| n > 0)
1217            .unwrap_or(usize::MAX);
1218        let chunk_diags = parallel_chunks_capped(&names, max_chunks, |chunk| {
1219            let mut checker = PointerChecker::with_shared(
1220                ScriptSource::Resident(compiled_scripts),
1221                registry,
1222                shared.clone(),
1223            );
1224            checker.set_source_cache(source);
1225            checker.validate_names(chunk)
1226        });
1227        let mut pointer_diags = DiagnosticsCollector::new();
1228        for d in chunk_diags {
1229            pointer_diags.merge(d);
1230        }
1231        let ptr_warnings = pointer_diags.warning_count();
1232        if ptr_warnings > 0 {
1233            info!("  Pointer check: {} warning(s)", ptr_warnings);
1234        }
1235        diagnostics.merge(pointer_diags);
1236    }
1237    #[cfg(feature = "memprof")]
1238    crate::pointer_checker::cfg_trace_report();
1239
1240    // Phase 4.6: Lint passes (optional)
1241    if lint {
1242        info!("Running lint passes (unused locals, unreachable code)...");
1243        let lint_diags = lints::run_lints(compiled_scripts, Some(source));
1244        let lint_warnings = lint_diags.warning_count();
1245        if lint_warnings > 0 {
1246            info!("  Lints: {} warning(s)", lint_warnings);
1247        }
1248        diagnostics.merge(lint_diags);
1249    }
1250}
1251
1252fn codegen(
1253    all_files: &[ParsedFile],
1254    registry: &SymbolRegistry,
1255    diagnostics: &mut DiagnosticsCollector,
1256) -> Vec<CompiledScript> {
1257    // Each file compiles independently against the read-only registry, so fan
1258    // the files out across threads. Output order doesn't matter — the caller
1259    // sorts compiled scripts by id — so a simple flatten of the per-file results
1260    // is enough.
1261    let mut compiled = Vec::new();
1262    for (scripts, errors) in parallel_map(all_files, |pf| compile_file(pf, registry)) {
1263        compiled.extend(scripts);
1264        diagnostics.merge(errors);
1265    }
1266    compiled
1267}
1268
1269/// Compile every (non-`command`) script in one file. Creating a `Compiler` per
1270/// file is cheap now that it only borrows the registry, and keeps the codegen
1271/// scratch state thread-local.
1272fn compile_file(
1273    pf: &ParsedFile,
1274    registry: &SymbolRegistry,
1275) -> (Vec<CompiledScript>, DiagnosticsCollector) {
1276    let mut diagnostics = DiagnosticsCollector::new();
1277    let mut scripts =
1278        crate::compiler::compile_file(&pf.file, &pf.path, registry, false, &mut diagnostics);
1279    for script in &mut scripts {
1280        script.source_path = pf.source_path.clone();
1281        script.shrink_to_fit();
1282    }
1283    (scripts, diagnostics)
1284}
1285
1286fn run_type_checker(
1287    diagnostics: &mut DiagnosticsCollector,
1288    all_files: &[ParsedFile],
1289    registry: &SymbolRegistry,
1290) {
1291    // Each file type-checks independently against the read-only registry, so
1292    // check chunks of files in parallel; per-chunk diagnostics are merged in
1293    // file order, matching a single-threaded run.
1294    let chunk_diags = parallel_chunks(all_files, |chunk| {
1295        let mut type_checker = TypeChecker::new(registry);
1296        for pf in chunk {
1297            type_checker.check_file_with_test_boundary(&pf.file, &pf.path, pf.testscript_line);
1298        }
1299        type_checker.diagnostics
1300    });
1301    for d in chunk_diags {
1302        diagnostics.merge(d);
1303    }
1304}
1305
1306/// Resolve the pack directory: explicit `--pack`, else the nearest ancestor's `pack/`, else
1307/// `content/pack` up the tree.
1308fn resolve_pack_dir(scripts_dir: &Path, pack_dir: Option<&Path>) -> Option<PathBuf> {
1309    pack_dir
1310        .map(PathBuf::from)
1311        .filter(|p| p.exists())
1312        .or_else(|| {
1313            scripts_dir.ancestors().find_map(|dir| {
1314                let pack = dir.join("pack");
1315                pack.is_dir().then_some(pack)
1316            })
1317        })
1318        .or_else(|| {
1319            scripts_dir
1320                .ancestors()
1321                .nth(3)
1322                .map(|p| p.join("content/pack"))
1323                .filter(|p| p.exists())
1324        })
1325}
1326
1327/// Build the part of the registry that does NOT depend on the parsed scripts:
1328/// pack files, `.constant`/`.var`/`.dbtable` configs, and engine command params.
1329/// This runs concurrently with parsing; `script.pack` (re)generation and script
1330/// registration — which need the parsed sources/ASTs — happen afterwards in
1331/// `register_scripts`.
1332fn build_symbol_registry(scripts_dir: &Path, resolved_pack_dir: Option<&Path>) -> SymbolRegistry {
1333    // Visibility reads every source and config, and only its indexing needs
1334    // the registry built meanwhile.
1335    let visibility = {
1336        let dir = scripts_dir.to_path_buf();
1337        thread::spawn(move || sources::Visibility::scan(&dir))
1338    };
1339    let mut registry = SymbolRegistry::new();
1340
1341    if let Some(pd) = resolved_pack_dir {
1342        info!("  Loading packs from {}", pd.display());
1343        symloader::load_packs(&mut registry, pd);
1344        info!(
1345            "  Loaded {} commands, {} game vars",
1346            registry.commands.len(),
1347            registry.game_vars.len(),
1348        );
1349    } else {
1350        info!("  No pack directory found - symbol resolution will be limited");
1351    }
1352
1353    // Walk the scripts tree once and let the constant/game-var/dbtable loaders
1354    // filter this shared list by extension, instead of each re-walking the tree
1355    // (previously 6 separate recursive traversals over the same directories).
1356    let mut config_files: Vec<PathBuf> = Vec::new();
1357    config_files.extend(sources::config_files(scripts_dir));
1358
1359    symloader::load_constant_files(&mut registry, &config_files);
1360
1361    info!(
1362        "  Loaded {} constants from .constant files",
1363        registry.constants.len()
1364    );
1365
1366    symloader::load_game_var_types(&mut registry, &config_files);
1367
1368    if let Some(pd) = resolved_pack_dir
1369        && let Err(error) = clientscript::load_clan_settings(&mut registry, &config_files, pd)
1370    {
1371        warn!("  Clan settings not loaded: {error}");
1372    }
1373
1374    if let Some(pd) = resolved_pack_dir {
1375        let dbtable_ids = symloader::load_dbtable_pack(&pd.join("dbtable.pack"));
1376        if !dbtable_ids.is_empty() {
1377            symloader::load_dbtable_configs(&mut registry, &config_files, &dbtable_ids);
1378            info!(
1379                "  Registered {} dbcolumn compound IDs",
1380                registry.dbcolumn_types.len()
1381            );
1382        }
1383    }
1384
1385    let engine_rs2 = scripts_dir.join("engine.rs2");
1386
1387    if engine_rs2.exists() {
1388        symloader::load_engine_command_params(&mut registry, &engine_rs2);
1389    } else {
1390        if let Ok(entries) = std::fs::read_dir(scripts_dir) {
1391            for entry in entries.flatten() {
1392                let candidate = entry.path().join("engine.rs2");
1393                if candidate.exists() {
1394                    crate::symloader::load_engine_command_params(&mut registry, &candidate);
1395                    break;
1396                }
1397            }
1398        }
1399    }
1400
1401    let client_dir = if scripts_dir
1402        .file_name()
1403        .is_some_and(|name| name == "runescript")
1404    {
1405        scripts_dir.with_file_name("clientscript")
1406    } else {
1407        scripts_dir.join("clientscript")
1408    };
1409    if client_dir.is_dir() {
1410        if let Some(pack) = resolved_pack_dir {
1411            symloader::load_clientscript_pack(&mut registry, &pack.join("clientscript.pack"));
1412        }
1413        symloader::load_clientscript_signatures(&mut registry, &client_dir);
1414    }
1415
1416    symloader::patch_command_return_types(&mut registry);
1417
1418    let mut visibility = visibility
1419        .join()
1420        .unwrap_or_else(|panic| panic::resume_unwind(panic));
1421    visibility.index_registry(&registry);
1422    registry.visibility = Arc::new(visibility);
1423
1424    registry
1425}
1426
1427/// Finalize the registry once parsing is done: regenerate `script.pack` from the
1428/// parsed sources, load pre-assigned script IDs, then register every parsed
1429/// script (emitting trigger/redeclaration/return diagnostics). Builds on the
1430/// registry produced concurrently by `build_symbol_registry`.
1431fn register_scripts(
1432    registry: &mut SymbolRegistry,
1433    scripts_dir: &Path,
1434    resolved_pack_dir: Option<&Path>,
1435    diagnostics: &mut DiagnosticsCollector,
1436    all_files: &[ParsedFile],
1437    raw_by_path: &HashMap<PathBuf, Arc<String>>,
1438) {
1439    if let Some(pd) = resolved_pack_dir {
1440        symloader::generate_script_pack(scripts_dir, pd, raw_by_path);
1441        symloader::load_script_ids(registry, pd);
1442    }
1443
1444    {
1445        use crate::diagnostic_messages as msg;
1446
1447        let mut registered_scripts: std::collections::HashSet<String> =
1448            std::collections::HashSet::new();
1449
1450        for pf in all_files.iter() {
1451            Arc::make_mut(&mut registry.visibility).declare_source(&pf.path, &pf.file);
1452            for script in &pf.file.scripts {
1453                if script.trigger == "command" {
1454                    continue;
1455                }
1456
1457                if !trigger_table::is_valid_trigger(&script.trigger) {
1458                    diagnostics.warning(
1459                        pf.path.clone(),
1460                        script.line,
1461                        0,
1462                        msg::fmt(msg::SCRIPT_TRIGGER_INVALID, &[&script.trigger]),
1463                        crate::diagnostics::Phase::SymbolRegistration,
1464                    );
1465                }
1466
1467                let key = format!("{}:{}", script.trigger, script.name);
1468                if !registered_scripts.insert(key.clone()) {
1469                    diagnostics.warning(
1470                        pf.path.clone(),
1471                        script.line,
1472                        0,
1473                        msg::fmt(msg::SCRIPT_REDECLARATION, &[&script.trigger, &script.name]),
1474                        crate::diagnostics::Phase::SymbolRegistration,
1475                    );
1476                }
1477
1478                if !script.return_types.is_empty()
1479                    && !trigger_table::allows_returns(&script.trigger)
1480                {
1481                    diagnostics.warning(
1482                        pf.path.clone(),
1483                        script.line,
1484                        0,
1485                        msg::fmt(msg::SCRIPT_TRIGGER_NO_RETURNS, &[&script.trigger]),
1486                        diagnostics::Phase::SymbolRegistration,
1487                    );
1488                }
1489
1490                let param_types: Vec<types::Type> =
1491                    script.params.iter().map(|p| p.param_type).collect();
1492                registry.register_script(
1493                    script.name.clone(),
1494                    script.trigger.clone(),
1495                    param_types,
1496                    script.return_types.clone(),
1497                );
1498
1499                if let Some(ts_line) = pf.testscript_line
1500                    && script.line >= ts_line
1501                {
1502                    registry.mark_test_script(&script.trigger, &script.name);
1503                }
1504            }
1505        }
1506    }
1507}
1508
1509// ── Streaming recompile pipeline (prototype, gated by RUNEC_RECOMPILE) ────────
1510// Never holds all compiled scripts. Pass 2 only type-checks; the pointer check
1511// re-compiles each script on demand (bounded cache — measured ~1.3x re-parse on
1512// rs-majula, the call graph has high file locality); output is produced by
1513// re-compiling file-by-file into a by-id buffer. Output + diagnostics are
1514// byte-identical to the batch path (deterministic re-compile, same id order).
1515
1516/// Re-compiles scripts on demand for the pointer checker, caching `Rc`s with a
1517/// bounded FIFO so the full compiled set is never resident. A cache miss
1518/// re-parses the script's file and compiles all its scripts at once (siblings
1519/// share the parse), keying each by its full `[trigger,name]` to its index.
1520struct RecompileStore<'a> {
1521    idx_to_path: Vec<PathBuf>,
1522    /// file path -> its script idxs in declaration order (command-skipped).
1523    /// Position-based so duplicate `[trigger,name]` declarations each map to the
1524    /// correct distinct idx.
1525    file_to_idxs: HashMap<PathBuf, Vec<usize>>,
1526    registry: &'a SymbolRegistry,
1527    include_tests: bool,
1528    cache: HashMap<usize, std::rc::Rc<CompiledScript>>,
1529    order: std::collections::VecDeque<usize>,
1530    cap: usize,
1531}
1532
1533impl RecompileStore<'_> {
1534    fn get(&mut self, idx: usize) -> std::rc::Rc<CompiledScript> {
1535        if let Some(rc) = self.cache.get(&idx) {
1536            return std::rc::Rc::clone(rc);
1537        }
1538        let path = self.idx_to_path[idx].clone();
1539        let parsed = parse_one_file(&path, self.include_tests);
1540        if let Ok(ps) = parsed.read
1541            && let ParseOutcome::Parsed { file, .. } = ps.outcome
1542        {
1543            // The j-th non-command declaration in this file maps to idxs[j],
1544            // matching how the metas were built (same parse, same skip rule).
1545            let idxs = self.file_to_idxs.get(&path).cloned().unwrap_or_default();
1546            let mut compiler = Compiler::new(self.registry);
1547            let mut j = 0usize;
1548            for script in &file.scripts {
1549                if script.trigger == "command" {
1550                    continue;
1551                }
1552                if let Some(&i) = idxs.get(j) {
1553                    let mut c = compiler.compile_script(script);
1554                    c.source_path = ps.cache_key.clone();
1555                    self.insert(i, std::rc::Rc::new(c));
1556                }
1557                j += 1;
1558            }
1559        }
1560        std::rc::Rc::clone(
1561            self.cache
1562                .get(&idx)
1563                .expect("recompile store: script not produced by its file"),
1564        )
1565    }
1566
1567    fn insert(&mut self, idx: usize, rc: std::rc::Rc<CompiledScript>) {
1568        if self.cache.insert(idx, rc).is_none() {
1569            self.order.push_back(idx);
1570            while self.order.len() > self.cap {
1571                if let Some(old) = self.order.pop_front()
1572                    && old != idx
1573                {
1574                    self.cache.remove(&old);
1575                }
1576            }
1577        }
1578    }
1579}
1580
1581/// Per-slot encoded output blobs (indexed by script id) plus accumulated
1582/// diagnostics — what the recompile core hands back before the bytes are written
1583/// to disk or assembled in memory.
1584type RecompiledOutput = (Vec<Vec<u8>>, DiagnosticsCollector);
1585
1586/// Core of the recompile (low-memory) pipeline: runs every phase re-compiling
1587/// scripts on demand — never holding all compiled bytecode at once — and returns
1588/// the per-slot encoded output blobs plus diagnostics, without writing or
1589/// printing. The disk and in-memory entry points (`compile_recompile`,
1590/// `compile_recompile_memory`) wrap this. On a phase error it prints diagnostics
1591/// and returns `Err`, matching the batch path.
1592fn recompile_to_encoded(
1593    scripts_dir: &Path,
1594    pack_dir: Option<&Path>,
1595    lint: bool,
1596    include_tests: bool,
1597) -> Result<RecompiledOutput, Box<dyn Error>> {
1598    let source_dir = sources::target_dir(scripts_dir, sources::Scope::Rune);
1599    let scripts_dir = source_dir.as_path();
1600    use crate::pointer_checker::{PointerChecker, ScriptSource};
1601
1602    let mut rs2_files: Vec<PathBuf> = get_rs2_files(scripts_dir, "rs2")?;
1603    rs2_files.sort();
1604    info!("Found {} script files", rs2_files.len());
1605    mem_mark("startup");
1606    let mut diagnostics = DiagnosticsCollector::new();
1607
1608    // Pass 1: signatures only (ASTs dropped in-worker); keep file paths for lazy
1609    // source + raw text for script.pack regeneration.
1610    info!("[recompile] Pass 1: parsing for signatures...");
1611    let sig_results = parallel_map(&rs2_files, |path| parse_one_file_sigs(path, include_tests));
1612    let mut source_paths: HashMap<String, PathBuf> = HashMap::new();
1613    let mut raw_by_path: HashMap<PathBuf, Arc<String>> = HashMap::new();
1614    let mut file_sigs: Vec<FileSigs> = Vec::new();
1615    for r in sig_results {
1616        let (cache_key, raw_source, outcome) = match r.read {
1617            Ok(t) => t,
1618            Err(e) => return Err(Box::new(e)),
1619        };
1620        source_paths.insert(cache_key, r.path.clone());
1621        raw_by_path.insert(r.path.clone(), Arc::new(raw_source));
1622        match outcome {
1623            SigOutcome::Parsed {
1624                sigs,
1625                testscript_line,
1626            } => file_sigs.push(FileSigs {
1627                path: r.path,
1628                testscript_line,
1629                sigs,
1630            }),
1631            SigOutcome::Error {
1632                line,
1633                position,
1634                message,
1635                phase,
1636            } => diagnostics.error(r.path, line, position, message, phase),
1637        }
1638    }
1639    if diagnostics.has_errors() {
1640        diagnostics.print_all();
1641        return Err(Box::new(error::CompilerError::TypeError(
1642            "Parsing failed".to_string(),
1643        )));
1644    }
1645    mem_mark("after parse");
1646
1647    // Registration.
1648    info!("[recompile] Registering scripts and building the symbol registry...");
1649    let resolved_pack_dir = resolve_pack_dir(scripts_dir, pack_dir);
1650    let mut registry = build_symbol_registry(scripts_dir, resolved_pack_dir.as_deref());
1651    register_scripts_from_sigs(
1652        &mut registry,
1653        scripts_dir,
1654        resolved_pack_dir.as_deref(),
1655        &mut diagnostics,
1656        &file_sigs,
1657        &raw_by_path,
1658    );
1659    info!("  Registered {} scripts", registry.scripts.len());
1660    drop(raw_by_path);
1661    registry.drop_registration_scratch();
1662    for fs in &file_sigs {
1663        for sig in &fs.sigs {
1664            if let Some(msg) =
1665                Compiler::validate_trigger_subject(&sig.trigger, &sig.name, &registry)
1666            {
1667                diagnostics.warning(
1668                    fs.path.clone(),
1669                    sig.line,
1670                    0,
1671                    msg,
1672                    diagnostics::Phase::SymbolRegistration,
1673                );
1674            }
1675        }
1676    }
1677    mem_mark("after register");
1678
1679    // Pass 2: type-check only (re-parse, no compiled scripts retained).
1680    info!("[recompile] Pass 2: type-check (streaming, no bytecode held)...");
1681    let tc_diags = parallel_map(&rs2_files, |path| {
1682        let parsed = match parse_one_file(path, include_tests).read {
1683            Ok(p) => p,
1684            Err(_) => return DiagnosticsCollector::new(),
1685        };
1686        let (file, testscript_line) = match parsed.outcome {
1687            ParseOutcome::Parsed {
1688                file,
1689                testscript_line,
1690            } => (file, testscript_line),
1691            ParseOutcome::Error { .. } => return DiagnosticsCollector::new(),
1692        };
1693        let mut type_checker = TypeChecker::new(&registry);
1694        type_checker.check_file_with_test_boundary(&file, path, testscript_line);
1695        if !type_checker.diagnostics.has_errors() {
1696            let mut compiler = Compiler::new(&registry);
1697            for script in &file.scripts {
1698                if script.trigger != "command" {
1699                    compiler.compile_checked(script, path, &mut type_checker.diagnostics);
1700                }
1701            }
1702        }
1703        type_checker.diagnostics
1704    });
1705    for d in tc_diags {
1706        diagnostics.merge(d);
1707    }
1708    if diagnostics.has_errors() {
1709        diagnostics.print_all();
1710        return Err(Box::new(error::CompilerError::TypeError(
1711            "Type checking failed".to_string(),
1712        )));
1713    }
1714    mem_mark("after typecheck");
1715
1716    // Build script metas in the same idx order as the batch path: file order
1717    // (file_sigs / declaration order, command-skipped), then a stable sort by id.
1718    // ids + names come from the registry — no bytecode needed.
1719    let mut pre: Vec<(i32, String, PathBuf)> = Vec::new();
1720    for fs in &file_sigs {
1721        for sig in &fs.sigs {
1722            if sig.trigger == "command" {
1723                continue;
1724            }
1725            let id = registry
1726                .script_id_for_trigger(&sig.trigger, &sig.name)
1727                .or_else(|| registry.script_id(&sig.name))
1728                .unwrap_or(-1);
1729            pre.push((
1730                id,
1731                format!("[{},{}]", sig.trigger, sig.name),
1732                fs.path.clone(),
1733            ));
1734        }
1735    }
1736    let n = pre.len();
1737    // Stable sort indices by id → the batch's idx order (file/decl pre-order
1738    // preserved among equal ids). `inv` maps a pre-order position back to its idx.
1739    let mut order: Vec<usize> = (0..n).collect();
1740    order.sort_by_key(|&p| pre[p].0);
1741    let idents: Vec<(i32, String)> = order
1742        .iter()
1743        .map(|&p| (pre[p].0, pre[p].1.clone()))
1744        .collect();
1745    let idx_to_path: Vec<PathBuf> = order.iter().map(|&p| pre[p].2.clone()).collect();
1746    let mut inv = vec![0usize; n];
1747    for (idx, &p) in order.iter().enumerate() {
1748        inv[p] = idx;
1749    }
1750    let mut file_to_idxs: HashMap<PathBuf, Vec<usize>> = HashMap::new();
1751    for (p, m) in pre.iter().enumerate() {
1752        file_to_idxs.entry(m.2.clone()).or_default().push(inv[p]);
1753    }
1754    let max_id = pre.iter().map(|m| m.0).max().unwrap_or(-1);
1755    let slot_count = if max_id < 0 { 0 } else { (max_id + 1) as usize };
1756    let len = n;
1757    drop(pre);
1758
1759    // Pointer check via on-demand re-compile (single-threaded: the store getter
1760    // is `FnMut`/`!Sync`). Diagnostics come out in id order, as in the batch path.
1761    info!("[recompile] Pointer check (re-compile on demand)...");
1762    let shared = PointerChecker::build_shared_idents(&idents, &registry);
1763    {
1764        // Cache cap must exceed the largest file's script count, or loading that
1765        // file would evict its own just-compiled scripts. A small margin over
1766        // that keeps cross-file locality high (measured ~1.1-1.3x re-parse).
1767        let cap = file_to_idxs
1768            .values()
1769            .map(|v| v.len())
1770            .max()
1771            .unwrap_or(0)
1772            .max(128);
1773        let mut store = RecompileStore {
1774            idx_to_path,
1775            file_to_idxs,
1776            registry: &registry,
1777            include_tests,
1778            cache: HashMap::new(),
1779            order: std::collections::VecDeque::new(),
1780            cap,
1781        };
1782        let getter: Box<dyn FnMut(usize) -> std::rc::Rc<CompiledScript>> =
1783            Box::new(move |idx| store.get(idx));
1784        let scripts = ScriptSource::Recompile {
1785            len,
1786            get: std::cell::RefCell::new(getter),
1787        };
1788        let mut checker = PointerChecker::with_shared(scripts, &registry, shared);
1789        checker.set_source_cache(SourceProvider::Lazy(&source_paths));
1790        let names: Vec<String> = idents.iter().map(|(_, n)| n.clone()).collect();
1791        let ptr_diags = checker.validate_names(&names);
1792        let w = ptr_diags.warning_count();
1793        if w > 0 {
1794            info!("  Pointer check: {} warning(s)", w);
1795        }
1796        diagnostics.merge(ptr_diags);
1797    }
1798    mem_mark("after pointercheck");
1799
1800    // Output: re-compile each file once, encode each script into its id slot,
1801    // optionally lint — never holding all compiled scripts at once. The caller
1802    // writes the blobs to disk or assembles them in memory.
1803    info!("[recompile] Re-compiling + encoding output...");
1804    let mut encoded: Vec<Vec<u8>> = vec![Vec::new(); slot_count];
1805    for path in &rs2_files {
1806        let parsed = match parse_one_file(path, include_tests).read {
1807            Ok(p) => p,
1808            Err(_) => continue,
1809        };
1810        let file = match parsed.outcome {
1811            ParseOutcome::Parsed { file, .. } => file,
1812            ParseOutcome::Error { .. } => continue,
1813        };
1814        let mut compiler = Compiler::new(&registry);
1815        let mut file_scripts: Vec<CompiledScript> = Vec::new();
1816        for script in &file.scripts {
1817            if script.trigger == "command" {
1818                continue;
1819            }
1820            let Some(mut c) = compiler.compile_checked(script, path, &mut diagnostics) else {
1821                continue;
1822            };
1823            c.source_path = parsed.cache_key.clone();
1824            file_scripts.push(c);
1825        }
1826        for c in &file_scripts {
1827            if c.id >= 0 && (c.id as usize) < slot_count {
1828                encoded[c.id as usize] = crate::writer::encode_script(c);
1829            }
1830        }
1831        if lint {
1832            diagnostics.merge(lints::run_lints(
1833                &file_scripts,
1834                Some(SourceProvider::Lazy(&source_paths)),
1835            ));
1836        }
1837    }
1838    mem_mark("after codegen");
1839    Ok((encoded, diagnostics))
1840}
1841
1842/// Full compile via the recompile (low-memory) pipeline: writes `script.dat` /
1843/// `script.idx` to `output_dir`, then prints diagnostics.
1844fn compile_recompile(
1845    scripts_dir: &Path,
1846    pack_dir: Option<&Path>,
1847    output_dir: &Path,
1848    lint: bool,
1849) -> Result<(), Box<dyn Error>> {
1850    let (encoded, diagnostics) = recompile_to_encoded(scripts_dir, pack_dir, lint, false)?;
1851    info!(
1852        "Writing output to {}...",
1853        output_dir.display().to_string().replace('\\', "/")
1854    );
1855    ScriptWriter::new(output_dir.display().to_string()).write_encoded(&encoded)?;
1856    diagnostics.print_all();
1857    info!("Compilation complete!");
1858    Ok(())
1859}
1860
1861/// In-memory compile via the recompile (low-memory) pipeline: returns the
1862/// `(script.dat, script.idx)` bytes, then prints diagnostics. Byte-identical to
1863/// the in-memory batch path (`compile_memory*`).
1864fn compile_recompile_memory(
1865    scripts_dir: &Path,
1866    pack_dir: Option<&Path>,
1867    lint: bool,
1868    include_tests: bool,
1869) -> Result<(Vec<u8>, Vec<u8>), Box<dyn Error>> {
1870    let (encoded, diagnostics) = recompile_to_encoded(scripts_dir, pack_dir, lint, include_tests)?;
1871    info!("Writing compiled output to memory...");
1872    let out = ScriptWriter::new("".into()).build_encoded(&encoded);
1873    diagnostics.print_all();
1874    info!("Compilation complete!");
1875    Ok(out)
1876}
1877
1878pub(crate) fn find_testscript_line(source: &str) -> Option<usize> {
1879    for (i, line) in source.lines().enumerate() {
1880        if line.trim() == "#testscript" {
1881            return Some(i + 1);
1882        }
1883    }
1884    None
1885}
1886
1887fn truncate_at_line(source: &str, line_no: usize) -> String {
1888    source
1889        .lines()
1890        .take(line_no - 1)
1891        .collect::<Vec<_>>()
1892        .join("\n")
1893}
1894
1895fn strip_testscript_annotation(source: &str) -> String {
1896    source
1897        .lines()
1898        .map(|line| {
1899            if line.trim() == "#testscript" {
1900                ""
1901            } else {
1902                line
1903            }
1904        })
1905        .collect::<Vec<_>>()
1906        .join("\n")
1907}
1908
1909fn get_rs2_files(dir: &Path, ext: &str) -> Result<Vec<PathBuf>, Box<dyn Error>> {
1910    Ok(sources::script_files(dir, ext)?)
1911}