Skip to main content

runec/
filesystem.rs

1use crate::bytecode::CompiledScript;
2use crate::clientscript::{self, format::Format};
3use crate::compiler::Compiler;
4use crate::diagnostics::DiagnosticsCollector;
5use crate::lexer::Lexer;
6#[cfg(feature = "memprof")]
7use crate::memprof;
8use crate::parallel::{parallel_chunks, parallel_chunks_capped, parallel_map};
9use crate::parser::{Parser, ScriptFile};
10use crate::project::{Options, Project, Target};
11use crate::sources::Scope;
12use crate::symbol::SymbolRegistry;
13use crate::typechecker::TypeChecker;
14use crate::writer::ScriptWriter;
15use crate::{diagnostics, error, lints, sources, symloader, trigger_table, types};
16use std::collections::HashMap;
17use std::error::Error;
18use std::fmt::Write;
19use std::fs;
20use std::path::{Path, PathBuf};
21use std::sync::Arc;
22use tracing::info;
23
24/// Build an editor project using the legacy directory conventions. No ID file
25/// is generated or modified; all further editing happens in memory.
26pub fn editor_project(
27    source: &Path,
28    pack: Option<&Path>,
29    client: bool,
30) -> Result<Project, Box<dyn Error>> {
31    let scope = if client { Scope::Client } else { Scope::Rune };
32    let dir = sources::target_dir(source, scope);
33    let pack = resolve_pack_dir(&dir, pack);
34    let mut registry = if client {
35        clientscript::registry(
36            &dir,
37            pack.as_deref()
38                .ok_or("Client opcode definitions require --pack")?,
39        )?
40    } else {
41        build_symbol_registry(&dir, pack.as_deref())
42    };
43    let target = if client {
44        registry.preloaded_script_ids.clear();
45        for (key, id) in clientscript::script_ids(pack.as_deref().unwrap())? {
46            let (trigger, name) = key
47                .trim_start_matches('[')
48                .trim_end_matches(']')
49                .split_once(',')
50                .ok_or("Invalid client ID")?;
51            registry.set_script_id(trigger, name, id);
52        }
53        Target::Client(Format::load(
54            &pack.as_ref().unwrap().join("cs2_command.pack"),
55        )?)
56    } else {
57        if let Some(pack) = &pack {
58            symloader::load_script_ids(&mut registry, pack);
59        }
60        Target::Rune
61    };
62    let mut project = Project::new(
63        target,
64        registry,
65        Options {
66            include_tests: true,
67            skip_shared_lints: client
68                && dir
69                    .parent()
70                    .is_some_and(|parent| parent.join("runescript").is_dir()),
71            ..Options::default()
72        },
73    );
74    for path in sources::script_files(&dir, if client { "cs2" } else { "rs2" })? {
75        let text = fs::read_to_string(&path)?;
76        let scope = if path.extension().is_some_and(|ext| ext == "gs2") {
77            Scope::Global
78        } else {
79            scope
80        };
81        let uri = file_uri(&path)?;
82        let id = project.open(uri, scope, 0, text);
83        if client {
84            let ids = project
85                .document(id)
86                .and_then(|doc| doc.syntax())
87                .into_iter()
88                .flat_map(|file| &file.scripts)
89                .filter_map(|script| {
90                    script
91                        .name
92                        .strip_prefix("clientscript_")
93                        .and_then(|id| id.parse::<i32>().ok())
94                        .map(|id| (script.trigger.clone(), script.name.clone(), id))
95                })
96                .collect::<Vec<_>>();
97            for (trigger, name, id) in ids {
98                project.registry.set_script_id(&trigger, &name, id);
99            }
100        }
101    }
102    if !client {
103        let client_dir = if dir.file_name().is_some_and(|name| name == "runescript") {
104            dir.with_file_name("clientscript")
105        } else {
106            dir.join("clientscript")
107        };
108        if client_dir.is_dir() {
109            for path in sources::script_files(&client_dir, "cs2")?
110                .into_iter()
111                .filter(|path| {
112                    path.extension().is_some_and(|ext| ext == "cs2")
113                        && path.file_name().is_some_and(|name| name != "engine.cs2")
114                })
115            {
116                project.open(
117                    file_uri(&path)?,
118                    Scope::Client,
119                    0,
120                    fs::read_to_string(&path)?,
121                );
122            }
123        }
124    }
125    let mut files = sources::config_files(&dir);
126    if let Some(pack) = &pack {
127        symloader::collect_all_files(pack, &mut files);
128        // Interfaces, world maps and defaults live beside the scripts they're for.
129        if let Some(content) = pack.parent() {
130            for (folder, extension) in [
131                ("interfaces", "if3"),
132                ("worldmap_data", "wma"),
133                ("defaults", "defaults"),
134            ] {
135                let Ok(entries) = fs::read_dir(content.join(folder)) else {
136                    continue;
137                };
138                files.extend(
139                    entries
140                        .flatten()
141                        .map(|entry| entry.path())
142                        .filter(|path| path.extension().is_some_and(|ext| ext == extension)),
143                );
144            }
145        }
146    }
147    files.sort();
148    files.dedup();
149    for path in files {
150        let Some(kind) = config_kind(&path) else {
151            continue;
152        };
153        let text = fs::read_to_string(&path)?;
154        let config_scope = if path
155            .components()
156            .any(|part| part.as_os_str() == "globalscript")
157        {
158            Scope::Global
159        } else {
160            scope
161        };
162        project.open_config(file_uri(&path)?, config_scope, 0, text, kind);
163    }
164    Ok(project)
165}
166
167pub(crate) fn config_kind(path: &Path) -> Option<String> {
168    let extension = path.extension()?.to_str()?;
169    if matches!(extension, "pack" | "order" | "hashes") {
170        return Some(format!("pack:{}", path.file_stem()?.to_str()?));
171    }
172    (matches!(extension, "constant" | "if" | "if3" | "defaults" | "anim")
173        || types::Type::from_name(extension).is_some()
174        || extension
175            .strip_prefix("config")
176            .is_some_and(|number| number.parse::<u32>().is_ok()))
177    .then(|| extension.to_string())
178}
179
180pub(crate) fn file_uri(path: &Path) -> Result<String, Box<dyn Error>> {
181    let absolute = fs::canonicalize(path)?
182        .to_string_lossy()
183        .trim_start_matches("\\\\?\\")
184        .replace('\\', "/");
185    let mut uri = String::from(if absolute.starts_with('/') {
186        "file://"
187    } else {
188        "file:///"
189    });
190    for byte in absolute.bytes() {
191        if byte.is_ascii_alphanumeric() || b"/-._~:".contains(&byte) {
192            uri.push(byte as char);
193        } else {
194            write!(uri, "%{byte:02X}")?;
195        }
196    }
197    Ok(uri)
198}
199
200/// How the diagnostic layer fetches a file's source text (by its cache-key path)
201/// when building rustc-style help/suggestions. The batch path keeps every source
202/// in memory; the streaming path keeps only the on-disk paths and re-reads the
203/// handful of files that actually produce a warning — so the ~4 MB of source
204/// text isn't resident through the compile's peak. Cheap to copy (just borrows).
205#[derive(Clone, Copy)]
206pub enum SourceProvider<'a> {
207    /// Sources already resident, keyed by cache key.
208    Eager(&'a HashMap<String, Arc<String>>),
209    /// Cache key → on-disk path; re-read on demand.
210    Lazy(&'a HashMap<String, PathBuf>),
211}
212
213impl SourceProvider<'_> {
214    /// Source text for `key`, if available. Returns an owned `Arc` — a cheap
215    /// refcount bump for `Eager`, a fresh disk read for `Lazy`. Called only while
216    /// emitting diagnostic help, which is rare, so the re-read cost is immaterial.
217    pub fn get(&self, key: &str) -> Option<Arc<String>> {
218        match self {
219            SourceProvider::Eager(m) => m.get(key).cloned(),
220            SourceProvider::Lazy(m) => fs::read_to_string(m.get(key)?).ok().map(Arc::new),
221        }
222    }
223}
224
225/// Log live heap + global peak at a pipeline phase boundary. No-op unless built
226/// with `--features memprof`. Lets us attribute the footprint to each phase.
227#[cfg(feature = "memprof")]
228fn mem_mark(label: &str) {
229    info!(
230        "[mem] {:<22} live {:>7.1} MB | peak {:>7.1} MB | allocs {}",
231        label,
232        memprof::current_bytes() as f64 / 1_048_576.0,
233        memprof::peak_bytes() as f64 / 1_048_576.0,
234        memprof::alloc_count(),
235    );
236}
237#[cfg(not(feature = "memprof"))]
238#[inline]
239fn mem_mark(_label: &str) {}
240
241/// Per-map breakdown of the symbol registry's heap, to see the irreducible
242/// floor for a streaming rebuild. Built only under `--features memprof`.
243/// Accounts three components per map: the hashbrown table (inline `(K,V)`
244/// buckets + 1 control byte each), the key strings, and value-side heap.
245#[cfg(feature = "memprof")]
246fn mem_stats_registry(reg: &crate::symbol::SymbolRegistry) {
247    use crate::symbol::{Symbol, SymbolKind};
248    use crate::types::Type;
249    use std::collections::{HashMap, HashSet};
250    let t = std::mem::size_of::<Type>();
251    let mb = |b: usize| b as f64 / 1_048_576.0;
252
253    // Allocated bucket count: hashbrown keeps `capacity()` ≈ 7/8 of a power-of-two
254    // bucket array; an empty map allocates nothing.
255    fn tbl<K, V>(m: &HashMap<K, V>) -> usize {
256        if m.capacity() == 0 {
257            return 0;
258        }
259        let buckets = (m.capacity() * 8 / 7).next_power_of_two();
260        buckets * (std::mem::size_of::<(K, V)>() + 1)
261    }
262    fn keys_cap<V>(m: &HashMap<String, V>) -> usize {
263        m.keys().map(|k| k.capacity()).sum()
264    }
265    let sym_heap = |s: &Symbol| -> usize {
266        s.name.capacity()
267            + match &s.kind {
268                SymbolKind::Script {
269                    trigger,
270                    param_types,
271                    return_types,
272                    ..
273                } => trigger.capacity() + (param_types.capacity() + return_types.capacity()) * t,
274                SymbolKind::Command {
275                    param_types,
276                    return_types,
277                    ..
278                } => (param_types.capacity() + return_types.capacity()) * t,
279                SymbolKind::GameVar { category, .. } => category.capacity(),
280                SymbolKind::Constant { string_value, .. } => {
281                    string_value.as_ref().map_or(0, |s| s.capacity())
282                }
283                _ => 0,
284            }
285    };
286    let sym_map = |m: &HashMap<String, Symbol>| -> usize {
287        tbl(m) + keys_cap(m) + m.values().map(&sym_heap).sum::<usize>()
288    };
289    let i32_map = |m: &HashMap<String, i32>| -> usize { tbl(m) + keys_cap(m) };
290
291    let mut rows: Vec<(&str, usize, usize)> = vec![
292        (
293            "scripts",
294            reg.scripts.len(),
295            tbl(&reg.scripts) + keys_cap(&reg.scripts),
296        ),
297        ("commands", reg.commands.len(), sym_map(&reg.commands)),
298        ("game_vars", reg.game_vars.len(), sym_map(&reg.game_vars)),
299        ("constants", reg.constants.len(), sym_map(&reg.constants)),
300        (
301            "entity_ids",
302            reg.entity_ids.len(),
303            tbl(&reg.entity_ids)
304                + keys_cap(&reg.entity_ids)
305                + reg
306                    .entity_ids
307                    .values()
308                    .map(|e| e.variants.capacity() * std::mem::size_of::<(Type, i32)>())
309                    .sum::<usize>(),
310        ),
311        (
312            "scripts_by_trigger",
313            reg.scripts_by_trigger.len(),
314            tbl(&reg.scripts_by_trigger)
315                + keys_cap(&reg.scripts_by_trigger)
316                + reg
317                    .scripts_by_trigger
318                    .values()
319                    .map(|s| 16 + sym_heap(s.as_ref()))
320                    .sum::<usize>(),
321        ),
322        (
323            "command_param_types",
324            reg.command_param_types.len(),
325            tbl(&reg.command_param_types)
326                + keys_cap(&reg.command_param_types)
327                + reg
328                    .command_param_types
329                    .values()
330                    .map(|v| v.capacity() * t)
331                    .sum::<usize>(),
332        ),
333        ("script_ids", reg.script_ids.len(), i32_map(&reg.script_ids)),
334        (
335            "proc_script_ids",
336            reg.proc_script_ids.len(),
337            i32_map(&reg.proc_script_ids),
338        ),
339        (
340            "label_script_ids",
341            reg.label_script_ids.len(),
342            i32_map(&reg.label_script_ids),
343        ),
344        (
345            "trigger_script_ids",
346            reg.trigger_script_ids.len(),
347            i32_map(&reg.trigger_script_ids),
348        ),
349        (
350            "preloaded_script_ids",
351            reg.preloaded_script_ids.len(),
352            i32_map(&reg.preloaded_script_ids),
353        ),
354        ("components", reg.components.len(), i32_map(&reg.components)),
355        (
356            "interface_ids",
357            reg.interface_ids.len(),
358            i32_map(&reg.interface_ids),
359        ),
360        ("type_chars", reg.type_chars.len(), i32_map(&reg.type_chars)),
361        (
362            "dbcolumn_types",
363            reg.dbcolumn_types.len(),
364            tbl(&reg.dbcolumn_types) + keys_cap(&reg.dbcolumn_types),
365        ),
366    ];
367    let ts_tbl = {
368        let s: &HashSet<String> = &reg.test_scripts;
369        if s.capacity() == 0 {
370            0
371        } else {
372            (s.capacity() * 8 / 7).next_power_of_two() * (std::mem::size_of::<String>() + 1)
373                + s.iter().map(|k| k.capacity()).sum::<usize>()
374        }
375    };
376    rows.push(("test_scripts", reg.test_scripts.len(), ts_tbl));
377
378    rows.sort_by_key(|row| std::cmp::Reverse(row.2));
379    let total: usize = rows.iter().map(|r| r.2).sum();
380    info!(
381        "[mem] registry ~{:.2} MB (sizeof Symbol={}B, SymbolKind={}B, Type={}B):",
382        mb(total),
383        std::mem::size_of::<Symbol>(),
384        std::mem::size_of::<SymbolKind>(),
385        t,
386    );
387    for (name, n, bytes) in &rows {
388        info!(
389            "[mem]   {:<20} {:>6} entries  {:>7.3} MB",
390            name,
391            n,
392            mb(*bytes)
393        );
394    }
395}
396
397/// One-off breakdown of where the compiled-bytecode heap goes, to target
398/// memory work. Built only under `--features memprof`.
399#[cfg(feature = "memprof")]
400fn mem_stats_compiled(scripts: &[CompiledScript]) {
401    use crate::bytecode::{Instruction, Opcode, Operand};
402    let (mut n_instr, mut n_str, mut str_bytes) = (0usize, 0usize, 0usize);
403    let (mut n_switch, mut switch_entries) = (0usize, 0usize);
404    let (mut source_path_bytes, mut name_bytes) = (0usize, 0usize);
405    for s in scripts {
406        n_instr += s.instructions.len();
407        source_path_bytes += s.source_path.capacity();
408        name_bytes += s.name.capacity() + s.trigger.capacity();
409        for st in &s.strings {
410            n_str += 1;
411            str_bytes += st.len();
412        }
413        for sw in &s.switch_tables {
414            n_switch += 1;
415            switch_entries += sw.len();
416        }
417    }
418    let sz = std::mem::size_of::<Instruction>();
419    let mb = |b: usize| b as f64 / 1_048_576.0;
420    info!(
421        "[mem] sizeof Instruction={}B Operand={}B Opcode={}B CompiledScript={}B",
422        sz,
423        std::mem::size_of::<Operand>(),
424        std::mem::size_of::<Opcode>(),
425        std::mem::size_of::<CompiledScript>(),
426    );
427    info!(
428        "[mem] compiled: {} scripts, {} instrs => {:.2} MB instr structs",
429        scripts.len(),
430        n_instr,
431        mb(n_instr * sz)
432    );
433    info!(
434        "[mem]   Str operands: {} ({:.2} MB content + {:.2} MB Box<str> headers)",
435        n_str,
436        mb(str_bytes),
437        mb(n_str * 16)
438    );
439    info!(
440        "[mem]   SwitchTable operands: {} ({} entries => {:.2} MB)",
441        n_switch,
442        switch_entries,
443        mb(switch_entries * 16)
444    );
445    info!(
446        "[mem]   source_path {:.2} MB, names+triggers {:.2} MB",
447        mb(source_path_bytes),
448        mb(name_bytes)
449    );
450}
451
452struct ParsedFile {
453    path: PathBuf,
454    file: ScriptFile,
455    testscript_line: Option<usize>,
456    /// Canonicalized, normalized source path. Computed once during parsing and
457    /// reused as each compiled script's `source_path` (it is byte-identical to
458    /// the `source_cache` key), avoiding a second `canonicalize` syscall per
459    /// file in codegen.
460    source_path: String,
461}
462
463/// Outcome of parsing one file off-thread. The owned source text and cache key
464/// travel back so the (single-threaded) merge can build the `Rc`-based caches
465/// without sending `Rc` across threads.
466struct FileParseResult {
467    path: PathBuf,
468    read: Result<ParsedSource, std::io::Error>,
469}
470
471struct ParsedSource {
472    cache_key: String,
473    raw_source: String,
474    outcome: ParseOutcome,
475}
476
477enum ParseOutcome {
478    Parsed {
479        file: ScriptFile,
480        testscript_line: Option<usize>,
481    },
482    Error {
483        line: usize,
484        position: usize,
485        message: String,
486        phase: diagnostics::Phase,
487    },
488}
489
490/// Read, lex and parse a single file off-thread. Pure: no shared state, so it
491/// is safe to run concurrently. Mirrors the original sequential body exactly —
492/// same source truncation, same cache-key normalization, same lex-then-parse
493/// error reporting — only restructured to return owned data for the merge.
494fn parse_one_file(path: &Path, include_tests: bool) -> FileParseResult {
495    let raw_source = match fs::read_to_string(path) {
496        Ok(s) => s,
497        Err(e) => {
498            return FileParseResult {
499                path: path.to_path_buf(),
500                read: Err(e),
501            };
502        }
503    };
504    let testscript_line = find_testscript_line(&raw_source);
505
506    let source_code = if !include_tests {
507        if let Some(line_no) = testscript_line {
508            truncate_at_line(&raw_source, line_no)
509        } else {
510            raw_source.clone()
511        }
512    } else {
513        strip_testscript_annotation(&raw_source)
514    };
515
516    let canonical = fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf());
517    let mut cache_key = canonical.to_string_lossy().into_owned();
518    if cache_key.starts_with("\\\\?\\") {
519        cache_key = cache_key[4..].to_string();
520    }
521    cache_key = cache_key.replace("\\Content\\", "\\content\\");
522
523    // Lexer/Parser borrow a `&PathBuf` for diagnostics; keep one alive locally.
524    let path_buf = path.to_path_buf();
525    let outcome = match Lexer::new(&source_code, &path_buf).tokenize() {
526        Err(e) => ParseOutcome::Error {
527            line: e.line,
528            position: e.position,
529            message: e.message,
530            phase: diagnostics::Phase::Lexing,
531        },
532        Ok(tokens) => match Parser::new(tokens, &path_buf).parse() {
533            Ok(file) => ParseOutcome::Parsed {
534                file,
535                testscript_line: if include_tests { testscript_line } else { None },
536            },
537            Err(e) => ParseOutcome::Error {
538                line: e.line,
539                position: e.position,
540                message: e.message,
541                phase: diagnostics::Phase::Parsing,
542            },
543        },
544    };
545
546    FileParseResult {
547        path: path_buf,
548        read: Ok(ParsedSource {
549            cache_key,
550            raw_source,
551            outcome,
552        }),
553    }
554}
555
556/// Compile scripts and write output. When `lint` is true, also run lint passes
557/// (unused locals, unreachable code) after code generation.
558pub fn compile_memory(
559    scripts_dir: &Path,
560    pack_dir: Option<&Path>,
561    lint: bool,
562) -> Result<(Vec<u8>, Vec<u8>), Box<dyn Error>> {
563    compile_memory_with_options(scripts_dir, pack_dir, lint, false, false)
564}
565
566/// Like `compile_memory` but includes `#testscript` sections when
567/// `include_tests` is true.
568pub fn compile_memory_with_tests(
569    scripts_dir: &Path,
570    pack_dir: Option<&Path>,
571    lint: bool,
572    include_tests: bool,
573) -> Result<(Vec<u8>, Vec<u8>), Box<dyn Error>> {
574    compile_memory_with_options(scripts_dir, pack_dir, lint, include_tests, false)
575}
576
577/// In-memory compile — returns the `(script.dat, script.idx)` bytes instead of
578/// writing them — choosing the compilation strategy.
579///
580/// With `low_mem`, the recompile pipeline is used: scripts are re-parsed and
581/// re-compiled on demand so the full compiled set is never resident (~16 MB heap
582/// vs ~93), at roughly 6× the time; the returned bytes are byte-identical to the
583/// default path. `include_tests` keeps `#testscript` sections. The
584/// `RUNEC_RECOMPILE` environment variable forces low-memory mode on as well.
585pub fn compile_memory_with_options(
586    scripts_dir: &Path,
587    pack_dir: Option<&Path>,
588    lint: bool,
589    include_tests: bool,
590    low_mem: bool,
591) -> Result<(Vec<u8>, Vec<u8>), Box<dyn Error>> {
592    if low_mem || std::env::var("RUNEC_RECOMPILE").is_ok_and(|v| !v.is_empty()) {
593        info!("[recompile] low-memory recompile pipeline enabled");
594        return compile_recompile_memory(scripts_dir, pack_dir, lint, include_tests);
595    }
596
597    let (compiled_scripts, diagnostics) = run_pipeline(scripts_dir, pack_dir, lint, include_tests)?;
598
599    info!("Writing compiled output to memory...");
600    let writer = ScriptWriter::new("".into());
601
602    diagnostics.print_all();
603    info!("Compilation complete!");
604
605    Ok(writer.build_all(&compiled_scripts)?)
606}
607
608/// Compile scripts and write output. When `lint` is true, also run lint passes
609/// (unused locals, unreachable code) after code generation.
610///
611/// Convenience wrapper over [`compile_with_options`] using the default (fast,
612/// fully-resident) pipeline. Kept signature-compatible for existing callers.
613pub fn compile(
614    scripts_dir: &Path,
615    pack_dir: Option<&Path>,
616    output_dir: &Path,
617    lint: bool,
618) -> Result<(), Box<dyn Error>> {
619    compile_with_options(scripts_dir, pack_dir, output_dir, lint, false)
620}
621
622/// Compile scripts and write output, choosing the compilation strategy.
623///
624/// With `low_mem`, the **recompile** pipeline is used: scripts are re-parsed and
625/// re-compiled on demand for the pointer check and written file-by-file, so the
626/// full set of compiled bytecode is never resident at once. That cuts peak usage
627/// to roughly 16 MB heap / 32 MB working set (from ~93 / ~111 MB) at about 6× the
628/// compile time. Output is byte-identical to the default path. The
629/// `RUNEC_RECOMPILE` environment variable forces this mode on as well.
630pub fn compile_with_options(
631    scripts_dir: &Path,
632    pack_dir: Option<&Path>,
633    output_dir: &Path,
634    lint: bool,
635    low_mem: bool,
636) -> Result<(), Box<dyn Error>> {
637    // The recompile pipeline never holds all compiled scripts (it re-compiles on
638    // demand for the pointer check + writes file-by-file). It does its own
639    // output, so it returns here. The default path below is unchanged.
640    if low_mem || std::env::var("RUNEC_RECOMPILE").is_ok_and(|v| !v.is_empty()) {
641        info!("[recompile] low-memory recompile pipeline enabled");
642        return compile_recompile(scripts_dir, pack_dir, output_dir, lint);
643    }
644
645    let (compiled_scripts, diagnostics) = run_pipeline(scripts_dir, pack_dir, lint, false)?;
646
647    let output = output_dir.display().to_string();
648    // Normalize separators for a clean, consistent log message (input paths may
649    // mix `/` and `\`, e.g. `data/pack\server`); the writer keeps the original.
650    info!("Writing output to {}...", output.replace('\\', "/"));
651    let writer = ScriptWriter::new(output);
652    writer.write_all(&compiled_scripts)?;
653
654    diagnostics.print_all();
655    info!("Compilation complete!");
656
657    Ok(())
658}
659
660/// Run all analysis passes (parse, type-check, codegen, pointer-check, lints)
661/// without writing output. Useful for editor tooling and CI lint gates.
662pub fn lint(scripts_dir: &Path, pack_dir: Option<&Path>) -> Result<(), Box<dyn Error>> {
663    let (_, diagnostics) = run_pipeline(scripts_dir, pack_dir, true, false)?;
664
665    diagnostics.print_all();
666    info!("Compilation complete!");
667
668    Ok(())
669}
670
671/// Runs the full compilation pipeline (parse, register, type-check, codegen,
672/// pointer-check, and optional lint passes), returning the compiled scripts
673/// and accumulated diagnostics for the caller to handle output.
674fn run_pipeline(
675    scripts_dir: &Path,
676    pack_dir: Option<&Path>,
677    lint: bool,
678    include_tests: bool,
679) -> Result<(Vec<CompiledScript>, DiagnosticsCollector), Box<dyn Error>> {
680    let source_dir = sources::target_dir(scripts_dir, sources::Scope::Rune);
681    let scripts_dir = source_dir.as_path();
682    if !scripts_dir.exists() || !scripts_dir.is_dir() {
683        return Err(Box::new(error::CompilerError::FileNotFound(
684            scripts_dir.display().to_string(),
685        )));
686    }
687
688    // Prototype: opt into the streaming pipeline (never holds all ASTs at once)
689    // with RUNEC_STREAM=1. The default batch path below is unchanged.
690    if std::env::var("RUNEC_STREAM").is_ok_and(|v| !v.is_empty()) {
691        info!("[stream] streaming pipeline enabled (RUNEC_STREAM)");
692        return run_pipeline_streaming(scripts_dir, pack_dir, lint, include_tests);
693    }
694
695    let mut rs2_files: Vec<PathBuf> = get_rs2_files(scripts_dir, "rs2")?;
696
697    rs2_files.sort();
698    info!("Found {} script files", rs2_files.len());
699    mem_mark("startup");
700
701    let mut diagnostics = DiagnosticsCollector::new();
702
703    // Phase 1: Parse all files. Reading, lexing and parsing are independent
704    // per file, so fan them out across threads; the results are merged back in
705    // input order so registration/ID assignment stays deterministic.
706    info!("Parsing script files into syntax trees...");
707    let parse_results = parallel_map(&rs2_files, |path| parse_one_file(path, include_tests));
708
709    let mut all_files: Vec<ParsedFile> = Vec::new();
710    let mut source_cache: HashMap<String, Arc<String>> = HashMap::new();
711    // Raw source text keyed by on-disk path, reused by `generate_script_pack`
712    // so it doesn't re-read every file from disk a second time.
713    let mut raw_by_path: HashMap<PathBuf, Arc<String>> = HashMap::new();
714    for result in parse_results {
715        let parsed = match result.read {
716            Ok(p) => p,
717            // A read failure aborts the whole compile, exactly as the previous
718            // `fs::read_to_string(path)?` did.
719            Err(e) => return Err(Box::new(e)),
720        };
721        let rc = Arc::new(parsed.raw_source);
722        // The cache key is the canonicalized source path; reuse it as each
723        // compiled script's `source_path` so codegen needn't canonicalize again.
724        let source_path = parsed.cache_key.clone();
725        source_cache.insert(parsed.cache_key, rc.clone());
726        raw_by_path.insert(result.path.clone(), rc);
727        match parsed.outcome {
728            ParseOutcome::Parsed {
729                file,
730                testscript_line,
731            } => all_files.push(ParsedFile {
732                path: result.path,
733                file,
734                testscript_line,
735                source_path,
736            }),
737            ParseOutcome::Error {
738                line,
739                position,
740                message,
741                phase,
742            } => diagnostics.error(result.path, line, position, message, phase),
743        }
744    }
745
746    if diagnostics.has_errors() {
747        diagnostics.print_all();
748        return Err(Box::new(error::CompilerError::TypeError(
749            "Parsing failed".to_string(),
750        )));
751    }
752
753    mem_mark("after parse");
754
755    // Phase 2: Register all scripts. Build the pack/config registry, then
756    // regenerate script.pack, load pre-assigned IDs, and register every script.
757    info!("Registering scripts and building the symbol registry...");
758    let resolved_pack_dir = resolve_pack_dir(scripts_dir, pack_dir);
759    let mut registry = build_symbol_registry(scripts_dir, resolved_pack_dir.as_deref());
760    register_scripts(
761        &mut registry,
762        scripts_dir,
763        resolved_pack_dir.as_deref(),
764        &mut diagnostics,
765        &all_files,
766        &raw_by_path,
767    );
768    info!("  Registered {} scripts", registry.scripts.len());
769    // `raw_by_path` was only needed to (re)generate script.pack inside
770    // `register_scripts`; the source text itself stays reachable via
771    // `source_cache` (shared `Arc`s). Drop the redundant index now.
772    drop(raw_by_path);
773    // Pre-assigned script ids were consumed during registration; free them
774    // before codegen (the compile's peak-memory phase) rather than at scope end.
775    registry.drop_registration_scratch();
776
777    for pf in &all_files {
778        for script in &pf.file.scripts {
779            if let Some(msg) =
780                Compiler::validate_trigger_subject(&script.trigger, &script.name, &registry)
781            {
782                diagnostics.warning(
783                    pf.path.clone(),
784                    script.line,
785                    0,
786                    msg,
787                    diagnostics::Phase::SymbolRegistration,
788                );
789            }
790        }
791    }
792
793    mem_mark("after register");
794    #[cfg(feature = "memprof")]
795    mem_stats_registry(&registry);
796
797    // Phase 3: Type checking
798    info!("Type checking {} files...", all_files.len());
799    run_type_checker(&mut diagnostics, &all_files, &registry);
800
801    if diagnostics.has_errors() {
802        diagnostics.print_all();
803        return Err(Box::new(error::CompilerError::TypeError(
804            "Type checking failed".to_string(),
805        )));
806    }
807
808    mem_mark("after typecheck");
809
810    // Phase 4: Code generation
811    info!("Generating bytecode from type-checked scripts...");
812    let mut compiled_scripts = codegen(&all_files, &registry, &mut diagnostics);
813    if diagnostics.has_errors() {
814        diagnostics.print_all();
815        return Err(Box::new(error::CompilerError::CodeGenError(
816            "Code generation failed".into(),
817        )));
818    }
819    info!("  Generated {} compiled scripts", compiled_scripts.len());
820    compiled_scripts.sort_by_key(|s| s.id);
821    compiled_scripts.shrink_to_fit();
822    mem_mark("after codegen");
823    #[cfg(feature = "memprof")]
824    mem_stats_compiled(&compiled_scripts);
825
826    // The ASTs are consumed by codegen and never read again — pointer checking
827    // and lints work purely off `compiled_scripts` + the registry + the source
828    // cache. Free the syntax trees now so they don't coexist with the pointer
829    // checker's (per-thread) scratch, which is the compile's peak-memory moment.
830    drop(all_files);
831    mem_mark("after drop asts");
832
833    // Phase 4.5: Pointer checking & lints
834    info!("Checking pointers in generated code...");
835    check_pointers(
836        lint,
837        &mut diagnostics,
838        SourceProvider::Eager(&source_cache),
839        &registry,
840        &compiled_scripts,
841    );
842    mem_mark("after pointercheck");
843
844    Ok((compiled_scripts, diagnostics))
845}
846
847// ── Streaming pipeline (prototype, gated by RUNEC_STREAM) ────────────────────
848// Goal: never materialize all ASTs (~59 MB) at once. Pass 1 parses in parallel
849// but keeps only per-script signatures, dropping each AST in-worker; the registry
850// is built from those. Pass 2 re-parses each file in parallel, type-checks and
851// compiles it, then drops the AST immediately — so syntax trees never accumulate.
852// Output is byte-identical to the batch path (same registration, same per-file
853// codegen, same stable id-sort). It still holds all compiled scripts (for the
854// cross-script pointer check + writer); shrinking that is the next increment.
855
856/// One script's registration-relevant signature — everything the registry and
857/// trigger diagnostics need — so the full AST can be dropped right after parsing.
858struct ScriptSig {
859    name: String,
860    trigger: String,
861    param_types: Vec<types::Type>,
862    return_types: Vec<types::Type>,
863    line: usize,
864}
865
866struct FileSigs {
867    path: PathBuf,
868    testscript_line: Option<usize>,
869    sigs: Vec<ScriptSig>,
870}
871
872enum SigOutcome {
873    Parsed {
874        sigs: Vec<ScriptSig>,
875        testscript_line: Option<usize>,
876    },
877    Error {
878        line: usize,
879        position: usize,
880        message: String,
881        phase: diagnostics::Phase,
882    },
883}
884
885struct FileSigsResult {
886    path: PathBuf,
887    /// `(cache_key, raw_source, outcome)` on success.
888    read: Result<(String, String, SigOutcome), std::io::Error>,
889}
890
891/// Parse one file, extract signatures, and DROP the AST in-worker, so a parallel
892/// sweep never holds every syntax tree at once.
893fn parse_one_file_sigs(path: &Path, include_tests: bool) -> FileSigsResult {
894    let r = parse_one_file(path, include_tests);
895    let read = r.read.map(|ps| {
896        let outcome = match ps.outcome {
897            ParseOutcome::Parsed {
898                file,
899                testscript_line,
900            } => {
901                let sigs = file
902                    .scripts
903                    .iter()
904                    .map(|d| ScriptSig {
905                        name: d.name.clone(),
906                        trigger: d.trigger.clone(),
907                        param_types: d.params.iter().map(|p| p.param_type).collect(),
908                        return_types: d.return_types.clone(),
909                        line: d.line,
910                    })
911                    .collect();
912                SigOutcome::Parsed {
913                    sigs,
914                    testscript_line,
915                }
916                // `file` (the AST) is dropped here — only the sigs travel back.
917            }
918            ParseOutcome::Error {
919                line,
920                position,
921                message,
922                phase,
923            } => SigOutcome::Error {
924                line,
925                position,
926                message,
927                phase,
928            },
929        };
930        (ps.cache_key, ps.raw_source, outcome)
931    });
932    FileSigsResult { path: r.path, read }
933}
934
935fn run_pipeline_streaming(
936    scripts_dir: &Path,
937    pack_dir: Option<&Path>,
938    lint: bool,
939    include_tests: bool,
940) -> Result<(Vec<CompiledScript>, DiagnosticsCollector), Box<dyn Error>> {
941    let mut rs2_files: Vec<PathBuf> = get_rs2_files(scripts_dir, "rs2")?;
942    rs2_files.sort();
943    info!("Found {} script files", rs2_files.len());
944    mem_mark("startup");
945
946    let mut diagnostics = DiagnosticsCollector::new();
947
948    // Pass 1: parse for signatures only (each AST dropped in its worker).
949    info!("[stream] Pass 1: parsing for signatures...");
950    let sig_results = parallel_map(&rs2_files, |path| parse_one_file_sigs(path, include_tests));
951
952    // Streaming keeps only `cache_key -> on-disk path`; source text is re-read
953    // lazily for the rare diagnostic that needs it, so the ~4 MB of sources isn't
954    // resident through the peak. `raw_by_path` still holds the text transiently
955    // for `script.pack` regeneration, then is dropped right after registration.
956    let mut source_paths: HashMap<String, PathBuf> = HashMap::new();
957    let mut raw_by_path: HashMap<PathBuf, Arc<String>> = HashMap::new();
958    let mut file_sigs: Vec<FileSigs> = Vec::new();
959    for r in sig_results {
960        let (cache_key, raw_source, outcome) = match r.read {
961            Ok(t) => t,
962            Err(e) => return Err(Box::new(e)),
963        };
964        source_paths.insert(cache_key, r.path.clone());
965        raw_by_path.insert(r.path.clone(), Arc::new(raw_source));
966        match outcome {
967            SigOutcome::Parsed {
968                sigs,
969                testscript_line,
970            } => file_sigs.push(FileSigs {
971                path: r.path,
972                testscript_line,
973                sigs,
974            }),
975            SigOutcome::Error {
976                line,
977                position,
978                message,
979                phase,
980            } => diagnostics.error(r.path, line, position, message, phase),
981        }
982    }
983
984    if diagnostics.has_errors() {
985        diagnostics.print_all();
986        return Err(Box::new(error::CompilerError::TypeError(
987            "Parsing failed".to_string(),
988        )));
989    }
990    mem_mark("after parse");
991
992    // Registration (from signatures, not ASTs).
993    info!("[stream] Registering scripts and building the symbol registry...");
994    let resolved_pack_dir = resolve_pack_dir(scripts_dir, pack_dir);
995    let mut registry = build_symbol_registry(scripts_dir, resolved_pack_dir.as_deref());
996    register_scripts_from_sigs(
997        &mut registry,
998        scripts_dir,
999        resolved_pack_dir.as_deref(),
1000        &mut diagnostics,
1001        &file_sigs,
1002        &raw_by_path,
1003    );
1004    info!("  Registered {} scripts", registry.scripts.len());
1005    drop(raw_by_path);
1006    registry.drop_registration_scratch();
1007
1008    for fs in &file_sigs {
1009        for sig in &fs.sigs {
1010            if let Some(msg) =
1011                Compiler::validate_trigger_subject(&sig.trigger, &sig.name, &registry)
1012            {
1013                diagnostics.warning(
1014                    fs.path.clone(),
1015                    sig.line,
1016                    0,
1017                    msg,
1018                    diagnostics::Phase::SymbolRegistration,
1019                );
1020            }
1021        }
1022    }
1023    // Signatures are done; only the registry + source cache carry forward.
1024    drop(file_sigs);
1025    mem_mark("after register");
1026    #[cfg(feature = "memprof")]
1027    mem_stats_registry(&registry);
1028
1029    // Pass 2: re-parse + type-check + compile per file, streaming. Parse can't
1030    // fail here (pass 1 validated), so a failed re-parse just yields nothing.
1031    info!("[stream] Pass 2: type-check + codegen (streaming)...");
1032    let per_file = parallel_map(&rs2_files, |path| {
1033        let parsed = match parse_one_file(path, include_tests).read {
1034            Ok(p) => p,
1035            Err(_) => return (Vec::new(), DiagnosticsCollector::new()),
1036        };
1037        let (file, testscript_line) = match parsed.outcome {
1038            ParseOutcome::Parsed {
1039                file,
1040                testscript_line,
1041            } => (file, testscript_line),
1042            ParseOutcome::Error { .. } => return (Vec::new(), DiagnosticsCollector::new()),
1043        };
1044        let mut type_checker = TypeChecker::new(&registry);
1045        type_checker.check_file_with_test_boundary(&file, path, testscript_line);
1046        let mut compiled = Vec::new();
1047        // Compile only when this file type-checks clean (codegen assumes
1048        // well-typed input). On any global error the whole compile aborts below,
1049        // so output produced here would be discarded anyway.
1050        if !type_checker.diagnostics.has_errors() {
1051            let mut compiler = Compiler::new(&registry);
1052            for script in &file.scripts {
1053                if script.trigger == "command" {
1054                    continue;
1055                }
1056                let Some(mut c) =
1057                    compiler.compile_checked(script, path, &mut type_checker.diagnostics)
1058                else {
1059                    continue;
1060                };
1061                c.source_path = parsed.cache_key.clone();
1062                c.shrink_to_fit();
1063                compiled.push(c);
1064            }
1065        }
1066        (compiled, type_checker.diagnostics)
1067        // `file` (the AST) is dropped here.
1068    });
1069
1070    let mut compiled_scripts: Vec<CompiledScript> = Vec::new();
1071    for (compiled, diags) in per_file {
1072        compiled_scripts.extend(compiled);
1073        diagnostics.merge(diags);
1074    }
1075    if diagnostics.has_errors() {
1076        diagnostics.print_all();
1077        return Err(Box::new(error::CompilerError::TypeError(
1078            "Type checking failed".to_string(),
1079        )));
1080    }
1081    // Stable sort by id matches the batch path (file order preserved among
1082    // equal ids → identical writer last-write-wins behaviour).
1083    compiled_scripts.sort_by_key(|s| s.id);
1084    compiled_scripts.shrink_to_fit();
1085    info!("  Generated {} compiled scripts", compiled_scripts.len());
1086    mem_mark("after codegen");
1087    #[cfg(feature = "memprof")]
1088    mem_stats_compiled(&compiled_scripts);
1089
1090    check_pointers(
1091        lint,
1092        &mut diagnostics,
1093        SourceProvider::Lazy(&source_paths),
1094        &registry,
1095        &compiled_scripts,
1096    );
1097    mem_mark("after pointercheck");
1098
1099    Ok((compiled_scripts, diagnostics))
1100}
1101
1102/// `register_scripts` for the streaming path: identical logic, driven by the
1103/// extracted signatures instead of the (already-dropped) ASTs.
1104fn register_scripts_from_sigs(
1105    registry: &mut SymbolRegistry,
1106    scripts_dir: &Path,
1107    resolved_pack_dir: Option<&Path>,
1108    diagnostics: &mut DiagnosticsCollector,
1109    file_sigs: &[FileSigs],
1110    raw_by_path: &HashMap<PathBuf, Arc<String>>,
1111) {
1112    if let Some(pd) = resolved_pack_dir {
1113        symloader::generate_script_pack(scripts_dir, pd, raw_by_path);
1114        symloader::load_script_ids(registry, pd);
1115    }
1116
1117    use crate::diagnostic_messages as msg;
1118    let mut registered_scripts: std::collections::HashSet<String> =
1119        std::collections::HashSet::new();
1120
1121    for fs in file_sigs.iter() {
1122        for sig in &fs.sigs {
1123            Arc::make_mut(&mut registry.visibility).declare_script(
1124                &fs.path,
1125                &sig.trigger,
1126                &sig.name,
1127            );
1128            if sig.trigger == "command" {
1129                continue;
1130            }
1131
1132            if !trigger_table::is_valid_trigger(&sig.trigger) {
1133                diagnostics.warning(
1134                    fs.path.clone(),
1135                    sig.line,
1136                    0,
1137                    msg::fmt(msg::SCRIPT_TRIGGER_INVALID, &[&sig.trigger]),
1138                    crate::diagnostics::Phase::SymbolRegistration,
1139                );
1140            }
1141
1142            let key = format!("{}:{}", sig.trigger, sig.name);
1143            if !registered_scripts.insert(key) {
1144                diagnostics.warning(
1145                    fs.path.clone(),
1146                    sig.line,
1147                    0,
1148                    msg::fmt(msg::SCRIPT_REDECLARATION, &[&sig.trigger, &sig.name]),
1149                    crate::diagnostics::Phase::SymbolRegistration,
1150                );
1151            }
1152
1153            if !sig.return_types.is_empty() && !trigger_table::allows_returns(&sig.trigger) {
1154                diagnostics.warning(
1155                    fs.path.clone(),
1156                    sig.line,
1157                    0,
1158                    msg::fmt(msg::SCRIPT_TRIGGER_NO_RETURNS, &[&sig.trigger]),
1159                    diagnostics::Phase::SymbolRegistration,
1160                );
1161            }
1162
1163            registry.register_script(
1164                sig.name.clone(),
1165                sig.trigger.clone(),
1166                sig.param_types.clone(),
1167                sig.return_types.clone(),
1168            );
1169
1170            if let Some(ts_line) = fs.testscript_line
1171                && sig.line >= ts_line
1172            {
1173                registry.mark_test_script(&sig.trigger, &sig.name);
1174            }
1175        }
1176    }
1177}
1178
1179fn check_pointers(
1180    lint: bool,
1181    diagnostics: &mut DiagnosticsCollector,
1182    source: SourceProvider,
1183    registry: &SymbolRegistry,
1184    compiled_scripts: &[CompiledScript],
1185) {
1186    {
1187        use crate::pointer_checker::{PointerChecker, ScriptSource};
1188        // Validate scripts across threads: each chunk gets its own checker that
1189        // shares the read-only scripts/registry/source cache but keeps its own
1190        // (purely memoizing) caches. Diagnostics are merged in script order, so
1191        // the result is identical to a single-threaded run.
1192        //
1193        // The read-only command/script lookup tables are built ONCE here and
1194        // shared (`Arc`) across every chunk checker, instead of each thread
1195        // rebuilding the full ~700-command pointer table + per-script reverse
1196        // maps. `names` is just the script order, so derive it directly rather
1197        // than spinning up a throwaway checker for `script_names()`.
1198        let shared = PointerChecker::build_shared(compiled_scripts, registry);
1199        let names: Vec<String> = compiled_scripts.iter().map(|s| s.name.clone()).collect();
1200        // Each chunk's checker retains its CFG caches until the chunk finishes,
1201        // so concurrency multiplies peak scratch. RUNEC_PTR_CHUNKS caps the
1202        // concurrent chunk count to trade pointer-check speed for memory; the
1203        // result is chunk-count-independent, so diagnostics stay byte-identical.
1204        let max_chunks = std::env::var("RUNEC_PTR_CHUNKS")
1205            .ok()
1206            .and_then(|s| s.parse::<usize>().ok())
1207            .filter(|&n| n > 0)
1208            .unwrap_or(usize::MAX);
1209        let chunk_diags = parallel_chunks_capped(&names, max_chunks, |chunk| {
1210            let mut checker = PointerChecker::with_shared(
1211                ScriptSource::Resident(compiled_scripts),
1212                registry,
1213                shared.clone(),
1214            );
1215            checker.set_source_cache(source);
1216            checker.validate_names(chunk)
1217        });
1218        let mut pointer_diags = DiagnosticsCollector::new();
1219        for d in chunk_diags {
1220            pointer_diags.merge(d);
1221        }
1222        let ptr_warnings = pointer_diags.warning_count();
1223        if ptr_warnings > 0 {
1224            info!("  Pointer check: {} warning(s)", ptr_warnings);
1225        }
1226        diagnostics.merge(pointer_diags);
1227    }
1228    #[cfg(feature = "memprof")]
1229    crate::pointer_checker::cfg_trace_report();
1230
1231    // Phase 4.6: Lint passes (optional)
1232    if lint {
1233        info!("Running lint passes (unused locals, unreachable code)...");
1234        let lint_diags = lints::run_lints(compiled_scripts, Some(source));
1235        let lint_warnings = lint_diags.warning_count();
1236        if lint_warnings > 0 {
1237            info!("  Lints: {} warning(s)", lint_warnings);
1238        }
1239        diagnostics.merge(lint_diags);
1240    }
1241}
1242
1243fn codegen(
1244    all_files: &[ParsedFile],
1245    registry: &SymbolRegistry,
1246    diagnostics: &mut DiagnosticsCollector,
1247) -> Vec<CompiledScript> {
1248    // Each file compiles independently against the read-only registry, so fan
1249    // the files out across threads. Output order doesn't matter — the caller
1250    // sorts compiled scripts by id — so a simple flatten of the per-file results
1251    // is enough.
1252    let mut compiled = Vec::new();
1253    for (scripts, errors) in parallel_map(all_files, |pf| compile_file(pf, registry)) {
1254        compiled.extend(scripts);
1255        diagnostics.merge(errors);
1256    }
1257    compiled
1258}
1259
1260/// Compile every (non-`command`) script in one file. Creating a `Compiler` per
1261/// file is cheap now that it only borrows the registry, and keeps the codegen
1262/// scratch state thread-local.
1263fn compile_file(
1264    pf: &ParsedFile,
1265    registry: &SymbolRegistry,
1266) -> (Vec<CompiledScript>, DiagnosticsCollector) {
1267    let mut diagnostics = DiagnosticsCollector::new();
1268    let mut scripts =
1269        crate::compiler::compile_file(&pf.file, &pf.path, registry, false, &mut diagnostics);
1270    for script in &mut scripts {
1271        script.source_path = pf.source_path.clone();
1272        script.shrink_to_fit();
1273    }
1274    (scripts, diagnostics)
1275}
1276
1277fn run_type_checker(
1278    diagnostics: &mut DiagnosticsCollector,
1279    all_files: &[ParsedFile],
1280    registry: &SymbolRegistry,
1281) {
1282    // Each file type-checks independently against the read-only registry, so
1283    // check chunks of files in parallel; per-chunk diagnostics are merged in
1284    // file order, matching a single-threaded run.
1285    let chunk_diags = parallel_chunks(all_files, |chunk| {
1286        let mut type_checker = TypeChecker::new(registry);
1287        for pf in chunk {
1288            type_checker.check_file_with_test_boundary(&pf.file, &pf.path, pf.testscript_line);
1289        }
1290        type_checker.diagnostics
1291    });
1292    for d in chunk_diags {
1293        diagnostics.merge(d);
1294    }
1295}
1296
1297/// Resolve the pack directory: explicit `--pack`, else the nearest ancestor's `pack/`, else
1298/// `content/pack` up the tree.
1299fn resolve_pack_dir(scripts_dir: &Path, pack_dir: Option<&Path>) -> Option<PathBuf> {
1300    pack_dir
1301        .map(PathBuf::from)
1302        .filter(|p| p.exists())
1303        .or_else(|| {
1304            scripts_dir.ancestors().find_map(|dir| {
1305                let pack = dir.join("pack");
1306                pack.is_dir().then_some(pack)
1307            })
1308        })
1309        .or_else(|| {
1310            scripts_dir
1311                .ancestors()
1312                .nth(3)
1313                .map(|p| p.join("content/pack"))
1314                .filter(|p| p.exists())
1315        })
1316}
1317
1318/// Build the part of the registry that does NOT depend on the parsed scripts:
1319/// pack files, `.constant`/`.var`/`.dbtable` configs, and engine command params.
1320/// This runs concurrently with parsing; `script.pack` (re)generation and script
1321/// registration — which need the parsed sources/ASTs — happen afterwards in
1322/// `register_scripts`.
1323fn build_symbol_registry(scripts_dir: &Path, resolved_pack_dir: Option<&Path>) -> SymbolRegistry {
1324    let mut registry = SymbolRegistry::new();
1325
1326    if let Some(pd) = resolved_pack_dir {
1327        info!("  Loading packs from {}", pd.display());
1328        symloader::load_packs(&mut registry, pd);
1329        info!(
1330            "  Loaded {} commands, {} game vars",
1331            registry.commands.len(),
1332            registry.game_vars.len(),
1333        );
1334    } else {
1335        info!("  No pack directory found - symbol resolution will be limited");
1336    }
1337
1338    // Walk the scripts tree once and let the constant/game-var/dbtable loaders
1339    // filter this shared list by extension, instead of each re-walking the tree
1340    // (previously 6 separate recursive traversals over the same directories).
1341    let mut config_files: Vec<PathBuf> = Vec::new();
1342    config_files.extend(sources::config_files(scripts_dir));
1343
1344    symloader::load_constant_files(&mut registry, &config_files);
1345
1346    info!(
1347        "  Loaded {} constants from .constant files",
1348        registry.constants.len()
1349    );
1350
1351    symloader::load_game_var_types(&mut registry, &config_files);
1352
1353    if let Some(pd) = resolved_pack_dir {
1354        let dbtable_ids = symloader::load_dbtable_pack(&pd.join("dbtable.pack"));
1355        if !dbtable_ids.is_empty() {
1356            symloader::load_dbtable_configs(&mut registry, &config_files, &dbtable_ids);
1357            info!(
1358                "  Registered {} dbcolumn compound IDs",
1359                registry.dbcolumn_types.len()
1360            );
1361        }
1362    }
1363
1364    let engine_rs2 = scripts_dir.join("engine.rs2");
1365
1366    if engine_rs2.exists() {
1367        symloader::load_engine_command_params(&mut registry, &engine_rs2);
1368    } else {
1369        if let Ok(entries) = std::fs::read_dir(scripts_dir) {
1370            for entry in entries.flatten() {
1371                let candidate = entry.path().join("engine.rs2");
1372                if candidate.exists() {
1373                    crate::symloader::load_engine_command_params(&mut registry, &candidate);
1374                    break;
1375                }
1376            }
1377        }
1378    }
1379
1380    let client_dir = if scripts_dir
1381        .file_name()
1382        .is_some_and(|name| name == "runescript")
1383    {
1384        scripts_dir.with_file_name("clientscript")
1385    } else {
1386        scripts_dir.join("clientscript")
1387    };
1388    if client_dir.is_dir() {
1389        if let Some(pack) = resolved_pack_dir {
1390            symloader::load_clientscript_pack(&mut registry, &pack.join("clientscript.pack"));
1391        }
1392        symloader::load_clientscript_signatures(&mut registry, &client_dir);
1393    }
1394
1395    symloader::patch_command_return_types(&mut registry);
1396
1397    registry.visibility = Arc::new(sources::Visibility::load(scripts_dir, &registry));
1398
1399    registry
1400}
1401
1402/// Finalize the registry once parsing is done: regenerate `script.pack` from the
1403/// parsed sources, load pre-assigned script IDs, then register every parsed
1404/// script (emitting trigger/redeclaration/return diagnostics). Builds on the
1405/// registry produced concurrently by `build_symbol_registry`.
1406fn register_scripts(
1407    registry: &mut SymbolRegistry,
1408    scripts_dir: &Path,
1409    resolved_pack_dir: Option<&Path>,
1410    diagnostics: &mut DiagnosticsCollector,
1411    all_files: &[ParsedFile],
1412    raw_by_path: &HashMap<PathBuf, Arc<String>>,
1413) {
1414    if let Some(pd) = resolved_pack_dir {
1415        symloader::generate_script_pack(scripts_dir, pd, raw_by_path);
1416        symloader::load_script_ids(registry, pd);
1417    }
1418
1419    {
1420        use crate::diagnostic_messages as msg;
1421
1422        let mut registered_scripts: std::collections::HashSet<String> =
1423            std::collections::HashSet::new();
1424
1425        for pf in all_files.iter() {
1426            Arc::make_mut(&mut registry.visibility).declare_source(&pf.path, &pf.file);
1427            for script in &pf.file.scripts {
1428                if script.trigger == "command" {
1429                    continue;
1430                }
1431
1432                if !trigger_table::is_valid_trigger(&script.trigger) {
1433                    diagnostics.warning(
1434                        pf.path.clone(),
1435                        script.line,
1436                        0,
1437                        msg::fmt(msg::SCRIPT_TRIGGER_INVALID, &[&script.trigger]),
1438                        crate::diagnostics::Phase::SymbolRegistration,
1439                    );
1440                }
1441
1442                let key = format!("{}:{}", script.trigger, script.name);
1443                if !registered_scripts.insert(key.clone()) {
1444                    diagnostics.warning(
1445                        pf.path.clone(),
1446                        script.line,
1447                        0,
1448                        msg::fmt(msg::SCRIPT_REDECLARATION, &[&script.trigger, &script.name]),
1449                        crate::diagnostics::Phase::SymbolRegistration,
1450                    );
1451                }
1452
1453                if !script.return_types.is_empty()
1454                    && !trigger_table::allows_returns(&script.trigger)
1455                {
1456                    diagnostics.warning(
1457                        pf.path.clone(),
1458                        script.line,
1459                        0,
1460                        msg::fmt(msg::SCRIPT_TRIGGER_NO_RETURNS, &[&script.trigger]),
1461                        diagnostics::Phase::SymbolRegistration,
1462                    );
1463                }
1464
1465                let param_types: Vec<types::Type> =
1466                    script.params.iter().map(|p| p.param_type).collect();
1467                registry.register_script(
1468                    script.name.clone(),
1469                    script.trigger.clone(),
1470                    param_types,
1471                    script.return_types.clone(),
1472                );
1473
1474                if let Some(ts_line) = pf.testscript_line
1475                    && script.line >= ts_line
1476                {
1477                    registry.mark_test_script(&script.trigger, &script.name);
1478                }
1479            }
1480        }
1481    }
1482}
1483
1484// ── Streaming recompile pipeline (prototype, gated by RUNEC_RECOMPILE) ────────
1485// Never holds all compiled scripts. Pass 2 only type-checks; the pointer check
1486// re-compiles each script on demand (bounded cache — measured ~1.3x re-parse on
1487// rs-majula, the call graph has high file locality); output is produced by
1488// re-compiling file-by-file into a by-id buffer. Output + diagnostics are
1489// byte-identical to the batch path (deterministic re-compile, same id order).
1490
1491/// Re-compiles scripts on demand for the pointer checker, caching `Rc`s with a
1492/// bounded FIFO so the full compiled set is never resident. A cache miss
1493/// re-parses the script's file and compiles all its scripts at once (siblings
1494/// share the parse), keying each by its full `[trigger,name]` to its index.
1495struct RecompileStore<'a> {
1496    idx_to_path: Vec<PathBuf>,
1497    /// file path -> its script idxs in declaration order (command-skipped).
1498    /// Position-based so duplicate `[trigger,name]` declarations each map to the
1499    /// correct distinct idx.
1500    file_to_idxs: HashMap<PathBuf, Vec<usize>>,
1501    registry: &'a SymbolRegistry,
1502    include_tests: bool,
1503    cache: HashMap<usize, std::rc::Rc<CompiledScript>>,
1504    order: std::collections::VecDeque<usize>,
1505    cap: usize,
1506}
1507
1508impl RecompileStore<'_> {
1509    fn get(&mut self, idx: usize) -> std::rc::Rc<CompiledScript> {
1510        if let Some(rc) = self.cache.get(&idx) {
1511            return std::rc::Rc::clone(rc);
1512        }
1513        let path = self.idx_to_path[idx].clone();
1514        let parsed = parse_one_file(&path, self.include_tests);
1515        if let Ok(ps) = parsed.read
1516            && let ParseOutcome::Parsed { file, .. } = ps.outcome
1517        {
1518            // The j-th non-command declaration in this file maps to idxs[j],
1519            // matching how the metas were built (same parse, same skip rule).
1520            let idxs = self.file_to_idxs.get(&path).cloned().unwrap_or_default();
1521            let mut compiler = Compiler::new(self.registry);
1522            let mut j = 0usize;
1523            for script in &file.scripts {
1524                if script.trigger == "command" {
1525                    continue;
1526                }
1527                if let Some(&i) = idxs.get(j) {
1528                    let mut c = compiler.compile_script(script);
1529                    c.source_path = ps.cache_key.clone();
1530                    self.insert(i, std::rc::Rc::new(c));
1531                }
1532                j += 1;
1533            }
1534        }
1535        std::rc::Rc::clone(
1536            self.cache
1537                .get(&idx)
1538                .expect("recompile store: script not produced by its file"),
1539        )
1540    }
1541
1542    fn insert(&mut self, idx: usize, rc: std::rc::Rc<CompiledScript>) {
1543        if self.cache.insert(idx, rc).is_none() {
1544            self.order.push_back(idx);
1545            while self.order.len() > self.cap {
1546                if let Some(old) = self.order.pop_front()
1547                    && old != idx
1548                {
1549                    self.cache.remove(&old);
1550                }
1551            }
1552        }
1553    }
1554}
1555
1556/// Per-slot encoded output blobs (indexed by script id) plus accumulated
1557/// diagnostics — what the recompile core hands back before the bytes are written
1558/// to disk or assembled in memory.
1559type RecompiledOutput = (Vec<Vec<u8>>, DiagnosticsCollector);
1560
1561/// Core of the recompile (low-memory) pipeline: runs every phase re-compiling
1562/// scripts on demand — never holding all compiled bytecode at once — and returns
1563/// the per-slot encoded output blobs plus diagnostics, without writing or
1564/// printing. The disk and in-memory entry points (`compile_recompile`,
1565/// `compile_recompile_memory`) wrap this. On a phase error it prints diagnostics
1566/// and returns `Err`, matching the batch path.
1567fn recompile_to_encoded(
1568    scripts_dir: &Path,
1569    pack_dir: Option<&Path>,
1570    lint: bool,
1571    include_tests: bool,
1572) -> Result<RecompiledOutput, Box<dyn Error>> {
1573    let source_dir = sources::target_dir(scripts_dir, sources::Scope::Rune);
1574    let scripts_dir = source_dir.as_path();
1575    use crate::pointer_checker::{PointerChecker, ScriptSource};
1576
1577    let mut rs2_files: Vec<PathBuf> = get_rs2_files(scripts_dir, "rs2")?;
1578    rs2_files.sort();
1579    info!("Found {} script files", rs2_files.len());
1580    mem_mark("startup");
1581    let mut diagnostics = DiagnosticsCollector::new();
1582
1583    // Pass 1: signatures only (ASTs dropped in-worker); keep file paths for lazy
1584    // source + raw text for script.pack regeneration.
1585    info!("[recompile] Pass 1: parsing for signatures...");
1586    let sig_results = parallel_map(&rs2_files, |path| parse_one_file_sigs(path, include_tests));
1587    let mut source_paths: HashMap<String, PathBuf> = HashMap::new();
1588    let mut raw_by_path: HashMap<PathBuf, Arc<String>> = HashMap::new();
1589    let mut file_sigs: Vec<FileSigs> = Vec::new();
1590    for r in sig_results {
1591        let (cache_key, raw_source, outcome) = match r.read {
1592            Ok(t) => t,
1593            Err(e) => return Err(Box::new(e)),
1594        };
1595        source_paths.insert(cache_key, r.path.clone());
1596        raw_by_path.insert(r.path.clone(), Arc::new(raw_source));
1597        match outcome {
1598            SigOutcome::Parsed {
1599                sigs,
1600                testscript_line,
1601            } => file_sigs.push(FileSigs {
1602                path: r.path,
1603                testscript_line,
1604                sigs,
1605            }),
1606            SigOutcome::Error {
1607                line,
1608                position,
1609                message,
1610                phase,
1611            } => diagnostics.error(r.path, line, position, message, phase),
1612        }
1613    }
1614    if diagnostics.has_errors() {
1615        diagnostics.print_all();
1616        return Err(Box::new(error::CompilerError::TypeError(
1617            "Parsing failed".to_string(),
1618        )));
1619    }
1620    mem_mark("after parse");
1621
1622    // Registration.
1623    info!("[recompile] Registering scripts and building the symbol registry...");
1624    let resolved_pack_dir = resolve_pack_dir(scripts_dir, pack_dir);
1625    let mut registry = build_symbol_registry(scripts_dir, resolved_pack_dir.as_deref());
1626    register_scripts_from_sigs(
1627        &mut registry,
1628        scripts_dir,
1629        resolved_pack_dir.as_deref(),
1630        &mut diagnostics,
1631        &file_sigs,
1632        &raw_by_path,
1633    );
1634    info!("  Registered {} scripts", registry.scripts.len());
1635    drop(raw_by_path);
1636    registry.drop_registration_scratch();
1637    for fs in &file_sigs {
1638        for sig in &fs.sigs {
1639            if let Some(msg) =
1640                Compiler::validate_trigger_subject(&sig.trigger, &sig.name, &registry)
1641            {
1642                diagnostics.warning(
1643                    fs.path.clone(),
1644                    sig.line,
1645                    0,
1646                    msg,
1647                    diagnostics::Phase::SymbolRegistration,
1648                );
1649            }
1650        }
1651    }
1652    mem_mark("after register");
1653
1654    // Pass 2: type-check only (re-parse, no compiled scripts retained).
1655    info!("[recompile] Pass 2: type-check (streaming, no bytecode held)...");
1656    let tc_diags = parallel_map(&rs2_files, |path| {
1657        let parsed = match parse_one_file(path, include_tests).read {
1658            Ok(p) => p,
1659            Err(_) => return DiagnosticsCollector::new(),
1660        };
1661        let (file, testscript_line) = match parsed.outcome {
1662            ParseOutcome::Parsed {
1663                file,
1664                testscript_line,
1665            } => (file, testscript_line),
1666            ParseOutcome::Error { .. } => return DiagnosticsCollector::new(),
1667        };
1668        let mut type_checker = TypeChecker::new(&registry);
1669        type_checker.check_file_with_test_boundary(&file, path, testscript_line);
1670        if !type_checker.diagnostics.has_errors() {
1671            let mut compiler = Compiler::new(&registry);
1672            for script in &file.scripts {
1673                if script.trigger != "command" {
1674                    compiler.compile_checked(script, path, &mut type_checker.diagnostics);
1675                }
1676            }
1677        }
1678        type_checker.diagnostics
1679    });
1680    for d in tc_diags {
1681        diagnostics.merge(d);
1682    }
1683    if diagnostics.has_errors() {
1684        diagnostics.print_all();
1685        return Err(Box::new(error::CompilerError::TypeError(
1686            "Type checking failed".to_string(),
1687        )));
1688    }
1689    mem_mark("after typecheck");
1690
1691    // Build script metas in the same idx order as the batch path: file order
1692    // (file_sigs / declaration order, command-skipped), then a stable sort by id.
1693    // ids + names come from the registry — no bytecode needed.
1694    let mut pre: Vec<(i32, String, PathBuf)> = Vec::new();
1695    for fs in &file_sigs {
1696        for sig in &fs.sigs {
1697            if sig.trigger == "command" {
1698                continue;
1699            }
1700            let id = registry
1701                .script_id_for_trigger(&sig.trigger, &sig.name)
1702                .or_else(|| registry.script_id(&sig.name))
1703                .unwrap_or(-1);
1704            pre.push((
1705                id,
1706                format!("[{},{}]", sig.trigger, sig.name),
1707                fs.path.clone(),
1708            ));
1709        }
1710    }
1711    let n = pre.len();
1712    // Stable sort indices by id → the batch's idx order (file/decl pre-order
1713    // preserved among equal ids). `inv` maps a pre-order position back to its idx.
1714    let mut order: Vec<usize> = (0..n).collect();
1715    order.sort_by_key(|&p| pre[p].0);
1716    let idents: Vec<(i32, String)> = order
1717        .iter()
1718        .map(|&p| (pre[p].0, pre[p].1.clone()))
1719        .collect();
1720    let idx_to_path: Vec<PathBuf> = order.iter().map(|&p| pre[p].2.clone()).collect();
1721    let mut inv = vec![0usize; n];
1722    for (idx, &p) in order.iter().enumerate() {
1723        inv[p] = idx;
1724    }
1725    let mut file_to_idxs: HashMap<PathBuf, Vec<usize>> = HashMap::new();
1726    for (p, m) in pre.iter().enumerate() {
1727        file_to_idxs.entry(m.2.clone()).or_default().push(inv[p]);
1728    }
1729    let max_id = pre.iter().map(|m| m.0).max().unwrap_or(-1);
1730    let slot_count = if max_id < 0 { 0 } else { (max_id + 1) as usize };
1731    let len = n;
1732    drop(pre);
1733
1734    // Pointer check via on-demand re-compile (single-threaded: the store getter
1735    // is `FnMut`/`!Sync`). Diagnostics come out in id order, as in the batch path.
1736    info!("[recompile] Pointer check (re-compile on demand)...");
1737    let shared = PointerChecker::build_shared_idents(&idents, &registry);
1738    {
1739        // Cache cap must exceed the largest file's script count, or loading that
1740        // file would evict its own just-compiled scripts. A small margin over
1741        // that keeps cross-file locality high (measured ~1.1-1.3x re-parse).
1742        let cap = file_to_idxs
1743            .values()
1744            .map(|v| v.len())
1745            .max()
1746            .unwrap_or(0)
1747            .max(128);
1748        let mut store = RecompileStore {
1749            idx_to_path,
1750            file_to_idxs,
1751            registry: &registry,
1752            include_tests,
1753            cache: HashMap::new(),
1754            order: std::collections::VecDeque::new(),
1755            cap,
1756        };
1757        let getter: Box<dyn FnMut(usize) -> std::rc::Rc<CompiledScript>> =
1758            Box::new(move |idx| store.get(idx));
1759        let scripts = ScriptSource::Recompile {
1760            len,
1761            get: std::cell::RefCell::new(getter),
1762        };
1763        let mut checker = PointerChecker::with_shared(scripts, &registry, shared);
1764        checker.set_source_cache(SourceProvider::Lazy(&source_paths));
1765        let names: Vec<String> = idents.iter().map(|(_, n)| n.clone()).collect();
1766        let ptr_diags = checker.validate_names(&names);
1767        let w = ptr_diags.warning_count();
1768        if w > 0 {
1769            info!("  Pointer check: {} warning(s)", w);
1770        }
1771        diagnostics.merge(ptr_diags);
1772    }
1773    mem_mark("after pointercheck");
1774
1775    // Output: re-compile each file once, encode each script into its id slot,
1776    // optionally lint — never holding all compiled scripts at once. The caller
1777    // writes the blobs to disk or assembles them in memory.
1778    info!("[recompile] Re-compiling + encoding output...");
1779    let mut encoded: Vec<Vec<u8>> = vec![Vec::new(); slot_count];
1780    for path in &rs2_files {
1781        let parsed = match parse_one_file(path, include_tests).read {
1782            Ok(p) => p,
1783            Err(_) => continue,
1784        };
1785        let file = match parsed.outcome {
1786            ParseOutcome::Parsed { file, .. } => file,
1787            ParseOutcome::Error { .. } => continue,
1788        };
1789        let mut compiler = Compiler::new(&registry);
1790        let mut file_scripts: Vec<CompiledScript> = Vec::new();
1791        for script in &file.scripts {
1792            if script.trigger == "command" {
1793                continue;
1794            }
1795            let Some(mut c) = compiler.compile_checked(script, path, &mut diagnostics) else {
1796                continue;
1797            };
1798            c.source_path = parsed.cache_key.clone();
1799            file_scripts.push(c);
1800        }
1801        for c in &file_scripts {
1802            if c.id >= 0 && (c.id as usize) < slot_count {
1803                encoded[c.id as usize] = crate::writer::encode_script(c);
1804            }
1805        }
1806        if lint {
1807            diagnostics.merge(lints::run_lints(
1808                &file_scripts,
1809                Some(SourceProvider::Lazy(&source_paths)),
1810            ));
1811        }
1812    }
1813    mem_mark("after codegen");
1814    Ok((encoded, diagnostics))
1815}
1816
1817/// Full compile via the recompile (low-memory) pipeline: writes `script.dat` /
1818/// `script.idx` to `output_dir`, then prints diagnostics.
1819fn compile_recompile(
1820    scripts_dir: &Path,
1821    pack_dir: Option<&Path>,
1822    output_dir: &Path,
1823    lint: bool,
1824) -> Result<(), Box<dyn Error>> {
1825    let (encoded, diagnostics) = recompile_to_encoded(scripts_dir, pack_dir, lint, false)?;
1826    info!(
1827        "Writing output to {}...",
1828        output_dir.display().to_string().replace('\\', "/")
1829    );
1830    ScriptWriter::new(output_dir.display().to_string()).write_encoded(&encoded)?;
1831    diagnostics.print_all();
1832    info!("Compilation complete!");
1833    Ok(())
1834}
1835
1836/// In-memory compile via the recompile (low-memory) pipeline: returns the
1837/// `(script.dat, script.idx)` bytes, then prints diagnostics. Byte-identical to
1838/// the in-memory batch path (`compile_memory*`).
1839fn compile_recompile_memory(
1840    scripts_dir: &Path,
1841    pack_dir: Option<&Path>,
1842    lint: bool,
1843    include_tests: bool,
1844) -> Result<(Vec<u8>, Vec<u8>), Box<dyn Error>> {
1845    let (encoded, diagnostics) = recompile_to_encoded(scripts_dir, pack_dir, lint, include_tests)?;
1846    info!("Writing compiled output to memory...");
1847    let out = ScriptWriter::new("".into()).build_encoded(&encoded);
1848    diagnostics.print_all();
1849    info!("Compilation complete!");
1850    Ok(out)
1851}
1852
1853pub(crate) fn find_testscript_line(source: &str) -> Option<usize> {
1854    for (i, line) in source.lines().enumerate() {
1855        if line.trim() == "#testscript" {
1856            return Some(i + 1);
1857        }
1858    }
1859    None
1860}
1861
1862fn truncate_at_line(source: &str, line_no: usize) -> String {
1863    source
1864        .lines()
1865        .take(line_no - 1)
1866        .collect::<Vec<_>>()
1867        .join("\n")
1868}
1869
1870fn strip_testscript_annotation(source: &str) -> String {
1871    source
1872        .lines()
1873        .map(|line| {
1874            if line.trim() == "#testscript" {
1875                ""
1876            } else {
1877                line
1878            }
1879        })
1880        .collect::<Vec<_>>()
1881        .join("\n")
1882}
1883
1884fn get_rs2_files(dir: &Path, ext: &str) -> Result<Vec<PathBuf>, Box<dyn Error>> {
1885    Ok(sources::script_files(dir, ext)?)
1886}