Skip to main content

runec/
filesystem.rs

1use crate::bytecode::CompiledScript;
2use crate::clientscript::{self, format::Format};
3use crate::compiler::Compiler;
4use crate::diagnostics::DiagnosticsCollector;
5use crate::lexer::Lexer;
6#[cfg(feature = "memprof")]
7use crate::memprof;
8use crate::parallel::{parallel_chunks, parallel_chunks_capped, parallel_map};
9use crate::parser::{Parser, ScriptFile};
10use crate::project::{Options, Project, Target};
11use crate::sources::Scope;
12use crate::symbol::SymbolRegistry;
13use crate::typechecker::TypeChecker;
14use crate::writer::ScriptWriter;
15use crate::{diagnostics, error, lints, sources, symloader, trigger_table, types};
16use std::collections::HashMap;
17use std::error::Error;
18use std::fmt::Write;
19use std::fs;
20use std::path::{Path, PathBuf};
21use std::sync::Arc;
22use tracing::info;
23
24/// Build an editor project using the legacy directory conventions. No ID file
25/// is generated or modified; all further editing happens in memory.
26pub fn editor_project(
27    source: &Path,
28    pack: Option<&Path>,
29    client: bool,
30) -> Result<Project, Box<dyn Error>> {
31    let scope = if client { Scope::Client } else { Scope::Rune };
32    let dir = sources::target_dir(source, scope);
33    let pack = resolve_pack_dir(&dir, pack);
34    let mut registry = if client {
35        clientscript::registry(
36            &dir,
37            pack.as_deref()
38                .ok_or("Client opcode definitions require --pack")?,
39        )?
40    } else {
41        build_symbol_registry(&dir, pack.as_deref())
42    };
43    let target = if client {
44        registry.preloaded_script_ids.clear();
45        for (key, id) in clientscript::script_ids(pack.as_deref().unwrap())? {
46            let (trigger, name) = key
47                .trim_start_matches('[')
48                .trim_end_matches(']')
49                .split_once(',')
50                .ok_or("Invalid client ID")?;
51            registry.set_script_id(trigger, name, id);
52        }
53        Target::Client(Format::load(
54            &pack.as_ref().unwrap().join("cs2_command.pack"),
55        )?)
56    } else {
57        if let Some(pack) = &pack {
58            symloader::load_script_ids(&mut registry, pack);
59        }
60        Target::Rune
61    };
62    let mut project = Project::new(
63        target,
64        registry,
65        Options {
66            include_tests: true,
67            skip_shared_lints: client
68                && dir
69                    .parent()
70                    .is_some_and(|parent| parent.join("runescript").is_dir()),
71            ..Options::default()
72        },
73    );
74    for path in sources::script_files(&dir, if client { "cs2" } else { "rs2" })? {
75        let text = fs::read_to_string(&path)?;
76        let scope = if path.extension().is_some_and(|ext| ext == "gs2") {
77            Scope::Global
78        } else {
79            scope
80        };
81        let uri = file_uri(&path)?;
82        let id = project.open(uri, scope, 0, text);
83        if client {
84            let ids = project
85                .document(id)
86                .and_then(|doc| doc.syntax())
87                .into_iter()
88                .flat_map(|file| &file.scripts)
89                .filter_map(|script| {
90                    script
91                        .name
92                        .strip_prefix("clientscript_")
93                        .and_then(|id| id.parse::<i32>().ok())
94                        .map(|id| (script.trigger.clone(), script.name.clone(), id))
95                })
96                .collect::<Vec<_>>();
97            for (trigger, name, id) in ids {
98                project.registry.set_script_id(&trigger, &name, id);
99            }
100        }
101    }
102    if !client {
103        let client_dir = if dir.file_name().is_some_and(|name| name == "runescript") {
104            dir.with_file_name("clientscript")
105        } else {
106            dir.join("clientscript")
107        };
108        if client_dir.is_dir() {
109            for path in sources::script_files(&client_dir, "cs2")?
110                .into_iter()
111                .filter(|path| {
112                    path.extension().is_some_and(|ext| ext == "cs2")
113                        && path.file_name().is_some_and(|name| name != "engine.cs2")
114                })
115            {
116                project.open(
117                    file_uri(&path)?,
118                    Scope::Client,
119                    0,
120                    fs::read_to_string(&path)?,
121                );
122            }
123        }
124    }
125    let mut files = sources::config_files(&dir);
126    if let Some(pack) = &pack {
127        symloader::collect_all_files(pack, &mut files);
128    }
129    files.sort();
130    files.dedup();
131    for path in files {
132        let Some(kind) = config_kind(&path) else {
133            continue;
134        };
135        let text = fs::read_to_string(&path)?;
136        let config_scope = if path
137            .components()
138            .any(|part| part.as_os_str() == "globalscript")
139        {
140            Scope::Global
141        } else {
142            scope
143        };
144        project.open_config(file_uri(&path)?, config_scope, 0, text, kind);
145    }
146    Ok(project)
147}
148
149pub(crate) fn config_kind(path: &Path) -> Option<String> {
150    let extension = path.extension()?.to_str()?;
151    if matches!(extension, "pack" | "order" | "hashes") {
152        return Some(format!("pack:{}", path.file_stem()?.to_str()?));
153    }
154    (matches!(
155        extension,
156        "constant"
157            | "if"
158            | "flo"
159            | "flu"
160            | "anim"
161            | "varc"
162            | "varcstr"
163            | "mel"
164            | "bas"
165            | "cursor"
166            | "light"
167            | "msi"
168            | "quest"
169            | "varclansetting"
170    ) || types::Type::from_name(extension).is_some()
171        || extension
172            .strip_prefix("config")
173            .is_some_and(|number| number.parse::<u32>().is_ok()))
174    .then(|| extension.to_string())
175}
176
177pub(crate) fn file_uri(path: &Path) -> Result<String, Box<dyn Error>> {
178    let absolute = fs::canonicalize(path)?
179        .to_string_lossy()
180        .trim_start_matches("\\\\?\\")
181        .replace('\\', "/");
182    let mut uri = String::from(if absolute.starts_with('/') {
183        "file://"
184    } else {
185        "file:///"
186    });
187    for byte in absolute.bytes() {
188        if byte.is_ascii_alphanumeric() || b"/-._~:".contains(&byte) {
189            uri.push(byte as char);
190        } else {
191            write!(uri, "%{byte:02X}")?;
192        }
193    }
194    Ok(uri)
195}
196
197/// How the diagnostic layer fetches a file's source text (by its cache-key path)
198/// when building rustc-style help/suggestions. The batch path keeps every source
199/// in memory; the streaming path keeps only the on-disk paths and re-reads the
200/// handful of files that actually produce a warning — so the ~4 MB of source
201/// text isn't resident through the compile's peak. Cheap to copy (just borrows).
202#[derive(Clone, Copy)]
203pub enum SourceProvider<'a> {
204    /// Sources already resident, keyed by cache key.
205    Eager(&'a HashMap<String, Arc<String>>),
206    /// Cache key → on-disk path; re-read on demand.
207    Lazy(&'a HashMap<String, PathBuf>),
208}
209
210impl SourceProvider<'_> {
211    /// Source text for `key`, if available. Returns an owned `Arc` — a cheap
212    /// refcount bump for `Eager`, a fresh disk read for `Lazy`. Called only while
213    /// emitting diagnostic help, which is rare, so the re-read cost is immaterial.
214    pub fn get(&self, key: &str) -> Option<Arc<String>> {
215        match self {
216            SourceProvider::Eager(m) => m.get(key).cloned(),
217            SourceProvider::Lazy(m) => fs::read_to_string(m.get(key)?).ok().map(Arc::new),
218        }
219    }
220}
221
222/// Log live heap + global peak at a pipeline phase boundary. No-op unless built
223/// with `--features memprof`. Lets us attribute the footprint to each phase.
224#[cfg(feature = "memprof")]
225fn mem_mark(label: &str) {
226    info!(
227        "[mem] {:<22} live {:>7.1} MB | peak {:>7.1} MB | allocs {}",
228        label,
229        memprof::current_bytes() as f64 / 1_048_576.0,
230        memprof::peak_bytes() as f64 / 1_048_576.0,
231        memprof::alloc_count(),
232    );
233}
234#[cfg(not(feature = "memprof"))]
235#[inline]
236fn mem_mark(_label: &str) {}
237
238/// Per-map breakdown of the symbol registry's heap, to see the irreducible
239/// floor for a streaming rebuild. Built only under `--features memprof`.
240/// Accounts three components per map: the hashbrown table (inline `(K,V)`
241/// buckets + 1 control byte each), the key strings, and value-side heap.
242#[cfg(feature = "memprof")]
243fn mem_stats_registry(reg: &crate::symbol::SymbolRegistry) {
244    use crate::symbol::{Symbol, SymbolKind};
245    use crate::types::Type;
246    use std::collections::{HashMap, HashSet};
247    let t = std::mem::size_of::<Type>();
248    let mb = |b: usize| b as f64 / 1_048_576.0;
249
250    // Allocated bucket count: hashbrown keeps `capacity()` ≈ 7/8 of a power-of-two
251    // bucket array; an empty map allocates nothing.
252    fn tbl<K, V>(m: &HashMap<K, V>) -> usize {
253        if m.capacity() == 0 {
254            return 0;
255        }
256        let buckets = (m.capacity() * 8 / 7).next_power_of_two();
257        buckets * (std::mem::size_of::<(K, V)>() + 1)
258    }
259    fn keys_cap<V>(m: &HashMap<String, V>) -> usize {
260        m.keys().map(|k| k.capacity()).sum()
261    }
262    let sym_heap = |s: &Symbol| -> usize {
263        s.name.capacity()
264            + match &s.kind {
265                SymbolKind::Script {
266                    trigger,
267                    param_types,
268                    return_types,
269                    ..
270                } => trigger.capacity() + (param_types.capacity() + return_types.capacity()) * t,
271                SymbolKind::Command {
272                    param_types,
273                    return_types,
274                    ..
275                } => (param_types.capacity() + return_types.capacity()) * t,
276                SymbolKind::GameVar { category, .. } => category.capacity(),
277                SymbolKind::Constant { string_value, .. } => {
278                    string_value.as_ref().map_or(0, |s| s.capacity())
279                }
280                _ => 0,
281            }
282    };
283    let sym_map = |m: &HashMap<String, Symbol>| -> usize {
284        tbl(m) + keys_cap(m) + m.values().map(&sym_heap).sum::<usize>()
285    };
286    let i32_map = |m: &HashMap<String, i32>| -> usize { tbl(m) + keys_cap(m) };
287
288    let mut rows: Vec<(&str, usize, usize)> = vec![
289        (
290            "scripts",
291            reg.scripts.len(),
292            tbl(&reg.scripts) + keys_cap(&reg.scripts),
293        ),
294        ("commands", reg.commands.len(), sym_map(&reg.commands)),
295        ("game_vars", reg.game_vars.len(), sym_map(&reg.game_vars)),
296        ("constants", reg.constants.len(), sym_map(&reg.constants)),
297        (
298            "entity_ids",
299            reg.entity_ids.len(),
300            tbl(&reg.entity_ids)
301                + keys_cap(&reg.entity_ids)
302                + reg
303                    .entity_ids
304                    .values()
305                    .map(|e| e.variants.capacity() * std::mem::size_of::<(Type, i32)>())
306                    .sum::<usize>(),
307        ),
308        (
309            "scripts_by_trigger",
310            reg.scripts_by_trigger.len(),
311            tbl(&reg.scripts_by_trigger)
312                + keys_cap(&reg.scripts_by_trigger)
313                + reg
314                    .scripts_by_trigger
315                    .values()
316                    .map(|s| 16 + sym_heap(s.as_ref()))
317                    .sum::<usize>(),
318        ),
319        (
320            "command_param_types",
321            reg.command_param_types.len(),
322            tbl(&reg.command_param_types)
323                + keys_cap(&reg.command_param_types)
324                + reg
325                    .command_param_types
326                    .values()
327                    .map(|v| v.capacity() * t)
328                    .sum::<usize>(),
329        ),
330        ("script_ids", reg.script_ids.len(), i32_map(&reg.script_ids)),
331        (
332            "proc_script_ids",
333            reg.proc_script_ids.len(),
334            i32_map(&reg.proc_script_ids),
335        ),
336        (
337            "label_script_ids",
338            reg.label_script_ids.len(),
339            i32_map(&reg.label_script_ids),
340        ),
341        (
342            "trigger_script_ids",
343            reg.trigger_script_ids.len(),
344            i32_map(&reg.trigger_script_ids),
345        ),
346        (
347            "preloaded_script_ids",
348            reg.preloaded_script_ids.len(),
349            i32_map(&reg.preloaded_script_ids),
350        ),
351        ("components", reg.components.len(), i32_map(&reg.components)),
352        (
353            "interface_ids",
354            reg.interface_ids.len(),
355            i32_map(&reg.interface_ids),
356        ),
357        ("type_chars", reg.type_chars.len(), i32_map(&reg.type_chars)),
358        (
359            "dbcolumn_types",
360            reg.dbcolumn_types.len(),
361            tbl(&reg.dbcolumn_types) + keys_cap(&reg.dbcolumn_types),
362        ),
363    ];
364    let ts_tbl = {
365        let s: &HashSet<String> = &reg.test_scripts;
366        if s.capacity() == 0 {
367            0
368        } else {
369            (s.capacity() * 8 / 7).next_power_of_two() * (std::mem::size_of::<String>() + 1)
370                + s.iter().map(|k| k.capacity()).sum::<usize>()
371        }
372    };
373    rows.push(("test_scripts", reg.test_scripts.len(), ts_tbl));
374
375    rows.sort_by_key(|row| std::cmp::Reverse(row.2));
376    let total: usize = rows.iter().map(|r| r.2).sum();
377    info!(
378        "[mem] registry ~{:.2} MB (sizeof Symbol={}B, SymbolKind={}B, Type={}B):",
379        mb(total),
380        std::mem::size_of::<Symbol>(),
381        std::mem::size_of::<SymbolKind>(),
382        t,
383    );
384    for (name, n, bytes) in &rows {
385        info!(
386            "[mem]   {:<20} {:>6} entries  {:>7.3} MB",
387            name,
388            n,
389            mb(*bytes)
390        );
391    }
392}
393
394/// One-off breakdown of where the compiled-bytecode heap goes, to target
395/// memory work. Built only under `--features memprof`.
396#[cfg(feature = "memprof")]
397fn mem_stats_compiled(scripts: &[CompiledScript]) {
398    use crate::bytecode::{Instruction, Opcode, Operand};
399    let (mut n_instr, mut n_str, mut str_bytes) = (0usize, 0usize, 0usize);
400    let (mut n_switch, mut switch_entries) = (0usize, 0usize);
401    let (mut source_path_bytes, mut name_bytes) = (0usize, 0usize);
402    for s in scripts {
403        n_instr += s.instructions.len();
404        source_path_bytes += s.source_path.capacity();
405        name_bytes += s.name.capacity() + s.trigger.capacity();
406        for st in &s.strings {
407            n_str += 1;
408            str_bytes += st.len();
409        }
410        for sw in &s.switch_tables {
411            n_switch += 1;
412            switch_entries += sw.len();
413        }
414    }
415    let sz = std::mem::size_of::<Instruction>();
416    let mb = |b: usize| b as f64 / 1_048_576.0;
417    info!(
418        "[mem] sizeof Instruction={}B Operand={}B Opcode={}B CompiledScript={}B",
419        sz,
420        std::mem::size_of::<Operand>(),
421        std::mem::size_of::<Opcode>(),
422        std::mem::size_of::<CompiledScript>(),
423    );
424    info!(
425        "[mem] compiled: {} scripts, {} instrs => {:.2} MB instr structs",
426        scripts.len(),
427        n_instr,
428        mb(n_instr * sz)
429    );
430    info!(
431        "[mem]   Str operands: {} ({:.2} MB content + {:.2} MB Box<str> headers)",
432        n_str,
433        mb(str_bytes),
434        mb(n_str * 16)
435    );
436    info!(
437        "[mem]   SwitchTable operands: {} ({} entries => {:.2} MB)",
438        n_switch,
439        switch_entries,
440        mb(switch_entries * 16)
441    );
442    info!(
443        "[mem]   source_path {:.2} MB, names+triggers {:.2} MB",
444        mb(source_path_bytes),
445        mb(name_bytes)
446    );
447}
448
449struct ParsedFile {
450    path: PathBuf,
451    file: ScriptFile,
452    testscript_line: Option<usize>,
453    /// Canonicalized, normalized source path. Computed once during parsing and
454    /// reused as each compiled script's `source_path` (it is byte-identical to
455    /// the `source_cache` key), avoiding a second `canonicalize` syscall per
456    /// file in codegen.
457    source_path: String,
458}
459
460/// Outcome of parsing one file off-thread. The owned source text and cache key
461/// travel back so the (single-threaded) merge can build the `Rc`-based caches
462/// without sending `Rc` across threads.
463struct FileParseResult {
464    path: PathBuf,
465    read: Result<ParsedSource, std::io::Error>,
466}
467
468struct ParsedSource {
469    cache_key: String,
470    raw_source: String,
471    outcome: ParseOutcome,
472}
473
474enum ParseOutcome {
475    Parsed {
476        file: ScriptFile,
477        testscript_line: Option<usize>,
478    },
479    Error {
480        line: usize,
481        position: usize,
482        message: String,
483        phase: diagnostics::Phase,
484    },
485}
486
487/// Read, lex and parse a single file off-thread. Pure: no shared state, so it
488/// is safe to run concurrently. Mirrors the original sequential body exactly —
489/// same source truncation, same cache-key normalization, same lex-then-parse
490/// error reporting — only restructured to return owned data for the merge.
491fn parse_one_file(path: &Path, include_tests: bool) -> FileParseResult {
492    let raw_source = match fs::read_to_string(path) {
493        Ok(s) => s,
494        Err(e) => {
495            return FileParseResult {
496                path: path.to_path_buf(),
497                read: Err(e),
498            };
499        }
500    };
501    let testscript_line = find_testscript_line(&raw_source);
502
503    let source_code = if !include_tests {
504        if let Some(line_no) = testscript_line {
505            truncate_at_line(&raw_source, line_no)
506        } else {
507            raw_source.clone()
508        }
509    } else {
510        strip_testscript_annotation(&raw_source)
511    };
512
513    let canonical = fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf());
514    let mut cache_key = canonical.to_string_lossy().into_owned();
515    if cache_key.starts_with("\\\\?\\") {
516        cache_key = cache_key[4..].to_string();
517    }
518    cache_key = cache_key.replace("\\Content\\", "\\content\\");
519
520    // Lexer/Parser borrow a `&PathBuf` for diagnostics; keep one alive locally.
521    let path_buf = path.to_path_buf();
522    let outcome = match Lexer::new(&source_code, &path_buf).tokenize() {
523        Err(e) => ParseOutcome::Error {
524            line: e.line,
525            position: e.position,
526            message: e.message,
527            phase: diagnostics::Phase::Lexing,
528        },
529        Ok(tokens) => match Parser::new(tokens, &path_buf).parse() {
530            Ok(file) => ParseOutcome::Parsed {
531                file,
532                testscript_line: if include_tests { testscript_line } else { None },
533            },
534            Err(e) => ParseOutcome::Error {
535                line: e.line,
536                position: e.position,
537                message: e.message,
538                phase: diagnostics::Phase::Parsing,
539            },
540        },
541    };
542
543    FileParseResult {
544        path: path_buf,
545        read: Ok(ParsedSource {
546            cache_key,
547            raw_source,
548            outcome,
549        }),
550    }
551}
552
553/// Compile scripts and write output. When `lint` is true, also run lint passes
554/// (unused locals, unreachable code) after code generation.
555pub fn compile_memory(
556    scripts_dir: &Path,
557    pack_dir: Option<&Path>,
558    lint: bool,
559) -> Result<(Vec<u8>, Vec<u8>), Box<dyn Error>> {
560    compile_memory_with_options(scripts_dir, pack_dir, lint, false, false)
561}
562
563/// Like `compile_memory` but includes `#testscript` sections when
564/// `include_tests` is true.
565pub fn compile_memory_with_tests(
566    scripts_dir: &Path,
567    pack_dir: Option<&Path>,
568    lint: bool,
569    include_tests: bool,
570) -> Result<(Vec<u8>, Vec<u8>), Box<dyn Error>> {
571    compile_memory_with_options(scripts_dir, pack_dir, lint, include_tests, false)
572}
573
574/// In-memory compile — returns the `(script.dat, script.idx)` bytes instead of
575/// writing them — choosing the compilation strategy.
576///
577/// With `low_mem`, the recompile pipeline is used: scripts are re-parsed and
578/// re-compiled on demand so the full compiled set is never resident (~16 MB heap
579/// vs ~93), at roughly 6× the time; the returned bytes are byte-identical to the
580/// default path. `include_tests` keeps `#testscript` sections. The
581/// `RUNEC_RECOMPILE` environment variable forces low-memory mode on as well.
582pub fn compile_memory_with_options(
583    scripts_dir: &Path,
584    pack_dir: Option<&Path>,
585    lint: bool,
586    include_tests: bool,
587    low_mem: bool,
588) -> Result<(Vec<u8>, Vec<u8>), Box<dyn Error>> {
589    if low_mem || std::env::var("RUNEC_RECOMPILE").is_ok_and(|v| !v.is_empty()) {
590        info!("[recompile] low-memory recompile pipeline enabled");
591        return compile_recompile_memory(scripts_dir, pack_dir, lint, include_tests);
592    }
593
594    let (compiled_scripts, diagnostics) = run_pipeline(scripts_dir, pack_dir, lint, include_tests)?;
595
596    info!("Writing compiled output to memory...");
597    let writer = ScriptWriter::new("".into());
598
599    diagnostics.print_all();
600    info!("Compilation complete!");
601
602    Ok(writer.build_all(&compiled_scripts)?)
603}
604
605/// Compile scripts and write output. When `lint` is true, also run lint passes
606/// (unused locals, unreachable code) after code generation.
607///
608/// Convenience wrapper over [`compile_with_options`] using the default (fast,
609/// fully-resident) pipeline. Kept signature-compatible for existing callers.
610pub fn compile(
611    scripts_dir: &Path,
612    pack_dir: Option<&Path>,
613    output_dir: &Path,
614    lint: bool,
615) -> Result<(), Box<dyn Error>> {
616    compile_with_options(scripts_dir, pack_dir, output_dir, lint, false)
617}
618
619/// Compile scripts and write output, choosing the compilation strategy.
620///
621/// With `low_mem`, the **recompile** pipeline is used: scripts are re-parsed and
622/// re-compiled on demand for the pointer check and written file-by-file, so the
623/// full set of compiled bytecode is never resident at once. That cuts peak usage
624/// to roughly 16 MB heap / 32 MB working set (from ~93 / ~111 MB) at about 6× the
625/// compile time. Output is byte-identical to the default path. The
626/// `RUNEC_RECOMPILE` environment variable forces this mode on as well.
627pub fn compile_with_options(
628    scripts_dir: &Path,
629    pack_dir: Option<&Path>,
630    output_dir: &Path,
631    lint: bool,
632    low_mem: bool,
633) -> Result<(), Box<dyn Error>> {
634    // The recompile pipeline never holds all compiled scripts (it re-compiles on
635    // demand for the pointer check + writes file-by-file). It does its own
636    // output, so it returns here. The default path below is unchanged.
637    if low_mem || std::env::var("RUNEC_RECOMPILE").is_ok_and(|v| !v.is_empty()) {
638        info!("[recompile] low-memory recompile pipeline enabled");
639        return compile_recompile(scripts_dir, pack_dir, output_dir, lint);
640    }
641
642    let (compiled_scripts, diagnostics) = run_pipeline(scripts_dir, pack_dir, lint, false)?;
643
644    let output = output_dir.display().to_string();
645    // Normalize separators for a clean, consistent log message (input paths may
646    // mix `/` and `\`, e.g. `data/pack\server`); the writer keeps the original.
647    info!("Writing output to {}...", output.replace('\\', "/"));
648    let writer = ScriptWriter::new(output);
649    writer.write_all(&compiled_scripts)?;
650
651    diagnostics.print_all();
652    info!("Compilation complete!");
653
654    Ok(())
655}
656
657/// Run all analysis passes (parse, type-check, codegen, pointer-check, lints)
658/// without writing output. Useful for editor tooling and CI lint gates.
659pub fn lint(scripts_dir: &Path, pack_dir: Option<&Path>) -> Result<(), Box<dyn Error>> {
660    let (_, diagnostics) = run_pipeline(scripts_dir, pack_dir, true, false)?;
661
662    diagnostics.print_all();
663    info!("Compilation complete!");
664
665    Ok(())
666}
667
668/// Runs the full compilation pipeline (parse, register, type-check, codegen,
669/// pointer-check, and optional lint passes), returning the compiled scripts
670/// and accumulated diagnostics for the caller to handle output.
671fn run_pipeline(
672    scripts_dir: &Path,
673    pack_dir: Option<&Path>,
674    lint: bool,
675    include_tests: bool,
676) -> Result<(Vec<CompiledScript>, DiagnosticsCollector), Box<dyn Error>> {
677    let source_dir = sources::target_dir(scripts_dir, sources::Scope::Rune);
678    let scripts_dir = source_dir.as_path();
679    if !scripts_dir.exists() || !scripts_dir.is_dir() {
680        return Err(Box::new(error::CompilerError::FileNotFound(
681            scripts_dir.display().to_string(),
682        )));
683    }
684
685    // Prototype: opt into the streaming pipeline (never holds all ASTs at once)
686    // with RUNEC_STREAM=1. The default batch path below is unchanged.
687    if std::env::var("RUNEC_STREAM").is_ok_and(|v| !v.is_empty()) {
688        info!("[stream] streaming pipeline enabled (RUNEC_STREAM)");
689        return run_pipeline_streaming(scripts_dir, pack_dir, lint, include_tests);
690    }
691
692    let mut rs2_files: Vec<PathBuf> = get_rs2_files(scripts_dir, "rs2")?;
693
694    rs2_files.sort();
695    info!("Found {} script files", rs2_files.len());
696    mem_mark("startup");
697
698    let mut diagnostics = DiagnosticsCollector::new();
699
700    // Phase 1: Parse all files. Reading, lexing and parsing are independent
701    // per file, so fan them out across threads; the results are merged back in
702    // input order so registration/ID assignment stays deterministic.
703    info!("Parsing script files into syntax trees...");
704    let parse_results = parallel_map(&rs2_files, |path| parse_one_file(path, include_tests));
705
706    let mut all_files: Vec<ParsedFile> = Vec::new();
707    let mut source_cache: HashMap<String, Arc<String>> = HashMap::new();
708    // Raw source text keyed by on-disk path, reused by `generate_script_pack`
709    // so it doesn't re-read every file from disk a second time.
710    let mut raw_by_path: HashMap<PathBuf, Arc<String>> = HashMap::new();
711    for result in parse_results {
712        let parsed = match result.read {
713            Ok(p) => p,
714            // A read failure aborts the whole compile, exactly as the previous
715            // `fs::read_to_string(path)?` did.
716            Err(e) => return Err(Box::new(e)),
717        };
718        let rc = Arc::new(parsed.raw_source);
719        // The cache key is the canonicalized source path; reuse it as each
720        // compiled script's `source_path` so codegen needn't canonicalize again.
721        let source_path = parsed.cache_key.clone();
722        source_cache.insert(parsed.cache_key, rc.clone());
723        raw_by_path.insert(result.path.clone(), rc);
724        match parsed.outcome {
725            ParseOutcome::Parsed {
726                file,
727                testscript_line,
728            } => all_files.push(ParsedFile {
729                path: result.path,
730                file,
731                testscript_line,
732                source_path,
733            }),
734            ParseOutcome::Error {
735                line,
736                position,
737                message,
738                phase,
739            } => diagnostics.error(result.path, line, position, message, phase),
740        }
741    }
742
743    if diagnostics.has_errors() {
744        diagnostics.print_all();
745        return Err(Box::new(error::CompilerError::TypeError(
746            "Parsing failed".to_string(),
747        )));
748    }
749
750    mem_mark("after parse");
751
752    // Phase 2: Register all scripts. Build the pack/config registry, then
753    // regenerate script.pack, load pre-assigned IDs, and register every script.
754    info!("Registering scripts and building the symbol registry...");
755    let resolved_pack_dir = resolve_pack_dir(scripts_dir, pack_dir);
756    let mut registry = build_symbol_registry(scripts_dir, resolved_pack_dir.as_deref());
757    register_scripts(
758        &mut registry,
759        scripts_dir,
760        resolved_pack_dir.as_deref(),
761        &mut diagnostics,
762        &all_files,
763        &raw_by_path,
764    );
765    info!("  Registered {} scripts", registry.scripts.len());
766    // `raw_by_path` was only needed to (re)generate script.pack inside
767    // `register_scripts`; the source text itself stays reachable via
768    // `source_cache` (shared `Arc`s). Drop the redundant index now.
769    drop(raw_by_path);
770    // Pre-assigned script ids were consumed during registration; free them
771    // before codegen (the compile's peak-memory phase) rather than at scope end.
772    registry.drop_registration_scratch();
773
774    for pf in &all_files {
775        for script in &pf.file.scripts {
776            if let Some(msg) =
777                Compiler::validate_trigger_subject(&script.trigger, &script.name, &registry)
778            {
779                diagnostics.warning(
780                    pf.path.clone(),
781                    script.line,
782                    0,
783                    msg,
784                    diagnostics::Phase::SymbolRegistration,
785                );
786            }
787        }
788    }
789
790    mem_mark("after register");
791    #[cfg(feature = "memprof")]
792    mem_stats_registry(&registry);
793
794    // Phase 3: Type checking
795    info!("Type checking {} files...", all_files.len());
796    run_type_checker(&mut diagnostics, &all_files, &registry);
797
798    if diagnostics.has_errors() {
799        diagnostics.print_all();
800        return Err(Box::new(error::CompilerError::TypeError(
801            "Type checking failed".to_string(),
802        )));
803    }
804
805    mem_mark("after typecheck");
806
807    // Phase 4: Code generation
808    info!("Generating bytecode from type-checked scripts...");
809    let mut compiled_scripts = codegen(&all_files, &registry, &mut diagnostics);
810    if diagnostics.has_errors() {
811        diagnostics.print_all();
812        return Err(Box::new(error::CompilerError::CodeGenError(
813            "Code generation failed".into(),
814        )));
815    }
816    info!("  Generated {} compiled scripts", compiled_scripts.len());
817    compiled_scripts.sort_by_key(|s| s.id);
818    compiled_scripts.shrink_to_fit();
819    mem_mark("after codegen");
820    #[cfg(feature = "memprof")]
821    mem_stats_compiled(&compiled_scripts);
822
823    // The ASTs are consumed by codegen and never read again — pointer checking
824    // and lints work purely off `compiled_scripts` + the registry + the source
825    // cache. Free the syntax trees now so they don't coexist with the pointer
826    // checker's (per-thread) scratch, which is the compile's peak-memory moment.
827    drop(all_files);
828    mem_mark("after drop asts");
829
830    // Phase 4.5: Pointer checking & lints
831    info!("Checking pointers in generated code...");
832    check_pointers(
833        lint,
834        &mut diagnostics,
835        SourceProvider::Eager(&source_cache),
836        &registry,
837        &compiled_scripts,
838    );
839    mem_mark("after pointercheck");
840
841    Ok((compiled_scripts, diagnostics))
842}
843
844// ── Streaming pipeline (prototype, gated by RUNEC_STREAM) ────────────────────
845// Goal: never materialize all ASTs (~59 MB) at once. Pass 1 parses in parallel
846// but keeps only per-script signatures, dropping each AST in-worker; the registry
847// is built from those. Pass 2 re-parses each file in parallel, type-checks and
848// compiles it, then drops the AST immediately — so syntax trees never accumulate.
849// Output is byte-identical to the batch path (same registration, same per-file
850// codegen, same stable id-sort). It still holds all compiled scripts (for the
851// cross-script pointer check + writer); shrinking that is the next increment.
852
853/// One script's registration-relevant signature — everything the registry and
854/// trigger diagnostics need — so the full AST can be dropped right after parsing.
855struct ScriptSig {
856    name: String,
857    trigger: String,
858    param_types: Vec<types::Type>,
859    return_types: Vec<types::Type>,
860    line: usize,
861}
862
863struct FileSigs {
864    path: PathBuf,
865    testscript_line: Option<usize>,
866    sigs: Vec<ScriptSig>,
867}
868
869enum SigOutcome {
870    Parsed {
871        sigs: Vec<ScriptSig>,
872        testscript_line: Option<usize>,
873    },
874    Error {
875        line: usize,
876        position: usize,
877        message: String,
878        phase: diagnostics::Phase,
879    },
880}
881
882struct FileSigsResult {
883    path: PathBuf,
884    /// `(cache_key, raw_source, outcome)` on success.
885    read: Result<(String, String, SigOutcome), std::io::Error>,
886}
887
888/// Parse one file, extract signatures, and DROP the AST in-worker, so a parallel
889/// sweep never holds every syntax tree at once.
890fn parse_one_file_sigs(path: &Path, include_tests: bool) -> FileSigsResult {
891    let r = parse_one_file(path, include_tests);
892    let read = r.read.map(|ps| {
893        let outcome = match ps.outcome {
894            ParseOutcome::Parsed {
895                file,
896                testscript_line,
897            } => {
898                let sigs = file
899                    .scripts
900                    .iter()
901                    .map(|d| ScriptSig {
902                        name: d.name.clone(),
903                        trigger: d.trigger.clone(),
904                        param_types: d.params.iter().map(|p| p.param_type).collect(),
905                        return_types: d.return_types.clone(),
906                        line: d.line,
907                    })
908                    .collect();
909                SigOutcome::Parsed {
910                    sigs,
911                    testscript_line,
912                }
913                // `file` (the AST) is dropped here — only the sigs travel back.
914            }
915            ParseOutcome::Error {
916                line,
917                position,
918                message,
919                phase,
920            } => SigOutcome::Error {
921                line,
922                position,
923                message,
924                phase,
925            },
926        };
927        (ps.cache_key, ps.raw_source, outcome)
928    });
929    FileSigsResult { path: r.path, read }
930}
931
932fn run_pipeline_streaming(
933    scripts_dir: &Path,
934    pack_dir: Option<&Path>,
935    lint: bool,
936    include_tests: bool,
937) -> Result<(Vec<CompiledScript>, DiagnosticsCollector), Box<dyn Error>> {
938    let mut rs2_files: Vec<PathBuf> = get_rs2_files(scripts_dir, "rs2")?;
939    rs2_files.sort();
940    info!("Found {} script files", rs2_files.len());
941    mem_mark("startup");
942
943    let mut diagnostics = DiagnosticsCollector::new();
944
945    // Pass 1: parse for signatures only (each AST dropped in its worker).
946    info!("[stream] Pass 1: parsing for signatures...");
947    let sig_results = parallel_map(&rs2_files, |path| parse_one_file_sigs(path, include_tests));
948
949    // Streaming keeps only `cache_key -> on-disk path`; source text is re-read
950    // lazily for the rare diagnostic that needs it, so the ~4 MB of sources isn't
951    // resident through the peak. `raw_by_path` still holds the text transiently
952    // for `script.pack` regeneration, then is dropped right after registration.
953    let mut source_paths: HashMap<String, PathBuf> = HashMap::new();
954    let mut raw_by_path: HashMap<PathBuf, Arc<String>> = HashMap::new();
955    let mut file_sigs: Vec<FileSigs> = Vec::new();
956    for r in sig_results {
957        let (cache_key, raw_source, outcome) = match r.read {
958            Ok(t) => t,
959            Err(e) => return Err(Box::new(e)),
960        };
961        source_paths.insert(cache_key, r.path.clone());
962        raw_by_path.insert(r.path.clone(), Arc::new(raw_source));
963        match outcome {
964            SigOutcome::Parsed {
965                sigs,
966                testscript_line,
967            } => file_sigs.push(FileSigs {
968                path: r.path,
969                testscript_line,
970                sigs,
971            }),
972            SigOutcome::Error {
973                line,
974                position,
975                message,
976                phase,
977            } => diagnostics.error(r.path, line, position, message, phase),
978        }
979    }
980
981    if diagnostics.has_errors() {
982        diagnostics.print_all();
983        return Err(Box::new(error::CompilerError::TypeError(
984            "Parsing failed".to_string(),
985        )));
986    }
987    mem_mark("after parse");
988
989    // Registration (from signatures, not ASTs).
990    info!("[stream] Registering scripts and building the symbol registry...");
991    let resolved_pack_dir = resolve_pack_dir(scripts_dir, pack_dir);
992    let mut registry = build_symbol_registry(scripts_dir, resolved_pack_dir.as_deref());
993    register_scripts_from_sigs(
994        &mut registry,
995        scripts_dir,
996        resolved_pack_dir.as_deref(),
997        &mut diagnostics,
998        &file_sigs,
999        &raw_by_path,
1000    );
1001    info!("  Registered {} scripts", registry.scripts.len());
1002    drop(raw_by_path);
1003    registry.drop_registration_scratch();
1004
1005    for fs in &file_sigs {
1006        for sig in &fs.sigs {
1007            if let Some(msg) =
1008                Compiler::validate_trigger_subject(&sig.trigger, &sig.name, &registry)
1009            {
1010                diagnostics.warning(
1011                    fs.path.clone(),
1012                    sig.line,
1013                    0,
1014                    msg,
1015                    diagnostics::Phase::SymbolRegistration,
1016                );
1017            }
1018        }
1019    }
1020    // Signatures are done; only the registry + source cache carry forward.
1021    drop(file_sigs);
1022    mem_mark("after register");
1023    #[cfg(feature = "memprof")]
1024    mem_stats_registry(&registry);
1025
1026    // Pass 2: re-parse + type-check + compile per file, streaming. Parse can't
1027    // fail here (pass 1 validated), so a failed re-parse just yields nothing.
1028    info!("[stream] Pass 2: type-check + codegen (streaming)...");
1029    let per_file = parallel_map(&rs2_files, |path| {
1030        let parsed = match parse_one_file(path, include_tests).read {
1031            Ok(p) => p,
1032            Err(_) => return (Vec::new(), DiagnosticsCollector::new()),
1033        };
1034        let (file, testscript_line) = match parsed.outcome {
1035            ParseOutcome::Parsed {
1036                file,
1037                testscript_line,
1038            } => (file, testscript_line),
1039            ParseOutcome::Error { .. } => return (Vec::new(), DiagnosticsCollector::new()),
1040        };
1041        let mut type_checker = TypeChecker::new(&registry);
1042        type_checker.check_file_with_test_boundary(&file, path, testscript_line);
1043        let mut compiled = Vec::new();
1044        // Compile only when this file type-checks clean (codegen assumes
1045        // well-typed input). On any global error the whole compile aborts below,
1046        // so output produced here would be discarded anyway.
1047        if !type_checker.diagnostics.has_errors() {
1048            let mut compiler = Compiler::new(&registry);
1049            for script in &file.scripts {
1050                if script.trigger == "command" {
1051                    continue;
1052                }
1053                let Some(mut c) =
1054                    compiler.compile_checked(script, path, &mut type_checker.diagnostics)
1055                else {
1056                    continue;
1057                };
1058                c.source_path = parsed.cache_key.clone();
1059                c.shrink_to_fit();
1060                compiled.push(c);
1061            }
1062        }
1063        (compiled, type_checker.diagnostics)
1064        // `file` (the AST) is dropped here.
1065    });
1066
1067    let mut compiled_scripts: Vec<CompiledScript> = Vec::new();
1068    for (compiled, diags) in per_file {
1069        compiled_scripts.extend(compiled);
1070        diagnostics.merge(diags);
1071    }
1072    if diagnostics.has_errors() {
1073        diagnostics.print_all();
1074        return Err(Box::new(error::CompilerError::TypeError(
1075            "Type checking failed".to_string(),
1076        )));
1077    }
1078    // Stable sort by id matches the batch path (file order preserved among
1079    // equal ids → identical writer last-write-wins behaviour).
1080    compiled_scripts.sort_by_key(|s| s.id);
1081    compiled_scripts.shrink_to_fit();
1082    info!("  Generated {} compiled scripts", compiled_scripts.len());
1083    mem_mark("after codegen");
1084    #[cfg(feature = "memprof")]
1085    mem_stats_compiled(&compiled_scripts);
1086
1087    check_pointers(
1088        lint,
1089        &mut diagnostics,
1090        SourceProvider::Lazy(&source_paths),
1091        &registry,
1092        &compiled_scripts,
1093    );
1094    mem_mark("after pointercheck");
1095
1096    Ok((compiled_scripts, diagnostics))
1097}
1098
1099/// `register_scripts` for the streaming path: identical logic, driven by the
1100/// extracted signatures instead of the (already-dropped) ASTs.
1101fn register_scripts_from_sigs(
1102    registry: &mut SymbolRegistry,
1103    scripts_dir: &Path,
1104    resolved_pack_dir: Option<&Path>,
1105    diagnostics: &mut DiagnosticsCollector,
1106    file_sigs: &[FileSigs],
1107    raw_by_path: &HashMap<PathBuf, Arc<String>>,
1108) {
1109    if let Some(pd) = resolved_pack_dir {
1110        symloader::generate_script_pack(scripts_dir, pd, raw_by_path);
1111        symloader::load_script_ids(registry, pd);
1112    }
1113
1114    use crate::diagnostic_messages as msg;
1115    let mut registered_scripts: std::collections::HashSet<String> =
1116        std::collections::HashSet::new();
1117
1118    for fs in file_sigs.iter() {
1119        for sig in &fs.sigs {
1120            Arc::make_mut(&mut registry.visibility).declare_script(
1121                &fs.path,
1122                &sig.trigger,
1123                &sig.name,
1124            );
1125            if sig.trigger == "command" {
1126                continue;
1127            }
1128
1129            if !trigger_table::is_valid_trigger(&sig.trigger) {
1130                diagnostics.warning(
1131                    fs.path.clone(),
1132                    sig.line,
1133                    0,
1134                    msg::fmt(msg::SCRIPT_TRIGGER_INVALID, &[&sig.trigger]),
1135                    crate::diagnostics::Phase::SymbolRegistration,
1136                );
1137            }
1138
1139            let key = format!("{}:{}", sig.trigger, sig.name);
1140            if !registered_scripts.insert(key) {
1141                diagnostics.warning(
1142                    fs.path.clone(),
1143                    sig.line,
1144                    0,
1145                    msg::fmt(msg::SCRIPT_REDECLARATION, &[&sig.trigger, &sig.name]),
1146                    crate::diagnostics::Phase::SymbolRegistration,
1147                );
1148            }
1149
1150            if !sig.return_types.is_empty() && !trigger_table::allows_returns(&sig.trigger) {
1151                diagnostics.warning(
1152                    fs.path.clone(),
1153                    sig.line,
1154                    0,
1155                    msg::fmt(msg::SCRIPT_TRIGGER_NO_RETURNS, &[&sig.trigger]),
1156                    diagnostics::Phase::SymbolRegistration,
1157                );
1158            }
1159
1160            registry.register_script(
1161                sig.name.clone(),
1162                sig.trigger.clone(),
1163                sig.param_types.clone(),
1164                sig.return_types.clone(),
1165            );
1166
1167            if let Some(ts_line) = fs.testscript_line
1168                && sig.line >= ts_line
1169            {
1170                registry.mark_test_script(&sig.trigger, &sig.name);
1171            }
1172        }
1173    }
1174}
1175
1176fn check_pointers(
1177    lint: bool,
1178    diagnostics: &mut DiagnosticsCollector,
1179    source: SourceProvider,
1180    registry: &SymbolRegistry,
1181    compiled_scripts: &[CompiledScript],
1182) {
1183    {
1184        use crate::pointer_checker::{PointerChecker, ScriptSource};
1185        // Validate scripts across threads: each chunk gets its own checker that
1186        // shares the read-only scripts/registry/source cache but keeps its own
1187        // (purely memoizing) caches. Diagnostics are merged in script order, so
1188        // the result is identical to a single-threaded run.
1189        //
1190        // The read-only command/script lookup tables are built ONCE here and
1191        // shared (`Arc`) across every chunk checker, instead of each thread
1192        // rebuilding the full ~700-command pointer table + per-script reverse
1193        // maps. `names` is just the script order, so derive it directly rather
1194        // than spinning up a throwaway checker for `script_names()`.
1195        let shared = PointerChecker::build_shared(compiled_scripts, registry);
1196        let names: Vec<String> = compiled_scripts.iter().map(|s| s.name.clone()).collect();
1197        // Each chunk's checker retains its CFG caches until the chunk finishes,
1198        // so concurrency multiplies peak scratch. RUNEC_PTR_CHUNKS caps the
1199        // concurrent chunk count to trade pointer-check speed for memory; the
1200        // result is chunk-count-independent, so diagnostics stay byte-identical.
1201        let max_chunks = std::env::var("RUNEC_PTR_CHUNKS")
1202            .ok()
1203            .and_then(|s| s.parse::<usize>().ok())
1204            .filter(|&n| n > 0)
1205            .unwrap_or(usize::MAX);
1206        let chunk_diags = parallel_chunks_capped(&names, max_chunks, |chunk| {
1207            let mut checker = PointerChecker::with_shared(
1208                ScriptSource::Resident(compiled_scripts),
1209                registry,
1210                shared.clone(),
1211            );
1212            checker.set_source_cache(source);
1213            checker.validate_names(chunk)
1214        });
1215        let mut pointer_diags = DiagnosticsCollector::new();
1216        for d in chunk_diags {
1217            pointer_diags.merge(d);
1218        }
1219        let ptr_warnings = pointer_diags.warning_count();
1220        if ptr_warnings > 0 {
1221            info!("  Pointer check: {} warning(s)", ptr_warnings);
1222        }
1223        diagnostics.merge(pointer_diags);
1224    }
1225    #[cfg(feature = "memprof")]
1226    crate::pointer_checker::cfg_trace_report();
1227
1228    // Phase 4.6: Lint passes (optional)
1229    if lint {
1230        info!("Running lint passes (unused locals, unreachable code)...");
1231        let lint_diags = lints::run_lints(compiled_scripts, Some(source));
1232        let lint_warnings = lint_diags.warning_count();
1233        if lint_warnings > 0 {
1234            info!("  Lints: {} warning(s)", lint_warnings);
1235        }
1236        diagnostics.merge(lint_diags);
1237    }
1238}
1239
1240fn codegen(
1241    all_files: &[ParsedFile],
1242    registry: &SymbolRegistry,
1243    diagnostics: &mut DiagnosticsCollector,
1244) -> Vec<CompiledScript> {
1245    // Each file compiles independently against the read-only registry, so fan
1246    // the files out across threads. Output order doesn't matter — the caller
1247    // sorts compiled scripts by id — so a simple flatten of the per-file results
1248    // is enough.
1249    let mut compiled = Vec::new();
1250    for (scripts, errors) in parallel_map(all_files, |pf| compile_file(pf, registry)) {
1251        compiled.extend(scripts);
1252        diagnostics.merge(errors);
1253    }
1254    compiled
1255}
1256
1257/// Compile every (non-`command`) script in one file. Creating a `Compiler` per
1258/// file is cheap now that it only borrows the registry, and keeps the codegen
1259/// scratch state thread-local.
1260fn compile_file(
1261    pf: &ParsedFile,
1262    registry: &SymbolRegistry,
1263) -> (Vec<CompiledScript>, DiagnosticsCollector) {
1264    let mut diagnostics = DiagnosticsCollector::new();
1265    let mut scripts =
1266        crate::compiler::compile_file(&pf.file, &pf.path, registry, false, &mut diagnostics);
1267    for script in &mut scripts {
1268        script.source_path = pf.source_path.clone();
1269        script.shrink_to_fit();
1270    }
1271    (scripts, diagnostics)
1272}
1273
1274fn run_type_checker(
1275    diagnostics: &mut DiagnosticsCollector,
1276    all_files: &[ParsedFile],
1277    registry: &SymbolRegistry,
1278) {
1279    // Each file type-checks independently against the read-only registry, so
1280    // check chunks of files in parallel; per-chunk diagnostics are merged in
1281    // file order, matching a single-threaded run.
1282    let chunk_diags = parallel_chunks(all_files, |chunk| {
1283        let mut type_checker = TypeChecker::new(registry);
1284        for pf in chunk {
1285            type_checker.check_file_with_test_boundary(&pf.file, &pf.path, pf.testscript_line);
1286        }
1287        type_checker.diagnostics
1288    });
1289    for d in chunk_diags {
1290        diagnostics.merge(d);
1291    }
1292}
1293
1294/// Resolve the pack directory: explicit `--pack`, else the nearest ancestor's `pack/`, else
1295/// `content/pack` up the tree.
1296fn resolve_pack_dir(scripts_dir: &Path, pack_dir: Option<&Path>) -> Option<PathBuf> {
1297    pack_dir
1298        .map(PathBuf::from)
1299        .filter(|p| p.exists())
1300        .or_else(|| {
1301            scripts_dir.ancestors().find_map(|dir| {
1302                let pack = dir.join("pack");
1303                pack.is_dir().then_some(pack)
1304            })
1305        })
1306        .or_else(|| {
1307            scripts_dir
1308                .ancestors()
1309                .nth(3)
1310                .map(|p| p.join("content/pack"))
1311                .filter(|p| p.exists())
1312        })
1313}
1314
1315/// Build the part of the registry that does NOT depend on the parsed scripts:
1316/// pack files, `.constant`/`.var`/`.dbtable` configs, and engine command params.
1317/// This runs concurrently with parsing; `script.pack` (re)generation and script
1318/// registration — which need the parsed sources/ASTs — happen afterwards in
1319/// `register_scripts`.
1320fn build_symbol_registry(scripts_dir: &Path, resolved_pack_dir: Option<&Path>) -> SymbolRegistry {
1321    let mut registry = SymbolRegistry::new();
1322
1323    if let Some(pd) = resolved_pack_dir {
1324        info!("  Loading packs from {}", pd.display());
1325        symloader::load_packs(&mut registry, pd);
1326        info!(
1327            "  Loaded {} commands, {} game vars",
1328            registry.commands.len(),
1329            registry.game_vars.len(),
1330        );
1331    } else {
1332        info!("  No pack directory found - symbol resolution will be limited");
1333    }
1334
1335    // Walk the scripts tree once and let the constant/game-var/dbtable loaders
1336    // filter this shared list by extension, instead of each re-walking the tree
1337    // (previously 6 separate recursive traversals over the same directories).
1338    let mut config_files: Vec<PathBuf> = Vec::new();
1339    config_files.extend(sources::config_files(scripts_dir));
1340
1341    symloader::load_constant_files(&mut registry, &config_files);
1342
1343    info!(
1344        "  Loaded {} constants from .constant files",
1345        registry.constants.len()
1346    );
1347
1348    symloader::load_game_var_types(&mut registry, &config_files);
1349
1350    if let Some(pd) = resolved_pack_dir {
1351        let dbtable_ids = symloader::load_dbtable_pack(&pd.join("dbtable.pack"));
1352        if !dbtable_ids.is_empty() {
1353            symloader::load_dbtable_configs(&mut registry, &config_files, &dbtable_ids);
1354            info!(
1355                "  Registered {} dbcolumn compound IDs",
1356                registry.dbcolumn_types.len()
1357            );
1358        }
1359    }
1360
1361    let engine_rs2 = scripts_dir.join("engine.rs2");
1362
1363    if engine_rs2.exists() {
1364        symloader::load_engine_command_params(&mut registry, &engine_rs2);
1365    } else {
1366        if let Ok(entries) = std::fs::read_dir(scripts_dir) {
1367            for entry in entries.flatten() {
1368                let candidate = entry.path().join("engine.rs2");
1369                if candidate.exists() {
1370                    crate::symloader::load_engine_command_params(&mut registry, &candidate);
1371                    break;
1372                }
1373            }
1374        }
1375    }
1376
1377    let client_dir = if scripts_dir
1378        .file_name()
1379        .is_some_and(|name| name == "runescript")
1380    {
1381        scripts_dir.with_file_name("clientscript")
1382    } else {
1383        scripts_dir.join("clientscript")
1384    };
1385    if client_dir.is_dir() {
1386        if let Some(pack) = resolved_pack_dir {
1387            symloader::load_clientscript_pack(&mut registry, &pack.join("clientscript.pack"));
1388        }
1389        symloader::load_clientscript_signatures(&mut registry, &client_dir);
1390    }
1391
1392    symloader::patch_command_return_types(&mut registry);
1393
1394    registry.visibility = Arc::new(sources::Visibility::load(scripts_dir, &registry));
1395
1396    registry
1397}
1398
1399/// Finalize the registry once parsing is done: regenerate `script.pack` from the
1400/// parsed sources, load pre-assigned script IDs, then register every parsed
1401/// script (emitting trigger/redeclaration/return diagnostics). Builds on the
1402/// registry produced concurrently by `build_symbol_registry`.
1403fn register_scripts(
1404    registry: &mut SymbolRegistry,
1405    scripts_dir: &Path,
1406    resolved_pack_dir: Option<&Path>,
1407    diagnostics: &mut DiagnosticsCollector,
1408    all_files: &[ParsedFile],
1409    raw_by_path: &HashMap<PathBuf, Arc<String>>,
1410) {
1411    if let Some(pd) = resolved_pack_dir {
1412        symloader::generate_script_pack(scripts_dir, pd, raw_by_path);
1413        symloader::load_script_ids(registry, pd);
1414    }
1415
1416    {
1417        use crate::diagnostic_messages as msg;
1418
1419        let mut registered_scripts: std::collections::HashSet<String> =
1420            std::collections::HashSet::new();
1421
1422        for pf in all_files.iter() {
1423            Arc::make_mut(&mut registry.visibility).declare_source(&pf.path, &pf.file);
1424            for script in &pf.file.scripts {
1425                if script.trigger == "command" {
1426                    continue;
1427                }
1428
1429                if !trigger_table::is_valid_trigger(&script.trigger) {
1430                    diagnostics.warning(
1431                        pf.path.clone(),
1432                        script.line,
1433                        0,
1434                        msg::fmt(msg::SCRIPT_TRIGGER_INVALID, &[&script.trigger]),
1435                        crate::diagnostics::Phase::SymbolRegistration,
1436                    );
1437                }
1438
1439                let key = format!("{}:{}", script.trigger, script.name);
1440                if !registered_scripts.insert(key.clone()) {
1441                    diagnostics.warning(
1442                        pf.path.clone(),
1443                        script.line,
1444                        0,
1445                        msg::fmt(msg::SCRIPT_REDECLARATION, &[&script.trigger, &script.name]),
1446                        crate::diagnostics::Phase::SymbolRegistration,
1447                    );
1448                }
1449
1450                if !script.return_types.is_empty()
1451                    && !trigger_table::allows_returns(&script.trigger)
1452                {
1453                    diagnostics.warning(
1454                        pf.path.clone(),
1455                        script.line,
1456                        0,
1457                        msg::fmt(msg::SCRIPT_TRIGGER_NO_RETURNS, &[&script.trigger]),
1458                        diagnostics::Phase::SymbolRegistration,
1459                    );
1460                }
1461
1462                let param_types: Vec<types::Type> =
1463                    script.params.iter().map(|p| p.param_type).collect();
1464                registry.register_script(
1465                    script.name.clone(),
1466                    script.trigger.clone(),
1467                    param_types,
1468                    script.return_types.clone(),
1469                );
1470
1471                if let Some(ts_line) = pf.testscript_line
1472                    && script.line >= ts_line
1473                {
1474                    registry.mark_test_script(&script.trigger, &script.name);
1475                }
1476            }
1477        }
1478    }
1479}
1480
1481// ── Streaming recompile pipeline (prototype, gated by RUNEC_RECOMPILE) ────────
1482// Never holds all compiled scripts. Pass 2 only type-checks; the pointer check
1483// re-compiles each script on demand (bounded cache — measured ~1.3x re-parse on
1484// rs-majula, the call graph has high file locality); output is produced by
1485// re-compiling file-by-file into a by-id buffer. Output + diagnostics are
1486// byte-identical to the batch path (deterministic re-compile, same id order).
1487
1488/// Re-compiles scripts on demand for the pointer checker, caching `Rc`s with a
1489/// bounded FIFO so the full compiled set is never resident. A cache miss
1490/// re-parses the script's file and compiles all its scripts at once (siblings
1491/// share the parse), keying each by its full `[trigger,name]` to its index.
1492struct RecompileStore<'a> {
1493    idx_to_path: Vec<PathBuf>,
1494    /// file path -> its script idxs in declaration order (command-skipped).
1495    /// Position-based so duplicate `[trigger,name]` declarations each map to the
1496    /// correct distinct idx.
1497    file_to_idxs: HashMap<PathBuf, Vec<usize>>,
1498    registry: &'a SymbolRegistry,
1499    include_tests: bool,
1500    cache: HashMap<usize, std::rc::Rc<CompiledScript>>,
1501    order: std::collections::VecDeque<usize>,
1502    cap: usize,
1503}
1504
1505impl RecompileStore<'_> {
1506    fn get(&mut self, idx: usize) -> std::rc::Rc<CompiledScript> {
1507        if let Some(rc) = self.cache.get(&idx) {
1508            return std::rc::Rc::clone(rc);
1509        }
1510        let path = self.idx_to_path[idx].clone();
1511        let parsed = parse_one_file(&path, self.include_tests);
1512        if let Ok(ps) = parsed.read
1513            && let ParseOutcome::Parsed { file, .. } = ps.outcome
1514        {
1515            // The j-th non-command declaration in this file maps to idxs[j],
1516            // matching how the metas were built (same parse, same skip rule).
1517            let idxs = self.file_to_idxs.get(&path).cloned().unwrap_or_default();
1518            let mut compiler = Compiler::new(self.registry);
1519            let mut j = 0usize;
1520            for script in &file.scripts {
1521                if script.trigger == "command" {
1522                    continue;
1523                }
1524                if let Some(&i) = idxs.get(j) {
1525                    let mut c = compiler.compile_script(script);
1526                    c.source_path = ps.cache_key.clone();
1527                    self.insert(i, std::rc::Rc::new(c));
1528                }
1529                j += 1;
1530            }
1531        }
1532        std::rc::Rc::clone(
1533            self.cache
1534                .get(&idx)
1535                .expect("recompile store: script not produced by its file"),
1536        )
1537    }
1538
1539    fn insert(&mut self, idx: usize, rc: std::rc::Rc<CompiledScript>) {
1540        if self.cache.insert(idx, rc).is_none() {
1541            self.order.push_back(idx);
1542            while self.order.len() > self.cap {
1543                if let Some(old) = self.order.pop_front()
1544                    && old != idx
1545                {
1546                    self.cache.remove(&old);
1547                }
1548            }
1549        }
1550    }
1551}
1552
1553/// Per-slot encoded output blobs (indexed by script id) plus accumulated
1554/// diagnostics — what the recompile core hands back before the bytes are written
1555/// to disk or assembled in memory.
1556type RecompiledOutput = (Vec<Vec<u8>>, DiagnosticsCollector);
1557
1558/// Core of the recompile (low-memory) pipeline: runs every phase re-compiling
1559/// scripts on demand — never holding all compiled bytecode at once — and returns
1560/// the per-slot encoded output blobs plus diagnostics, without writing or
1561/// printing. The disk and in-memory entry points (`compile_recompile`,
1562/// `compile_recompile_memory`) wrap this. On a phase error it prints diagnostics
1563/// and returns `Err`, matching the batch path.
1564fn recompile_to_encoded(
1565    scripts_dir: &Path,
1566    pack_dir: Option<&Path>,
1567    lint: bool,
1568    include_tests: bool,
1569) -> Result<RecompiledOutput, Box<dyn Error>> {
1570    let source_dir = sources::target_dir(scripts_dir, sources::Scope::Rune);
1571    let scripts_dir = source_dir.as_path();
1572    use crate::pointer_checker::{PointerChecker, ScriptSource};
1573
1574    let mut rs2_files: Vec<PathBuf> = get_rs2_files(scripts_dir, "rs2")?;
1575    rs2_files.sort();
1576    info!("Found {} script files", rs2_files.len());
1577    mem_mark("startup");
1578    let mut diagnostics = DiagnosticsCollector::new();
1579
1580    // Pass 1: signatures only (ASTs dropped in-worker); keep file paths for lazy
1581    // source + raw text for script.pack regeneration.
1582    info!("[recompile] Pass 1: parsing for signatures...");
1583    let sig_results = parallel_map(&rs2_files, |path| parse_one_file_sigs(path, include_tests));
1584    let mut source_paths: HashMap<String, PathBuf> = HashMap::new();
1585    let mut raw_by_path: HashMap<PathBuf, Arc<String>> = HashMap::new();
1586    let mut file_sigs: Vec<FileSigs> = Vec::new();
1587    for r in sig_results {
1588        let (cache_key, raw_source, outcome) = match r.read {
1589            Ok(t) => t,
1590            Err(e) => return Err(Box::new(e)),
1591        };
1592        source_paths.insert(cache_key, r.path.clone());
1593        raw_by_path.insert(r.path.clone(), Arc::new(raw_source));
1594        match outcome {
1595            SigOutcome::Parsed {
1596                sigs,
1597                testscript_line,
1598            } => file_sigs.push(FileSigs {
1599                path: r.path,
1600                testscript_line,
1601                sigs,
1602            }),
1603            SigOutcome::Error {
1604                line,
1605                position,
1606                message,
1607                phase,
1608            } => diagnostics.error(r.path, line, position, message, phase),
1609        }
1610    }
1611    if diagnostics.has_errors() {
1612        diagnostics.print_all();
1613        return Err(Box::new(error::CompilerError::TypeError(
1614            "Parsing failed".to_string(),
1615        )));
1616    }
1617    mem_mark("after parse");
1618
1619    // Registration.
1620    info!("[recompile] Registering scripts and building the symbol registry...");
1621    let resolved_pack_dir = resolve_pack_dir(scripts_dir, pack_dir);
1622    let mut registry = build_symbol_registry(scripts_dir, resolved_pack_dir.as_deref());
1623    register_scripts_from_sigs(
1624        &mut registry,
1625        scripts_dir,
1626        resolved_pack_dir.as_deref(),
1627        &mut diagnostics,
1628        &file_sigs,
1629        &raw_by_path,
1630    );
1631    info!("  Registered {} scripts", registry.scripts.len());
1632    drop(raw_by_path);
1633    registry.drop_registration_scratch();
1634    for fs in &file_sigs {
1635        for sig in &fs.sigs {
1636            if let Some(msg) =
1637                Compiler::validate_trigger_subject(&sig.trigger, &sig.name, &registry)
1638            {
1639                diagnostics.warning(
1640                    fs.path.clone(),
1641                    sig.line,
1642                    0,
1643                    msg,
1644                    diagnostics::Phase::SymbolRegistration,
1645                );
1646            }
1647        }
1648    }
1649    mem_mark("after register");
1650
1651    // Pass 2: type-check only (re-parse, no compiled scripts retained).
1652    info!("[recompile] Pass 2: type-check (streaming, no bytecode held)...");
1653    let tc_diags = parallel_map(&rs2_files, |path| {
1654        let parsed = match parse_one_file(path, include_tests).read {
1655            Ok(p) => p,
1656            Err(_) => return DiagnosticsCollector::new(),
1657        };
1658        let (file, testscript_line) = match parsed.outcome {
1659            ParseOutcome::Parsed {
1660                file,
1661                testscript_line,
1662            } => (file, testscript_line),
1663            ParseOutcome::Error { .. } => return DiagnosticsCollector::new(),
1664        };
1665        let mut type_checker = TypeChecker::new(&registry);
1666        type_checker.check_file_with_test_boundary(&file, path, testscript_line);
1667        if !type_checker.diagnostics.has_errors() {
1668            let mut compiler = Compiler::new(&registry);
1669            for script in &file.scripts {
1670                if script.trigger != "command" {
1671                    compiler.compile_checked(script, path, &mut type_checker.diagnostics);
1672                }
1673            }
1674        }
1675        type_checker.diagnostics
1676    });
1677    for d in tc_diags {
1678        diagnostics.merge(d);
1679    }
1680    if diagnostics.has_errors() {
1681        diagnostics.print_all();
1682        return Err(Box::new(error::CompilerError::TypeError(
1683            "Type checking failed".to_string(),
1684        )));
1685    }
1686    mem_mark("after typecheck");
1687
1688    // Build script metas in the same idx order as the batch path: file order
1689    // (file_sigs / declaration order, command-skipped), then a stable sort by id.
1690    // ids + names come from the registry — no bytecode needed.
1691    let mut pre: Vec<(i32, String, PathBuf)> = Vec::new();
1692    for fs in &file_sigs {
1693        for sig in &fs.sigs {
1694            if sig.trigger == "command" {
1695                continue;
1696            }
1697            let id = registry
1698                .script_id_for_trigger(&sig.trigger, &sig.name)
1699                .or_else(|| registry.script_id(&sig.name))
1700                .unwrap_or(-1);
1701            pre.push((
1702                id,
1703                format!("[{},{}]", sig.trigger, sig.name),
1704                fs.path.clone(),
1705            ));
1706        }
1707    }
1708    let n = pre.len();
1709    // Stable sort indices by id → the batch's idx order (file/decl pre-order
1710    // preserved among equal ids). `inv` maps a pre-order position back to its idx.
1711    let mut order: Vec<usize> = (0..n).collect();
1712    order.sort_by_key(|&p| pre[p].0);
1713    let idents: Vec<(i32, String)> = order
1714        .iter()
1715        .map(|&p| (pre[p].0, pre[p].1.clone()))
1716        .collect();
1717    let idx_to_path: Vec<PathBuf> = order.iter().map(|&p| pre[p].2.clone()).collect();
1718    let mut inv = vec![0usize; n];
1719    for (idx, &p) in order.iter().enumerate() {
1720        inv[p] = idx;
1721    }
1722    let mut file_to_idxs: HashMap<PathBuf, Vec<usize>> = HashMap::new();
1723    for (p, m) in pre.iter().enumerate() {
1724        file_to_idxs.entry(m.2.clone()).or_default().push(inv[p]);
1725    }
1726    let max_id = pre.iter().map(|m| m.0).max().unwrap_or(-1);
1727    let slot_count = if max_id < 0 { 0 } else { (max_id + 1) as usize };
1728    let len = n;
1729    drop(pre);
1730
1731    // Pointer check via on-demand re-compile (single-threaded: the store getter
1732    // is `FnMut`/`!Sync`). Diagnostics come out in id order, as in the batch path.
1733    info!("[recompile] Pointer check (re-compile on demand)...");
1734    let shared = PointerChecker::build_shared_idents(&idents, &registry);
1735    {
1736        // Cache cap must exceed the largest file's script count, or loading that
1737        // file would evict its own just-compiled scripts. A small margin over
1738        // that keeps cross-file locality high (measured ~1.1-1.3x re-parse).
1739        let cap = file_to_idxs
1740            .values()
1741            .map(|v| v.len())
1742            .max()
1743            .unwrap_or(0)
1744            .max(128);
1745        let mut store = RecompileStore {
1746            idx_to_path,
1747            file_to_idxs,
1748            registry: &registry,
1749            include_tests,
1750            cache: HashMap::new(),
1751            order: std::collections::VecDeque::new(),
1752            cap,
1753        };
1754        let getter: Box<dyn FnMut(usize) -> std::rc::Rc<CompiledScript>> =
1755            Box::new(move |idx| store.get(idx));
1756        let scripts = ScriptSource::Recompile {
1757            len,
1758            get: std::cell::RefCell::new(getter),
1759        };
1760        let mut checker = PointerChecker::with_shared(scripts, &registry, shared);
1761        checker.set_source_cache(SourceProvider::Lazy(&source_paths));
1762        let names: Vec<String> = idents.iter().map(|(_, n)| n.clone()).collect();
1763        let ptr_diags = checker.validate_names(&names);
1764        let w = ptr_diags.warning_count();
1765        if w > 0 {
1766            info!("  Pointer check: {} warning(s)", w);
1767        }
1768        diagnostics.merge(ptr_diags);
1769    }
1770    mem_mark("after pointercheck");
1771
1772    // Output: re-compile each file once, encode each script into its id slot,
1773    // optionally lint — never holding all compiled scripts at once. The caller
1774    // writes the blobs to disk or assembles them in memory.
1775    info!("[recompile] Re-compiling + encoding output...");
1776    let mut encoded: Vec<Vec<u8>> = vec![Vec::new(); slot_count];
1777    for path in &rs2_files {
1778        let parsed = match parse_one_file(path, include_tests).read {
1779            Ok(p) => p,
1780            Err(_) => continue,
1781        };
1782        let file = match parsed.outcome {
1783            ParseOutcome::Parsed { file, .. } => file,
1784            ParseOutcome::Error { .. } => continue,
1785        };
1786        let mut compiler = Compiler::new(&registry);
1787        let mut file_scripts: Vec<CompiledScript> = Vec::new();
1788        for script in &file.scripts {
1789            if script.trigger == "command" {
1790                continue;
1791            }
1792            let Some(mut c) = compiler.compile_checked(script, path, &mut diagnostics) else {
1793                continue;
1794            };
1795            c.source_path = parsed.cache_key.clone();
1796            file_scripts.push(c);
1797        }
1798        for c in &file_scripts {
1799            if c.id >= 0 && (c.id as usize) < slot_count {
1800                encoded[c.id as usize] = crate::writer::encode_script(c);
1801            }
1802        }
1803        if lint {
1804            diagnostics.merge(lints::run_lints(
1805                &file_scripts,
1806                Some(SourceProvider::Lazy(&source_paths)),
1807            ));
1808        }
1809    }
1810    mem_mark("after codegen");
1811    Ok((encoded, diagnostics))
1812}
1813
1814/// Full compile via the recompile (low-memory) pipeline: writes `script.dat` /
1815/// `script.idx` to `output_dir`, then prints diagnostics.
1816fn compile_recompile(
1817    scripts_dir: &Path,
1818    pack_dir: Option<&Path>,
1819    output_dir: &Path,
1820    lint: bool,
1821) -> Result<(), Box<dyn Error>> {
1822    let (encoded, diagnostics) = recompile_to_encoded(scripts_dir, pack_dir, lint, false)?;
1823    info!(
1824        "Writing output to {}...",
1825        output_dir.display().to_string().replace('\\', "/")
1826    );
1827    ScriptWriter::new(output_dir.display().to_string()).write_encoded(&encoded)?;
1828    diagnostics.print_all();
1829    info!("Compilation complete!");
1830    Ok(())
1831}
1832
1833/// In-memory compile via the recompile (low-memory) pipeline: returns the
1834/// `(script.dat, script.idx)` bytes, then prints diagnostics. Byte-identical to
1835/// the in-memory batch path (`compile_memory*`).
1836fn compile_recompile_memory(
1837    scripts_dir: &Path,
1838    pack_dir: Option<&Path>,
1839    lint: bool,
1840    include_tests: bool,
1841) -> Result<(Vec<u8>, Vec<u8>), Box<dyn Error>> {
1842    let (encoded, diagnostics) = recompile_to_encoded(scripts_dir, pack_dir, lint, include_tests)?;
1843    info!("Writing compiled output to memory...");
1844    let out = ScriptWriter::new("".into()).build_encoded(&encoded);
1845    diagnostics.print_all();
1846    info!("Compilation complete!");
1847    Ok(out)
1848}
1849
1850pub(crate) fn find_testscript_line(source: &str) -> Option<usize> {
1851    for (i, line) in source.lines().enumerate() {
1852        if line.trim() == "#testscript" {
1853            return Some(i + 1);
1854        }
1855    }
1856    None
1857}
1858
1859fn truncate_at_line(source: &str, line_no: usize) -> String {
1860    source
1861        .lines()
1862        .take(line_no - 1)
1863        .collect::<Vec<_>>()
1864        .join("\n")
1865}
1866
1867fn strip_testscript_annotation(source: &str) -> String {
1868    source
1869        .lines()
1870        .map(|line| {
1871            if line.trim() == "#testscript" {
1872                ""
1873            } else {
1874                line
1875            }
1876        })
1877        .collect::<Vec<_>>()
1878        .join("\n")
1879}
1880
1881fn get_rs2_files(dir: &Path, ext: &str) -> Result<Vec<PathBuf>, Box<dyn Error>> {
1882    Ok(sources::script_files(dir, ext)?)
1883}