Skip to main content

runec/
filesystem.rs

1use crate::bytecode::CompiledScript;
2use crate::clientscript::{self, format::Format};
3use crate::compiler::Compiler;
4use crate::diagnostics::DiagnosticsCollector;
5use crate::lexer::Lexer;
6#[cfg(feature = "memprof")]
7use crate::memprof;
8use crate::parallel::{parallel_chunks, parallel_chunks_capped, parallel_map};
9use crate::parser::{Parser, ScriptFile};
10use crate::project::{Options, Project, Target};
11use crate::sources::Scope;
12use crate::symbol::SymbolRegistry;
13use crate::typechecker::TypeChecker;
14use crate::writer::ScriptWriter;
15use crate::{diagnostics, error, lints, sources, symloader, trigger_table, types};
16use std::collections::HashMap;
17use std::error::Error;
18use std::fmt::Write;
19use std::fs;
20use std::path::{Path, PathBuf};
21use std::sync::Arc;
22use tracing::{info, warn};
23
24/// Build an editor project using the legacy directory conventions. No ID file
25/// is generated or modified; all further editing happens in memory.
26pub fn editor_project(
27    source: &Path,
28    pack: Option<&Path>,
29    client: bool,
30) -> Result<Project, Box<dyn Error>> {
31    let scope = if client { Scope::Client } else { Scope::Rune };
32    let dir = sources::target_dir(source, scope);
33    let pack = resolve_pack_dir(&dir, pack);
34    let mut registry = if client {
35        clientscript::registry(
36            &dir,
37            pack.as_deref()
38                .ok_or("Client opcode definitions require --pack")?,
39        )?
40    } else {
41        build_symbol_registry(&dir, pack.as_deref())
42    };
43    let target = if client {
44        registry.preloaded_script_ids.clear();
45        for (key, id) in clientscript::script_ids(pack.as_deref().unwrap())? {
46            let (trigger, name) = key
47                .trim_start_matches('[')
48                .trim_end_matches(']')
49                .split_once(',')
50                .ok_or("Invalid client ID")?;
51            registry.set_script_id(trigger, name, id);
52        }
53        Target::Client(Format::load(
54            &pack.as_ref().unwrap().join("cs2_command.pack"),
55        )?)
56    } else {
57        if let Some(pack) = &pack {
58            symloader::load_script_ids(&mut registry, pack);
59        }
60        Target::Rune
61    };
62    let mut project = Project::new(
63        target,
64        registry,
65        Options {
66            include_tests: true,
67            skip_shared_lints: client
68                && dir
69                    .parent()
70                    .is_some_and(|parent| parent.join("runescript").is_dir()),
71            ..Options::default()
72        },
73    );
74    for path in sources::script_files(&dir, if client { "cs2" } else { "rs2" })? {
75        let text = fs::read_to_string(&path)?;
76        let scope = if path.extension().is_some_and(|ext| ext == "gs2") {
77            Scope::Global
78        } else {
79            scope
80        };
81        let uri = file_uri(&path)?;
82        let id = project.open(uri, scope, 0, text);
83        if client {
84            let ids = project
85                .document(id)
86                .and_then(|doc| doc.syntax())
87                .into_iter()
88                .flat_map(|file| &file.scripts)
89                .filter_map(|script| {
90                    script
91                        .name
92                        .strip_prefix("clientscript_")
93                        .and_then(|id| id.parse::<i32>().ok())
94                        .map(|id| (script.trigger.clone(), script.name.clone(), id))
95                })
96                .collect::<Vec<_>>();
97            for (trigger, name, id) in ids {
98                project.registry.set_script_id(&trigger, &name, id);
99            }
100        }
101    }
102    if !client {
103        let client_dir = if dir.file_name().is_some_and(|name| name == "runescript") {
104            dir.with_file_name("clientscript")
105        } else {
106            dir.join("clientscript")
107        };
108        if client_dir.is_dir() {
109            for path in sources::script_files(&client_dir, "cs2")?
110                .into_iter()
111                .filter(|path| {
112                    path.extension().is_some_and(|ext| ext == "cs2")
113                        && path.file_name().is_some_and(|name| name != "engine.cs2")
114                })
115            {
116                project.open(
117                    file_uri(&path)?,
118                    Scope::Client,
119                    0,
120                    fs::read_to_string(&path)?,
121                );
122            }
123        }
124    }
125    let mut files = sources::config_files(&dir);
126    if let Some(pack) = &pack {
127        symloader::collect_all_files(pack, &mut files);
128        // Interfaces and defaults live beside the scripts they're for.
129        if let Some(content) = pack.parent() {
130            for (folder, extension) in [("interfaces", "if3"), ("defaults", "defaults")] {
131                let Ok(entries) = fs::read_dir(content.join(folder)) else {
132                    continue;
133                };
134                files.extend(
135                    entries
136                        .flatten()
137                        .map(|entry| entry.path())
138                        .filter(|path| path.extension().is_some_and(|ext| ext == extension)),
139                );
140            }
141        }
142    }
143    files.sort();
144    files.dedup();
145    for path in files {
146        let Some(kind) = config_kind(&path) else {
147            continue;
148        };
149        let text = fs::read_to_string(&path)?;
150        let config_scope = if path
151            .components()
152            .any(|part| part.as_os_str() == "globalscript")
153        {
154            Scope::Global
155        } else {
156            scope
157        };
158        project.open_config(file_uri(&path)?, config_scope, 0, text, kind);
159    }
160    Ok(project)
161}
162
163pub(crate) fn config_kind(path: &Path) -> Option<String> {
164    let extension = path.extension()?.to_str()?;
165    if matches!(extension, "pack" | "order" | "hashes") {
166        return Some(format!("pack:{}", path.file_stem()?.to_str()?));
167    }
168    (matches!(extension, "constant" | "if" | "if3" | "defaults" | "anim")
169        || types::Type::from_name(extension).is_some()
170        || extension
171            .strip_prefix("config")
172            .is_some_and(|number| number.parse::<u32>().is_ok()))
173    .then(|| extension.to_string())
174}
175
176pub(crate) fn file_uri(path: &Path) -> Result<String, Box<dyn Error>> {
177    let absolute = fs::canonicalize(path)?
178        .to_string_lossy()
179        .trim_start_matches("\\\\?\\")
180        .replace('\\', "/");
181    let mut uri = String::from(if absolute.starts_with('/') {
182        "file://"
183    } else {
184        "file:///"
185    });
186    for byte in absolute.bytes() {
187        if byte.is_ascii_alphanumeric() || b"/-._~:".contains(&byte) {
188            uri.push(byte as char);
189        } else {
190            write!(uri, "%{byte:02X}")?;
191        }
192    }
193    Ok(uri)
194}
195
196/// How the diagnostic layer fetches a file's source text (by its cache-key path)
197/// when building rustc-style help/suggestions. The batch path keeps every source
198/// in memory; the streaming path keeps only the on-disk paths and re-reads the
199/// handful of files that actually produce a warning — so the ~4 MB of source
200/// text isn't resident through the compile's peak. Cheap to copy (just borrows).
201#[derive(Clone, Copy)]
202pub enum SourceProvider<'a> {
203    /// Sources already resident, keyed by cache key.
204    Eager(&'a HashMap<String, Arc<String>>),
205    /// Cache key → on-disk path; re-read on demand.
206    Lazy(&'a HashMap<String, PathBuf>),
207}
208
209impl SourceProvider<'_> {
210    /// Source text for `key`, if available. Returns an owned `Arc` — a cheap
211    /// refcount bump for `Eager`, a fresh disk read for `Lazy`. Called only while
212    /// emitting diagnostic help, which is rare, so the re-read cost is immaterial.
213    pub fn get(&self, key: &str) -> Option<Arc<String>> {
214        match self {
215            SourceProvider::Eager(m) => m.get(key).cloned(),
216            SourceProvider::Lazy(m) => fs::read_to_string(m.get(key)?).ok().map(Arc::new),
217        }
218    }
219}
220
221/// Log live heap + global peak at a pipeline phase boundary. No-op unless built
222/// with `--features memprof`. Lets us attribute the footprint to each phase.
223#[cfg(feature = "memprof")]
224fn mem_mark(label: &str) {
225    info!(
226        "[mem] {:<22} live {:>7.1} MB | peak {:>7.1} MB | allocs {}",
227        label,
228        memprof::current_bytes() as f64 / 1_048_576.0,
229        memprof::peak_bytes() as f64 / 1_048_576.0,
230        memprof::alloc_count(),
231    );
232}
233#[cfg(not(feature = "memprof"))]
234#[inline]
235fn mem_mark(_label: &str) {}
236
237/// Per-map breakdown of the symbol registry's heap, to see the irreducible
238/// floor for a streaming rebuild. Built only under `--features memprof`.
239/// Accounts three components per map: the hashbrown table (inline `(K,V)`
240/// buckets + 1 control byte each), the key strings, and value-side heap.
241#[cfg(feature = "memprof")]
242fn mem_stats_registry(reg: &crate::symbol::SymbolRegistry) {
243    use crate::symbol::{Symbol, SymbolKind};
244    use crate::types::Type;
245    use std::collections::{HashMap, HashSet};
246    let t = std::mem::size_of::<Type>();
247    let mb = |b: usize| b as f64 / 1_048_576.0;
248
249    // Allocated bucket count: hashbrown keeps `capacity()` ≈ 7/8 of a power-of-two
250    // bucket array; an empty map allocates nothing.
251    fn tbl<K, V>(m: &HashMap<K, V>) -> usize {
252        if m.capacity() == 0 {
253            return 0;
254        }
255        let buckets = (m.capacity() * 8 / 7).next_power_of_two();
256        buckets * (std::mem::size_of::<(K, V)>() + 1)
257    }
258    fn keys_cap<V>(m: &HashMap<String, V>) -> usize {
259        m.keys().map(|k| k.capacity()).sum()
260    }
261    let sym_heap = |s: &Symbol| -> usize {
262        s.name.capacity()
263            + match &s.kind {
264                SymbolKind::Script {
265                    trigger,
266                    param_types,
267                    return_types,
268                    ..
269                } => trigger.capacity() + (param_types.capacity() + return_types.capacity()) * t,
270                SymbolKind::Command {
271                    param_types,
272                    return_types,
273                    ..
274                } => (param_types.capacity() + return_types.capacity()) * t,
275                SymbolKind::GameVar { category, .. } => category.capacity(),
276                SymbolKind::Constant { string_value, .. } => {
277                    string_value.as_ref().map_or(0, |s| s.capacity())
278                }
279                _ => 0,
280            }
281    };
282    let sym_map = |m: &HashMap<String, Symbol>| -> usize {
283        tbl(m) + keys_cap(m) + m.values().map(&sym_heap).sum::<usize>()
284    };
285    let i32_map = |m: &HashMap<String, i32>| -> usize { tbl(m) + keys_cap(m) };
286
287    let mut rows: Vec<(&str, usize, usize)> = vec![
288        (
289            "scripts",
290            reg.scripts.len(),
291            tbl(&reg.scripts) + keys_cap(&reg.scripts),
292        ),
293        ("commands", reg.commands.len(), sym_map(&reg.commands)),
294        ("game_vars", reg.game_vars.len(), sym_map(&reg.game_vars)),
295        ("constants", reg.constants.len(), sym_map(&reg.constants)),
296        (
297            "entity_ids",
298            reg.entity_ids.len(),
299            tbl(&reg.entity_ids)
300                + keys_cap(&reg.entity_ids)
301                + reg
302                    .entity_ids
303                    .values()
304                    .map(|e| e.variants.capacity() * std::mem::size_of::<(Type, i32)>())
305                    .sum::<usize>(),
306        ),
307        (
308            "scripts_by_trigger",
309            reg.scripts_by_trigger.len(),
310            tbl(&reg.scripts_by_trigger)
311                + keys_cap(&reg.scripts_by_trigger)
312                + reg
313                    .scripts_by_trigger
314                    .values()
315                    .map(|s| 16 + sym_heap(s.as_ref()))
316                    .sum::<usize>(),
317        ),
318        (
319            "command_param_types",
320            reg.command_param_types.len(),
321            tbl(&reg.command_param_types)
322                + keys_cap(&reg.command_param_types)
323                + reg
324                    .command_param_types
325                    .values()
326                    .map(|v| v.capacity() * t)
327                    .sum::<usize>(),
328        ),
329        ("script_ids", reg.script_ids.len(), i32_map(&reg.script_ids)),
330        (
331            "proc_script_ids",
332            reg.proc_script_ids.len(),
333            i32_map(&reg.proc_script_ids),
334        ),
335        (
336            "label_script_ids",
337            reg.label_script_ids.len(),
338            i32_map(&reg.label_script_ids),
339        ),
340        (
341            "trigger_script_ids",
342            reg.trigger_script_ids.len(),
343            i32_map(&reg.trigger_script_ids),
344        ),
345        (
346            "preloaded_script_ids",
347            reg.preloaded_script_ids.len(),
348            i32_map(&reg.preloaded_script_ids),
349        ),
350        ("components", reg.components.len(), i32_map(&reg.components)),
351        (
352            "interface_ids",
353            reg.interface_ids.len(),
354            i32_map(&reg.interface_ids),
355        ),
356        ("type_chars", reg.type_chars.len(), i32_map(&reg.type_chars)),
357        (
358            "dbcolumn_types",
359            reg.dbcolumn_types.len(),
360            tbl(&reg.dbcolumn_types) + keys_cap(&reg.dbcolumn_types),
361        ),
362    ];
363    let ts_tbl = {
364        let s: &HashSet<String> = &reg.test_scripts;
365        if s.capacity() == 0 {
366            0
367        } else {
368            (s.capacity() * 8 / 7).next_power_of_two() * (std::mem::size_of::<String>() + 1)
369                + s.iter().map(|k| k.capacity()).sum::<usize>()
370        }
371    };
372    rows.push(("test_scripts", reg.test_scripts.len(), ts_tbl));
373
374    rows.sort_by_key(|row| std::cmp::Reverse(row.2));
375    let total: usize = rows.iter().map(|r| r.2).sum();
376    info!(
377        "[mem] registry ~{:.2} MB (sizeof Symbol={}B, SymbolKind={}B, Type={}B):",
378        mb(total),
379        std::mem::size_of::<Symbol>(),
380        std::mem::size_of::<SymbolKind>(),
381        t,
382    );
383    for (name, n, bytes) in &rows {
384        info!(
385            "[mem]   {:<20} {:>6} entries  {:>7.3} MB",
386            name,
387            n,
388            mb(*bytes)
389        );
390    }
391}
392
393/// One-off breakdown of where the compiled-bytecode heap goes, to target
394/// memory work. Built only under `--features memprof`.
395#[cfg(feature = "memprof")]
396fn mem_stats_compiled(scripts: &[CompiledScript]) {
397    use crate::bytecode::{Instruction, Opcode, Operand};
398    let (mut n_instr, mut n_str, mut str_bytes) = (0usize, 0usize, 0usize);
399    let (mut n_switch, mut switch_entries) = (0usize, 0usize);
400    let (mut source_path_bytes, mut name_bytes) = (0usize, 0usize);
401    for s in scripts {
402        n_instr += s.instructions.len();
403        source_path_bytes += s.source_path.capacity();
404        name_bytes += s.name.capacity() + s.trigger.capacity();
405        for st in &s.strings {
406            n_str += 1;
407            str_bytes += st.len();
408        }
409        for sw in &s.switch_tables {
410            n_switch += 1;
411            switch_entries += sw.len();
412        }
413    }
414    let sz = std::mem::size_of::<Instruction>();
415    let mb = |b: usize| b as f64 / 1_048_576.0;
416    info!(
417        "[mem] sizeof Instruction={}B Operand={}B Opcode={}B CompiledScript={}B",
418        sz,
419        std::mem::size_of::<Operand>(),
420        std::mem::size_of::<Opcode>(),
421        std::mem::size_of::<CompiledScript>(),
422    );
423    info!(
424        "[mem] compiled: {} scripts, {} instrs => {:.2} MB instr structs",
425        scripts.len(),
426        n_instr,
427        mb(n_instr * sz)
428    );
429    info!(
430        "[mem]   Str operands: {} ({:.2} MB content + {:.2} MB Box<str> headers)",
431        n_str,
432        mb(str_bytes),
433        mb(n_str * 16)
434    );
435    info!(
436        "[mem]   SwitchTable operands: {} ({} entries => {:.2} MB)",
437        n_switch,
438        switch_entries,
439        mb(switch_entries * 16)
440    );
441    info!(
442        "[mem]   source_path {:.2} MB, names+triggers {:.2} MB",
443        mb(source_path_bytes),
444        mb(name_bytes)
445    );
446}
447
448struct ParsedFile {
449    path: PathBuf,
450    file: ScriptFile,
451    testscript_line: Option<usize>,
452    /// Canonicalized, normalized source path. Computed once during parsing and
453    /// reused as each compiled script's `source_path` (it is byte-identical to
454    /// the `source_cache` key), avoiding a second `canonicalize` syscall per
455    /// file in codegen.
456    source_path: String,
457}
458
459/// Outcome of parsing one file off-thread. The owned source text and cache key
460/// travel back so the (single-threaded) merge can build the `Rc`-based caches
461/// without sending `Rc` across threads.
462struct FileParseResult {
463    path: PathBuf,
464    read: Result<ParsedSource, std::io::Error>,
465}
466
467struct ParsedSource {
468    cache_key: String,
469    raw_source: String,
470    outcome: ParseOutcome,
471}
472
473enum ParseOutcome {
474    Parsed {
475        file: ScriptFile,
476        testscript_line: Option<usize>,
477    },
478    Error {
479        line: usize,
480        position: usize,
481        message: String,
482        phase: diagnostics::Phase,
483    },
484}
485
486/// Read, lex and parse a single file off-thread. Pure: no shared state, so it
487/// is safe to run concurrently. Mirrors the original sequential body exactly —
488/// same source truncation, same cache-key normalization, same lex-then-parse
489/// error reporting — only restructured to return owned data for the merge.
490fn parse_one_file(path: &Path, include_tests: bool) -> FileParseResult {
491    let raw_source = match fs::read_to_string(path) {
492        Ok(s) => s,
493        Err(e) => {
494            return FileParseResult {
495                path: path.to_path_buf(),
496                read: Err(e),
497            };
498        }
499    };
500    let testscript_line = find_testscript_line(&raw_source);
501
502    let source_code = if !include_tests {
503        if let Some(line_no) = testscript_line {
504            truncate_at_line(&raw_source, line_no)
505        } else {
506            raw_source.clone()
507        }
508    } else {
509        strip_testscript_annotation(&raw_source)
510    };
511
512    let canonical = fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf());
513    let mut cache_key = canonical.to_string_lossy().into_owned();
514    if cache_key.starts_with("\\\\?\\") {
515        cache_key = cache_key[4..].to_string();
516    }
517    cache_key = cache_key.replace("\\Content\\", "\\content\\");
518
519    // Lexer/Parser borrow a `&PathBuf` for diagnostics; keep one alive locally.
520    let path_buf = path.to_path_buf();
521    let outcome = match Lexer::new(&source_code, &path_buf).tokenize() {
522        Err(e) => ParseOutcome::Error {
523            line: e.line,
524            position: e.position,
525            message: e.message,
526            phase: diagnostics::Phase::Lexing,
527        },
528        Ok(tokens) => match Parser::new(tokens, &path_buf).parse() {
529            Ok(file) => ParseOutcome::Parsed {
530                file,
531                testscript_line: if include_tests { testscript_line } else { None },
532            },
533            Err(e) => ParseOutcome::Error {
534                line: e.line,
535                position: e.position,
536                message: e.message,
537                phase: diagnostics::Phase::Parsing,
538            },
539        },
540    };
541
542    FileParseResult {
543        path: path_buf,
544        read: Ok(ParsedSource {
545            cache_key,
546            raw_source,
547            outcome,
548        }),
549    }
550}
551
552/// Compile scripts and write output. When `lint` is true, also run lint passes
553/// (unused locals, unreachable code) after code generation.
554pub fn compile_memory(
555    scripts_dir: &Path,
556    pack_dir: Option<&Path>,
557    lint: bool,
558) -> Result<(Vec<u8>, Vec<u8>), Box<dyn Error>> {
559    compile_memory_with_options(scripts_dir, pack_dir, lint, false, false)
560}
561
562/// Like `compile_memory` but includes `#testscript` sections when
563/// `include_tests` is true.
564pub fn compile_memory_with_tests(
565    scripts_dir: &Path,
566    pack_dir: Option<&Path>,
567    lint: bool,
568    include_tests: bool,
569) -> Result<(Vec<u8>, Vec<u8>), Box<dyn Error>> {
570    compile_memory_with_options(scripts_dir, pack_dir, lint, include_tests, false)
571}
572
573/// In-memory compile — returns the `(script.dat, script.idx)` bytes instead of
574/// writing them — choosing the compilation strategy.
575///
576/// With `low_mem`, the recompile pipeline is used: scripts are re-parsed and
577/// re-compiled on demand so the full compiled set is never resident (~16 MB heap
578/// vs ~93), at roughly 6× the time; the returned bytes are byte-identical to the
579/// default path. `include_tests` keeps `#testscript` sections. The
580/// `RUNEC_RECOMPILE` environment variable forces low-memory mode on as well.
581pub fn compile_memory_with_options(
582    scripts_dir: &Path,
583    pack_dir: Option<&Path>,
584    lint: bool,
585    include_tests: bool,
586    low_mem: bool,
587) -> Result<(Vec<u8>, Vec<u8>), Box<dyn Error>> {
588    if low_mem || std::env::var("RUNEC_RECOMPILE").is_ok_and(|v| !v.is_empty()) {
589        info!("[recompile] low-memory recompile pipeline enabled");
590        return compile_recompile_memory(scripts_dir, pack_dir, lint, include_tests);
591    }
592
593    let (compiled_scripts, diagnostics) = run_pipeline(scripts_dir, pack_dir, lint, include_tests)?;
594
595    info!("Writing compiled output to memory...");
596    let writer = ScriptWriter::new("".into());
597
598    diagnostics.print_all();
599    info!("Compilation complete!");
600
601    Ok(writer.build_all(&compiled_scripts)?)
602}
603
604/// Compile scripts and write output. When `lint` is true, also run lint passes
605/// (unused locals, unreachable code) after code generation.
606///
607/// Convenience wrapper over [`compile_with_options`] using the default (fast,
608/// fully-resident) pipeline. Kept signature-compatible for existing callers.
609pub fn compile(
610    scripts_dir: &Path,
611    pack_dir: Option<&Path>,
612    output_dir: &Path,
613    lint: bool,
614) -> Result<(), Box<dyn Error>> {
615    compile_with_options(scripts_dir, pack_dir, output_dir, lint, false)
616}
617
618/// Compile scripts and write output, choosing the compilation strategy.
619///
620/// With `low_mem`, the **recompile** pipeline is used: scripts are re-parsed and
621/// re-compiled on demand for the pointer check and written file-by-file, so the
622/// full set of compiled bytecode is never resident at once. That cuts peak usage
623/// to roughly 16 MB heap / 32 MB working set (from ~93 / ~111 MB) at about 6× the
624/// compile time. Output is byte-identical to the default path. The
625/// `RUNEC_RECOMPILE` environment variable forces this mode on as well.
626pub fn compile_with_options(
627    scripts_dir: &Path,
628    pack_dir: Option<&Path>,
629    output_dir: &Path,
630    lint: bool,
631    low_mem: bool,
632) -> Result<(), Box<dyn Error>> {
633    // The recompile pipeline never holds all compiled scripts (it re-compiles on
634    // demand for the pointer check + writes file-by-file). It does its own
635    // output, so it returns here. The default path below is unchanged.
636    if low_mem || std::env::var("RUNEC_RECOMPILE").is_ok_and(|v| !v.is_empty()) {
637        info!("[recompile] low-memory recompile pipeline enabled");
638        return compile_recompile(scripts_dir, pack_dir, output_dir, lint);
639    }
640
641    let (compiled_scripts, diagnostics) = run_pipeline(scripts_dir, pack_dir, lint, false)?;
642
643    let output = output_dir.display().to_string();
644    // Normalize separators for a clean, consistent log message (input paths may
645    // mix `/` and `\`, e.g. `data/pack\server`); the writer keeps the original.
646    info!("Writing output to {}...", output.replace('\\', "/"));
647    let writer = ScriptWriter::new(output);
648    writer.write_all(&compiled_scripts)?;
649
650    diagnostics.print_all();
651    info!("Compilation complete!");
652
653    Ok(())
654}
655
656/// Run all analysis passes (parse, type-check, codegen, pointer-check, lints)
657/// without writing output. Useful for editor tooling and CI lint gates.
658pub fn lint(scripts_dir: &Path, pack_dir: Option<&Path>) -> Result<(), Box<dyn Error>> {
659    let (_, diagnostics) = run_pipeline(scripts_dir, pack_dir, true, false)?;
660
661    diagnostics.print_all();
662    info!("Compilation complete!");
663
664    Ok(())
665}
666
667/// Runs the full compilation pipeline (parse, register, type-check, codegen,
668/// pointer-check, and optional lint passes), returning the compiled scripts
669/// and accumulated diagnostics for the caller to handle output.
670fn run_pipeline(
671    scripts_dir: &Path,
672    pack_dir: Option<&Path>,
673    lint: bool,
674    include_tests: bool,
675) -> Result<(Vec<CompiledScript>, DiagnosticsCollector), Box<dyn Error>> {
676    let source_dir = sources::target_dir(scripts_dir, sources::Scope::Rune);
677    let scripts_dir = source_dir.as_path();
678    if !scripts_dir.exists() || !scripts_dir.is_dir() {
679        return Err(Box::new(error::CompilerError::FileNotFound(
680            scripts_dir.display().to_string(),
681        )));
682    }
683
684    // Prototype: opt into the streaming pipeline (never holds all ASTs at once)
685    // with RUNEC_STREAM=1. The default batch path below is unchanged.
686    if std::env::var("RUNEC_STREAM").is_ok_and(|v| !v.is_empty()) {
687        info!("[stream] streaming pipeline enabled (RUNEC_STREAM)");
688        return run_pipeline_streaming(scripts_dir, pack_dir, lint, include_tests);
689    }
690
691    let mut rs2_files: Vec<PathBuf> = get_rs2_files(scripts_dir, "rs2")?;
692
693    rs2_files.sort();
694    info!("Found {} script files", rs2_files.len());
695    mem_mark("startup");
696
697    let mut diagnostics = DiagnosticsCollector::new();
698
699    // Phase 1: Parse all files. Reading, lexing and parsing are independent
700    // per file, so fan them out across threads; the results are merged back in
701    // input order so registration/ID assignment stays deterministic.
702    info!("Parsing script files into syntax trees...");
703    let parse_results = parallel_map(&rs2_files, |path| parse_one_file(path, include_tests));
704
705    let mut all_files: Vec<ParsedFile> = Vec::new();
706    let mut source_cache: HashMap<String, Arc<String>> = HashMap::new();
707    // Raw source text keyed by on-disk path, reused by `generate_script_pack`
708    // so it doesn't re-read every file from disk a second time.
709    let mut raw_by_path: HashMap<PathBuf, Arc<String>> = HashMap::new();
710    for result in parse_results {
711        let parsed = match result.read {
712            Ok(p) => p,
713            // A read failure aborts the whole compile, exactly as the previous
714            // `fs::read_to_string(path)?` did.
715            Err(e) => return Err(Box::new(e)),
716        };
717        let rc = Arc::new(parsed.raw_source);
718        // The cache key is the canonicalized source path; reuse it as each
719        // compiled script's `source_path` so codegen needn't canonicalize again.
720        let source_path = parsed.cache_key.clone();
721        source_cache.insert(parsed.cache_key, rc.clone());
722        raw_by_path.insert(result.path.clone(), rc);
723        match parsed.outcome {
724            ParseOutcome::Parsed {
725                file,
726                testscript_line,
727            } => all_files.push(ParsedFile {
728                path: result.path,
729                file,
730                testscript_line,
731                source_path,
732            }),
733            ParseOutcome::Error {
734                line,
735                position,
736                message,
737                phase,
738            } => diagnostics.error(result.path, line, position, message, phase),
739        }
740    }
741
742    if diagnostics.has_errors() {
743        diagnostics.print_all();
744        return Err(Box::new(error::CompilerError::TypeError(
745            "Parsing failed".to_string(),
746        )));
747    }
748
749    mem_mark("after parse");
750
751    // Phase 2: Register all scripts. Build the pack/config registry, then
752    // regenerate script.pack, load pre-assigned IDs, and register every script.
753    info!("Registering scripts and building the symbol registry...");
754    let resolved_pack_dir = resolve_pack_dir(scripts_dir, pack_dir);
755    let mut registry = build_symbol_registry(scripts_dir, resolved_pack_dir.as_deref());
756    register_scripts(
757        &mut registry,
758        scripts_dir,
759        resolved_pack_dir.as_deref(),
760        &mut diagnostics,
761        &all_files,
762        &raw_by_path,
763    );
764    info!("  Registered {} scripts", registry.scripts.len());
765    // `raw_by_path` was only needed to (re)generate script.pack inside
766    // `register_scripts`; the source text itself stays reachable via
767    // `source_cache` (shared `Arc`s). Drop the redundant index now.
768    drop(raw_by_path);
769    // Pre-assigned script ids were consumed during registration; free them
770    // before codegen (the compile's peak-memory phase) rather than at scope end.
771    registry.drop_registration_scratch();
772
773    for pf in &all_files {
774        for script in &pf.file.scripts {
775            if let Some(msg) =
776                Compiler::validate_trigger_subject(&script.trigger, &script.name, &registry)
777            {
778                diagnostics.warning(
779                    pf.path.clone(),
780                    script.line,
781                    0,
782                    msg,
783                    diagnostics::Phase::SymbolRegistration,
784                );
785            }
786        }
787    }
788
789    mem_mark("after register");
790    #[cfg(feature = "memprof")]
791    mem_stats_registry(&registry);
792
793    // Phase 3: Type checking
794    info!("Type checking {} files...", all_files.len());
795    run_type_checker(&mut diagnostics, &all_files, &registry);
796
797    if diagnostics.has_errors() {
798        diagnostics.print_all();
799        return Err(Box::new(error::CompilerError::TypeError(
800            "Type checking failed".to_string(),
801        )));
802    }
803
804    mem_mark("after typecheck");
805
806    // Phase 4: Code generation
807    info!("Generating bytecode from type-checked scripts...");
808    let mut compiled_scripts = codegen(&all_files, &registry, &mut diagnostics);
809    if diagnostics.has_errors() {
810        diagnostics.print_all();
811        return Err(Box::new(error::CompilerError::CodeGenError(
812            "Code generation failed".into(),
813        )));
814    }
815    info!("  Generated {} compiled scripts", compiled_scripts.len());
816    compiled_scripts.sort_by_key(|s| s.id);
817    compiled_scripts.shrink_to_fit();
818    mem_mark("after codegen");
819    #[cfg(feature = "memprof")]
820    mem_stats_compiled(&compiled_scripts);
821
822    // The ASTs are consumed by codegen and never read again — pointer checking
823    // and lints work purely off `compiled_scripts` + the registry + the source
824    // cache. Free the syntax trees now so they don't coexist with the pointer
825    // checker's (per-thread) scratch, which is the compile's peak-memory moment.
826    drop(all_files);
827    mem_mark("after drop asts");
828
829    // Phase 4.5: Pointer checking & lints
830    info!("Checking pointers in generated code...");
831    check_pointers(
832        lint,
833        &mut diagnostics,
834        SourceProvider::Eager(&source_cache),
835        &registry,
836        &compiled_scripts,
837    );
838    mem_mark("after pointercheck");
839
840    Ok((compiled_scripts, diagnostics))
841}
842
843// ── Streaming pipeline (prototype, gated by RUNEC_STREAM) ────────────────────
844// Goal: never materialize all ASTs (~59 MB) at once. Pass 1 parses in parallel
845// but keeps only per-script signatures, dropping each AST in-worker; the registry
846// is built from those. Pass 2 re-parses each file in parallel, type-checks and
847// compiles it, then drops the AST immediately — so syntax trees never accumulate.
848// Output is byte-identical to the batch path (same registration, same per-file
849// codegen, same stable id-sort). It still holds all compiled scripts (for the
850// cross-script pointer check + writer); shrinking that is the next increment.
851
852/// One script's registration-relevant signature — everything the registry and
853/// trigger diagnostics need — so the full AST can be dropped right after parsing.
854struct ScriptSig {
855    name: String,
856    trigger: String,
857    param_types: Vec<types::Type>,
858    return_types: Vec<types::Type>,
859    line: usize,
860}
861
862struct FileSigs {
863    path: PathBuf,
864    testscript_line: Option<usize>,
865    sigs: Vec<ScriptSig>,
866}
867
868enum SigOutcome {
869    Parsed {
870        sigs: Vec<ScriptSig>,
871        testscript_line: Option<usize>,
872    },
873    Error {
874        line: usize,
875        position: usize,
876        message: String,
877        phase: diagnostics::Phase,
878    },
879}
880
881struct FileSigsResult {
882    path: PathBuf,
883    /// `(cache_key, raw_source, outcome)` on success.
884    read: Result<(String, String, SigOutcome), std::io::Error>,
885}
886
887/// Parse one file, extract signatures, and DROP the AST in-worker, so a parallel
888/// sweep never holds every syntax tree at once.
889fn parse_one_file_sigs(path: &Path, include_tests: bool) -> FileSigsResult {
890    let r = parse_one_file(path, include_tests);
891    let read = r.read.map(|ps| {
892        let outcome = match ps.outcome {
893            ParseOutcome::Parsed {
894                file,
895                testscript_line,
896            } => {
897                let sigs = file
898                    .scripts
899                    .iter()
900                    .map(|d| ScriptSig {
901                        name: d.name.clone(),
902                        trigger: d.trigger.clone(),
903                        param_types: d.params.iter().map(|p| p.param_type).collect(),
904                        return_types: d.return_types.clone(),
905                        line: d.line,
906                    })
907                    .collect();
908                SigOutcome::Parsed {
909                    sigs,
910                    testscript_line,
911                }
912                // `file` (the AST) is dropped here — only the sigs travel back.
913            }
914            ParseOutcome::Error {
915                line,
916                position,
917                message,
918                phase,
919            } => SigOutcome::Error {
920                line,
921                position,
922                message,
923                phase,
924            },
925        };
926        (ps.cache_key, ps.raw_source, outcome)
927    });
928    FileSigsResult { path: r.path, read }
929}
930
931fn run_pipeline_streaming(
932    scripts_dir: &Path,
933    pack_dir: Option<&Path>,
934    lint: bool,
935    include_tests: bool,
936) -> Result<(Vec<CompiledScript>, DiagnosticsCollector), Box<dyn Error>> {
937    let mut rs2_files: Vec<PathBuf> = get_rs2_files(scripts_dir, "rs2")?;
938    rs2_files.sort();
939    info!("Found {} script files", rs2_files.len());
940    mem_mark("startup");
941
942    let mut diagnostics = DiagnosticsCollector::new();
943
944    // Pass 1: parse for signatures only (each AST dropped in its worker).
945    info!("[stream] Pass 1: parsing for signatures...");
946    let sig_results = parallel_map(&rs2_files, |path| parse_one_file_sigs(path, include_tests));
947
948    // Streaming keeps only `cache_key -> on-disk path`; source text is re-read
949    // lazily for the rare diagnostic that needs it, so the ~4 MB of sources isn't
950    // resident through the peak. `raw_by_path` still holds the text transiently
951    // for `script.pack` regeneration, then is dropped right after registration.
952    let mut source_paths: HashMap<String, PathBuf> = HashMap::new();
953    let mut raw_by_path: HashMap<PathBuf, Arc<String>> = HashMap::new();
954    let mut file_sigs: Vec<FileSigs> = Vec::new();
955    for r in sig_results {
956        let (cache_key, raw_source, outcome) = match r.read {
957            Ok(t) => t,
958            Err(e) => return Err(Box::new(e)),
959        };
960        source_paths.insert(cache_key, r.path.clone());
961        raw_by_path.insert(r.path.clone(), Arc::new(raw_source));
962        match outcome {
963            SigOutcome::Parsed {
964                sigs,
965                testscript_line,
966            } => file_sigs.push(FileSigs {
967                path: r.path,
968                testscript_line,
969                sigs,
970            }),
971            SigOutcome::Error {
972                line,
973                position,
974                message,
975                phase,
976            } => diagnostics.error(r.path, line, position, message, phase),
977        }
978    }
979
980    if diagnostics.has_errors() {
981        diagnostics.print_all();
982        return Err(Box::new(error::CompilerError::TypeError(
983            "Parsing failed".to_string(),
984        )));
985    }
986    mem_mark("after parse");
987
988    // Registration (from signatures, not ASTs).
989    info!("[stream] Registering scripts and building the symbol registry...");
990    let resolved_pack_dir = resolve_pack_dir(scripts_dir, pack_dir);
991    let mut registry = build_symbol_registry(scripts_dir, resolved_pack_dir.as_deref());
992    register_scripts_from_sigs(
993        &mut registry,
994        scripts_dir,
995        resolved_pack_dir.as_deref(),
996        &mut diagnostics,
997        &file_sigs,
998        &raw_by_path,
999    );
1000    info!("  Registered {} scripts", registry.scripts.len());
1001    drop(raw_by_path);
1002    registry.drop_registration_scratch();
1003
1004    for fs in &file_sigs {
1005        for sig in &fs.sigs {
1006            if let Some(msg) =
1007                Compiler::validate_trigger_subject(&sig.trigger, &sig.name, &registry)
1008            {
1009                diagnostics.warning(
1010                    fs.path.clone(),
1011                    sig.line,
1012                    0,
1013                    msg,
1014                    diagnostics::Phase::SymbolRegistration,
1015                );
1016            }
1017        }
1018    }
1019    // Signatures are done; only the registry + source cache carry forward.
1020    drop(file_sigs);
1021    mem_mark("after register");
1022    #[cfg(feature = "memprof")]
1023    mem_stats_registry(&registry);
1024
1025    // Pass 2: re-parse + type-check + compile per file, streaming. Parse can't
1026    // fail here (pass 1 validated), so a failed re-parse just yields nothing.
1027    info!("[stream] Pass 2: type-check + codegen (streaming)...");
1028    let per_file = parallel_map(&rs2_files, |path| {
1029        let parsed = match parse_one_file(path, include_tests).read {
1030            Ok(p) => p,
1031            Err(_) => return (Vec::new(), DiagnosticsCollector::new()),
1032        };
1033        let (file, testscript_line) = match parsed.outcome {
1034            ParseOutcome::Parsed {
1035                file,
1036                testscript_line,
1037            } => (file, testscript_line),
1038            ParseOutcome::Error { .. } => return (Vec::new(), DiagnosticsCollector::new()),
1039        };
1040        let mut type_checker = TypeChecker::new(&registry);
1041        type_checker.check_file_with_test_boundary(&file, path, testscript_line);
1042        let mut compiled = Vec::new();
1043        // Compile only when this file type-checks clean (codegen assumes
1044        // well-typed input). On any global error the whole compile aborts below,
1045        // so output produced here would be discarded anyway.
1046        if !type_checker.diagnostics.has_errors() {
1047            let mut compiler = Compiler::new(&registry);
1048            for script in &file.scripts {
1049                if script.trigger == "command" {
1050                    continue;
1051                }
1052                let Some(mut c) =
1053                    compiler.compile_checked(script, path, &mut type_checker.diagnostics)
1054                else {
1055                    continue;
1056                };
1057                c.source_path = parsed.cache_key.clone();
1058                c.shrink_to_fit();
1059                compiled.push(c);
1060            }
1061        }
1062        (compiled, type_checker.diagnostics)
1063        // `file` (the AST) is dropped here.
1064    });
1065
1066    let mut compiled_scripts: Vec<CompiledScript> = Vec::new();
1067    for (compiled, diags) in per_file {
1068        compiled_scripts.extend(compiled);
1069        diagnostics.merge(diags);
1070    }
1071    if diagnostics.has_errors() {
1072        diagnostics.print_all();
1073        return Err(Box::new(error::CompilerError::TypeError(
1074            "Type checking failed".to_string(),
1075        )));
1076    }
1077    // Stable sort by id matches the batch path (file order preserved among
1078    // equal ids → identical writer last-write-wins behaviour).
1079    compiled_scripts.sort_by_key(|s| s.id);
1080    compiled_scripts.shrink_to_fit();
1081    info!("  Generated {} compiled scripts", compiled_scripts.len());
1082    mem_mark("after codegen");
1083    #[cfg(feature = "memprof")]
1084    mem_stats_compiled(&compiled_scripts);
1085
1086    check_pointers(
1087        lint,
1088        &mut diagnostics,
1089        SourceProvider::Lazy(&source_paths),
1090        &registry,
1091        &compiled_scripts,
1092    );
1093    mem_mark("after pointercheck");
1094
1095    Ok((compiled_scripts, diagnostics))
1096}
1097
1098/// `register_scripts` for the streaming path: identical logic, driven by the
1099/// extracted signatures instead of the (already-dropped) ASTs.
1100fn register_scripts_from_sigs(
1101    registry: &mut SymbolRegistry,
1102    scripts_dir: &Path,
1103    resolved_pack_dir: Option<&Path>,
1104    diagnostics: &mut DiagnosticsCollector,
1105    file_sigs: &[FileSigs],
1106    raw_by_path: &HashMap<PathBuf, Arc<String>>,
1107) {
1108    if let Some(pd) = resolved_pack_dir {
1109        symloader::generate_script_pack(scripts_dir, pd, raw_by_path);
1110        symloader::load_script_ids(registry, pd);
1111    }
1112
1113    use crate::diagnostic_messages as msg;
1114    let mut registered_scripts: std::collections::HashSet<String> =
1115        std::collections::HashSet::new();
1116
1117    for fs in file_sigs.iter() {
1118        for sig in &fs.sigs {
1119            Arc::make_mut(&mut registry.visibility).declare_script(
1120                &fs.path,
1121                &sig.trigger,
1122                &sig.name,
1123            );
1124            if sig.trigger == "command" {
1125                continue;
1126            }
1127
1128            if !trigger_table::is_valid_trigger(&sig.trigger) {
1129                diagnostics.warning(
1130                    fs.path.clone(),
1131                    sig.line,
1132                    0,
1133                    msg::fmt(msg::SCRIPT_TRIGGER_INVALID, &[&sig.trigger]),
1134                    crate::diagnostics::Phase::SymbolRegistration,
1135                );
1136            }
1137
1138            let key = format!("{}:{}", sig.trigger, sig.name);
1139            if !registered_scripts.insert(key) {
1140                diagnostics.warning(
1141                    fs.path.clone(),
1142                    sig.line,
1143                    0,
1144                    msg::fmt(msg::SCRIPT_REDECLARATION, &[&sig.trigger, &sig.name]),
1145                    crate::diagnostics::Phase::SymbolRegistration,
1146                );
1147            }
1148
1149            if !sig.return_types.is_empty() && !trigger_table::allows_returns(&sig.trigger) {
1150                diagnostics.warning(
1151                    fs.path.clone(),
1152                    sig.line,
1153                    0,
1154                    msg::fmt(msg::SCRIPT_TRIGGER_NO_RETURNS, &[&sig.trigger]),
1155                    diagnostics::Phase::SymbolRegistration,
1156                );
1157            }
1158
1159            registry.register_script(
1160                sig.name.clone(),
1161                sig.trigger.clone(),
1162                sig.param_types.clone(),
1163                sig.return_types.clone(),
1164            );
1165
1166            if let Some(ts_line) = fs.testscript_line
1167                && sig.line >= ts_line
1168            {
1169                registry.mark_test_script(&sig.trigger, &sig.name);
1170            }
1171        }
1172    }
1173}
1174
1175fn check_pointers(
1176    lint: bool,
1177    diagnostics: &mut DiagnosticsCollector,
1178    source: SourceProvider,
1179    registry: &SymbolRegistry,
1180    compiled_scripts: &[CompiledScript],
1181) {
1182    {
1183        use crate::pointer_checker::{PointerChecker, ScriptSource};
1184        // Validate scripts across threads: each chunk gets its own checker that
1185        // shares the read-only scripts/registry/source cache but keeps its own
1186        // (purely memoizing) caches. Diagnostics are merged in script order, so
1187        // the result is identical to a single-threaded run.
1188        //
1189        // The read-only command/script lookup tables are built ONCE here and
1190        // shared (`Arc`) across every chunk checker, instead of each thread
1191        // rebuilding the full ~700-command pointer table + per-script reverse
1192        // maps. `names` is just the script order, so derive it directly rather
1193        // than spinning up a throwaway checker for `script_names()`.
1194        let shared = PointerChecker::build_shared(compiled_scripts, registry);
1195        let names: Vec<String> = compiled_scripts.iter().map(|s| s.name.clone()).collect();
1196        // Each chunk's checker retains its CFG caches until the chunk finishes,
1197        // so concurrency multiplies peak scratch. RUNEC_PTR_CHUNKS caps the
1198        // concurrent chunk count to trade pointer-check speed for memory; the
1199        // result is chunk-count-independent, so diagnostics stay byte-identical.
1200        let max_chunks = std::env::var("RUNEC_PTR_CHUNKS")
1201            .ok()
1202            .and_then(|s| s.parse::<usize>().ok())
1203            .filter(|&n| n > 0)
1204            .unwrap_or(usize::MAX);
1205        let chunk_diags = parallel_chunks_capped(&names, max_chunks, |chunk| {
1206            let mut checker = PointerChecker::with_shared(
1207                ScriptSource::Resident(compiled_scripts),
1208                registry,
1209                shared.clone(),
1210            );
1211            checker.set_source_cache(source);
1212            checker.validate_names(chunk)
1213        });
1214        let mut pointer_diags = DiagnosticsCollector::new();
1215        for d in chunk_diags {
1216            pointer_diags.merge(d);
1217        }
1218        let ptr_warnings = pointer_diags.warning_count();
1219        if ptr_warnings > 0 {
1220            info!("  Pointer check: {} warning(s)", ptr_warnings);
1221        }
1222        diagnostics.merge(pointer_diags);
1223    }
1224    #[cfg(feature = "memprof")]
1225    crate::pointer_checker::cfg_trace_report();
1226
1227    // Phase 4.6: Lint passes (optional)
1228    if lint {
1229        info!("Running lint passes (unused locals, unreachable code)...");
1230        let lint_diags = lints::run_lints(compiled_scripts, Some(source));
1231        let lint_warnings = lint_diags.warning_count();
1232        if lint_warnings > 0 {
1233            info!("  Lints: {} warning(s)", lint_warnings);
1234        }
1235        diagnostics.merge(lint_diags);
1236    }
1237}
1238
1239fn codegen(
1240    all_files: &[ParsedFile],
1241    registry: &SymbolRegistry,
1242    diagnostics: &mut DiagnosticsCollector,
1243) -> Vec<CompiledScript> {
1244    // Each file compiles independently against the read-only registry, so fan
1245    // the files out across threads. Output order doesn't matter — the caller
1246    // sorts compiled scripts by id — so a simple flatten of the per-file results
1247    // is enough.
1248    let mut compiled = Vec::new();
1249    for (scripts, errors) in parallel_map(all_files, |pf| compile_file(pf, registry)) {
1250        compiled.extend(scripts);
1251        diagnostics.merge(errors);
1252    }
1253    compiled
1254}
1255
1256/// Compile every (non-`command`) script in one file. Creating a `Compiler` per
1257/// file is cheap now that it only borrows the registry, and keeps the codegen
1258/// scratch state thread-local.
1259fn compile_file(
1260    pf: &ParsedFile,
1261    registry: &SymbolRegistry,
1262) -> (Vec<CompiledScript>, DiagnosticsCollector) {
1263    let mut diagnostics = DiagnosticsCollector::new();
1264    let mut scripts =
1265        crate::compiler::compile_file(&pf.file, &pf.path, registry, false, &mut diagnostics);
1266    for script in &mut scripts {
1267        script.source_path = pf.source_path.clone();
1268        script.shrink_to_fit();
1269    }
1270    (scripts, diagnostics)
1271}
1272
1273fn run_type_checker(
1274    diagnostics: &mut DiagnosticsCollector,
1275    all_files: &[ParsedFile],
1276    registry: &SymbolRegistry,
1277) {
1278    // Each file type-checks independently against the read-only registry, so
1279    // check chunks of files in parallel; per-chunk diagnostics are merged in
1280    // file order, matching a single-threaded run.
1281    let chunk_diags = parallel_chunks(all_files, |chunk| {
1282        let mut type_checker = TypeChecker::new(registry);
1283        for pf in chunk {
1284            type_checker.check_file_with_test_boundary(&pf.file, &pf.path, pf.testscript_line);
1285        }
1286        type_checker.diagnostics
1287    });
1288    for d in chunk_diags {
1289        diagnostics.merge(d);
1290    }
1291}
1292
1293/// Resolve the pack directory: explicit `--pack`, else the nearest ancestor's `pack/`, else
1294/// `content/pack` up the tree.
1295fn resolve_pack_dir(scripts_dir: &Path, pack_dir: Option<&Path>) -> Option<PathBuf> {
1296    pack_dir
1297        .map(PathBuf::from)
1298        .filter(|p| p.exists())
1299        .or_else(|| {
1300            scripts_dir.ancestors().find_map(|dir| {
1301                let pack = dir.join("pack");
1302                pack.is_dir().then_some(pack)
1303            })
1304        })
1305        .or_else(|| {
1306            scripts_dir
1307                .ancestors()
1308                .nth(3)
1309                .map(|p| p.join("content/pack"))
1310                .filter(|p| p.exists())
1311        })
1312}
1313
1314/// Build the part of the registry that does NOT depend on the parsed scripts:
1315/// pack files, `.constant`/`.var`/`.dbtable` configs, and engine command params.
1316/// This runs concurrently with parsing; `script.pack` (re)generation and script
1317/// registration — which need the parsed sources/ASTs — happen afterwards in
1318/// `register_scripts`.
1319fn build_symbol_registry(scripts_dir: &Path, resolved_pack_dir: Option<&Path>) -> SymbolRegistry {
1320    let mut registry = SymbolRegistry::new();
1321
1322    if let Some(pd) = resolved_pack_dir {
1323        info!("  Loading packs from {}", pd.display());
1324        symloader::load_packs(&mut registry, pd);
1325        info!(
1326            "  Loaded {} commands, {} game vars",
1327            registry.commands.len(),
1328            registry.game_vars.len(),
1329        );
1330    } else {
1331        info!("  No pack directory found - symbol resolution will be limited");
1332    }
1333
1334    // Walk the scripts tree once and let the constant/game-var/dbtable loaders
1335    // filter this shared list by extension, instead of each re-walking the tree
1336    // (previously 6 separate recursive traversals over the same directories).
1337    let mut config_files: Vec<PathBuf> = Vec::new();
1338    config_files.extend(sources::config_files(scripts_dir));
1339
1340    symloader::load_constant_files(&mut registry, &config_files);
1341
1342    info!(
1343        "  Loaded {} constants from .constant files",
1344        registry.constants.len()
1345    );
1346
1347    symloader::load_game_var_types(&mut registry, &config_files);
1348
1349    if let Some(pd) = resolved_pack_dir
1350        && let Err(error) = clientscript::load_clan_settings(&mut registry, &config_files, pd)
1351    {
1352        warn!("  Clan settings not loaded: {error}");
1353    }
1354
1355    if let Some(pd) = resolved_pack_dir {
1356        let dbtable_ids = symloader::load_dbtable_pack(&pd.join("dbtable.pack"));
1357        if !dbtable_ids.is_empty() {
1358            symloader::load_dbtable_configs(&mut registry, &config_files, &dbtable_ids);
1359            info!(
1360                "  Registered {} dbcolumn compound IDs",
1361                registry.dbcolumn_types.len()
1362            );
1363        }
1364    }
1365
1366    let engine_rs2 = scripts_dir.join("engine.rs2");
1367
1368    if engine_rs2.exists() {
1369        symloader::load_engine_command_params(&mut registry, &engine_rs2);
1370    } else {
1371        if let Ok(entries) = std::fs::read_dir(scripts_dir) {
1372            for entry in entries.flatten() {
1373                let candidate = entry.path().join("engine.rs2");
1374                if candidate.exists() {
1375                    crate::symloader::load_engine_command_params(&mut registry, &candidate);
1376                    break;
1377                }
1378            }
1379        }
1380    }
1381
1382    let client_dir = if scripts_dir
1383        .file_name()
1384        .is_some_and(|name| name == "runescript")
1385    {
1386        scripts_dir.with_file_name("clientscript")
1387    } else {
1388        scripts_dir.join("clientscript")
1389    };
1390    if client_dir.is_dir() {
1391        if let Some(pack) = resolved_pack_dir {
1392            symloader::load_clientscript_pack(&mut registry, &pack.join("clientscript.pack"));
1393        }
1394        symloader::load_clientscript_signatures(&mut registry, &client_dir);
1395    }
1396
1397    symloader::patch_command_return_types(&mut registry);
1398
1399    registry.visibility = Arc::new(sources::Visibility::load(scripts_dir, &registry));
1400
1401    registry
1402}
1403
1404/// Finalize the registry once parsing is done: regenerate `script.pack` from the
1405/// parsed sources, load pre-assigned script IDs, then register every parsed
1406/// script (emitting trigger/redeclaration/return diagnostics). Builds on the
1407/// registry produced concurrently by `build_symbol_registry`.
1408fn register_scripts(
1409    registry: &mut SymbolRegistry,
1410    scripts_dir: &Path,
1411    resolved_pack_dir: Option<&Path>,
1412    diagnostics: &mut DiagnosticsCollector,
1413    all_files: &[ParsedFile],
1414    raw_by_path: &HashMap<PathBuf, Arc<String>>,
1415) {
1416    if let Some(pd) = resolved_pack_dir {
1417        symloader::generate_script_pack(scripts_dir, pd, raw_by_path);
1418        symloader::load_script_ids(registry, pd);
1419    }
1420
1421    {
1422        use crate::diagnostic_messages as msg;
1423
1424        let mut registered_scripts: std::collections::HashSet<String> =
1425            std::collections::HashSet::new();
1426
1427        for pf in all_files.iter() {
1428            Arc::make_mut(&mut registry.visibility).declare_source(&pf.path, &pf.file);
1429            for script in &pf.file.scripts {
1430                if script.trigger == "command" {
1431                    continue;
1432                }
1433
1434                if !trigger_table::is_valid_trigger(&script.trigger) {
1435                    diagnostics.warning(
1436                        pf.path.clone(),
1437                        script.line,
1438                        0,
1439                        msg::fmt(msg::SCRIPT_TRIGGER_INVALID, &[&script.trigger]),
1440                        crate::diagnostics::Phase::SymbolRegistration,
1441                    );
1442                }
1443
1444                let key = format!("{}:{}", script.trigger, script.name);
1445                if !registered_scripts.insert(key.clone()) {
1446                    diagnostics.warning(
1447                        pf.path.clone(),
1448                        script.line,
1449                        0,
1450                        msg::fmt(msg::SCRIPT_REDECLARATION, &[&script.trigger, &script.name]),
1451                        crate::diagnostics::Phase::SymbolRegistration,
1452                    );
1453                }
1454
1455                if !script.return_types.is_empty()
1456                    && !trigger_table::allows_returns(&script.trigger)
1457                {
1458                    diagnostics.warning(
1459                        pf.path.clone(),
1460                        script.line,
1461                        0,
1462                        msg::fmt(msg::SCRIPT_TRIGGER_NO_RETURNS, &[&script.trigger]),
1463                        diagnostics::Phase::SymbolRegistration,
1464                    );
1465                }
1466
1467                let param_types: Vec<types::Type> =
1468                    script.params.iter().map(|p| p.param_type).collect();
1469                registry.register_script(
1470                    script.name.clone(),
1471                    script.trigger.clone(),
1472                    param_types,
1473                    script.return_types.clone(),
1474                );
1475
1476                if let Some(ts_line) = pf.testscript_line
1477                    && script.line >= ts_line
1478                {
1479                    registry.mark_test_script(&script.trigger, &script.name);
1480                }
1481            }
1482        }
1483    }
1484}
1485
1486// ── Streaming recompile pipeline (prototype, gated by RUNEC_RECOMPILE) ────────
1487// Never holds all compiled scripts. Pass 2 only type-checks; the pointer check
1488// re-compiles each script on demand (bounded cache — measured ~1.3x re-parse on
1489// rs-majula, the call graph has high file locality); output is produced by
1490// re-compiling file-by-file into a by-id buffer. Output + diagnostics are
1491// byte-identical to the batch path (deterministic re-compile, same id order).
1492
1493/// Re-compiles scripts on demand for the pointer checker, caching `Rc`s with a
1494/// bounded FIFO so the full compiled set is never resident. A cache miss
1495/// re-parses the script's file and compiles all its scripts at once (siblings
1496/// share the parse), keying each by its full `[trigger,name]` to its index.
1497struct RecompileStore<'a> {
1498    idx_to_path: Vec<PathBuf>,
1499    /// file path -> its script idxs in declaration order (command-skipped).
1500    /// Position-based so duplicate `[trigger,name]` declarations each map to the
1501    /// correct distinct idx.
1502    file_to_idxs: HashMap<PathBuf, Vec<usize>>,
1503    registry: &'a SymbolRegistry,
1504    include_tests: bool,
1505    cache: HashMap<usize, std::rc::Rc<CompiledScript>>,
1506    order: std::collections::VecDeque<usize>,
1507    cap: usize,
1508}
1509
1510impl RecompileStore<'_> {
1511    fn get(&mut self, idx: usize) -> std::rc::Rc<CompiledScript> {
1512        if let Some(rc) = self.cache.get(&idx) {
1513            return std::rc::Rc::clone(rc);
1514        }
1515        let path = self.idx_to_path[idx].clone();
1516        let parsed = parse_one_file(&path, self.include_tests);
1517        if let Ok(ps) = parsed.read
1518            && let ParseOutcome::Parsed { file, .. } = ps.outcome
1519        {
1520            // The j-th non-command declaration in this file maps to idxs[j],
1521            // matching how the metas were built (same parse, same skip rule).
1522            let idxs = self.file_to_idxs.get(&path).cloned().unwrap_or_default();
1523            let mut compiler = Compiler::new(self.registry);
1524            let mut j = 0usize;
1525            for script in &file.scripts {
1526                if script.trigger == "command" {
1527                    continue;
1528                }
1529                if let Some(&i) = idxs.get(j) {
1530                    let mut c = compiler.compile_script(script);
1531                    c.source_path = ps.cache_key.clone();
1532                    self.insert(i, std::rc::Rc::new(c));
1533                }
1534                j += 1;
1535            }
1536        }
1537        std::rc::Rc::clone(
1538            self.cache
1539                .get(&idx)
1540                .expect("recompile store: script not produced by its file"),
1541        )
1542    }
1543
1544    fn insert(&mut self, idx: usize, rc: std::rc::Rc<CompiledScript>) {
1545        if self.cache.insert(idx, rc).is_none() {
1546            self.order.push_back(idx);
1547            while self.order.len() > self.cap {
1548                if let Some(old) = self.order.pop_front()
1549                    && old != idx
1550                {
1551                    self.cache.remove(&old);
1552                }
1553            }
1554        }
1555    }
1556}
1557
1558/// Per-slot encoded output blobs (indexed by script id) plus accumulated
1559/// diagnostics — what the recompile core hands back before the bytes are written
1560/// to disk or assembled in memory.
1561type RecompiledOutput = (Vec<Vec<u8>>, DiagnosticsCollector);
1562
1563/// Core of the recompile (low-memory) pipeline: runs every phase re-compiling
1564/// scripts on demand — never holding all compiled bytecode at once — and returns
1565/// the per-slot encoded output blobs plus diagnostics, without writing or
1566/// printing. The disk and in-memory entry points (`compile_recompile`,
1567/// `compile_recompile_memory`) wrap this. On a phase error it prints diagnostics
1568/// and returns `Err`, matching the batch path.
1569fn recompile_to_encoded(
1570    scripts_dir: &Path,
1571    pack_dir: Option<&Path>,
1572    lint: bool,
1573    include_tests: bool,
1574) -> Result<RecompiledOutput, Box<dyn Error>> {
1575    let source_dir = sources::target_dir(scripts_dir, sources::Scope::Rune);
1576    let scripts_dir = source_dir.as_path();
1577    use crate::pointer_checker::{PointerChecker, ScriptSource};
1578
1579    let mut rs2_files: Vec<PathBuf> = get_rs2_files(scripts_dir, "rs2")?;
1580    rs2_files.sort();
1581    info!("Found {} script files", rs2_files.len());
1582    mem_mark("startup");
1583    let mut diagnostics = DiagnosticsCollector::new();
1584
1585    // Pass 1: signatures only (ASTs dropped in-worker); keep file paths for lazy
1586    // source + raw text for script.pack regeneration.
1587    info!("[recompile] Pass 1: parsing for signatures...");
1588    let sig_results = parallel_map(&rs2_files, |path| parse_one_file_sigs(path, include_tests));
1589    let mut source_paths: HashMap<String, PathBuf> = HashMap::new();
1590    let mut raw_by_path: HashMap<PathBuf, Arc<String>> = HashMap::new();
1591    let mut file_sigs: Vec<FileSigs> = Vec::new();
1592    for r in sig_results {
1593        let (cache_key, raw_source, outcome) = match r.read {
1594            Ok(t) => t,
1595            Err(e) => return Err(Box::new(e)),
1596        };
1597        source_paths.insert(cache_key, r.path.clone());
1598        raw_by_path.insert(r.path.clone(), Arc::new(raw_source));
1599        match outcome {
1600            SigOutcome::Parsed {
1601                sigs,
1602                testscript_line,
1603            } => file_sigs.push(FileSigs {
1604                path: r.path,
1605                testscript_line,
1606                sigs,
1607            }),
1608            SigOutcome::Error {
1609                line,
1610                position,
1611                message,
1612                phase,
1613            } => diagnostics.error(r.path, line, position, message, phase),
1614        }
1615    }
1616    if diagnostics.has_errors() {
1617        diagnostics.print_all();
1618        return Err(Box::new(error::CompilerError::TypeError(
1619            "Parsing failed".to_string(),
1620        )));
1621    }
1622    mem_mark("after parse");
1623
1624    // Registration.
1625    info!("[recompile] Registering scripts and building the symbol registry...");
1626    let resolved_pack_dir = resolve_pack_dir(scripts_dir, pack_dir);
1627    let mut registry = build_symbol_registry(scripts_dir, resolved_pack_dir.as_deref());
1628    register_scripts_from_sigs(
1629        &mut registry,
1630        scripts_dir,
1631        resolved_pack_dir.as_deref(),
1632        &mut diagnostics,
1633        &file_sigs,
1634        &raw_by_path,
1635    );
1636    info!("  Registered {} scripts", registry.scripts.len());
1637    drop(raw_by_path);
1638    registry.drop_registration_scratch();
1639    for fs in &file_sigs {
1640        for sig in &fs.sigs {
1641            if let Some(msg) =
1642                Compiler::validate_trigger_subject(&sig.trigger, &sig.name, &registry)
1643            {
1644                diagnostics.warning(
1645                    fs.path.clone(),
1646                    sig.line,
1647                    0,
1648                    msg,
1649                    diagnostics::Phase::SymbolRegistration,
1650                );
1651            }
1652        }
1653    }
1654    mem_mark("after register");
1655
1656    // Pass 2: type-check only (re-parse, no compiled scripts retained).
1657    info!("[recompile] Pass 2: type-check (streaming, no bytecode held)...");
1658    let tc_diags = parallel_map(&rs2_files, |path| {
1659        let parsed = match parse_one_file(path, include_tests).read {
1660            Ok(p) => p,
1661            Err(_) => return DiagnosticsCollector::new(),
1662        };
1663        let (file, testscript_line) = match parsed.outcome {
1664            ParseOutcome::Parsed {
1665                file,
1666                testscript_line,
1667            } => (file, testscript_line),
1668            ParseOutcome::Error { .. } => return DiagnosticsCollector::new(),
1669        };
1670        let mut type_checker = TypeChecker::new(&registry);
1671        type_checker.check_file_with_test_boundary(&file, path, testscript_line);
1672        if !type_checker.diagnostics.has_errors() {
1673            let mut compiler = Compiler::new(&registry);
1674            for script in &file.scripts {
1675                if script.trigger != "command" {
1676                    compiler.compile_checked(script, path, &mut type_checker.diagnostics);
1677                }
1678            }
1679        }
1680        type_checker.diagnostics
1681    });
1682    for d in tc_diags {
1683        diagnostics.merge(d);
1684    }
1685    if diagnostics.has_errors() {
1686        diagnostics.print_all();
1687        return Err(Box::new(error::CompilerError::TypeError(
1688            "Type checking failed".to_string(),
1689        )));
1690    }
1691    mem_mark("after typecheck");
1692
1693    // Build script metas in the same idx order as the batch path: file order
1694    // (file_sigs / declaration order, command-skipped), then a stable sort by id.
1695    // ids + names come from the registry — no bytecode needed.
1696    let mut pre: Vec<(i32, String, PathBuf)> = Vec::new();
1697    for fs in &file_sigs {
1698        for sig in &fs.sigs {
1699            if sig.trigger == "command" {
1700                continue;
1701            }
1702            let id = registry
1703                .script_id_for_trigger(&sig.trigger, &sig.name)
1704                .or_else(|| registry.script_id(&sig.name))
1705                .unwrap_or(-1);
1706            pre.push((
1707                id,
1708                format!("[{},{}]", sig.trigger, sig.name),
1709                fs.path.clone(),
1710            ));
1711        }
1712    }
1713    let n = pre.len();
1714    // Stable sort indices by id → the batch's idx order (file/decl pre-order
1715    // preserved among equal ids). `inv` maps a pre-order position back to its idx.
1716    let mut order: Vec<usize> = (0..n).collect();
1717    order.sort_by_key(|&p| pre[p].0);
1718    let idents: Vec<(i32, String)> = order
1719        .iter()
1720        .map(|&p| (pre[p].0, pre[p].1.clone()))
1721        .collect();
1722    let idx_to_path: Vec<PathBuf> = order.iter().map(|&p| pre[p].2.clone()).collect();
1723    let mut inv = vec![0usize; n];
1724    for (idx, &p) in order.iter().enumerate() {
1725        inv[p] = idx;
1726    }
1727    let mut file_to_idxs: HashMap<PathBuf, Vec<usize>> = HashMap::new();
1728    for (p, m) in pre.iter().enumerate() {
1729        file_to_idxs.entry(m.2.clone()).or_default().push(inv[p]);
1730    }
1731    let max_id = pre.iter().map(|m| m.0).max().unwrap_or(-1);
1732    let slot_count = if max_id < 0 { 0 } else { (max_id + 1) as usize };
1733    let len = n;
1734    drop(pre);
1735
1736    // Pointer check via on-demand re-compile (single-threaded: the store getter
1737    // is `FnMut`/`!Sync`). Diagnostics come out in id order, as in the batch path.
1738    info!("[recompile] Pointer check (re-compile on demand)...");
1739    let shared = PointerChecker::build_shared_idents(&idents, &registry);
1740    {
1741        // Cache cap must exceed the largest file's script count, or loading that
1742        // file would evict its own just-compiled scripts. A small margin over
1743        // that keeps cross-file locality high (measured ~1.1-1.3x re-parse).
1744        let cap = file_to_idxs
1745            .values()
1746            .map(|v| v.len())
1747            .max()
1748            .unwrap_or(0)
1749            .max(128);
1750        let mut store = RecompileStore {
1751            idx_to_path,
1752            file_to_idxs,
1753            registry: &registry,
1754            include_tests,
1755            cache: HashMap::new(),
1756            order: std::collections::VecDeque::new(),
1757            cap,
1758        };
1759        let getter: Box<dyn FnMut(usize) -> std::rc::Rc<CompiledScript>> =
1760            Box::new(move |idx| store.get(idx));
1761        let scripts = ScriptSource::Recompile {
1762            len,
1763            get: std::cell::RefCell::new(getter),
1764        };
1765        let mut checker = PointerChecker::with_shared(scripts, &registry, shared);
1766        checker.set_source_cache(SourceProvider::Lazy(&source_paths));
1767        let names: Vec<String> = idents.iter().map(|(_, n)| n.clone()).collect();
1768        let ptr_diags = checker.validate_names(&names);
1769        let w = ptr_diags.warning_count();
1770        if w > 0 {
1771            info!("  Pointer check: {} warning(s)", w);
1772        }
1773        diagnostics.merge(ptr_diags);
1774    }
1775    mem_mark("after pointercheck");
1776
1777    // Output: re-compile each file once, encode each script into its id slot,
1778    // optionally lint — never holding all compiled scripts at once. The caller
1779    // writes the blobs to disk or assembles them in memory.
1780    info!("[recompile] Re-compiling + encoding output...");
1781    let mut encoded: Vec<Vec<u8>> = vec![Vec::new(); slot_count];
1782    for path in &rs2_files {
1783        let parsed = match parse_one_file(path, include_tests).read {
1784            Ok(p) => p,
1785            Err(_) => continue,
1786        };
1787        let file = match parsed.outcome {
1788            ParseOutcome::Parsed { file, .. } => file,
1789            ParseOutcome::Error { .. } => continue,
1790        };
1791        let mut compiler = Compiler::new(&registry);
1792        let mut file_scripts: Vec<CompiledScript> = Vec::new();
1793        for script in &file.scripts {
1794            if script.trigger == "command" {
1795                continue;
1796            }
1797            let Some(mut c) = compiler.compile_checked(script, path, &mut diagnostics) else {
1798                continue;
1799            };
1800            c.source_path = parsed.cache_key.clone();
1801            file_scripts.push(c);
1802        }
1803        for c in &file_scripts {
1804            if c.id >= 0 && (c.id as usize) < slot_count {
1805                encoded[c.id as usize] = crate::writer::encode_script(c);
1806            }
1807        }
1808        if lint {
1809            diagnostics.merge(lints::run_lints(
1810                &file_scripts,
1811                Some(SourceProvider::Lazy(&source_paths)),
1812            ));
1813        }
1814    }
1815    mem_mark("after codegen");
1816    Ok((encoded, diagnostics))
1817}
1818
1819/// Full compile via the recompile (low-memory) pipeline: writes `script.dat` /
1820/// `script.idx` to `output_dir`, then prints diagnostics.
1821fn compile_recompile(
1822    scripts_dir: &Path,
1823    pack_dir: Option<&Path>,
1824    output_dir: &Path,
1825    lint: bool,
1826) -> Result<(), Box<dyn Error>> {
1827    let (encoded, diagnostics) = recompile_to_encoded(scripts_dir, pack_dir, lint, false)?;
1828    info!(
1829        "Writing output to {}...",
1830        output_dir.display().to_string().replace('\\', "/")
1831    );
1832    ScriptWriter::new(output_dir.display().to_string()).write_encoded(&encoded)?;
1833    diagnostics.print_all();
1834    info!("Compilation complete!");
1835    Ok(())
1836}
1837
1838/// In-memory compile via the recompile (low-memory) pipeline: returns the
1839/// `(script.dat, script.idx)` bytes, then prints diagnostics. Byte-identical to
1840/// the in-memory batch path (`compile_memory*`).
1841fn compile_recompile_memory(
1842    scripts_dir: &Path,
1843    pack_dir: Option<&Path>,
1844    lint: bool,
1845    include_tests: bool,
1846) -> Result<(Vec<u8>, Vec<u8>), Box<dyn Error>> {
1847    let (encoded, diagnostics) = recompile_to_encoded(scripts_dir, pack_dir, lint, include_tests)?;
1848    info!("Writing compiled output to memory...");
1849    let out = ScriptWriter::new("".into()).build_encoded(&encoded);
1850    diagnostics.print_all();
1851    info!("Compilation complete!");
1852    Ok(out)
1853}
1854
1855pub(crate) fn find_testscript_line(source: &str) -> Option<usize> {
1856    for (i, line) in source.lines().enumerate() {
1857        if line.trim() == "#testscript" {
1858            return Some(i + 1);
1859        }
1860    }
1861    None
1862}
1863
1864fn truncate_at_line(source: &str, line_no: usize) -> String {
1865    source
1866        .lines()
1867        .take(line_no - 1)
1868        .collect::<Vec<_>>()
1869        .join("\n")
1870}
1871
1872fn strip_testscript_annotation(source: &str) -> String {
1873    source
1874        .lines()
1875        .map(|line| {
1876            if line.trim() == "#testscript" {
1877                ""
1878            } else {
1879                line
1880            }
1881        })
1882        .collect::<Vec<_>>()
1883        .join("\n")
1884}
1885
1886fn get_rs2_files(dir: &Path, ext: &str) -> Result<Vec<PathBuf>, Box<dyn Error>> {
1887    Ok(sources::script_files(dir, ext)?)
1888}