rustledger_loader/

lib.rs

//! Beancount file loader with include resolution.
//!
//! This crate handles loading beancount files, resolving includes,
//! and collecting options. It builds on the parser to provide a
//! complete loading pipeline.
//!
//! # Features
//!
//! - Recursive include resolution with cycle detection
//! - Options collection and parsing
//! - Plugin directive collection
//! - Source map for error reporting
//! - Push/pop tag and metadata handling
//! - Automatic GPG decryption for encrypted files (`.gpg`, `.asc`)
//!
//! # Example
//!
//! ```ignore
//! use rustledger_loader::Loader;
//! use std::path::Path;
//!
//! let result = Loader::new().load(Path::new("ledger.beancount"))?;
//! for directive in result.directives {
//!     println!("{:?}", directive);
//! }
//! ```

#![forbid(unsafe_code)]
#![warn(missing_docs)]
// Never-panic surface: the loader resolves attacker-controlled `include` paths
// and parser output, so production code must not `unwrap`/`expect`. `not(test)`
// scopes the deny to non-test builds, so this crate's own `#[cfg(test)]` /
// `#[test]` code (incl. `#[cfg(all(test, feature = ...))]` modules) is exempt — it
// compiles with `cfg(test)`. (Integration tests under `tests/` are separate crates
// and aren't governed by this attribute either way.) Proven-safe production sites
// carry an audited `#[allow]` with a justification.
#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used))]

#[cfg(feature = "cache")]
pub mod cache;
mod dedup;
pub mod discover;
mod options;
mod phase;
mod process;
mod source_map;
mod vfs;

pub use phase::{
    Booked, Directives, EarlyValidated, Finalized, LateValidated, Phase, Raw,
    RegularPluginsApplied, Sorted, Synthed,
};
// Note: `FailedBookings` is NOT re-exported. It's internal to the
// pipeline (flowing from `book` to `finalize`) and accessed via the
// `crate::phase::FailedBookings` path within the crate.

#[cfg(feature = "cache")]
pub use cache::{
    CACHE_FILENAME_ENV, CacheEntry, CachedOptions, CachedPlugin, DISABLE_CACHE_ENV,
    cache_disabled_by_env, cache_path, default_cache_path, invalidate_cache, load_cache_entry,
    save_cache_entry,
};
pub use dedup::{reintern_directives, reintern_plain_directives};
pub use discover::{COMMON_ROOT_NAMES, discover_journal_file, discover_journal_upward};
pub use options::Options;
pub use source_map::{SourceFile, SourceMap};
pub use vfs::{DiskFileSystem, FileSystem, VirtualFileSystem};

// Re-export processing API when features are enabled
/// Shared option→validation mapping and document-dir resolution — single source
/// of truth so the LSP/MCP diagnostics cannot drift from `check` (issue #1648).
#[cfg(feature = "validation")]
pub use process::{document_source_dirs, validation_options_from_options};

/// Resolve `documents` roots against the ledger's directory.
///
/// Ungated: the loader itself calls these to raise E7006, and it needs to do so
/// whether or not `validation` is on.
pub use process::{document_root_warnings, resolve_document_dirs};

/// Whether an `include`/glob path contains glob metacharacters (`*`, `?`, `[`).
///
/// Single source of truth shared with the LSP's document-link resolver so the
/// two agree on what counts as a glob — a literal-path existence check on a glob
/// wrongly reports "File not found" (issue #1647).
#[must_use]
pub fn is_glob_pattern(path: &str) -> bool {
    path.contains(['*', '?', '['])
}
pub use process::{
    ErrorLocation, ErrorSeverity, ExtraPlugin, Ledger, LedgerError, LoadOptions, ProcessError,
    load, load_raw, load_with_fs, process,
};
#[cfg(feature = "plugins")]
pub use process::{PluginPass, run_plugins};

use rustledger_core::{Directive, DisplayContext};
use rustledger_parser::{ParseError, Span, Spanned};
use std::collections::HashSet;
use std::path::{Path, PathBuf};
use std::process::Command;
use thiserror::Error;

// Path normalization lives in a single place: `FileSystem::normalize`
// (`DiskFileSystem::normalize` is the disk implementation, `VirtualFileSystem`
// the in-memory one). The free `normalize_path` that used to duplicate the disk
// body was removed — all callers go through the injected filesystem so the
// include path-traversal guard normalizes consistently in one namespace.

/// Errors that can occur during loading.
#[derive(Debug, Error)]
pub enum LoadError {
    /// IO error reading a file.
    #[error("failed to read file {path}: {source}")]
    Io {
        /// The path that failed to read.
        path: PathBuf,
        /// The underlying IO error.
        #[source]
        source: std::io::Error,
    },

    /// Include cycle detected.
    ///
    /// The Display string intentionally begins with `Duplicate filename
    /// parsed:` to match Python beancount's wording for the same
    /// condition. The pta-standards `include-cycle-detection`
    /// conformance test asserts on the substring `"Duplicate filename"`,
    /// so this wording is load-bearing (#765). The full cycle path is
    /// preserved in a trailing parenthetical for debuggability.
    #[error(
        "Duplicate filename parsed: \"{}\" (include cycle: {})",
        .cycle.last().map_or("", String::as_str),
        .cycle.join(" -> ")
    )]
    IncludeCycle {
        /// The cycle of file paths. The last element is the
        /// re-encountered filename (equal to one of the earlier
        /// entries), and it's the one quoted in the `"Duplicate
        /// filename parsed:"` prefix.
        cycle: Vec<String>,
    },

    /// The same file was reached twice in the include graph without forming a
    /// cycle — a "diamond", e.g. a shared prices file included from two
    /// monthly journals.
    ///
    /// The file's directives are loaded ONCE, as beancount does; this records
    /// that the duplicate happened. The wording matches beancount's for the
    /// same condition, and deliberately omits the `(include cycle: …)`
    /// parenthetical of [`LoadError::IncludeCycle`], which this is not.
    ///
    /// Previously the second encounter returned `Ok(())` in silence, so a
    /// ledger beancount rejects with `Duplicate filename parsed` loaded here
    /// without a word (#2084 follow-up).
    #[error("Duplicate filename parsed: \"{path}\"")]
    DuplicateInclude {
        /// The path reached a second time.
        path: String,
    },

    /// Parse errors occurred.
    #[error("parse errors in {path}")]
    ParseErrors {
        /// The file with parse errors.
        path: PathBuf,
        /// The parse errors.
        errors: Vec<ParseError>,
    },

    /// Path traversal attempt detected.
    #[error("path traversal not allowed: {include_path} escapes base directory {base_dir}")]
    PathTraversal {
        /// The include path that attempted traversal.
        include_path: String,
        /// The base directory.
        base_dir: PathBuf,
    },

    /// GPG decryption failed.
    #[error("failed to decrypt {path}: {message}")]
    Decryption {
        /// The encrypted file path.
        path: PathBuf,
        /// Error message from GPG.
        message: String,
    },

    /// Glob pattern did not match any files.
    #[error("include pattern \"{pattern}\" does not match any files")]
    GlobNoMatch {
        /// The glob pattern that matched nothing.
        pattern: String,
    },

    /// Glob pattern expansion failed.
    #[error("failed to expand include pattern \"{pattern}\": {message}")]
    GlobError {
        /// The glob pattern that failed.
        pattern: String,
        /// The error message.
        message: String,
    },

    /// More files were referenced than the 16-bit file-id space allows.
    ///
    /// File ids are `u16` on every `Spanned` value, so a ledger may reference at
    /// most `u16::MAX` files via includes/globs. Reported as an error rather
    /// than panicking, since `load` is a never-panic-on-input surface.
    #[error("too many files: a ledger may reference at most {limit} files (16-bit file ids)")]
    TooManyFiles {
        /// The maximum number of files supported.
        limit: usize,
    },
}

/// Convert a 0-based file index to the `u16` file id stored on `Spanned`
/// values, returning [`LoadError::TooManyFiles`] instead of panicking when a
/// ledger references more files than the id space allows.
///
/// `SYNTHESIZED_FILE_ID` (`u16::MAX`) is reserved as the plugin-synthesized
/// sentinel, so a real file id must be strictly below it — we reject `>=` it,
/// not merely `> u16::MAX`, otherwise the 65,535th file would alias onto the
/// sentinel. (This is also the boundary `SourceMap::add_file` asserts, so the
/// loader must check it *before* calling `add_file`.)
const fn file_id_to_u16(file_id: usize) -> Result<u16, LoadError> {
    if file_id >= rustledger_parser::SYNTHESIZED_FILE_ID as usize {
        return Err(LoadError::TooManyFiles {
            limit: rustledger_parser::SYNTHESIZED_FILE_ID as usize,
        });
    }
    Ok(file_id as u16)
}

/// Result of loading a beancount file.
#[derive(Debug)]
pub struct LoadResult {
    /// All directives from all files, in order.
    pub directives: Vec<Spanned<Directive>>,
    /// Parsed options.
    pub options: Options,
    /// Plugins to load.
    pub plugins: Vec<Plugin>,
    /// Source map for error reporting.
    pub source_map: SourceMap,
    /// All errors encountered during loading.
    pub errors: Vec<LoadError>,
    /// Display context for formatting numbers (tracks precision per currency).
    pub display_context: DisplayContext,
}

/// A plugin directive.
#[derive(Debug, Clone)]
pub struct Plugin {
    /// Plugin module name (with any `python:` prefix stripped).
    pub name: String,
    /// Optional configuration string.
    pub config: Option<String>,
    /// Source location.
    pub span: Span,
    /// File this plugin was declared in.
    pub file_id: usize,
    /// Whether the `python:` prefix was used to force Python execution.
    pub force_python: bool,
}

/// Decrypt a GPG-encrypted file using the system `gpg` command.
///
/// This uses `gpg --batch --decrypt` which will use the user's
/// GPG keyring and gpg-agent for passphrase handling.
pub(crate) fn decrypt_gpg_file(path: &Path) -> Result<String, LoadError> {
    let output = Command::new("gpg")
        .args(["--batch", "--decrypt"])
        .arg(path)
        .output()
        .map_err(|e| LoadError::Decryption {
            path: path.to_path_buf(),
            message: format!("failed to run gpg: {e}"),
        })?;

    if !output.status.success() {
        return Err(LoadError::Decryption {
            path: path.to_path_buf(),
            message: String::from_utf8_lossy(&output.stderr).trim().to_string(),
        });
    }

    String::from_utf8(output.stdout).map_err(|e| LoadError::Decryption {
        path: path.to_path_buf(),
        message: format!("decrypted content is not valid UTF-8: {e}"),
    })
}

/// Beancount file loader.
#[derive(Debug)]
pub struct Loader {
    /// Files that have been loaded (for cycle detection).
    loaded_files: HashSet<PathBuf>,
    /// Stack for cycle detection during loading (maintains order for error messages).
    include_stack: Vec<PathBuf>,
    /// Set for O(1) cycle detection (mirrors `include_stack`).
    include_stack_set: HashSet<PathBuf>,
    /// Root directory for path traversal protection.
    /// If set, includes must resolve to paths within this directory.
    root_dir: Option<PathBuf>,
    /// Whether to enforce path traversal protection.
    enforce_path_security: bool,
    /// Filesystem abstraction for reading files.
    fs: Box<dyn FileSystem>,
}

impl Default for Loader {
    fn default() -> Self {
        Self {
            loaded_files: HashSet::new(),
            include_stack: Vec::new(),
            include_stack_set: HashSet::new(),
            root_dir: None,
            enforce_path_security: false,
            fs: Box::new(DiskFileSystem),
        }
    }
}

impl Loader {
    /// Create a new loader.
    #[must_use]
    pub fn new() -> Self {
        Self::default()
    }

    /// Enable path traversal protection.
    ///
    /// When enabled, include directives cannot escape the root directory
    /// of the main beancount file. This prevents malicious ledger files
    /// from accessing sensitive files outside the ledger directory.
    ///
    /// # Example
    ///
    /// ```ignore
    /// let result = Loader::new()
    ///     .with_path_security(true)
    ///     .load(Path::new("ledger.beancount"))?;
    /// ```
    #[must_use]
    pub const fn with_path_security(mut self, enabled: bool) -> Self {
        self.enforce_path_security = enabled;
        self
    }

    /// Set a custom root directory for path security.
    ///
    /// By default, the root directory is the parent directory of the main file.
    /// This method allows overriding that to a custom directory.
    #[must_use]
    pub fn with_root_dir(mut self, root: PathBuf) -> Self {
        self.root_dir = Some(root);
        self.enforce_path_security = true;
        self
    }

    /// Set a custom filesystem for file loading.
    ///
    /// This allows using a virtual filesystem (e.g., for WASM) instead of
    /// the default disk filesystem.
    ///
    /// # Example
    ///
    /// ```
    /// use rustledger_loader::{Loader, VirtualFileSystem};
    ///
    /// let mut vfs = VirtualFileSystem::new();
    /// vfs.add_file("main.beancount", "2024-01-01 open Assets:Bank USD");
    ///
    /// let loader = Loader::new().with_filesystem(Box::new(vfs));
    /// ```
    #[must_use]
    pub fn with_filesystem(mut self, fs: Box<dyn FileSystem>) -> Self {
        self.fs = fs;
        self
    }

    /// Load a beancount file and all its includes.
    ///
    /// Uses parallel file parsing when multiple files are discovered via
    /// include directives. The root file is parsed first to resolve the
    /// include tree, then all included files are read and parsed in
    /// parallel using rayon.
    ///
    /// # Errors
    ///
    /// Returns [`LoadError`] in the following cases:
    ///
    /// - [`LoadError::Io`] - Failed to read the file or an included file
    /// - [`LoadError::IncludeCycle`] - Circular include detected
    ///
    /// Note: Parse errors and path traversal errors are collected in
    /// [`LoadResult::errors`] rather than returned directly, allowing
    /// partial results to be returned.
    pub fn load(&mut self, path: &Path) -> Result<LoadResult, LoadError> {
        let mut directives = Vec::new();
        let mut options = Options::default();
        let mut plugins = Vec::new();
        let mut source_map = SourceMap::new();
        let mut errors = Vec::new();

        // Get normalized path (uses filesystem-specific normalization)
        let canonical = self.fs.normalize(path);

        // Set root directory for path security if enabled but not explicitly set
        if self.enforce_path_security && self.root_dir.is_none() {
            self.root_dir = canonical.parent().map(Path::to_path_buf);
        }
        // Normalize the root through the SAME filesystem namespace as the paths
        // it is compared against. An explicit `with_root_dir(...)` may be
        // relative/un-normalized, and on disk `normalize` makes paths absolute —
        // so without this, a relative root never `starts_with` a normalized path
        // and every include would be falsely rejected as a traversal. (The
        // default-derived root is already normalized, so re-normalizing is a
        // no-op there.)
        if let Some(root) = self.root_dir.take() {
            self.root_dir = Some(self.fs.normalize(&root));
        }

        // Phase 1: Parse the root file to discover includes.
        // The root file is typically small (just includes + options).
        self.load_recursive(
            &canonical,
            None,
            &mut directives,
            &mut options,
            &mut plugins,
            &mut source_map,
            &mut errors,
        )?;

        // Deduplicate every `InternedStr` reachable from a directive
        // across files. Each file parses with its own per-file
        // `StringInterner`, so identical strings — accounts,
        // currencies, tags, links, payees, narrations — appearing in
        // two included files land in two different `Arc<str>`
        // allocations, defeating the `Arc::ptr_eq` fast path in
        // `InternedStr`'s `PartialEq` and forcing all cross-file
        // equality through byte comparison.
        //
        // This is the fresh-parse path, and it is the only one that needs
        // the walk. A cache hit has no per-file split to repair: it
        // deserializes one archive under an `InternScope` (see
        // `cache::load_cache_entry`), which interns as it builds and so
        // arrives at the same postcondition without a second pass. The
        // WASM parsed-ledger constructor has no scope either and calls
        // `reintern_plain_directives` for itself.
        //
        // Either way every consumer of `LoadResult` sees a deduplicated
        // directive list regardless of how it was produced. Closes #1071.
        dedup::reintern_directives(&mut directives);

        // Build display context from directives and options
        let display_context = build_display_context(&directives, &options);

        // E7006: `option "documents"` roots that do not exist.
        //
        // This runs here rather than in option parsing because a relative
        // `documents` path is relative to the LEDGER FILE — the same rule
        // beancount and `include` use — and only the source map knows where
        // that file is. Doing it during parsing meant `Path::new(value)`,
        // which asked about the process CWD: `rledger check sub/ledger.bean`
        // reported a missing document root that was present, and the same
        // ledger checked clean from inside `sub/` (#1999).
        let base_dir = source_map.files().first().and_then(|f| f.path.parent());
        let doc_warnings =
            process::document_root_warnings(&options.documents, base_dir, self.fs.as_ref());
        options.warnings.extend(doc_warnings);

        Ok(LoadResult {
            directives,
            options,
            plugins,
            source_map,
            errors,
            display_context,
        })
    }

    #[allow(clippy::too_many_arguments)]
    fn load_recursive(
        &mut self,
        path: &Path,
        pre_parsed: Option<(std::sync::Arc<str>, rustledger_parser::ParseResult)>,
        directives: &mut Vec<Spanned<Directive>>,
        options: &mut Options,
        plugins: &mut Vec<Plugin>,
        source_map: &mut SourceMap,
        errors: &mut Vec<LoadError>,
    ) -> Result<(), LoadError> {
        // Allocate path once for reuse
        let path_buf = path.to_path_buf();

        // Check for cycles using O(1) HashSet lookup
        if self.include_stack_set.contains(&path_buf) {
            // `collect::<Vec<_>>()` on a chain of two `ExactSizeIterator`s
            // preallocates the exact capacity via `size_hint`, so an
            // explicit `Vec::with_capacity(...)` + `extend` + `push` is
            // equivalent and noisier. This is the cycle-error cold path
            // anyway — readability wins over micro-optimization.
            let cycle: Vec<String> = self
                .include_stack
                .iter()
                .map(|p| p.display().to_string())
                .chain(std::iter::once(path.display().to_string()))
                .collect();
            return Err(LoadError::IncludeCycle { cycle });
        }

        // Check if already loaded. Beancount reports this and carries on with
        // the single copy it already has, and `bean-check` exits non-zero for
        // it; record the same and do the same.
        if self.loaded_files.contains(&path_buf) {
            errors.push(LoadError::DuplicateInclude {
                path: path.display().to_string(),
            });
            return Ok(());
        }

        // Use pre-parsed data if available (from parallel loading path),
        // otherwise read and parse the file.
        let (source, result) = if let Some(pre) = pre_parsed {
            pre
        } else {
            let src: std::sync::Arc<str> = if self.fs.is_encrypted(path) {
                // Route decryption through the filesystem so a sandboxed fs (the
                // WASI component) can delegate to a host capability (#1667); the
                // default impl shells out to gpg.
                self.fs.decrypt(path)?
            } else {
                self.fs.read(path)?
            };
            // The processing pipeline never reads currency/account occurrences
            // (LSP-only); skip collecting them — see `parse_without_occurrences`.
            let parsed = rustledger_parser::parse_without_occurrences(&src);
            (src, parsed)
        };

        // Validate the prospective file id BEFORE `add_file` — `add_file`
        // asserts `id < SYNTHESIZED_FILE_ID` and would panic, but `load` must
        // never panic on input. The next id `add_file` assigns is the current
        // file count; reject it here (collected into `LoadResult::errors` by the
        // caller) so an over-large include/glob set fails loudly without a panic.
        let fid_u16 = file_id_to_u16(source_map.files().len())?;
        // Add to source map (Arc::clone is cheap - just increments refcount)
        let file_id = source_map.add_file(path_buf.clone(), std::sync::Arc::clone(&source));
        debug_assert_eq!(file_id, fid_u16 as usize);

        // Mark as loading (update both stack and set)
        self.include_stack_set.insert(path_buf.clone());
        self.include_stack.push(path_buf.clone());
        self.loaded_files.insert(path_buf);

        // Collect parse errors
        if !result.errors.is_empty() {
            errors.push(LoadError::ParseErrors {
                path: path.to_path_buf(),
                errors: result.errors,
            });
        }

        // Process options.
        //
        // `include_stack` was pushed with this file just above, so a length of
        // one means we are in the top-level ledger. Options outside
        // `ACCUMULATE_ACROSS_INCLUDES` are taken from that file only: an
        // included sub-ledger must not silently change how the whole tree
        // books or what counts as balanced (#2151).
        let top_level = self.include_stack.len() == 1;
        for (key, value, _span) in result.options {
            options.set_scoped(&key, &value, top_level);
        }

        // Process plugins
        for (name, config, span) in result.plugins {
            // Check for "python:" prefix to force Python execution
            let (actual_name, force_python) = if let Some(stripped) = name.strip_prefix("python:") {
                (stripped.to_string(), true)
            } else {
                (name, false)
            };
            plugins.push(Plugin {
                name: actual_name,
                config,
                span,
                file_id,
                force_python,
            });
        }

        // Process includes (with glob pattern support)
        let base_dir = path.parent().unwrap_or(Path::new("."));
        for (include_path, _span) in &result.includes {
            // Check if the include path contains glob metacharacters
            // (check on include_path, not full_path, to avoid false positives from directory names)
            let has_glob = is_glob_pattern(include_path);

            let full_path = base_dir.join(include_path);

            // Path traversal protection: check BEFORE glob expansion to avoid
            // enumerating files outside the allowed root directory
            if self.enforce_path_security
                && let Some(ref root) = self.root_dir
            {
                // For glob patterns, extract and check the non-glob prefix
                let path_to_check = if has_glob {
                    // Find where the first glob metacharacter is
                    let glob_start = include_path
                        .find(['*', '?', '['])
                        .unwrap_or(include_path.len());
                    // Get the directory prefix before the glob
                    let prefix = &include_path[..glob_start];
                    let prefix_path = if let Some(last_sep) = prefix.rfind('/') {
                        base_dir.join(&include_path[..=last_sep])
                    } else {
                        base_dir.to_path_buf()
                    };
                    // Normalize via the injected filesystem (not a hardcoded
                    // disk path fn), so this pre-glob traversal guard and the
                    // per-matched-file guard below resolve in the SAME namespace
                    // — under a `VirtualFileSystem` the disk `normalize_path`
                    // would have compared a disk-canonicalized prefix against a
                    // pure-string root.
                    self.fs.normalize(&prefix_path)
                } else {
                    self.fs.normalize(&full_path)
                };

                if !path_to_check.starts_with(root) {
                    errors.push(LoadError::PathTraversal {
                        include_path: include_path.clone(),
                        base_dir: root.clone(),
                    });
                    continue;
                }
            }

            let full_path_str = full_path.to_string_lossy();

            // Expand glob patterns or use literal path
            let paths_to_load: Vec<PathBuf> = if has_glob {
                match self.fs.glob(&full_path_str) {
                    Ok(matched) => matched,
                    Err(e) => {
                        errors.push(LoadError::GlobError {
                            pattern: include_path.clone(),
                            message: e,
                        });
                        continue;
                    }
                }
            } else {
                vec![full_path.clone()]
            };

            // Check if glob matched nothing
            if has_glob && paths_to_load.is_empty() {
                errors.push(LoadError::GlobNoMatch {
                    pattern: include_path.clone(),
                });
                continue;
            }

            // Normalize and security-check all matched paths first.
            let mut valid_paths = Vec::with_capacity(paths_to_load.len());
            for matched_path in paths_to_load {
                let canonical = self.fs.normalize(&matched_path);

                // Security check: glob could match files outside root via symlinks
                if self.enforce_path_security
                    && let Some(ref root) = self.root_dir
                    && !canonical.starts_with(root)
                {
                    errors.push(LoadError::PathTraversal {
                        include_path: matched_path.to_string_lossy().into_owned(),
                        base_dir: root.clone(),
                    });
                    continue;
                }

                valid_paths.push(canonical);
            }

            // Parallel optimization: when loading multiple sibling includes
            // from disk, read and parse them in parallel. The expensive work
            // (I/O + tokenize + parse) runs on rayon's thread pool while the
            // main thread coordinates the include tree walk.
            //
            // Each file is read and parsed independently. Results are then
            // merged sequentially to preserve include order and process any
            // nested includes via recursive calls.
            if valid_paths.len() > 1 && self.fs.supports_parallel_read() {
                use rayon::prelude::*;

                // Read + parse non-encrypted files in parallel, preserving
                // original include order. Each entry becomes either
                // Some((source, parsed)) for successful reads, or None for
                // encrypted/failed files (which fall back to sequential).
                //
                // We keep the original index to merge results in order,
                // ensuring option/directive precedence matches the declared
                // include sequence.
                let fs = &*self.fs;
                let pre_parsed: Vec<Option<(std::sync::Arc<str>, rustledger_parser::ParseResult)>> =
                    valid_paths
                        .par_iter()
                        .map(|p| {
                            // Skip encrypted files — they need sequential GPG decryption
                            if fs.is_encrypted(p) {
                                return None;
                            }
                            // Read through the FileSystem trait so all I/O goes
                            // through one code path (UTF-8 handling, error types, etc.)
                            let source = fs.read(p).ok()?;
                            // Occurrences are LSP-only; skip them on the load path.
                            let parsed = rustledger_parser::parse_without_occurrences(&source);
                            Some((source, parsed))
                        })
                        .collect();

                // Merge in original include order. Files that were
                // pre-parsed pass their data to load_recursive; files
                // that weren't (encrypted or I/O error) are loaded
                // sequentially as a fallback.
                for (canonical, pre) in valid_paths.iter().zip(pre_parsed) {
                    if let Err(e) = self.load_recursive(
                        canonical, pre, directives, options, plugins, source_map, errors,
                    ) {
                        errors.push(e);
                    }
                }
            } else {
                // Sequential fallback: single file or VFS.
                for canonical in valid_paths {
                    if let Err(e) = self.load_recursive(
                        &canonical, None, directives, options, plugins, source_map, errors,
                    ) {
                        errors.push(e);
                    }
                }
            }
        }

        // Add directives from this file, setting the file_id on the outer
        // Spanned<Directive> and on each inner Spanned<Posting> inside
        // transactions. Postings inside an included file share that file's
        // ID; this keeps inner spans consistent with their containing
        // directive so consumers don't need to traverse parent pointers.
        //
        // file_id is `u16` everywhere (see `Spanned::file_id` rustdoc). `fid_u16`
        // was validated above (before this file was added to the source map), so
        // no overflow/panic is possible here.
        directives.extend(result.directives.into_iter().map(|d| {
            let mut d = d.with_file_id(file_id);
            if let rustledger_core::Directive::Transaction(ref mut txn) = d.value {
                for p in &mut txn.postings {
                    p.file_id = fid_u16;
                }
            }
            d
        }));

        // Pop from stack and set
        if let Some(popped) = self.include_stack.pop() {
            self.include_stack_set.remove(&popped);
        }

        Ok(())
    }
}

/// Build a display context from loaded directives and options.
///
/// Thin wrapper over the canonical builder
/// [`DisplayContext::from_directives`] (moved to `rustledger-core` so the
/// FFI component's `session.format` shares the exact sampling rules —
/// #1766): amount-scan inference, then `option "display_precision"`
/// overrides, then per-commodity `precision:` metadata. Only
/// `render_commas` — presentation policy, not precision — is applied here.
fn build_display_context(directives: &[Spanned<Directive>], options: &Options) -> DisplayContext {
    DisplayContext::from_directives(
        directives.iter().map(|s| &s.value),
        options
            .display_precision
            .iter()
            .map(|(c, p)| (c.as_str(), *p)),
        options.render_commas,
    )
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::io::Write;
    use tempfile::NamedTempFile;

    #[test]
    fn file_id_to_u16_rejects_the_reserved_sentinel_not_panic() {
        let sentinel = rustledger_parser::SYNTHESIZED_FILE_ID as usize;
        // The last valid real id is one BELOW the reserved sentinel.
        assert_eq!(file_id_to_u16(0).unwrap(), 0);
        assert_eq!(
            file_id_to_u16(sentinel - 1).unwrap(),
            rustledger_parser::SYNTHESIZED_FILE_ID - 1
        );
        // The sentinel itself (and beyond) is rejected with a LoadError — NOT a
        // panic, and NOT aliased onto SYNTHESIZED_FILE_ID. `load` is a
        // never-panic-on-input surface; the old `expect`/`assert` would have
        // aborted the whole embedder / wasm module at this boundary.
        assert!(matches!(
            file_id_to_u16(sentinel),
            Err(LoadError::TooManyFiles { limit }) if limit == sentinel
        ));
        assert!(matches!(
            file_id_to_u16(sentinel + 1),
            Err(LoadError::TooManyFiles { .. })
        ));
    }

    #[test]
    fn test_is_encrypted_file_gpg_extension() {
        let fs = DiskFileSystem;
        let path = Path::new("test.beancount.gpg");
        assert!(fs.is_encrypted(path));
    }

    #[test]
    fn test_is_encrypted_file_plain_beancount() {
        let fs = DiskFileSystem;
        let path = Path::new("test.beancount");
        assert!(!fs.is_encrypted(path));
    }

    #[test]
    fn test_is_encrypted_file_asc_with_pgp_header() {
        let fs = DiskFileSystem;
        let mut file = NamedTempFile::with_suffix(".asc").unwrap();
        writeln!(file, "-----BEGIN PGP MESSAGE-----").unwrap();
        writeln!(file, "some encrypted content").unwrap();
        writeln!(file, "-----END PGP MESSAGE-----").unwrap();
        file.flush().unwrap();

        assert!(fs.is_encrypted(file.path()));
    }

    #[test]
    fn test_is_encrypted_file_asc_without_pgp_header() {
        let fs = DiskFileSystem;
        let mut file = NamedTempFile::with_suffix(".asc").unwrap();
        writeln!(file, "This is just a plain text file").unwrap();
        writeln!(file, "with .asc extension but no PGP content").unwrap();
        file.flush().unwrap();

        assert!(!fs.is_encrypted(file.path()));
    }

    #[test]
    fn test_decrypt_gpg_file_missing_gpg() {
        // Create a fake .gpg file
        let mut file = NamedTempFile::with_suffix(".gpg").unwrap();
        writeln!(file, "fake encrypted content").unwrap();
        file.flush().unwrap();

        // This will fail because the content isn't actually GPG-encrypted
        // (or gpg isn't installed, or there's no matching key)
        let result = decrypt_gpg_file(file.path());
        assert!(result.is_err());

        if let Err(LoadError::Decryption { path, message }) = result {
            assert_eq!(path, file.path().to_path_buf());
            assert!(!message.is_empty());
        } else {
            panic!("Expected Decryption error");
        }
    }

    #[test]
    fn test_plugin_force_python_prefix() {
        let mut file = NamedTempFile::with_suffix(".beancount").unwrap();
        writeln!(file, r#"plugin "python:my_plugin""#).unwrap();
        writeln!(file, r#"plugin "regular_plugin""#).unwrap();
        file.flush().unwrap();

        let result = Loader::new().load(file.path()).unwrap();

        assert_eq!(result.plugins.len(), 2);

        // First plugin should have force_python = true and name without prefix
        assert_eq!(result.plugins[0].name, "my_plugin");
        assert!(result.plugins[0].force_python);

        // Second plugin should have force_python = false
        assert_eq!(result.plugins[1].name, "regular_plugin");
        assert!(!result.plugins[1].force_python);
    }

    #[test]
    fn test_plugin_force_python_with_config() {
        let mut file = NamedTempFile::with_suffix(".beancount").unwrap();
        writeln!(file, r#"plugin "python:my_plugin" "config_value""#).unwrap();
        file.flush().unwrap();

        let result = Loader::new().load(file.path()).unwrap();

        assert_eq!(result.plugins.len(), 1);
        assert_eq!(result.plugins[0].name, "my_plugin");
        assert!(result.plugins[0].force_python);
        assert_eq!(result.plugins[0].config, Some("config_value".to_string()));
    }

    #[test]
    fn test_virtual_filesystem_include_resolution() {
        // Create a virtual filesystem with multiple files
        let mut vfs = VirtualFileSystem::new();
        vfs.add_file(
            "main.beancount",
            r#"
include "accounts.beancount"

2024-01-15 * "Coffee"
  Expenses:Food  5.00 USD
  Assets:Bank   -5.00 USD
"#,
        );
        vfs.add_file(
            "accounts.beancount",
            r"
2024-01-01 open Assets:Bank USD
2024-01-01 open Expenses:Food USD
",
        );

        // Load with virtual filesystem
        let result = Loader::new()
            .with_filesystem(Box::new(vfs))
            .load(Path::new("main.beancount"))
            .unwrap();

        // Should have 3 directives: 2 opens + 1 transaction
        assert_eq!(result.directives.len(), 3);
        assert!(result.errors.is_empty());

        // Verify directive types
        let directive_types: Vec<_> = result
            .directives
            .iter()
            .map(|d| match &d.value {
                rustledger_core::Directive::Open(_) => "open",
                rustledger_core::Directive::Transaction(_) => "txn",
                _ => "other",
            })
            .collect();
        assert_eq!(directive_types, vec!["open", "open", "txn"]);
    }

    #[test]
    fn test_virtual_filesystem_nested_includes() {
        // Test deeply nested includes
        let mut vfs = VirtualFileSystem::new();
        vfs.add_file("main.beancount", r#"include "level1.beancount""#);
        vfs.add_file(
            "level1.beancount",
            r#"
include "level2.beancount"
2024-01-01 open Assets:Level1 USD
"#,
        );
        vfs.add_file("level2.beancount", "2024-01-01 open Assets:Level2 USD");

        let result = Loader::new()
            .with_filesystem(Box::new(vfs))
            .load(Path::new("main.beancount"))
            .unwrap();

        // Should have 2 open directives from nested includes
        assert_eq!(result.directives.len(), 2);
        assert!(result.errors.is_empty());
    }

    #[test]
    fn test_virtual_filesystem_missing_include() {
        let mut vfs = VirtualFileSystem::new();
        vfs.add_file("main.beancount", r#"include "nonexistent.beancount""#);

        let result = Loader::new()
            .with_filesystem(Box::new(vfs))
            .load(Path::new("main.beancount"))
            .unwrap();

        // Should have an error for missing file
        assert!(!result.errors.is_empty());
        let error_msg = result.errors[0].to_string();
        assert!(error_msg.contains("not found") || error_msg.contains("Io"));
    }

    #[test]
    fn test_virtual_filesystem_glob_include() {
        let mut vfs = VirtualFileSystem::new();
        vfs.add_file(
            "main.beancount",
            r#"
include "transactions/*.beancount"

2024-01-01 open Assets:Bank USD
"#,
        );
        vfs.add_file(
            "transactions/2024.beancount",
            r#"
2024-01-01 open Expenses:Food USD

2024-06-15 * "Groceries"
  Expenses:Food  50.00 USD
  Assets:Bank   -50.00 USD
"#,
        );
        vfs.add_file(
            "transactions/2025.beancount",
            r#"
2025-01-01 open Expenses:Rent USD

2025-02-01 * "Rent"
  Expenses:Rent  1000.00 USD
  Assets:Bank   -1000.00 USD
"#,
        );
        // This file should NOT be matched by the glob
        vfs.add_file(
            "other/ignored.beancount",
            "2024-01-01 open Expenses:Other USD",
        );

        let result = Loader::new()
            .with_filesystem(Box::new(vfs))
            .load(Path::new("main.beancount"))
            .unwrap();

        // Should have: 1 open from main + 2 opens from transactions + 2 txns
        let opens = result
            .directives
            .iter()
            .filter(|d| matches!(d.value, rustledger_core::Directive::Open(_)))
            .count();
        assert_eq!(
            opens, 3,
            "expected 3 open directives (1 main + 2 transactions)"
        );

        let txns = result
            .directives
            .iter()
            .filter(|d| matches!(d.value, rustledger_core::Directive::Transaction(_)))
            .count();
        assert_eq!(txns, 2, "expected 2 transactions from glob-matched files");

        assert!(
            result.errors.is_empty(),
            "expected no errors, got: {:?}",
            result.errors
        );
    }

    #[test]
    fn test_virtual_filesystem_glob_dot_slash_prefix() {
        let mut vfs = VirtualFileSystem::new();
        vfs.add_file(
            "main.beancount",
            r#"
include "./transactions/*.beancount"

2024-01-01 open Assets:Bank USD
"#,
        );
        vfs.add_file(
            "transactions/2024.beancount",
            r#"
2024-01-01 open Expenses:Food USD

2024-06-15 * "Groceries"
  Expenses:Food  50.00 USD
  Assets:Bank   -50.00 USD
"#,
        );
        vfs.add_file(
            "transactions/2025.beancount",
            r#"
2025-01-01 open Expenses:Rent USD

2025-02-01 * "Rent"
  Expenses:Rent  1000.00 USD
  Assets:Bank   -1000.00 USD
"#,
        );

        let result = Loader::new()
            .with_filesystem(Box::new(vfs))
            .load(Path::new("main.beancount"))
            .unwrap();

        // Should have: 1 open from main + 2 opens from transactions + 2 txns
        let opens = result
            .directives
            .iter()
            .filter(|d| matches!(d.value, rustledger_core::Directive::Open(_)))
            .count();
        assert_eq!(
            opens, 3,
            "expected 3 open directives (1 main + 2 transactions), ./ prefix should be normalized"
        );

        let txns = result
            .directives
            .iter()
            .filter(|d| matches!(d.value, rustledger_core::Directive::Transaction(_)))
            .count();
        assert_eq!(
            txns, 2,
            "expected 2 transactions from glob-matched files despite ./ prefix"
        );

        assert!(
            result.errors.is_empty(),
            "expected no errors, got: {:?}",
            result.errors
        );
    }

    #[test]
    fn test_virtual_filesystem_glob_no_match() {
        let mut vfs = VirtualFileSystem::new();
        vfs.add_file("main.beancount", r#"include "nonexistent/*.beancount""#);

        let result = Loader::new()
            .with_filesystem(Box::new(vfs))
            .load(Path::new("main.beancount"))
            .unwrap();

        // Should have a GlobNoMatch error
        let has_glob_error = result
            .errors
            .iter()
            .any(|e| matches!(e, LoadError::GlobNoMatch { .. }));
        assert!(
            has_glob_error,
            "expected GlobNoMatch error, got: {:?}",
            result.errors
        );
    }

    /// Regression: with path security under a `VirtualFileSystem`, the pre-glob
    /// traversal guard used the disk `normalize_path` while the per-file guard
    /// used `self.fs.normalize` — different namespaces. So a LEGITIMATE glob
    /// include within the root got its prefix disk-normalized to a real-CWD path
    /// that didn't `starts_with` the VFS root, and was falsely rejected as a
    /// path traversal. Both guards now normalize through the injected filesystem.
    #[test]
    fn test_vfs_glob_include_within_root_not_flagged_as_traversal() {
        let mut vfs = VirtualFileSystem::new();
        vfs.add_file("ledger/main.beancount", r#"include "sub/*.beancount""#);
        vfs.add_file(
            "ledger/sub/a.beancount",
            "2024-01-01 open Assets:Cash USD\n",
        );

        let result = Loader::new()
            .with_filesystem(Box::new(vfs))
            .with_root_dir(PathBuf::from("ledger")) // enables path security
            .load(Path::new("ledger/main.beancount"))
            .unwrap();

        assert!(
            !result
                .errors
                .iter()
                .any(|e| matches!(e, LoadError::PathTraversal { .. })),
            "legit in-root VFS glob include wrongly flagged as traversal: {:?}",
            result.errors
        );
        // The included file actually loaded (its `open` is present).
        assert!(
            !result.directives.is_empty(),
            "the in-root include should have loaded; errors: {:?}",
            result.errors
        );
    }

    /// Regression test for #1071: a fresh multi-file parse must produce
    /// deduplicated `InternedStr` values, so two `Posting`s referencing
    /// the same account from different files share one `Arc<str>`.
    /// Pre-fix the per-file `StringInterner` kept the two `Arc`s
    /// distinct and `Arc::ptr_eq` fell through to byte comparison.
    #[test]
    fn test_fresh_parse_deduplicates_internedstr_across_files() {
        let mut vfs = VirtualFileSystem::new();
        vfs.add_file(
            "main.beancount",
            r#"
2024-01-01 open Assets:Bank USD
include "transactions.beancount"
"#,
        );
        vfs.add_file(
            "transactions.beancount",
            r#"
2024-01-15 * "Coffee"
  Assets:Bank   -5.00 USD
  Expenses:Coffee  5.00 USD

2024-01-16 open Expenses:Coffee
"#,
        );

        let result = Loader::new()
            .with_filesystem(Box::new(vfs))
            .load(Path::new("main.beancount"))
            .unwrap();

        // Collect every `Assets:Bank` `Account` (one from `open`, one
        // from the posting). They originate in different files, so
        // pre-fix they had distinct `Arc<str>` allocations.
        let bank_accounts: Vec<&rustledger_core::Account> = result
            .directives
            .iter()
            .filter_map(|s| match &s.value {
                rustledger_core::Directive::Open(o) if o.account.as_str() == "Assets:Bank" => {
                    Some(&o.account)
                }
                rustledger_core::Directive::Transaction(t) => t
                    .postings
                    .iter()
                    .find(|p| p.account.as_str() == "Assets:Bank")
                    .map(|p| &p.account),
                _ => None,
            })
            .collect();

        assert_eq!(
            bank_accounts.len(),
            2,
            "expected one Open and one posting for Assets:Bank"
        );
        assert!(
            bank_accounts[0]
                .as_interned()
                .ptr_eq(bank_accounts[1].as_interned()),
            "Assets:Bank from cross-file open/posting must share the same Arc<str> \
             after Loader::load runs reintern_directives"
        );
    }

    /// Companion to the previous test — covers the Transaction-level
    /// `InternedStr` fields (payee, narration, tags, links) that the
    /// pre-Copilot version of `reintern_directive` silently skipped
    /// (Copilot review on PR #1081). Two transactions in different
    /// files share the same payee + tag; after `Loader::load` they
    /// must share one `Arc<str>` per string.
    #[test]
    fn test_fresh_parse_deduplicates_transaction_fields_across_files() {
        let mut vfs = VirtualFileSystem::new();
        vfs.add_file(
            "main.beancount",
            r#"
2024-01-01 open Assets:Bank USD
2024-01-01 open Expenses:Coffee

2024-01-15 * "Cafe Bench" "Latte" #morning
  Assets:Bank   -5.00 USD
  Expenses:Coffee  5.00 USD

include "more.beancount"
"#,
        );
        vfs.add_file(
            "more.beancount",
            r#"
2024-01-16 * "Cafe Bench" "Espresso" #morning
  Assets:Bank   -3.00 USD
  Expenses:Coffee  3.00 USD
"#,
        );

        let result = Loader::new()
            .with_filesystem(Box::new(vfs))
            .load(Path::new("main.beancount"))
            .unwrap();

        let txns: Vec<&rustledger_core::Transaction> = result
            .directives
            .iter()
            .filter_map(|s| match &s.value {
                rustledger_core::Directive::Transaction(t) => Some(t),
                _ => None,
            })
            .collect();

        assert_eq!(txns.len(), 2, "expected the two transactions");
        let p1 = txns[0].payee.as_ref().expect("first txn has payee");
        let p2 = txns[1].payee.as_ref().expect("second txn has payee");
        assert!(
            p1.ptr_eq(p2),
            "Identical payee \"Cafe Bench\" across files must share one Arc<str>"
        );

        assert!(!txns[0].tags.is_empty() && !txns[1].tags.is_empty());
        assert!(
            txns[0].tags[0].ptr_eq(&txns[1].tags[0]),
            "Identical tag #morning across files must share one Arc<str>"
        );
    }

    /// Regression test responding to Copilot review on PR #1174: the
    /// dedup pass must walk every interned payload type inside
    /// `Metadata` maps — `MetaValue::{Account, Currency, Tag, Link,
    /// Amount.currency}` — at both the transaction level and the
    /// posting level. Before the meta walk was added, cross-file
    /// metadata values held distinct `Arc<str>` allocations even when
    /// they referenced identical strings.
    ///
    /// One multi-file fixture exercises all five variants in a single
    /// load to keep the test focused on the dedup invariant rather
    /// than the parse machinery.
    #[test]
    fn test_fresh_parse_deduplicates_metavalue_across_files() {
        use rustledger_core::MetaValue;

        let mut vfs = VirtualFileSystem::new();
        vfs.add_file(
            "main.beancount",
            r#"
2024-01-01 open Assets:Bank USD
2024-01-01 open Expenses:Coffee

2024-01-15 * "Latte"
  counterparty_account: Assets:Bank
  preferred_currency: USD
  category_tag: #coffee
  receipt_link: ^receipt-2024
  fee_amount: 0.50 USD
  Assets:Bank   -5.00 USD
    settled_with: Assets:Bank
  Expenses:Coffee  5.00 USD

include "more.beancount"
"#,
        );
        vfs.add_file(
            "more.beancount",
            r#"
2024-01-16 * "Espresso"
  counterparty_account: Assets:Bank
  preferred_currency: USD
  category_tag: #coffee
  receipt_link: ^receipt-2024
  fee_amount: 0.50 USD
  Assets:Bank   -3.00 USD
    settled_with: Assets:Bank
  Expenses:Coffee  3.00 USD
"#,
        );

        let result = Loader::new()
            .with_filesystem(Box::new(vfs))
            .load(Path::new("main.beancount"))
            .unwrap();

        let txns: Vec<&rustledger_core::Transaction> = result
            .directives
            .iter()
            .filter_map(|s| match &s.value {
                rustledger_core::Directive::Transaction(t) => Some(t),
                _ => None,
            })
            .collect();
        assert_eq!(txns.len(), 2);

        // --- Transaction-level meta: all four typed variants + Amount.currency ---

        let MetaValue::Account(a1) = &txns[0].meta["counterparty_account"] else {
            panic!("expected MetaValue::Account");
        };
        let MetaValue::Account(a2) = &txns[1].meta["counterparty_account"] else {
            panic!("expected MetaValue::Account");
        };
        assert!(
            a1.ptr_eq(a2),
            "MetaValue::Account in cross-file meta must share Arc<str>"
        );

        let MetaValue::Currency(c1) = &txns[0].meta["preferred_currency"] else {
            panic!("expected MetaValue::Currency");
        };
        let MetaValue::Currency(c2) = &txns[1].meta["preferred_currency"] else {
            panic!("expected MetaValue::Currency");
        };
        assert!(
            c1.ptr_eq(c2),
            "MetaValue::Currency in cross-file meta must share Arc<str>"
        );

        let MetaValue::Tag(t1) = &txns[0].meta["category_tag"] else {
            panic!("expected MetaValue::Tag");
        };
        let MetaValue::Tag(t2) = &txns[1].meta["category_tag"] else {
            panic!("expected MetaValue::Tag");
        };
        assert!(
            t1.ptr_eq(t2),
            "MetaValue::Tag in cross-file meta must share Arc<str>"
        );

        let MetaValue::Link(l1) = &txns[0].meta["receipt_link"] else {
            panic!("expected MetaValue::Link");
        };
        let MetaValue::Link(l2) = &txns[1].meta["receipt_link"] else {
            panic!("expected MetaValue::Link");
        };
        assert!(
            l1.ptr_eq(l2),
            "MetaValue::Link in cross-file meta must share Arc<str>"
        );

        let MetaValue::Amount(am1) = &txns[0].meta["fee_amount"] else {
            panic!("expected MetaValue::Amount");
        };
        let MetaValue::Amount(am2) = &txns[1].meta["fee_amount"] else {
            panic!("expected MetaValue::Amount");
        };
        assert!(
            am1.currency.ptr_eq(&am2.currency),
            "MetaValue::Amount.currency in cross-file meta must share Arc<str>"
        );

        // --- Posting-level meta: the per-posting `intern_meta` call ---

        let first_posting_0 = &txns[0].postings[0].value;
        let first_posting_1 = &txns[1].postings[0].value;
        let MetaValue::Account(p1) = &first_posting_0.meta["settled_with"] else {
            panic!("expected MetaValue::Account in posting meta");
        };
        let MetaValue::Account(p2) = &first_posting_1.meta["settled_with"] else {
            panic!("expected MetaValue::Account in posting meta");
        };
        assert!(
            p1.ptr_eq(p2),
            "Posting-level MetaValue::Account in cross-file meta must share Arc<str> \
             (verifies the per-posting `intern_meta` call, not just the directive-level one)"
        );
    }
}