rustledger_loader/

process.rs

//! Processing pipeline: sort → synth-plugins → Early → book → regular-plugins → Late → finalize.
//!
//! This module orchestrates the full processing pipeline for a beancount ledger,
//! equivalent to Python's `loader.load_file()` function.

use crate::{LoadError, LoadResult, Options, Plugin, SourceMap};
use rustledger_core::{BookingMethod, Directive, DisplayContext};
use rustledger_parser::Spanned;
use std::path::Path;
use thiserror::Error;

/// Options for loading and processing a ledger.
#[derive(Debug, Clone)]
pub struct LoadOptions {
    /// Booking method for lot matching (default: Strict).
    pub booking_method: BookingMethod,
    /// Run plugins declared in the file (default: true).
    pub run_plugins: bool,
    /// Run `auto_accounts` plugin (default: false).
    pub auto_accounts: bool,
    /// Additional native plugins to run (by name).
    pub extra_plugins: Vec<String>,
    /// Plugin configurations for extra plugins.
    pub extra_plugin_configs: Vec<Option<String>>,
    /// Run validation after processing (default: true).
    pub validate: bool,
    /// Enable path security (prevent include traversal).
    pub path_security: bool,
}

impl Default for LoadOptions {
    fn default() -> Self {
        Self {
            booking_method: BookingMethod::Strict,
            run_plugins: true,
            auto_accounts: false,
            extra_plugins: Vec::new(),
            extra_plugin_configs: Vec::new(),
            validate: true,
            path_security: false,
        }
    }
}

impl LoadOptions {
    /// Create options for raw loading (no booking, no plugins, no validation).
    #[must_use]
    pub const fn raw() -> Self {
        Self {
            booking_method: BookingMethod::Strict,
            run_plugins: false,
            auto_accounts: false,
            extra_plugins: Vec::new(),
            extra_plugin_configs: Vec::new(),
            validate: false,
            path_security: false,
        }
    }
}

/// Errors that can occur during ledger processing.
#[derive(Debug, Error)]
pub enum ProcessError {
    /// Loading failed.
    #[error("loading failed: {0}")]
    Load(#[from] LoadError),

    /// Booking/interpolation error.
    #[cfg(feature = "booking")]
    #[error("booking error: {message}")]
    Booking {
        /// Error message.
        message: String,
        /// Date of the transaction.
        date: rustledger_core::NaiveDate,
        /// Narration of the transaction.
        narration: String,
    },

    /// Plugin execution error.
    #[cfg(feature = "plugins")]
    #[error("plugin error: {0}")]
    Plugin(String),

    /// Validation error.
    #[cfg(feature = "validation")]
    #[error("validation error: {0}")]
    Validation(String),

    /// Plugin output conversion error.
    #[cfg(feature = "plugins")]
    #[error("failed to convert plugin output: {0}")]
    PluginConversion(String),
}

/// A fully processed ledger.
///
/// This is the result of loading and processing a beancount file,
/// equivalent to the tuple returned by Python's `loader.load_file()`.
#[derive(Debug)]
pub struct Ledger {
    /// Processed directives (sorted, booked, plugins applied).
    pub directives: Vec<Spanned<Directive>>,
    /// Options parsed from the file.
    pub options: Options,
    /// Plugins declared in the file.
    pub plugins: Vec<Plugin>,
    /// Source map for error reporting.
    pub source_map: SourceMap,
    /// Errors encountered during processing.
    pub errors: Vec<LedgerError>,
    /// Display context for formatting numbers.
    pub display_context: DisplayContext,
}

/// Unified error type for ledger processing.
///
/// This encompasses all error types that can occur during loading,
/// booking, plugin execution, and validation.
#[derive(Debug)]
#[non_exhaustive]
pub struct LedgerError {
    /// Error severity.
    pub severity: ErrorSeverity,
    /// Error code (e.g., "E0001", "W8002").
    pub code: String,
    /// Human-readable error message.
    pub message: String,
    /// Source location, if available.
    pub location: Option<ErrorLocation>,
    /// Byte span (inclusive start, exclusive end) in the source file,
    /// used by rich renderers (e.g. miette) to draw a snippet around
    /// the offending directive. Consumers that only need `file:line:col`
    /// should use `location`; those that want to show the surrounding
    /// source text want this.
    pub source_span: Option<(usize, usize)>,
    /// Source file ID — index into the ledger's [`SourceMap`]. Used
    /// alongside `source_span` for snippet rendering.
    pub file_id: Option<u16>,
    /// Processing phase that produced this error: "parse", "validate", or "plugin".
    pub phase: String,
}

/// Error severity level.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ErrorSeverity {
    /// Error - indicates a problem that should be fixed.
    Error,
    /// Warning - indicates a potential issue.
    Warning,
}

/// Source location for an error.
#[derive(Debug, Clone)]
pub struct ErrorLocation {
    /// File path.
    pub file: std::path::PathBuf,
    /// Line number (1-indexed).
    pub line: usize,
    /// Column number (1-indexed).
    pub column: usize,
}

impl LedgerError {
    /// Create a new error with the given phase.
    pub fn error(code: impl Into<String>, message: impl Into<String>) -> Self {
        Self {
            severity: ErrorSeverity::Error,
            code: code.into(),
            message: message.into(),
            location: None,
            source_span: None,
            file_id: None,
            phase: "validate".to_string(),
        }
    }

    /// Create a new warning.
    pub fn warning(code: impl Into<String>, message: impl Into<String>) -> Self {
        Self {
            severity: ErrorSeverity::Warning,
            code: code.into(),
            message: message.into(),
            location: None,
            source_span: None,
            file_id: None,
            phase: "validate".to_string(),
        }
    }

    /// Attach a source span and file ID so rich renderers can draw a snippet.
    #[must_use]
    pub const fn with_source_span(mut self, span: (usize, usize), file_id: u16) -> Self {
        self.source_span = Some(span);
        self.file_id = Some(file_id);
        self
    }

    /// Set the processing phase for this error.
    #[must_use]
    pub fn with_phase(mut self, phase: impl Into<String>) -> Self {
        self.phase = phase.into();
        self
    }

    /// Add a location to this error.
    #[must_use]
    pub fn with_location(mut self, location: ErrorLocation) -> Self {
        self.location = Some(location);
        self
    }
}

/// Process a raw load result into a fully processed ledger.
///
/// Pipeline (see numbered comments below for the rationale of each step):
///
/// ```text
///   1. sort                         (canonical display order)
///   2. synth plugins                (auto_accounts, document_discovery)
///   3. Early validation             (account presence, structural, lifecycle)
///   4. booking                      (cost spec resolution, interpolation)
///   5. partition                    (set aside failed-booking txns)
///   6. regular plugins              (file plugins + extras, on booked only)
///   7. Late validation              (balance, currency, inventory, on booked only)
///   8. finalize                     (unused-pad warnings)
///   9. re-merge                     (booked + failed → final Ledger.directives)
/// ```
pub fn process(raw: LoadResult, options: &LoadOptions) -> Result<Ledger, ProcessError> {
    let mut directives = raw.directives;
    let mut errors: Vec<LedgerError> = Vec::new();

    // Convert load errors to ledger errors (parse phase)
    for load_err in raw.errors {
        errors.push(LedgerError::error("LOAD", load_err.to_string()).with_phase("parse"));
    }

    // 1. Sort once into canonical display order: `(date, priority, file_id,
    //    span.start)`. This is what BQL / JSON / format output expect and
    //    what Python beancount produces via `(date, type_priority, lineno)`.
    //    `span.start` is a byte offset that orders within a file the same
    //    way line numbers would; `file_id` preserves include order across
    //    files (issue #1049 — same rows, different tie-break would diverge
    //    BQL output on same-date augmentation+reduction fixtures).
    //
    //    Booking needs a different iteration order — augmentations BEFORE
    //    reductions on the same `(date, priority)` so lots exist when
    //    matched — but it doesn't need the underlying vec reordered.
    //    `run_booking` walks the vec via a transient `Vec<usize>` index
    //    that adds `has_cost_reduction` as an extra tiebreaker; this
    //    avoids a second full sort of `Vec<Spanned<Directive>>` (large
    //    structs) after booking just to put display order back.
    directives.sort_by_key(|d| (d.value.date(), d.value.priority(), d.file_id, d.span.start));

    // 2. Synth-only plugins — run BEFORE early validation so the
    // synthesizers (`auto_accounts` and `document_discovery`) inject
    // Opens / Documents that Early checks depend on (E1001 account
    // presence, E5001 missing-document file). Only this narrow synth
    // subset runs here; everything else waits until after booking
    // (step 5) so cost-spec-reading plugins see filled-in
    // `cost.number_per` values. See `PluginPass` rustdoc for the
    // detailed split rationale.
    #[cfg(feature = "plugins")]
    if options.run_plugins || options.auto_accounts {
        run_plugins(
            &mut directives,
            &raw.plugins,
            &raw.options,
            options,
            &raw.source_map,
            &mut errors,
            PluginPass::PreBookingSynth,
        )?;
    }

    // 3. Validation (early phase) — runs on pre-booking directives,
    // AFTER plugins so account-presence checks (E1001) see any Opens
    // that plugins like `auto_accounts` injected.
    //
    // This is what lets booking match Python's "prune zero-interp
    // postings" behavior in step 4 without losing E1001 on the
    // elided-zero-to-unopened-account case (rustledger#877).
    //
    // The `ValidationSession` carries state (open accounts,
    // commodities, pending pads, accumulated tolerances) into the late
    // phase at step 5 so balance assertions and inventory updates see
    // everything the early phase recorded.
    #[cfg(feature = "validation")]
    let mut validation_session = if options.validate {
        Some(rustledger_validate::ValidationSession::new(
            build_validation_options(&raw.options, &raw.source_map),
        ))
    } else {
        None
    };

    // Compute `today` once for both phases — avoids a midnight-crossing
    // race where Early and Late could disagree on what day it is, and
    // gives `FutureDate` warnings a single coherent reference point.
    #[cfg(feature = "validation")]
    let today = jiff::Zoned::now().date();

    #[cfg(feature = "validation")]
    if let Some(session) = validation_session.as_mut() {
        let phase_errors =
            session.run_phase_spanned(&directives, rustledger_validate::Phase::Early, today);
        ledger_errors_extend(&mut errors, phase_errors, &raw.source_map);
    }

    // 4. Booking/interpolation
    //
    // The booking method comes from two sources: the API-level
    // `LoadOptions.booking_method` and the file-level `option
    // "booking_method"`. The file-level option takes precedence only
    // when the file explicitly set it AND the caller hasn't overridden
    // the API-level default. This matches Python beancount, where
    // `option "booking_method" "FIFO"` sets the default for all accounts
    // without an explicit method on their `open` directive.
    //
    // We check `set_options` (not `booking_method.is_empty()`) because
    // `Options::new()` defaults `booking_method` to "STRICT", so the
    // string is never empty.
    //
    // Booking drops zero-value interpolated postings as part of
    // `interpolate()` — see the comment in
    // `rustledger-booking/src/interpolate.rs`. The early validation
    // pass above already caught E1001 on any unopened-account
    // references, so it's safe to prune now (the now-removed
    // `INTERPOLATED_MARKER` workaround in #1114 is obsolete).
    // Run booking and receive the directives partitioned into
    // `(booked, failed)`. Failed transactions are in pre-booking shape
    // (unresolved cost specs, unfilled elided slots, possibly
    // unbalanced); they don't flow into regular plugins or Late
    // validation — booking already reported the root cause and the
    // downstream checks would cascade misleading errors. They get
    // re-merged for the final `Ledger.directives` so the user still
    // sees their original input.
    #[cfg(feature = "booking")]
    let (mut booked, failed): (Vec<Spanned<Directive>>, Vec<Spanned<Directive>>) = {
        let file_set_booking = raw.options.set_options.contains("booking_method");
        let effective_method = if file_set_booking {
            raw.options
                .booking_method
                .parse()
                .unwrap_or(options.booking_method)
        } else {
            options.booking_method
        };
        run_booking(directives, effective_method, &mut errors)
    };
    #[cfg(not(feature = "booking"))]
    let (mut booked, failed): (Vec<Spanned<Directive>>, Vec<Spanned<Directive>>) =
        (directives, Vec::new());

    // 5. Post-booking plugins — file-declared plugins + CLI extras.
    // Runs AFTER booking so cost-spec-reading plugins
    // (`implicit_prices`, `capital_gains_classifier`,
    // `check_average_cost`, `sell_gains`, `unrealized`, `valuation`)
    // see filled-in `cost.number_per` values. This matches Python
    // beancount's plugins-after-booking ordering and closes
    // rustledger#1117. Failed transactions were partitioned out
    // above; plugins only see successfully-booked input.
    #[cfg(feature = "plugins")]
    if options.run_plugins || !options.extra_plugins.is_empty() {
        run_plugins(
            &mut booked,
            &raw.plugins,
            &raw.options,
            options,
            &raw.source_map,
            &mut errors,
            PluginPass::PostBooking,
        )?;
    }

    // 6. Validation (late phase) — runs on booked + plugin-processed
    // directives. Reuses the `ValidationSession` from step 2 so
    // account/commodity/pad bookkeeping carries forward.
    #[cfg(feature = "validation")]
    if let Some(mut session) = validation_session {
        let phase_errors =
            session.run_phase_spanned(&booked, rustledger_validate::Phase::Late, today);
        ledger_errors_extend(&mut errors, phase_errors, &raw.source_map);
        let finalize_errors = session.finalize();
        ledger_errors_extend(&mut errors, finalize_errors, &raw.source_map);
    }

    // 7. Re-merge failed transactions back into the directive list
    // for output. The user wrote them and expects to see them in the
    // resulting `Ledger.directives`; we just kept them isolated from
    // post-booking processing. Re-sort to restore canonical display
    // order (booked retained order during plugin transformation; the
    // sort restores the failed entries' positions).
    let mut directives = booked;
    directives.extend(failed);
    directives.sort_by_key(|d| (d.value.date(), d.value.priority(), d.file_id, d.span.start));

    Ok(Ledger {
        directives,
        options: raw.options,
        plugins: raw.plugins,
        source_map: raw.source_map,
        errors,
        display_context: raw.display_context,
    })
}

/// Run booking and interpolation on transactions, returning the
/// directives partitioned into `(booked, failed)`.
///
/// The caller has already sorted `directives` into canonical display
/// order `(date, priority, file_id, span.start)`. Booking needs the
/// extra constraint that cost-reduction transactions process AFTER
/// augmentations on the same `(date, priority)` so lots exist when
/// matched. Rather than re-sorting the whole vec, we walk it via a
/// transient `Vec<usize>` of indices sorted by booking order. Stable
/// sort preserves display-order tiebreaks between transactions with
/// the same `has_cost_reduction` flag.
///
/// Failed transactions are partitioned out into the second return
/// value so they don't flow into regular plugins or Late validation
/// (they're in pre-booking shape — postings have unresolved cost
/// specs and unfilled elided slots, so downstream processing would
/// cascade misleading errors). The caller is responsible for
/// re-merging `failed` into the final `Ledger.directives` for output
/// so the user still sees their original input.
#[cfg(feature = "booking")]
fn run_booking(
    mut directives: Vec<Spanned<Directive>>,
    booking_method: BookingMethod,
    errors: &mut Vec<LedgerError>,
) -> (Vec<Spanned<Directive>>, Vec<Spanned<Directive>>) {
    use rustledger_booking::BookingEngine;

    let mut engine = BookingEngine::with_method(booking_method);
    engine.register_account_methods(directives.iter().map(|s| &s.value));

    // Build an index ordered for booking: stable sort by
    // `has_cost_reduction` only (display order — `(date, priority,
    // file_id, span.start)` — is already encoded in the existing
    // positional order, and stable_sort preserves that as the tiebreak).
    let mut order: Vec<usize> = (0..directives.len()).collect();
    order.sort_by_key(|&i| {
        let d = &directives[i].value;
        (d.date(), d.priority(), d.has_cost_reduction())
    });

    let mut failed_indices: Vec<usize> = Vec::new();
    for &i in &order {
        let spanned = &mut directives[i];
        if let Directive::Transaction(txn) = &mut spanned.value {
            match engine.book_and_interpolate(txn) {
                Ok(result) => {
                    engine.apply(&result.transaction);
                    *txn = result.transaction;
                }
                Err(e) => {
                    errors.push(LedgerError::error(
                        "BOOK",
                        format!("{} ({}, \"{}\")", e, txn.date, txn.narration),
                    ));
                    failed_indices.push(i);
                }
            }
        }
    }

    // Partition into (booked, failed). Indices are valid in the current
    // `directives` vec (no mutation has happened since they were
    // collected); after this consuming iteration the vec is gone and
    // partition is fait accompli — no window where a caller could
    // accidentally mutate between collection and partition.
    let failed_set: rustc_hash::FxHashSet<usize> = failed_indices.iter().copied().collect();
    let mut booked = Vec::with_capacity(directives.len() - failed_indices.len());
    let mut failed = Vec::with_capacity(failed_indices.len());
    for (i, d) in directives.into_iter().enumerate() {
        if failed_set.contains(&i) {
            failed.push(d);
        } else {
            booked.push(d);
        }
    }
    (booked, failed)
}

/// Which subset of plugins to run.
///
/// The loader pipeline calls `run_plugins` twice: once with
/// [`PluginPass::PreBookingSynth`] before the Early validation phase
/// (so synthesizers can inject Opens / Documents that early checks
/// depend on), and once with [`PluginPass::PostBooking`] after booking
/// (so cost-spec-reading plugins like `implicit_prices`,
/// `capital_gains_classifier`, `check_average_cost`, `sell_gains`,
/// `unrealized`, and `valuation` see filled-in `cost.number_per`
/// values).
///
/// Standalone callers (LSP, FFI, tests) that operate on already-booked
/// input should pass [`PluginPass::All`] for the historical single-pass
/// behavior.
#[cfg(feature = "plugins")]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PluginPass {
    /// Only plugins that synthesize directives the Early validator
    /// depends on: `auto_accounts` (synthesizes Open directives) and
    /// the built-in document discovery walker (synthesizes Document
    /// directives the early phase checks for missing files).
    PreBookingSynth,
    /// All file-declared plugins and CLI `extra_plugins`, EXCLUDING
    /// `auto_accounts` and `document_discovery` (those ran pre-booking).
    /// Includes the 28 plugins that don't depend on synth state but
    /// may depend on booked cost specs.
    PostBooking,
    /// Every plugin — historical single-pass behavior. Used by callers
    /// (LSP, FFI, standalone tests) that don't run booking themselves
    /// or that work on already-booked input.
    All,
}

/// Run plugins on directives.
///
/// Executes native plugins (and document discovery) on the given directives,
/// modifying them in-place. Plugin errors are appended to `errors`.
///
/// `pass` selects which subset of plugins to run — see [`PluginPass`].
/// The loader pipeline calls this twice (synth pass before Early,
/// regular pass after booking). LSP / FFI / standalone callers pass
/// `PluginPass::All` for the historical behavior.
#[cfg(feature = "plugins")]
pub fn run_plugins(
    directives: &mut Vec<Spanned<Directive>>,
    file_plugins: &[Plugin],
    file_options: &Options,
    options: &LoadOptions,
    source_map: &SourceMap,
    errors: &mut Vec<LedgerError>,
    pass: PluginPass,
) -> Result<(), ProcessError> {
    use rustledger_plugin::{
        DocumentDiscoveryPlugin, NativePlugin, NativePluginRegistry, PluginInput, PluginOptions,
    };

    // Resolve document directories relative to the main file's directory
    // Document discovery only runs when run_plugins is true (respects raw mode)
    let base_dir = source_map
        .files()
        .first()
        .and_then(|f| f.path.parent())
        .unwrap_or_else(|| std::path::Path::new("."));

    // `document_discovery` is a synthesizer — runs in PreBookingSynth
    // and All, skipped in PostBooking (it already injected directives
    // during the synth pass).
    let run_doc_discovery = matches!(pass, PluginPass::PreBookingSynth | PluginPass::All)
        && options.run_plugins
        && !file_options.documents.is_empty();
    let has_document_dirs = run_doc_discovery;
    let resolved_documents: Vec<String> = if has_document_dirs {
        file_options
            .documents
            .iter()
            .map(|d| {
                let path = std::path::Path::new(d);
                if path.is_absolute() {
                    d.clone()
                } else {
                    base_dir.join(path).to_string_lossy().to_string()
                }
            })
            .collect()
    } else {
        Vec::new()
    };

    // Build the native plugin registry up front so we can ask each
    // plugin whether it's a synthesizer (via `NativePlugin::is_synth`)
    // during the classification step below. Constructing the registry
    // is O(n_plugins) and just instantiates the plugin structs; it's
    // cheap to do before we know whether any plugins will actually
    // run.
    let registry = NativePluginRegistry::new();

    // Collect raw plugin names first (we'll resolve them with the registry later)
    // Tuple: (name, config, force_python)
    let mut raw_plugins: Vec<(String, Option<String>, bool)> = Vec::new();

    // Classify a plugin by name. Self-classification lives on the
    // `NativePlugin::is_synth` trait method (see
    // `rustledger-plugin/src/native/mod.rs`). Plugins not in the
    // native registry (WASM, Python) default to non-synth — they
    // run post-booking like file-authored beancount plugins.
    let is_synth = |name: &str| -> bool { registry.find(name).is_some_and(NativePlugin::is_synth) };

    // The API-level `options.auto_accounts` flag is a synth source.
    if options.auto_accounts && matches!(pass, PluginPass::PreBookingSynth | PluginPass::All) {
        raw_plugins.push(("auto_accounts".to_string(), None, false));
    }

    // File-declared plugins: synth plugins go in PreBookingSynth,
    // everything else (including the 6 cost-spec-reading ones) goes in
    // PostBooking. `PluginPass::All` runs everything for standalone
    // callers (LSP / FFI / tests on already-booked input).
    if options.run_plugins {
        for plugin in file_plugins {
            let synth = is_synth(&plugin.name);
            let in_pass = match pass {
                PluginPass::PreBookingSynth => synth,
                PluginPass::PostBooking => !synth,
                PluginPass::All => true,
            };
            if in_pass {
                raw_plugins.push((
                    plugin.name.clone(),
                    plugin.config.clone(),
                    plugin.force_python,
                ));
            }
        }
    }

    // CLI extras: same synth/regular split as file plugins.
    for (i, plugin_name) in options.extra_plugins.iter().enumerate() {
        let synth = is_synth(plugin_name);
        let in_pass = match pass {
            PluginPass::PreBookingSynth => synth,
            PluginPass::PostBooking => !synth,
            PluginPass::All => true,
        };
        if in_pass {
            let config = options.extra_plugin_configs.get(i).cloned().flatten();
            raw_plugins.push((plugin_name.clone(), config, false));
        }
    }

    // Check if we have any work to do - early return before creating registry
    if raw_plugins.is_empty() && !has_document_dirs {
        return Ok(());
    }

    let plugin_options = PluginOptions {
        operating_currencies: file_options.operating_currency.clone(),
        title: file_options.title.clone(),
    };

    // Run document discovery plugin if documents directories are configured.
    // Each plugin call builds wrappers freshly from the current `directives`,
    // sends them to the plugin, receives `PluginOp`s, and applies the ops
    // to update `directives` — spans on `Keep` / `Modify` ops are inherited
    // from the original `directives` entry by index, so plugin-transformed
    // directives retain byte-precise source locations.
    if has_document_dirs {
        let doc_plugin = DocumentDiscoveryPlugin::new(resolved_documents, base_dir.to_path_buf());
        let wrappers = build_wrappers(directives, source_map);
        let input = PluginInput {
            directives: wrappers,
            options: plugin_options.clone(),
            config: None,
        };
        let output = doc_plugin.process(input);
        record_plugin_errors(errors, output.errors, source_map);
        apply_plugin_ops(directives, output.ops, errors, source_map)?;
    }

    // Run each plugin (registry was constructed earlier for the
    // synth classification step).
    if !raw_plugins.is_empty() {
        for (raw_name, plugin_config, force_python) in &raw_plugins {
            // Resolve the plugin name - try direct match first, then prefixed variants.
            // Skip native resolution when force_python is set (plugin "python:..." prefix).
            let resolved_name = if *force_python {
                None
            } else if registry.find(raw_name).is_some() {
                Some(raw_name.as_str())
            } else if let Some(short_name) = raw_name.strip_prefix("beancount.plugins.") {
                registry.find(short_name).is_some().then_some(short_name)
            } else if let Some(short_name) = raw_name.strip_prefix("beancount_reds_plugins.") {
                registry.find(short_name).is_some().then_some(short_name)
            } else if let Some(short_name) = raw_name.strip_prefix("beancount_lazy_plugins.") {
                registry.find(short_name).is_some().then_some(short_name)
            } else {
                None
            };

            if let Some(name) = resolved_name
                && let Some(plugin) = registry.find(name)
            {
                let wrappers = build_wrappers(directives, source_map);
                let input = PluginInput {
                    directives: wrappers,
                    options: plugin_options.clone(),
                    config: plugin_config.clone(),
                };
                let output = plugin.process(input);
                record_plugin_errors(errors, output.errors, source_map);
                apply_plugin_ops(directives, output.ops, errors, source_map)?;
            } else {
                // Not a native plugin — categorize and handle
                let plugin_path = std::path::Path::new(raw_name);
                let ext = plugin_path
                    .extension()
                    .and_then(|e| e.to_str())
                    .unwrap_or("")
                    .to_lowercase();

                // The closure is only invoked from inside the wasm-plugins /
                // python-plugins cfg blocks below. The whole function is
                // already `#[cfg(feature = "plugins")]`, so this only matters
                // when `plugins` is enabled but neither child feature is
                // (e.g. `--features native-plugins`). Allow `unused_variables`
                // for exactly that configuration. Underscore-prefixing the
                // binding would have been the wrong fix because we DO call
                // the closure in builds with one of the features enabled,
                // which would trip `no_effect_underscore_binding` instead.
                #[cfg_attr(
                    not(any(feature = "wasm-plugins", feature = "python-plugins")),
                    allow(unused_variables)
                )]
                let resolve_path = |name: &str| -> Result<std::path::PathBuf, String> {
                    let p = std::path::Path::new(name);
                    let resolved = if p.is_absolute() {
                        p.to_path_buf()
                    } else {
                        base_dir.join(name)
                    };

                    // Path security: prevent plugins from outside the ledger directory
                    if options.path_security
                        && let (Ok(canon_base), Ok(canon_plugin)) =
                            (base_dir.canonicalize(), resolved.canonicalize())
                        && !canon_plugin.starts_with(&canon_base)
                    {
                        return Err(format!(
                            "plugin path '{name}' is outside the ledger directory"
                        ));
                    }

                    Ok(resolved)
                };

                if ext == "wasm" {
                    // WASM plugin
                    #[cfg(feature = "wasm-plugins")]
                    {
                        let wasm_path = match resolve_path(raw_name) {
                            Ok(p) => p,
                            Err(e) => {
                                errors.push(LedgerError::error("PLUGIN", e).with_phase("plugin"));
                                continue;
                            }
                        };
                        let wrappers = build_wrappers(directives, source_map);
                        match run_wasm_plugin(&wasm_path, &wrappers, &plugin_options, plugin_config)
                        {
                            Ok((ops, plugin_errors)) => {
                                for err in plugin_errors {
                                    errors.push(err);
                                }
                                apply_plugin_ops(directives, ops, errors, source_map)?;
                            }
                            Err(e) => {
                                errors.push(
                                    LedgerError::error(
                                        "PLUGIN",
                                        format!("WASM plugin {} failed: {e}", wasm_path.display()),
                                    )
                                    .with_phase("plugin"),
                                );
                            }
                        }
                    }
                    #[cfg(not(feature = "wasm-plugins"))]
                    {
                        errors.push(
                            LedgerError::error(
                                "PLUGIN",
                                format!(
                                    "WASM plugin '{raw_name}' requires the wasm-plugins feature",
                                ),
                            )
                            .with_phase("plugin"),
                        );
                    }
                } else if *force_python
                    || ext == "py"
                    || raw_name.contains(std::path::MAIN_SEPARATOR)
                    || raw_name.contains('.')
                {
                    // Python module or file-based plugin (or force_python via "python:" prefix)
                    #[cfg(feature = "python-plugins")]
                    {
                        let resolved = match resolve_path(raw_name) {
                            Ok(p) => p,
                            Err(e) => {
                                errors.push(LedgerError::error("PLUGIN", e).with_phase("plugin"));
                                continue;
                            }
                        };
                        let wrappers = build_wrappers(directives, source_map);
                        match run_python_plugin(
                            raw_name,
                            &resolved,
                            base_dir,
                            &wrappers,
                            &plugin_options,
                            plugin_config,
                        ) {
                            Ok((ops, plugin_errors)) => {
                                for err in plugin_errors {
                                    errors.push(err);
                                }
                                apply_plugin_ops(directives, ops, errors, source_map)?;
                            }
                            Err(e) => {
                                errors.push(LedgerError::error("E8002", e).with_phase("plugin"));
                            }
                        }
                    }
                    #[cfg(not(feature = "python-plugins"))]
                    {
                        errors.push(
                            LedgerError::error(
                                "E8005",
                                format!(
                                    "Python plugin \"{raw_name}\" requires the python-plugins feature",
                                ),
                            )
                            .with_phase("plugin"),
                        );
                    }
                } else {
                    // Completely unknown plugin name — try to suggest a module path
                    #[cfg(feature = "python-plugins")]
                    {
                        use rustledger_plugin::python::{is_python_available, suggest_module_path};
                        let suggestion = if is_python_available() {
                            suggest_module_path(raw_name)
                        } else {
                            None
                        };
                        if let Some(module_path) = suggestion {
                            errors.push(
                                LedgerError::error(
                                    "E8004",
                                    format!(
                                        "Cannot resolve Python module '{raw_name}'. Replace with: plugin \"{module_path}\""
                                    ),
                                )
                                .with_phase("plugin"),
                            );
                        } else {
                            errors.push(
                                LedgerError::error(
                                    "E8001",
                                    format!("Plugin not found: \"{raw_name}\""),
                                )
                                .with_phase("plugin"),
                            );
                        }
                    }
                    #[cfg(not(feature = "python-plugins"))]
                    {
                        errors.push(
                            LedgerError::error(
                                "E8001",
                                format!("Plugin not found: \"{raw_name}\""),
                            )
                            .with_phase("plugin"),
                        );
                    }
                }
            }
        }
    }

    // No final wrapper→directive conversion needed: `apply_plugin_ops`
    // updates `directives` in place after each plugin call, preserving
    // original spans on Keep/Modify ops. Plugin-synthesized directives
    // (Insert ops) get `SYNTHESIZED_FILE_ID` and a zero span.
    Ok(())
}

/// Build a fresh `Vec<DirectiveWrapper>` from the current directives,
/// carrying filename + line number for plugin-side error reporting.
/// Spans don't need to round-trip through the wrappers — the loader
/// preserves them via `apply_plugin_ops` matching on op index.
#[cfg(feature = "plugins")]
fn build_wrappers(
    directives: &[Spanned<Directive>],
    source_map: &SourceMap,
) -> Vec<rustledger_plugin::DirectiveWrapper> {
    use rustledger_plugin::directive_to_wrapper_with_location;

    directives
        .iter()
        .map(|spanned| {
            let (filename, lineno) = if let Some(file) = source_map.get(spanned.file_id as usize) {
                let (line, _col) = file.line_col(spanned.span.start);
                (Some(file.path.display().to_string()), Some(line as u32))
            } else {
                (None, None)
            };
            directive_to_wrapper_with_location(&spanned.value, filename, lineno)
        })
        .collect()
}

/// Push plugin errors into the ledger's error stream, tagged with
/// `phase: "plugin"` and — when the plugin set `source_file` /
/// `line_number` on the error — an attached `ErrorLocation` so
/// downstream renderers (CLI, LSP, JSON output) can pinpoint where
/// the plugin objected.
///
/// Source-location resolution: if the wrapper's `source_file` resolves
/// to a real file in the source map, use that for `ErrorLocation.file`
/// and treat `line_number` as the line index. Plugin-synthesized
/// filenames (e.g. `"<auto_accounts>"`) that don't match any real
/// file are passed through as `PathBuf::from(name)` so the rendered
/// location still attributes the error to the originating plugin —
/// better than silently dropping the field.
#[cfg(feature = "plugins")]
fn record_plugin_errors(
    errors: &mut Vec<LedgerError>,
    plugin_errors: Vec<rustledger_plugin::PluginError>,
    source_map: &SourceMap,
) {
    for err in plugin_errors {
        let mut ledger_err = match err.severity {
            rustledger_plugin::PluginErrorSeverity::Error => {
                LedgerError::error("PLUGIN", err.message).with_phase("plugin")
            }
            rustledger_plugin::PluginErrorSeverity::Warning => {
                LedgerError::warning("PLUGIN", err.message).with_phase("plugin")
            }
        };
        // Propagate plugin-set source location into `ErrorLocation`.
        // Column defaults to 1 — plugin errors don't carry column info
        // through the wrapper protocol.
        if let (Some(file), Some(line)) = (&err.source_file, err.line_number) {
            let resolved_path = source_map
                .get_by_path(std::path::Path::new(file))
                .map_or_else(|| std::path::PathBuf::from(file), |f| f.path.clone());
            ledger_err = ledger_err.with_location(ErrorLocation {
                file: resolved_path,
                line: line as usize,
                column: 1,
            });
        }
        errors.push(ledger_err);
    }
}

/// Apply a plugin's `Vec<PluginOp>` to `directives` in place.
///
/// Validates that the op set forms a complete partition of the input
/// indices (each input index appears in exactly one `Keep` / `Modify` /
/// `Delete` op). Protocol violations produce a `PLUGIN` error in
/// `errors` and leave `directives` untouched.
///
/// For `Keep(i)` / `Modify(i, w)`, the resulting `Spanned<Directive>`
/// inherits `directives[i]`'s span and `file_id` — this is the core of
/// the ops protocol's correctness guarantee (plugin-transformed
/// directives keep their original source identity for error reporting).
/// `Insert(w)` directives get `(Span::new(0, 0), SYNTHESIZED_FILE_ID)`.
#[cfg(feature = "plugins")]
fn apply_plugin_ops(
    directives: &mut Vec<Spanned<Directive>>,
    ops: Vec<rustledger_plugin::PluginOp>,
    errors: &mut Vec<LedgerError>,
    source_map: &SourceMap,
) -> Result<(), ProcessError> {
    use rustledger_plugin::PluginOp;
    use rustledger_plugin::wrapper_to_directive;

    let n = directives.len();

    // Validate: every input index in {Keep, Modify, Delete} exactly once.
    let mut seen = vec![false; n];
    for op in &ops {
        let idx = match op {
            PluginOp::Keep(i) | PluginOp::Modify(i, _) | PluginOp::Delete(i) => Some(*i),
            PluginOp::Insert(_) => None,
        };
        if let Some(i) = idx {
            if i >= n {
                errors.push(
                    LedgerError::error(
                        "PLUGIN",
                        format!(
                            "plugin op references out-of-bounds input index {i} (input has {n} directives)"
                        ),
                    )
                    .with_phase("plugin"),
                );
                return Ok(());
            }
            if seen[i] {
                errors.push(
                    LedgerError::error(
                        "PLUGIN",
                        format!("plugin op references input index {i} more than once"),
                    )
                    .with_phase("plugin"),
                );
                return Ok(());
            }
            seen[i] = true;
        }
    }
    for (i, was_seen) in seen.iter().enumerate() {
        if !was_seen {
            errors.push(
                LedgerError::error(
                    "PLUGIN",
                    format!(
                        "plugin omitted input directive {i} (must appear in exactly one of Keep/Modify/Delete)"
                    ),
                )
                .with_phase("plugin"),
            );
            return Ok(());
        }
    }

    // Materialize new directives, preserving spans for Keep/Modify.
    let mut new_directives = Vec::with_capacity(ops.len());
    for op in ops {
        match op {
            PluginOp::Keep(i) => {
                new_directives.push(directives[i].clone());
            }
            PluginOp::Modify(i, wrapper) => {
                let directive = wrapper_to_directive(&wrapper)
                    .map_err(|e| ProcessError::PluginConversion(e.to_string()))?;
                new_directives.push(Spanned {
                    value: directive,
                    span: directives[i].span,
                    file_id: directives[i].file_id,
                });
            }
            PluginOp::Insert(wrapper) => {
                // Resolve the wrapper's filename + line number, if set,
                // into a real (file_id, span) when the filename
                // corresponds to a loaded source file. Falls back to
                // SYNTHESIZED_FILE_ID + zero span otherwise — including
                // for plugin-only attribution like `"<auto_accounts>"`
                // (which never matches a loaded file).
                let (span, file_id) = match (&wrapper.filename, wrapper.lineno) {
                    (Some(filename), Some(lineno)) => {
                        if let Some(file) = source_map.get_by_path(std::path::Path::new(filename)) {
                            let span_start = file.line_start(lineno as usize).unwrap_or(0);
                            (
                                rustledger_parser::Span::new(span_start, span_start),
                                file.id as u16,
                            )
                        } else {
                            (
                                rustledger_parser::Span::new(0, 0),
                                rustledger_parser::SYNTHESIZED_FILE_ID,
                            )
                        }
                    }
                    _ => (
                        rustledger_parser::Span::new(0, 0),
                        rustledger_parser::SYNTHESIZED_FILE_ID,
                    ),
                };
                let directive = wrapper_to_directive(&wrapper)
                    .map_err(|e| ProcessError::PluginConversion(e.to_string()))?;
                new_directives.push(Spanned::new(directive, span).with_file_id(file_id as usize));
            }
            PluginOp::Delete(_) => {}
        }
    }

    *directives = new_directives;
    Ok(())
}

/// Build a [`ValidationOptions`] from loader-level file options.
///
/// Factored out of the old `run_validation` so both the early and
/// late phases in `process()` can share the same `ValidationSession`
/// configuration. Document-dir resolution is relative to the main
/// file's parent directory.
#[cfg(feature = "validation")]
fn build_validation_options(
    file_options: &Options,
    source_map: &SourceMap,
) -> rustledger_validate::ValidationOptions {
    use rustledger_validate::ValidationOptions;

    // Resolve document directories relative to the main file's
    // directory. Absolute paths pass through; relative paths are
    // joined onto the source map's first file's parent. Matches the
    // pre-refactor `run_validation` behavior exactly.
    let base_dir = source_map
        .files()
        .first()
        .and_then(|f| f.path.parent())
        .unwrap_or_else(|| std::path::Path::new("."));

    let resolved_document_dirs: Vec<std::path::PathBuf> = file_options
        .documents
        .iter()
        .map(|d| {
            let path = std::path::Path::new(d);
            if path.is_absolute() {
                path.to_path_buf()
            } else {
                base_dir.join(path)
            }
        })
        .collect();

    let account_types: Vec<String> = file_options
        .account_types()
        .iter()
        .map(|s| (*s).to_string())
        .collect();

    ValidationOptions::default()
        .with_account_types(account_types)
        .with_document_dirs(resolved_document_dirs)
        .with_infer_tolerance_from_cost(file_options.infer_tolerance_from_cost)
        .with_tolerance_multiplier(file_options.inferred_tolerance_multiplier)
        .with_inferred_tolerance_default(file_options.inferred_tolerance_default.clone())
}

/// Convert a batch of [`rustledger_validate::ValidationError`]s into
/// loader-level [`LedgerError`]s (with resolved `file:line:column`
/// locations) and append to the existing list.
///
/// Factored out so both validation phases in `process()` share the
/// same conversion path.
#[cfg(feature = "validation")]
fn ledger_errors_extend(
    errors: &mut Vec<LedgerError>,
    validation_errors: Vec<rustledger_validate::ValidationError>,
    source_map: &SourceMap,
) {
    for err in validation_errors {
        let phase = if err.code.is_parse_phase() {
            "parse"
        } else {
            "validate"
        };
        let severity_level = if err.code.is_warning() {
            ErrorSeverity::Warning
        } else {
            ErrorSeverity::Error
        };
        // Fold the advisory note (if any) into the message so it propagates
        // through every downstream format (LedgerError, JSON diagnostic, CLI
        // report, LSP diagnostic) without each one needing a dedicated field.
        let message = match &err.note {
            Some(note) => format!("{err}\n  note: {note}"),
            None => err.to_string(),
        };
        // Resolve span + file_id into a file/line/column triple so CLI and
        // LSP consumers can render `file:line:col` headers without having
        // to do the lookup themselves (issue #901).
        let location = err.span.and_then(|span| {
            let fid = err.file_id? as usize;
            let file = source_map.get(fid)?;
            let (line, column) = file.line_col(span.start);
            Some(ErrorLocation {
                file: file.path.clone(),
                line,
                column,
            })
        });
        errors.push(LedgerError {
            severity: severity_level,
            code: err.code.code().to_string(),
            message,
            location,
            source_span: err.span.map(|s| (s.start, s.end)),
            file_id: err.file_id,
            phase: phase.to_string(),
        });
    }
}

/// Load and fully process a beancount file.
///
/// This is the main entry point, equivalent to Python's `loader.load_file()`.
/// It performs: parse → sort → synth-plugins → Early → book → regular-plugins → Late → finalize.
///
/// # Example
///
/// ```ignore
/// use rustledger_loader::{load, LoadOptions};
/// use std::path::Path;
///
/// let ledger = load(Path::new("ledger.beancount"), LoadOptions::default())?;
/// for error in &ledger.errors {
///     eprintln!("{}: {}", error.code, error.message);
/// }
/// ```
pub fn load(path: &Path, options: &LoadOptions) -> Result<Ledger, ProcessError> {
    let mut loader = crate::Loader::new();

    if options.path_security {
        loader = loader.with_path_security(true);
    }

    let raw = loader.load(path)?;
    process(raw, options)
}

/// Load a beancount file without processing.
///
/// This returns raw directives without sorting, booking, or plugins.
/// Use this when you need the original parse output.
pub fn load_raw(path: &Path) -> Result<LoadResult, LoadError> {
    crate::Loader::new().load(path)
}

/// Run a WASM plugin and return its output ops and errors.
#[cfg(feature = "wasm-plugins")]
fn run_wasm_plugin(
    wasm_path: &std::path::Path,
    directives: &[rustledger_plugin::DirectiveWrapper],
    options: &rustledger_plugin::PluginOptions,
    config: &Option<String>,
) -> Result<(Vec<rustledger_plugin::PluginOp>, Vec<LedgerError>), String> {
    use rustledger_plugin::{PluginInput, PluginManager};

    let mut mgr = PluginManager::new();
    let plugin_idx = mgr
        .load(wasm_path)
        .map_err(|e| format!("failed to load: {e}"))?;

    let input = PluginInput {
        directives: directives.to_vec(),
        options: options.clone(),
        config: config.clone(),
    };

    let output = mgr
        .execute(plugin_idx, &input)
        .map_err(|e| format!("execution failed: {e}"))?;

    let mut errors = Vec::new();
    for err in output.errors {
        let ledger_err = match err.severity {
            rustledger_plugin::PluginErrorSeverity::Error => {
                LedgerError::error("PLUGIN", err.message).with_phase("plugin")
            }
            rustledger_plugin::PluginErrorSeverity::Warning => {
                LedgerError::warning("PLUGIN", err.message).with_phase("plugin")
            }
        };
        errors.push(ledger_err);
    }

    Ok((output.ops, errors))
}

/// Run a Python module plugin via the WASI-based Python runtime.
#[cfg(feature = "python-plugins")]
fn run_python_plugin(
    module_name: &str,
    resolved_path: &std::path::Path,
    base_dir: &std::path::Path,
    directives: &[rustledger_plugin::DirectiveWrapper],
    options: &rustledger_plugin::PluginOptions,
    config: &Option<String>,
) -> Result<(Vec<rustledger_plugin::PluginOp>, Vec<LedgerError>), String> {
    use rustledger_plugin::{PluginInput, python::PythonRuntime};

    let runtime = PythonRuntime::new().map_err(|e| format!("Python runtime unavailable: {e}"))?;

    let input = PluginInput {
        directives: directives.to_vec(),
        options: options.clone(),
        config: config.clone(),
    };

    // Try file-based execution first, then module-based
    let is_file = resolved_path.exists()
        || std::path::Path::new(module_name)
            .extension()
            .is_some_and(|ext| ext.eq_ignore_ascii_case("py"))
        || module_name.contains(std::path::MAIN_SEPARATOR);

    let output = if is_file {
        runtime
            .execute_module(module_name, &input, Some(base_dir))
            .map_err(|e| format!("Python plugin execution failed: {e}"))?
    } else {
        runtime
            .execute_module(module_name, &input, Some(base_dir))
            .map_err(|e| format!("Python plugin '{module_name}' execution failed: {e}"))?
    };

    let mut errors = Vec::new();
    for err in output.errors {
        let ledger_err = match err.severity {
            rustledger_plugin::PluginErrorSeverity::Error => {
                LedgerError::error("PLUGIN", err.message).with_phase("plugin")
            }
            rustledger_plugin::PluginErrorSeverity::Warning => {
                LedgerError::warning("PLUGIN", err.message).with_phase("plugin")
            }
        };
        errors.push(ledger_err);
    }

    Ok((output.ops, errors))
}