dumpfs 0.1.0

#![allow(clippy::too_many_arguments)]
//! A module for scanning filesystem directories, collecting metadata,
//! and optionally file content, using parallel processing.

use crossbeam_channel::{Receiver, Sender, bounded};
use ignore::overrides::OverrideBuilder;
use ignore::{WalkBuilder, WalkState};
use indicatif::{ProgressBar, ProgressStyle};
use rayon::ThreadPoolBuilder;
use tracing::{debug, error, info, instrument, trace, warn};

use std::collections::HashMap;
use std::fs;
use std::io::{self, BufReader, Read};
#[cfg(unix)]
use std::os::unix::fs::PermissionsExt;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use std::thread;

mod error;
mod opts;
mod output;
mod stats;

use crate::tk;
pub use output::*;

use super::nodes::*;
use error::*;
use stats::*;

pub use error::ScannerError;
pub use opts::FsScannerOpts;
pub use stats::{FsScanReportFormat, FsScanReportOpts};

#[derive(Debug)]
pub struct FsScanner {
    root: PathBuf,
    opts: FsScannerOpts,
    cancel_flag: Arc<AtomicBool>,
    progress: Option<ProgressBar>, // Optional progress reporting handle
}

// --- Constants ---
const WALKER_PROGRESS_UPDATE_INTERVAL: usize = 200;
const TEXT_READER_BUFFER_SIZE: usize = 64 * 1024;
const DEFAULT_CHANNEL_CAPACITY: usize = 1000;

#[derive(Clone)]
struct ProcessorWorkerArgs {
    id: usize,
    receiver: Receiver<PathBuf>,
    opts: Arc<FsScannerOpts>,
    root: Arc<PathBuf>,
    stats: Arc<FsScanStats>,
    results: Arc<Mutex<Vec<FsNode>>>,
    cancel: Arc<AtomicBool>,
    progress: Option<ProgressBar>,
    critical_error: Arc<Mutex<Option<ScannerError>>>,
    tokenizer: Option<Arc<Box<dyn tk::TokenCounter>>>,
}

impl FsScanner {
    #[instrument(level = "debug", skip(opts, progress), fields(root = %root.as_ref().display()))]
    pub fn new(
        root: impl AsRef<Path>,
        mut opts: FsScannerOpts,
        progress: Option<ProgressBar>,
    ) -> Result<Self, ScannerError> {
        let root_path = root.as_ref();
        debug!("Initializing scanner...");

        // Validate and sanitize channel capacity
        if opts.channel_capacity == 0 {
            warn!(
                "opts.channel_capacity cannot be 0, setting to default: {}",
                DEFAULT_CHANNEL_CAPACITY
            );
            opts.channel_capacity = DEFAULT_CHANNEL_CAPACITY;
        }

        // Perform internal setup and validation
        Self::try_new_internal(root_path, opts, progress)
    }

    fn try_new_internal(
        r: &Path,
        o: FsScannerOpts,
        p: Option<ProgressBar>,
    ) -> Result<Self, ScannerError> {
        let canonical_root = fs::canonicalize(r).map_err(|e| ScannerError::CanonicalizeError {
            path: r.to_path_buf(),
            source: e,
        })?;

        if !canonical_root.is_dir() {
            return Err(ScannerError::InvalidRoot(canonical_root));
        }

        debug!(path = %canonical_root.display(), "Validated root path");

        trace!(options = ?o, "Scanner configured");

        Ok(Self {
            root: canonical_root,
            opts: o,
            cancel_flag: Arc::new(AtomicBool::new(false)), // Initialize cancel flag
            progress: p,
        })
    }

    #[allow(dead_code)]
    pub fn cancel_handle(&self) -> Arc<AtomicBool> {
        self.cancel_flag.clone()
    }

    #[instrument(level = "info", skip(self), fields(root = %self.root.display()))]
    pub fn scan(self) -> Result<FsScanOutput, ScannerError> {
        info!("Starting scan...");
        let start_time = std::time::Instant::now();
        let tokenizer: Option<Box<dyn tk::TokenCounter>> = if let Some(model) = self.opts.model {
            if self.opts.skip_content {
                warn!("Token counting requested (--model) but --skip-content is set.");
            }
            let project_dir_str = self.root.to_str().unwrap_or("."); // Fallback for cache path
            match tk::create(model, project_dir_str) {
                Ok(t) => {
                    debug!(?model, "Tokenizer created successfully.");
                    Some(t)
                }
                Err(e) => {
                    error!(?model, error = %e, "Failed to create tokenizer. Token counting will be disabled.");
                    None
                }
            }
        } else {
            None
        };

        // --- Pre-Scan Checks ---
        if self.cancel_flag.load(Ordering::Relaxed) {
            debug!("Scan cancelled before starting");
            return Err(ScannerError::Cancelled);
        }

        // --- Setup Shared State ---
        let stats = Arc::new(FsScanStats::default());
        // Use Mutex for collecting results; alternatives like concurrent collections exist but Mutex is simpler here.
        let results: Arc<Mutex<Vec<FsNode>>> = Arc::new(Mutex::new(Vec::new()));
        let opts = Arc::new(self.opts); // Clone opts into Arc for sharing
        let root = Arc::new(self.root.clone()); // Clone root into Arc for sharing
        let cancel_flag = self.cancel_flag.clone(); // Move ownership of the Arc
        let progress = self.progress; // Move ownership of the Option<ProgressBar>
        let tokenizer = tokenizer.map(Arc::new); // Wrap tokenizer in Arc for sharing
        // Stores critical errors that should halt the scan (e.g., channel errors, panics)
        let critical_error: Arc<Mutex<Option<ScannerError>>> = Arc::new(Mutex::new(None));

        // --- Create Communication Channel ---
        let (path_sender, path_receiver) = bounded::<PathBuf>(opts.channel_capacity);
        debug!(capacity = opts.channel_capacity, "Created path channel");

        // --- Build Processor Thread Pool ---
        let processor_thread_pool = ThreadPoolBuilder::new()
            .num_threads(opts.num_processor_threads.unwrap_or(0)) // 0 lets Rayon choose
            .build() // This can return ThreadPoolBuildError
            .map_err(ScannerError::from)?; // Convert error using From impl

        debug!(
            threads = processor_thread_pool.current_num_threads(),
            "Built Rayon processor thread pool."
        );

        // --- Setup Progress Bar ---
        let walker_paths_sent_counter = Arc::new(AtomicUsize::new(0)); // Separate counter for walker progress display
        if let Some(p) = &progress {
            p.set_style(
                ProgressStyle::default_spinner()
                    // P = Processed Items, W = Walked Paths Sent
                    .template("{spinner:.green} [{elapsed_precise}] Processed: {pos:>7} Walked: {msg} {wide_msg}")
                    .expect("Failed to set progress style template"),
            );
            p.reset_elapsed();
            p.set_position(0); // Tracks processed items
            p.set_message("0"); // Tracks walked paths sent
        }

        // --- Run Walker and Processors Concurrently ---
        info!("Starting parallel walk and processing...");
        // Use crossbeam::scope for structured concurrency. Ensures walker thread is joined.
        let processing_result = crossbeam::scope(|scope| {
            // --- Spawn Walker Thread ---
            let walker_handle = {
                // Clone shared state Arcs needed for the walker thread
                let walker_opts = Arc::clone(&opts);
                let walker_root = Arc::clone(&root);
                let walker_stats = Arc::clone(&stats);
                let walker_cancel = Arc::clone(&cancel_flag);
                let walker_progress = progress.clone(); // ProgressBar is clonable
                let walker_path_count = Arc::clone(&walker_paths_sent_counter);
                let walker_crit_err = Arc::clone(&critical_error);
                // Move the sender into the thread closure
                let sender = path_sender;

                scope
                    .builder()
                    .name("scanner-walker".into())
                    .spawn(move |_| {
                        Self::run_walker(
                            walker_opts,
                            walker_root,
                            walker_stats,
                            sender, // Takes ownership
                            walker_cancel,
                            walker_progress,
                            walker_path_count,
                            walker_crit_err,
                        )
                    })
                    .expect("FATAL: Failed to spawn scanner-walker thread") // Panic if spawn fails
            };

            // --- Run Processor Tasks within Rayon Pool ---
            // Use pool.install to ensure Rayon tasks run on the configured pool.
            let proc_result = processor_thread_pool.install(|| {
                Self::run_processors(
                    // Clone shared state Arcs needed for processors
                    Arc::clone(&opts),
                    Arc::clone(&root),
                    Arc::clone(&stats),
                    Arc::clone(&results), // Share the results Vec
                    path_receiver,        // Move the receiver
                    Arc::clone(&cancel_flag),
                    progress.clone(), // Clone progress bar again
                    Arc::clone(&critical_error),
                    tokenizer.clone(), // Clone Arc<tokenizer>
                )
            });

            // --- Wait for Walker Thread ---
            // Important: Join the walker *inside* the crossbeam scope.
            debug!("Waiting for walker thread to finish...");
            match walker_handle.join() {
                Ok(Ok(())) => {
                    debug!("Walker thread finished successfully.");
                }
                Ok(Err(e)) => {
                    // Walker function returned an error (e.g., build error, initial IO error)
                    error!(error = %e, "Walker thread returned a critical error");
                    // Store the critical error if one hasn't been stored already
                    let mut guard = critical_error
                        .lock()
                        .expect("Critical error mutex poisoned while handling walker error");
                    if guard.is_none() {
                        *guard = Some(e);
                    }
                }
                Err(_) => {
                    // Walker thread panicked
                    error!("Walker thread panicked!");
                    let mut guard = critical_error
                        .lock()
                        .expect("Critical error mutex poisoned while handling walker panic");
                    if guard.is_none() {
                        *guard = Some(ScannerError::WalkerThreadPanic);
                    }
                }
            };
            debug!("Walker thread joined.");

            // Return the result from the processor setup/scope execution (e.g. error within rayon::scope)
            // Note: Errors within individual processor tasks are handled via the critical_error mutex.
            proc_result
        })
        .map_err(|_scope_panic| {
            // This error occurs if the crossbeam::scope itself panicked, which is less likely
            // now that we explicitly handle walker thread panics via join().
            error!("Crossbeam scope panicked unexpectedly!");
            // Ensure critical_error reflects this if not already set
            let mut guard = critical_error
                .lock()
                .expect("Mutex poisoned handling scope panic");
            if guard.is_none() {
                *guard = Some(ScannerError::ParallelProcessingError(
                    "Crossbeam scope panicked".into(),
                ));
            }
            // Return the error captured in the mutex
            guard.take().unwrap() // Should always be Some here
        })?;

        // --- Check Results and Errors (Post-Scope) ---
        debug!("Walker and Processor scope finished. Checking for critical errors...");

        // Check if a critical error was recorded during the scan
        Self::check_scan_errors(&critical_error, &progress)?;

        // Check if the processor block itself returned an error (less common now)
        processing_result?;

        // --- Final Tree Construction ---
        if self.cancel_flag.load(Ordering::Relaxed) {
            debug!("Scan cancelled before tree construction");
            // Finish progress gracefully if cancelled here
            if let Some(p) = &progress {
                p.abandon_with_message("Scan cancelled");
            }
            return Err(ScannerError::Cancelled);
        }

        info!("Constructing final node tree...");
        if let Some(p) = &progress {
            p.set_length(0); // Switch from item count to indeterminate spinner
            p.set_message("Building tree...");
            p.tick(); // Ensure spinner updates
        }

        // Retrieve the collected nodes. Requires exclusive access now.
        let final_nodes = Arc::try_unwrap(results)
            .map_err(|_| {
                ScannerError::ParallelProcessingError(
                    "Failed to obtain exclusive ownership of results Arc".into(),
                )
            })?
            .into_inner()
            .map_err(|_| {
                ScannerError::ParallelProcessingError(
                    "Results mutex was poisoned before final tree construction".into(),
                )
            })?;

        debug!(
            node_count = final_nodes.len(),
            "Extracted final nodes for tree building"
        );

        let mut root_node =
            Self::construct_tree_from_flat_nodes(&root, final_nodes).map_err(|e| {
                error!(error=%e, "Tree construction failed");
                if let Some(p) = &progress {
                    p.abandon_with_message("Tree construction failed");
                }
                e
            })?;
        debug!("Tree construction complete.");
        root_node.path = self.root;

        // --- Finalize ---
        let final_stats = Arc::try_unwrap(stats).map_err(|_| {
            ScannerError::ParallelProcessingError(
                "Failed to obtain exclusive ownership of stats Arc".into(),
            )
        })?;

        let duration = start_time.elapsed();
        let processed_count = final_stats.total_entries_processed();
        info!(
            duration = ?duration,
            processed = processed_count,
            skipped = final_stats.total_entries_skipped(),
            "Scan finished successfully."
        );

        if let Some(p) = &progress {
            p.finish_with_message(format!(
                "Scan complete! {} entries processed ({:.2?})",
                processed_count, duration
            ));
        }

        Ok(FsScanOutput {
            root: root_node,
            stats: final_stats,
            duration, // Include duration in the output
        })
    }

    fn check_scan_errors(
        critical_error: &Arc<Mutex<Option<ScannerError>>>,
        progress: &Option<ProgressBar>,
    ) -> Result<(), ScannerError> {
        match critical_error.lock() {
            Ok(mut guard) => {
                if let Some(err) = guard.take() {
                    // Take ownership of the error
                    error!(error = %err, "Critical error occurred during scan");
                    if let Some(p) = progress {
                        p.abandon_with_message(format!("Scan failed: {}", err));
                    }
                    return Err(err); // Return the captured critical error
                }
            }
            Err(poisoned) => {
                // Mutex poisoned - this is also a critical failure
                error!("Critical error mutex was poisoned! {}", poisoned);
                if let Some(p) = progress {
                    p.abandon_with_message("Scan failed: Internal mutex poisoned");
                }
                return Err(ScannerError::ParallelProcessingError(
                    "Critical error mutex poisoned".into(),
                ));
            }
        }
        // No critical error found
        Ok(())
    }

    #[instrument(level = "debug", skip_all, fields(walker_id=format!("{:?}", thread::current().id())))]
    fn run_walker(
        opts: Arc<FsScannerOpts>,
        root: Arc<PathBuf>,
        stats: Arc<FsScanStats>,
        path_sender: Sender<PathBuf>, // Takes ownership of the sender
        cancel_flag: Arc<AtomicBool>,
        progress: Option<ProgressBar>,
        walker_paths_sent_counter: Arc<AtomicUsize>,
        critical_error: Arc<Mutex<Option<ScannerError>>>,
    ) -> Result<(), ScannerError> {
        debug!("Walker thread starting.");

        let mut walker_builder = WalkBuilder::new(&*root); // Deref Arc<PathBuf> to get &Path
        walker_builder
            .standard_filters(!opts.no_gitignore)
            .git_global(!opts.no_git_global)
            .git_exclude(!opts.no_git_exclude)
            .follow_links(opts.follow_symlinks)
            .max_depth(opts.max_depth)
            .threads(opts.num_walker_threads.unwrap_or_else(num_cpus::get))
            .skip_stdout(true);

        if let Some(custom_ignore) = &opts.custom_ignore_path {
            walker_builder.add_custom_ignore_filename(custom_ignore);
        }

        // Apply include/exclude overrides (potential WalkBuildError)
        if !opts.ignore_patterns.is_empty() || !opts.include_patterns.is_empty() {
            debug!("Walker building path overrides...");
            let mut override_builder = OverrideBuilder::new(&*root); // Relative to root
            for pattern in &opts.ignore_patterns {
                // Use `?` to propagate ignore::Error, converted to ScannerError::WalkBuildError
                // ignore patterns are negated when added to OverrideBuilder
                override_builder.add(&format!("!{}", pattern))?;
            }
            for pattern in &opts.include_patterns {
                override_builder.add(pattern)?; // Add include patterns directly
            }
            let overrides = override_builder.build()?; // Can return ignore::Error
            trace!("Walker applying path overrides.");
            walker_builder.overrides(overrides);
        }

        // Build the parallel walker itself
        let walker = walker_builder.build_parallel();
        debug!("Walker built, starting parallel walk execution...");

        // --- Run Walk ---
        // Atomic flag to signal other walker threads to stop if channel send fails.
        let send_error_occurred = AtomicBool::new(false);

        walker.run(|| {
            // Clone shared Arcs and Sender for the closure executed by each walker thread
            let sender = path_sender.clone();
            let cancel = Arc::clone(&cancel_flag);
            let stats_local = Arc::clone(&stats); // Use different name to avoid confusion
            let path_count_local = Arc::clone(&walker_paths_sent_counter);
            let prog_local = progress.clone();
            // Borrow the atomic bool - must use a reference inside the closure
            let send_err_flag_ref = &send_error_occurred;

            Box::new(move |entry_result| {
                // Check cancellation and critical channel error flags first
                if cancel.load(Ordering::Relaxed) || send_err_flag_ref.load(Ordering::Relaxed) {
                    return WalkState::Quit; // Stop walking immediately
                }

                match entry_result {
                    Ok(entry) => {
                        // We usually don't process the root directory itself (depth 0) here,
                        // only its contents. Processing starts from depth 1.
                        if entry.depth() > 0 {
                            let path = entry.into_path(); // Take ownership

                            // Try sending the path to the processors
                            if sender.send(path).is_err() {
                                // Error means receiver is dropped (processors likely finished/panicked)
                                trace!(
                                    "Walker detected channel receiver dropped, signalling quit."
                                );
                                // Set the shared flag so other walker threads also quit quickly
                                send_err_flag_ref.store(true, Ordering::Relaxed);
                                return WalkState::Quit;
                            } else {
                                // Successfully sent, increment counters and update progress message periodically
                                let count = path_count_local.fetch_add(1, Ordering::Relaxed) + 1;
                                if count % WALKER_PROGRESS_UPDATE_INTERVAL == 0 {
                                    if let Some(p) = &prog_local {
                                        // Update the message part of the progress bar
                                        p.set_message(format!("{}", count));
                                    }
                                }

                                stats_local
                                    .walker_paths_sent
                                    .fetch_add(1, Ordering::Relaxed);
                            }
                        } else {
                            // Trace if we encounter the root, but generally do nothing with it here.
                            trace!("Walker skipping root path entry (depth 0).");
                        }
                    }
                    Err(e) => {
                        // An error occurred traversing *this specific path* (e.g., permission denied on a dir).
                        // The walk continues for other paths unless it's a fatal error.
                        warn!(error = %e, "Walker encountered error traversing filesystem entry");
                        // Increment skip count for errors encountered by the walker itself
                        stats_local
                            .entries_skipped_error
                            .fetch_add(1, Ordering::Relaxed);
                    }
                }
                // Continue walking unless Quit was returned
                WalkState::Continue
            })
        });

        // --- Finalize Walker ---
        // Update the progress message one last time with the final count
        if let Some(p) = &progress {
            p.set_message(format!(
                "{}",
                walker_paths_sent_counter.load(Ordering::Relaxed)
            ));
            p.tick(); // Ensure message update is displayed
        }
        debug!("Walker thread finished walk execution loop.");

        // After the walk, check if the send error flag was set
        if send_error_occurred.load(Ordering::Relaxed) {
            // Lock the critical error mutex to report the channel error, if not already set
            let mut guard = critical_error
                .lock()
                .expect("Critical error mutex poisoned after walker finish");
            if guard.is_none() {
                warn!(
                    "Walker stopped prematurely due to channel send error (receiver disconnected)."
                );
                *guard = Some(ScannerError::ChannelSendError);
            }
        }

        // If the walker setup succeeded and the walk finished (or quit gracefully), return Ok.
        // Specific file errors during walk are counted in stats.
        // Critical errors (build, channel) are returned or set in critical_error.
        Ok(())
    }

    #[instrument(level = "debug", skip_all)]
    fn run_processors(
        opts: Arc<FsScannerOpts>,
        root: Arc<PathBuf>,
        stats: Arc<FsScanStats>,
        results: Arc<Mutex<Vec<FsNode>>>,
        path_receiver: Receiver<PathBuf>, // Takes ownership
        cancel_flag: Arc<AtomicBool>,
        progress: Option<ProgressBar>,
        critical_error: Arc<Mutex<Option<ScannerError>>>,
        tokenizer: Option<Arc<Box<dyn tk::TokenCounter>>>, // Receive tokenizer Arc
    ) -> Result<(), ScannerError> {
        // Errors primarily relate to scope setup or panic
        debug!("Processor scope starting...");

        // Use rayon::scope to manage the lifecycle of worker tasks running on the pool.
        rayon::scope(|s| {
            let num_workers = rayon::current_num_threads(); // Get actual number of threads in the pool
            debug!(workers = num_workers, "Spawning processor worker tasks.");

            for i in 0..num_workers {
                let worker_args = ProcessorWorkerArgs {
                    id: i,
                    receiver: path_receiver.clone(), // Crucial: Clone receiver for multi-consumer
                    opts: Arc::clone(&opts),
                    root: Arc::clone(&root),
                    stats: Arc::clone(&stats),
                    results: Arc::clone(&results),
                    cancel: Arc::clone(&cancel_flag),
                    progress: progress.clone(),
                    critical_error: Arc::clone(&critical_error),
                    tokenizer: tokenizer.clone(), // Clone tokenizer Arc for worker
                };

                // Spawn a task within the Rayon scope
                s.spawn(move |_| {
                    // `move` captures the `worker_args` struct
                    Self::processor_worker_loop(worker_args); // Call the loop logic
                });
            }

            // Scope implicitly waits for all spawned tasks to complete here.
        });

        debug!("Processor scope finished execution.");

        // Check for critical errors again *after* the scope finishes.
        // This catches mutex poisoning that might occur right at the end.
        if critical_error
            .lock()
            .expect("Mutex poisoned checking after processor scope")
            .is_some()
        {
            warn!("Processing scope completed, but a critical error was detected.");
            // Let the caller (scan method) handle the error via check_scan_errors
        }

        // If the scope itself completes without panicking, return Ok.
        // Individual processing errors are logged/counted, critical errors are in the mutex.
        Ok(())
    }

    fn processor_worker_loop(args: ProcessorWorkerArgs) {
        // Takes ownership of cloned args
        let worker_id_str = format!("proc-{}", args.id);
        // Create a span for logging within this specific worker context
        let _span = tracing::debug_span!("processor_worker", id = %worker_id_str).entered();
        debug!("Worker started.");

        loop {
            // --- Check for Exit Conditions (Cancellation / Critical Error) ---
            // Use Relaxed ordering: if we miss one check, the next loop iteration or lock attempt will catch it.
            if args.cancel.load(Ordering::Relaxed) {
                trace!("Cancellation detected, worker exiting.");
                break;
            }
            // Check critical error using a lock. Expect is okay here as poisoning is handled by the caller scope.
            if args
                .critical_error
                .lock()
                .expect("Critical error mutex poisoned in worker check")
                .is_some()
            {
                trace!("Critical error detected by another thread, worker exiting.");
                break;
            }

            // --- Receive Path from Channel ---
            // `recv()` blocks until a path is available or the channel is closed.
            match args.receiver.recv() {
                Ok(abs_path) => {
                    // Received a path to process
                    let path_str_for_display = abs_path.display().to_string();
                    // Trace span for processing a single path
                    let processing_span =
                        tracing::trace_span!("process_path", path=%path_str_for_display);
                    let _enter = processing_span.enter();

                    // --- Process Path ---
                    // Call the main processing logic function, passing the tokenizer
                    let process_result = Self::process_path_to_node(
                        &abs_path,
                        &args.opts,
                        &args.stats,
                        &args.root,
                        args.tokenizer.as_deref().map(|v| &**v),
                    );

                    // --- Update Progress (Crucial: AFTER processing attempt) ---
                    // Increment the main position counter of the progress bar for every item processed (or skipped/errored).
                    // Briefly update message to show the currently processed path. This might flicker fast.
                    if let Some(p) = &args.progress {
                        // This updates the {wide_msg} part
                        p.set_message(path_str_for_display);
                        // This updates the {pos} part (Processed count)
                        p.inc(1);
                    }

                    // --- Handle Processing Result ---
                    match process_result {
                        Ok(Some(node)) => {
                            // Successfully processed into a Node
                            // Lock the results mutex to add the node
                            match args.results.lock() {
                                Ok(mut guard) => {
                                    guard.push(node);
                                    // Drop guard automatically unlocks mutex
                                }
                                Err(poisoned) => {
                                    // FATAL for this worker: Mutex is poisoned.
                                    error!(
                                        "Results mutex poisoned during node push! Worker exiting. Error: {}",
                                        poisoned
                                    );
                                    // Report critical error globally
                                    let mut crit_guard = args
                                        .critical_error
                                        .lock()
                                        .expect("CRITICAL+RESULTS mutexes poisoned!");
                                    if crit_guard.is_none() {
                                        *crit_guard = Some(ScannerError::ParallelProcessingError(
                                            "Results mutex poisoned".into(),
                                        ));
                                    }
                                    // Stop this worker's loop. Other workers will see the critical error flag.
                                    break;
                                }
                            }
                        }
                        Ok(None) => {
                            // Path was validly processed but filtered out (e.g., by date, perms) or was 'Other' type.
                            trace!("Path skipped by processing logic (filter/type)");
                        }
                        Err(e) => {
                            // An error occurred during processing this specific path (e.g., IO error reading metadata/content).
                            // The error is logged by process_path_to_node/helpers. Stat already incremented.
                            warn!(path = %abs_path.display(), error = %e, "Non-critical error processing path");
                            // No 'break' needed here; we continue processing other paths.
                            // `entries_skipped_error` stat is incremented within process_path_to_node or its helpers.
                        }
                    } // End match process_result
                }
                Err(_) => {
                    // `RecvError` means the channel is empty *and* all Senders have been dropped.
                    // This is the primary signal that the walker has finished.
                    trace!("Channel closed (walker finished), worker exiting.");
                    break; // Exit the loop naturally
                }
            } // End match recv
        } // End loop
        debug!("Worker finished processing loop.");
    } // End processor_worker_loop

    #[instrument(level = "trace", skip_all, fields(path = %abs_path.display()))]
    fn process_path_to_node(
        abs_path: &Path,
        opts: &FsScannerOpts,
        stats: &FsScanStats,
        root: &Path,
        tokenizer: Option<&dyn tk::TokenCounter>,
    ) -> Result<Option<FsNode>, ScannerError> {
        // Convert absolute path to relative path based on the scan root.
        // This can fail if abs_path doesn't start with root (shouldn't happen with ignore::Walk).
        let rel_path = match abs_path.strip_prefix(root) {
            Ok(p) => p.to_path_buf(),
            Err(_) => {
                // This is unexpected if the walker starts correctly. Treat as processing error.
                stats.entries_skipped_error.fetch_add(1, Ordering::Relaxed);
                warn!(path=%abs_path.display(), root=%root.display(), "Failed to strip root prefix");
                return Err(ScannerError::StripPrefixError {
                    prefix: root.to_path_buf(),
                    path: abs_path.to_path_buf(),
                });
            }
        };

        // --- Get Metadata (using symlink_metadata to handle links correctly) ---
        let entry_meta = match fs::symlink_metadata(abs_path) {
            Ok(m) => m,
            Err(e) => {
                // Failed to get metadata - skip this entry and count as error.
                stats.entries_skipped_error.fetch_add(1, Ordering::Relaxed);
                // Log using the macro which adds context
                return Err(map_meta_err(abs_path, e));
            }
        };
        let file_type = entry_meta.file_type(); // Use std::fs::FileType for basic type checks

        // --- Filtering ---
        // Filter by modification time
        match entry_meta.modified() {
            Ok(modified_time) => {
                if let Some(after) = opts.modified_after {
                    if modified_time < after {
                        trace!(filter = "mod_after", path=%abs_path.display());
                        stats.entries_skipped_date.fetch_add(1, Ordering::Relaxed);
                        return Ok(None); // Filtered out
                    }
                }
                if let Some(before) = opts.modified_before {
                    if modified_time > before {
                        trace!(filter = "mod_before", path=%abs_path.display());
                        stats.entries_skipped_date.fetch_add(1, Ordering::Relaxed);
                        return Ok(None); // Filtered out
                    }
                }
            }
            Err(e) => {
                // Failed to get modification time - treat as metadata error.
                stats.entries_skipped_error.fetch_add(1, Ordering::Relaxed);
                return Err(map_meta_err(abs_path, e));
            }
        }

        // Filter by permissions (Unix only for now)
        #[cfg(unix)]
        if let Some(filter_perms) = opts.permissions_filter {
            // Mask mode bits to compare relevant parts (owner/group/other R/W/X)
            let mode = entry_meta.permissions().mode() & 0o777; // Consider just standard rwx bits
            if mode != filter_perms {
                trace!(filter = "perm", path=%abs_path.display(), mode=format!("{:#o}", mode), expected=format!("{:#o}", filter_perms));
                stats
                    .entries_skipped_permission
                    .fetch_add(1, Ordering::Relaxed);
                return Ok(None); // Filtered out
            }
        }
        #[cfg(not(unix))]
        if opts.permissions_filter.is_some() {
            // Warn if permission filtering is requested on non-Unix systems, as it won't work.
            // Do this only once maybe? Or rely on documentation. For now, no warning.
            // Simply ignore the filter on non-Unix.
        }

        // --- Extract Filename ---
        // Ensure filename is valid UTF-8. If not, skip the entry.
        let name = match abs_path.file_name().and_then(|n| n.to_str()) {
            Some(n) => n.to_string(),
            None => {
                warn!(path=%abs_path.display(), "Skipping entry with non-UTF8 filename");
                stats.entries_skipped_error.fetch_add(1, Ordering::Relaxed);
                return Ok(None); // Filtered out due to bad name encoding
            }
        };

        // --- Node Creation based on FileType ---
        let node = if file_type.is_symlink() {
            // Process Symlink
            let link_metadata_result = Self::create_metadata_struct(&entry_meta); // Get metadata of the link itself
            let link_metadata = match link_metadata_result {
                Ok(md) => md,
                Err(e) => {
                    stats.entries_skipped_error.fetch_add(1, Ordering::Relaxed);
                    return Err(map_meta_err(abs_path, e));
                }
            };

            let target_path_result = fs::read_link(abs_path);
            let target_path = match target_path_result {
                Ok(tp) => tp,
                Err(e) => {
                    stats.entries_skipped_error.fetch_add(1, Ordering::Relaxed);
                    return Err(map_symlink_err(abs_path, e)); // Use symlink error macro
                }
            };

            // Optionally check if target exists/is_dir (only if following symlinks)
            let (target_exists, target_is_dir) = if opts.follow_symlinks {
                // Use `fs::metadata` which resolves the link.
                // Distinguish between Not Found and other errors.
                match fs::metadata(abs_path) {
                    Ok(target_meta) => (Some(true), Some(target_meta.is_dir())),
                    Err(e) if e.kind() == io::ErrorKind::NotFound => (Some(false), None),
                    Err(e) => {
                        trace!(path=%abs_path.display(), target=%target_path.display(), error=%e, "Could not resolve symlink target metadata");
                        (None, None) // Indicate error occurred trying to resolve
                    }
                }
            } else {
                (None, None) // Not following links, don't check target status
            };

            // Update stats for processed symlink
            stats.symlinks_processed.fetch_add(1, Ordering::Relaxed);
            stats
                .total_bytes
                .fetch_add(link_metadata.size, Ordering::Relaxed); // Use link's size? Or target size if resolved? Using link meta size here.

            FsNode::Symlink(FsSymlinkNode {
                name,
                path: rel_path,
                metadata: link_metadata,
                target: target_path.to_string_lossy().to_string(), // Store target path lossily
                target_is_dir,
                target_exists,
            })
        } else if file_type.is_dir() {
            // Process Directory
            let dir_metadata_result = Self::create_metadata_struct(&entry_meta);
            let dir_metadata = match dir_metadata_result {
                Ok(md) => md,
                Err(e) => {
                    stats.entries_skipped_error.fetch_add(1, Ordering::Relaxed);
                    return Err(map_meta_err(abs_path, e));
                }
            };

            stats.directories_processed.fetch_add(1, Ordering::Relaxed);
            stats
                .total_bytes
                .fetch_add(dir_metadata.size, Ordering::Relaxed);

            FsNode::Directory(FsDirectoryNode {
                name,
                path: rel_path,
                metadata: dir_metadata,
                contents: Vec::new(), // Contents populated during tree construction phase
            })
        } else if file_type.is_file() {
            // Process Regular File (requires heuristic)
            let initial_metadata_result = Self::create_metadata_struct(&entry_meta);
            let mut file_metadata = match initial_metadata_result {
                Ok(md) => md,
                Err(e) => {
                    stats.entries_skipped_error.fetch_add(1, Ordering::Relaxed);
                    return Err(map_meta_err(abs_path, e));
                }
            };

            // Determine if Text or Binary using heuristic (can return IO error)
            // This needs the *absolute* path for file reading.
            file_metadata.file_type =
                match Self::determine_file_type_heuristic(abs_path, &file_metadata, opts, stats) {
                    Ok(ft) => ft, // Returns FileType::TextFile or FileType::BinaryFile
                    Err(e) => {
                        // Heuristic failed (e.g., couldn't open/read file for detection).
                        // Stat already incremented in helper. Return the error.
                        return Err(e);
                    }
                };

            match file_metadata.file_type {
                FsFileType::TextFile => {
                    // Initialize optional fields
                    let mut content: Option<String> = None;
                    let mut lines: Option<usize> = None;
                    let mut chars: Option<usize> = None;
                    let mut token_count: Option<usize> = None; // Initialize token count

                    // Check if file is too large for content reading
                    let is_too_large = opts
                        .max_file_size_for_content
                        .is_some_and(|max_size| file_metadata.size > max_size);

                    if is_too_large {
                        stats
                            .files_skipped_large_content
                            .fetch_add(1, Ordering::Relaxed);
                        trace!(path=%abs_path.display(), size=file_metadata.size, limit=opts.max_file_size_for_content, "Skipping content read due to size limit");
                    }

                    // Attempt to read content if enabled, not too large, and potentially non-empty
                    if !opts.skip_content && !is_too_large {
                        // Handle zero-byte files explicitly as empty content
                        if file_metadata.size == 0 {
                            content = Some(String::new());
                            lines = Some(0);
                            chars = Some(0);
                            token_count = Some(0);
                            trace!(path=%abs_path.display(), "Processed zero-byte file as empty text content.");
                        } else {
                            // Use the content reading function (needs absolute path)
                            match Self::read_text_content_details(abs_path) {
                                Ok((mut read_content, mut line_count, mut char_count)) => {
                                    // Made mutable
                                    // --- Potentially remove inline test modules ---
                                    if opts.ignore_inline_tests
                                        && abs_path.extension().is_some_and(|ext| ext == "rs")
                                        && !read_content.is_empty()
                                    {
                                        let modified_content =
                                            Self::remove_rust_test_modules(&read_content);
                                        if modified_content.len() != read_content.len() {
                                            read_content = modified_content;
                                            // Recalculate lines and chars for the modified content
                                            char_count = read_content.chars().count();
                                            line_count = read_content.lines().count();
                                            trace!(path=%abs_path.display(), "Removed inline test modules, new lines: {}, new chars: {}", line_count, char_count);
                                        }
                                    }
                                    // --- End of test module removal ---

                                    content = Some(read_content); // Use potentially modified content
                                    lines = Some(line_count);
                                    chars = Some(char_count);
                                    // Store details in the stats map
                                    stats.file_details.insert(
                                        rel_path.clone(),
                                        FileDetail {
                                            lines: line_count,
                                            chars: char_count,
                                        },
                                    );
                                    stats.total_lines.fetch_add(line_count, Ordering::Relaxed);
                                    stats.total_chars.fetch_add(char_count, Ordering::Relaxed);

                                    // --- Count Tokens (if tokenizer available and content read) ---
                                    if let (Some(tok), Some(cont)) = (tokenizer, &content) {
                                        if !cont.is_empty() {
                                            // Avoid counting tokens for empty strings
                                            match tok.count_tokens(cont) {
                                                Ok(count_result) => {
                                                    token_count = Some(count_result.tokens);
                                                    stats.total_tokens.fetch_add(
                                                        count_result.tokens,
                                                        Ordering::Relaxed,
                                                    );
                                                    trace!(path=%abs_path.display(), tokens=count_result.tokens, cached=?count_result.cached, "Tokens counted");
                                                }
                                                Err(e) => {
                                                    warn!(path=%abs_path.display(), error=%e, "Failed to count tokens for file");
                                                }
                                            }
                                        } else {
                                            token_count = Some(0);
                                            trace!(path=%abs_path.display(), "Token count is 0 for empty file");
                                        }
                                    }
                                }
                                Err(read_err) => {
                                    // Failed to read content (IO error, UTF8 error).
                                    // Log the error. Stat for read errors incremented below.
                                    warn!(path=%abs_path.display(), error=%read_err, "Failed to read or decode text file content");
                                    // Increment stat specifically for read errors on files assumed to be text.
                                    stats
                                        .files_skipped_read_error
                                        .fetch_add(1, Ordering::Relaxed);
                                    // Also count as a general processing error.
                                    stats.entries_skipped_error.fetch_add(1, Ordering::Relaxed);
                                    // Keep content/lines/chars as None. Proceed to create the FileNode without content.
                                }
                            }
                        }
                    }

                    // Update overall stats for text files
                    stats.text_files_processed.fetch_add(1, Ordering::Relaxed);
                    stats
                        .total_bytes
                        .fetch_add(file_metadata.size, Ordering::Relaxed);

                    FsNode::File(FsFileNode {
                        name,
                        path: rel_path,
                        metadata: file_metadata, // Contains updated file_type
                        content,
                        lines,
                        chars,
                        token_count, // Add token_count field
                    })
                }
                FsFileType::BinaryFile => {
                    // Update stats for binary files
                    stats.binary_files_processed.fetch_add(1, Ordering::Relaxed);
                    stats
                        .total_bytes
                        .fetch_add(file_metadata.size, Ordering::Relaxed);

                    FsNode::Binary(FsBinaryNode {
                        name,
                        path: rel_path,
                        metadata: file_metadata, // Contains updated file_type
                    })
                }
                // These cases should not be reachable if heuristic works correctly for is_file()
                FsFileType::Directory | FsFileType::Symlink | FsFileType::Other => {
                    unreachable!(
                        "Heuristic determined Text/Binary, but got {:?}",
                        file_metadata.file_type
                    )
                }
            }
        } else {
            // Entry is not a regular file, directory, or symlink (e.g., socket, device).
            // Skip these types silently.
            trace!(path=%abs_path.display(), "Skipping entry of unsupported type: {:?}", file_type);
            stats.entries_skipped_error.fetch_add(1, Ordering::Relaxed); // Optionally count as skipped/error
            return Ok(None); // Filtered out
        };

        // If we reached here, a valid node was created
        Ok(Some(node))
    }

    // --- Helper Functions (Internal to Scanner impl) ---

    fn create_metadata_struct(fs_meta: &fs::Metadata) -> Result<FsMetadata, io::Error> {
        // Determine base FileType from std::fs::FileType
        let ft = fs_meta.file_type();
        let base_file_type = if ft.is_symlink() {
            FsFileType::Symlink
        } else if ft.is_dir() {
            FsFileType::Directory
        } else if ft.is_file() {
            FsFileType::BinaryFile
        } else {
            FsFileType::Other
        };

        // Extract permissions (conditionally compiled for Unix)
        let permissions = {
            #[cfg(unix)]
            {
                // Mask to get standard user/group/other permissions + sticky/setuid/setgid
                fs_meta.permissions().mode() & 0o170777
            }
            #[cfg(not(unix))]
            {
                // Provide a sensible default (0 or perhaps a constant indicating unknown/unsupported)
                // Using 0 for simplicity. Users on non-Unix cannot filter by permissions reliably.
                0
            }
        };

        Ok(FsMetadata {
            size: fs_meta.len(),
            // Propagate IO error if modification time retrieval fails
            modified: fs_meta.modified()?,
            permissions,
            file_type: base_file_type,
        })
    }

    #[instrument(level = "trace", skip(path, metadata, opts, stats), fields(path = %path.display()))]
    fn determine_file_type_heuristic(
        path: &Path,           // Absolute path needed for reading
        metadata: &FsMetadata, // Provide existing metadata for size check
        opts: &FsScannerOpts,
        stats: &FsScanStats,
    ) -> Result<FsFileType, ScannerError> {
        // Returns IO errors related to detection read
        trace!("Running heuristic file type determination...");

        // Handle obvious cases first
        if metadata.size == 0 {
            trace!("Classified as TextFile (zero size)");
            return Ok(FsFileType::TextFile); // Empty file is considered text
        }
        if opts
            .max_file_size_for_content // Reuse this option for heuristic buffer check? Or add a separate one?
            .is_some_and(|max| metadata.size > max)
        {
            // If we wouldn't even *read* the content due to size, classify as binary preemptively.
            // Avoids reading large files just for the heuristic.
            trace!("Classified as BinaryFile preemptively (exceeds max size limit)");
            return Ok(FsFileType::BinaryFile);
        }

        // Determine buffer size to read for heuristic check
        let buffer_size_to_read = std::cmp::min(
            opts.text_detection_buffer_size,
            metadata.size.try_into().unwrap_or(usize::MAX), // Read up to buffer_size or file size
        );

        // If calculated buffer size is 0 (e.g., tiny file fits in usize::MAX but opt buffer is huge)
        // but metadata.size > 0, we still consider it text as we read nothing.
        if buffer_size_to_read == 0 {
            trace!("Classified as TextFile (effective buffer size 0, but file not empty)");
            return Ok(FsFileType::TextFile);
        }

        // --- Read Initial Chunk ---
        let mut buffer = vec![0; buffer_size_to_read];
        let mut file = match fs::File::open(path) {
            Ok(f) => f,
            Err(e) => {
                warn!(path=%path.display(), error = %e, "Failed to open file for type detection heuristic");
                stats.entries_skipped_error.fetch_add(1, Ordering::Relaxed);
                // Return a specific IO error context
                return Err(ScannerError::Io {
                    path: path.to_path_buf(),
                    source: e,
                });
            }
        };

        match file.read(&mut buffer) {
            Ok(0) => {
                // Read 0 bytes even though size > 0 and buffer > 0? Strange, but treat as text.
                trace!(
                    "Read 0 bytes during heuristic check despite non-zero size, classifying as TextFile"
                );
                Ok(FsFileType::TextFile)
            }
            Ok(bytes_read) => {
                // We only care about the bytes actually read
                let data_slice = &buffer[..bytes_read];

                // Check 1: Presence of NULL bytes is a strong indicator of binary data.
                if memchr::memchr(b'\0', data_slice).is_some() {
                    trace!("Classified as BinaryFile (found null byte)");
                    return Ok(FsFileType::BinaryFile);
                }

                // Check 2: UTF-8 validity and character ratio (only if no null bytes found)
                match std::str::from_utf8(data_slice) {
                    Ok(s) => {
                        // It's valid UTF-8 up to this point. Check character properties.
                        let total_chars = s.chars().count();
                        if total_chars == 0 {
                            // Valid UTF-8 but no characters? Possible with only BOM? Treat as text.
                            trace!("Classified as TextFile (valid UTF-8 chunk with 0 chars)");
                            return Ok(FsFileType::TextFile);
                        }

                        // Count non-control characters (excluding whitespace which is allowed in text)
                        let printable_chars = s
                            .chars()
                            .filter(|c| !c.is_control() || c.is_whitespace())
                            .count();

                        // Calculate ratio
                        let ratio = printable_chars as f32 / total_chars as f32;
                        trace!(
                            printable = printable_chars,
                            total = total_chars,
                            ratio = ratio,
                            threshold = opts.text_detection_ratio,
                            "Heuristic ratio calculated"
                        );

                        if ratio >= opts.text_detection_ratio {
                            trace!("Classified as TextFile (ratio >= threshold)");
                            Ok(FsFileType::TextFile)
                        } else {
                            trace!("Classified as BinaryFile (ratio < threshold)");
                            Ok(FsFileType::BinaryFile)
                        }
                    }
                    Err(_) => {
                        // If the initial chunk is not valid UTF-8, it's binary.
                        trace!("Classified as BinaryFile (invalid UTF-8 sequence)");
                        Ok(FsFileType::BinaryFile)
                    }
                }
            }
            Err(e) => {
                // Error reading from the file during heuristic check
                warn!(path=%path.display(), error = %e, "Failed to read initial chunk for type detection heuristic");
                stats.entries_skipped_error.fetch_add(1, Ordering::Relaxed);
                // Return specific IO error
                Err(ScannerError::Io {
                    path: path.to_path_buf(),
                    source: e,
                })
            }
        }
    }

    #[instrument(level = "trace", skip(path), fields(path = %path.display()))]
    fn read_text_content_details(path: &Path) -> Result<(String, usize, usize), ScannerError> {
        trace!("Reading text file content and details (chunked)...");

        // --- File Opening ---
        let file = match fs::File::open(path) {
            Ok(f) => f,
            Err(e) => {
                warn!(path=%path.display(), error=%e, "Failed to open text file for content reading");
                return Err(ScannerError::Io {
                    path: path.to_path_buf(),
                    source: e,
                });
            }
        };
        // Hint for initial String capacity (best effort)
        let file_size = file.metadata().map(|m| m.len()).unwrap_or(0);
        let initial_capacity = file_size.try_into().unwrap_or(1024); // Default capacity if size conversion fails

        // Use BufReader for efficient underlying I/O
        let mut reader = BufReader::with_capacity(TEXT_READER_BUFFER_SIZE, file);
        let mut buffer = vec![0; TEXT_READER_BUFFER_SIZE];

        // --- State Variables ---
        let mut content = String::with_capacity(initial_capacity);
        let mut line_count: usize = 0;
        let mut char_count: usize = 0;
        let mut leftover: Vec<u8> = Vec::new(); // Stores incomplete lines between chunks

        // --- Chunked Reading Loop ---
        loop {
            // Read a chunk into the buffer
            let bytes_read = match reader.read(&mut buffer) {
                Ok(0) => break, // End Of File
                Ok(n) => n,
                Err(ref e) if e.kind() == io::ErrorKind::Interrupted => continue, // Retry on interrupt
                Err(e) => {
                    warn!(path=%path.display(), error = %e, "Error reading chunk from text file");
                    return Err(ScannerError::Io {
                        path: path.to_path_buf(),
                        source: e,
                    });
                }
            };

            // Create a slice representing the new data read in this iteration
            let current_data_slice = &buffer[..bytes_read];

            // Create a vector to hold data to process in this iteration
            // If we have leftover data, combine it with the current chunk
            let data_to_process = if leftover.is_empty() {
                // No leftover data, we can use the current chunk directly as a slice
                current_data_slice.to_vec()
            } else {
                // We have leftover data from previous iterations, need to combine
                let mut combined = Vec::with_capacity(leftover.len() + bytes_read);
                combined.extend_from_slice(&leftover);
                combined.extend_from_slice(current_data_slice);
                combined
            };

            // Find the position of the last newline character in the data
            let last_newline_pos = memchr::memrchr(b'\n', &data_to_process);

            // Clear leftover before updating it with new data
            leftover.clear();

            let process_now = match last_newline_pos {
                Some(pos) => {
                    // Found a newline - process up to and including the newline
                    // Store everything after the last newline as leftover for next iteration
                    leftover.extend_from_slice(&data_to_process[(pos + 1)..]);
                    &data_to_process[..(pos + 1)]
                }
                None => {
                    // No newline found - keep everything as leftover
                    leftover.extend_from_slice(&data_to_process);
                    &[] as &[u8] // Empty slice to process
                }
            };

            // --- Process Complete Lines ---
            if !process_now.is_empty() {
                // Count lines efficiently using bytecount
                line_count += bytecount::count(process_now, b'\n');

                // Attempt to decode as UTF-8
                match std::str::from_utf8(process_now) {
                    Ok(s) => {
                        char_count += s.chars().count();
                        content.push_str(s);
                    }
                    Err(e) => {
                        warn!(path = %path.display(), error = %e, utf8_error_len = e.valid_up_to(),
                                "Invalid UTF-8 sequence detected in text file content chunk");
                        return Err(ScannerError::Io {
                            path: path.to_path_buf(),
                            source: io::Error::new(io::ErrorKind::InvalidData, e),
                        });
                    }
                }
            }
        }

        // --- Process Final Leftover Chunk ---
        // After the loop (EOF reached), any remaining data in `leftover` represents
        // the last part of the file, which might not end with a newline.
        if !leftover.is_empty() {
            // If there's leftover data, it constitutes one final line (even if empty after trim).
            line_count += 1; // Count the final partial line.

            // Decode the final chunk as UTF-8.
            match std::str::from_utf8(&leftover) {
                Ok(s) => {
                    char_count += s.chars().count();
                    content.push_str(s);
                }
                Err(e) => {
                    warn!(path = %path.display(), error = %e, utf8_error_len = e.valid_up_to(),
                            "Invalid UTF-8 sequence detected in final text file chunk");
                    return Err(ScannerError::Io {
                        path: path.to_path_buf(),
                        source: io::Error::new(io::ErrorKind::InvalidData, e),
                    });
                }
            }
        }

        // Release potentially over-allocated memory in the content string.
        content.shrink_to_fit();

        trace!(
            path=%path.display(),
            lines = line_count,
            chars = char_count,
            bytes = content.len(),
            "Finished reading text content and details (chunked)"
        );
        Ok((content, line_count, char_count))
    }

    // --- Tree Construction (Sequential, after parallel processing) ---

    #[instrument(level = "debug", skip(absolute_root_path, flat_nodes), fields(node_count=flat_nodes.len()))]
    fn construct_tree_from_flat_nodes(
        absolute_root_path: &Path,
        flat_nodes: Vec<FsNode>,
    ) -> Result<FsDirectoryNode, ScannerError> {
        debug!("Constructing node tree from collected nodes...");

        // --- Step 1: Organize nodes by parent path and collect directory info ---
        // Map: Parent Path -> Vec<Child Node>
        let mut nodes_by_parent: HashMap<PathBuf, Vec<FsNode>> =
            HashMap::with_capacity(flat_nodes.len() / 4 + 1); // Heuristic capacity
        // Map: Directory Path -> (Directory Name, Directory Metadata)
        let mut dir_info: HashMap<PathBuf, (String, FsMetadata)> =
            HashMap::with_capacity(flat_nodes.len() / 10 + 1); // Heuristic capacity

        trace!("Populating parent map and extracting directory info...");
        for node in flat_nodes {
            // Takes ownership of nodes in vec
            let node_path = node.path(); // Borrow path from node

            // Get parent path relative to root (empty path "" for nodes directly under root)
            let parent_path = node_path
                .parent()
                .map(|p| p.to_path_buf())
                .unwrap_or_else(PathBuf::new); // Root's children have parent ""

            // Store directory info if the node is a directory
            if let FsNode::Directory(d) = &node {
                dir_info.insert(node_path.clone(), (d.name.clone(), d.metadata.clone()));
            }

            // Add the node to the list of children for its parent path
            nodes_by_parent.entry(parent_path).or_default().push(node);
        }
        trace!(
            parent_map_size = nodes_by_parent.len(),
            dir_info_size = dir_info.len(),
            "Map population complete."
        );

        // --- Step 2: Ensure root directory information is present ---
        let root_rel_path = PathBuf::new(); // Empty path represents the root relative to itself
        if !dir_info.contains_key(&root_rel_path) {
            // The root directory node itself might not have been processed if scan started inside it.
            trace!("Root directory info not found in processed nodes, fetching explicitly.");
            let root_fs_meta = fs::metadata(absolute_root_path).map_err(|e| {
                 error!(path=%absolute_root_path.display(), error=%e, "Failed to get metadata for absolute root path during tree construction");
                 ScannerError::MetadataError {
                     path: absolute_root_path.to_path_buf(),
                     source: e,
                 }
             })?;
            let root_metadata = Self::create_metadata_struct(&root_fs_meta).map_err(|e| {
                 error!(path=%absolute_root_path.display(), error=%e, "Failed to create metadata struct for absolute root");
                 ScannerError::MetadataError { // Map IO error from modified() call
                     path: absolute_root_path.to_path_buf(),
                     source: e,
                 }
             })?;

            // Extract root name
            let root_name = absolute_root_path
                .file_name()
                .map_or_else(|| "/".to_string(), |n| n.to_string_lossy().to_string());

            // Insert the root's info into the dir_info map
            dir_info.insert(root_rel_path.clone(), (root_name, root_metadata));
            trace!(root_name=%dir_info[&root_rel_path].0, "Added explicit root directory info.");

            // Note: The root directory is present in the tree but might not be counted in stats
            // This is handled in the test by adjusting expectations
        }

        // --- Step 3: Build the tree recursively starting from the root ---
        trace!("Starting recursive tree build from root...");
        let root_node = Self::build_recursive(&root_rel_path, &mut nodes_by_parent, &dir_info)?;

        // --- Step 4: Check for orphaned nodes (optional sanity check) ---
        if !nodes_by_parent.is_empty() {
            // This shouldn't happen if all paths were processed correctly.
            // It might indicate issues with parent path calculation or missing directories.
            warn!(
                count = nodes_by_parent.len(),
                orphaned_parents = ?nodes_by_parent.keys().collect::<Vec<_>>(),
                "Found orphaned nodes map entries after tree construction. This might indicate missing directory entries or path issues."
            );
            // Depending on requirements, this could be an error:
            // return Err(ScannerError::TreeConstructionError("Orphaned nodes found".into()));
        }

        debug!("Tree construction successful.");
        Ok(root_node)
    }

    #[instrument(level = "trace", skip(nodes_by_parent, dir_info), fields(dir = %current_rel_path.display()))]
    fn build_recursive(
        current_rel_path: &Path,
        nodes_by_parent: &mut HashMap<PathBuf, Vec<FsNode>>, // Mut borrow needed for `remove`
        dir_info: &HashMap<PathBuf, (String, FsMetadata)>,   // Read-only borrow
    ) -> Result<FsDirectoryNode, ScannerError> {
        // --- 1. Get current directory's name and metadata ---
        let (current_name, current_metadata) =
            dir_info.get(current_rel_path).ok_or_else(|| {
                // This directory's info wasn't collected, which is an error.
                error!(path=%current_rel_path.display(), "Metadata missing for directory during tree construction");
                 ScannerError::TreeConstructionError(format!(
                    "Metadata missing for directory: {}",
                     current_rel_path.display()
                 ))
            })?;

        // --- 2. Process children ---
        let mut contents: Vec<FsNode> = Vec::new();
        if let Some(children) = nodes_by_parent.remove(current_rel_path) {
            // Reserve capacity for potentially faster pushes
            contents.reserve(children.len());
            trace!(child_count = children.len(), "Processing children");

            // Iterate through the children that belonged to this directory
            for child_node in children {
                // Takes ownership of nodes in vec
                match child_node {
                    FsNode::Directory(dir_placeholder) => {
                        // If the child is a directory, recursively build its subtree.
                        // The dir_placeholder only contained the path/name/meta, not contents yet.
                        trace!(subdir=%dir_placeholder.path.display(), "Recursing into subdirectory");
                        let sub_dir_node = Self::build_recursive(
                            &dir_placeholder.path, // Use the relative path of the child dir
                            nodes_by_parent,       // Pass mutable map down
                            dir_info,              // Pass dir info down
                        )?; // Propagate errors from recursive calls
                        // Add the fully constructed DirectoryNode to contents
                        contents.push(FsNode::Directory(sub_dir_node));
                    }
                    // For files, symlinks, binary files - just add them directly.
                    other_node => {
                        trace!(child_name=%other_node.name(), child_type=?other_node.metadata().file_type, "Adding non-directory child");
                        contents.push(other_node)
                    }
                }
            }
        } else {
            trace!("No children found for this directory path.");
        }

        // --- 3. Sort contents ---
        // Sort directory contents alphabetically by name for consistent output.
        // Use unstable sort as order doesn't need to be preserved beyond the key comparison.
        contents.sort_unstable_by(|a, b| a.name().cmp(b.name()));
        trace!(
            final_content_count = contents.len(),
            "Finished processing children, contents sorted."
        );

        // --- 4. Construct and return the DirectoryNode ---
        Ok(FsDirectoryNode {
            name: current_name.clone(),           // Clone name String
            path: current_rel_path.to_path_buf(), // Clone path PathBuf
            metadata: current_metadata.clone(),   // Clone metadata
            contents,                             // Move the collected & sorted contents vec
        })
    }

    /// Removes rust `#[cfg(test)]` annotated modules from the given content.
    /// This is a heuristic and might not cover all edge cases of Rust syntax,
    /// especially with complex macros or conditional compilation within the test module.
    fn remove_rust_test_modules(content: &str) -> String {
        let mut new_content = String::new();
        let lines: Vec<&str> = content.lines().collect();
        let mut i = 0;
        while i < lines.len() {
            let line = lines[i];
            let trimmed_line = line.trim();

            if trimmed_line.starts_with("#[cfg(test)]") {
                // Potential start of a test module. Look for 'mod <name> {'
                let mut next_line_idx = i + 1;
                let mut mod_block_start_line_idx: Option<usize> = None;

                // Find the 'mod <name>' line, skipping comments and empty lines,
                // then find the line where the module block (e.g. '{') starts.
                while next_line_idx < lines.len() {
                    let current_line_trimmed = lines[next_line_idx].trim();
                    if current_line_trimmed.is_empty() || current_line_trimmed.starts_with("//") {
                        next_line_idx += 1;
                        continue;
                    }
                    if current_line_trimmed.starts_with("mod ") {
                        // Found 'mod <name>'. Now check if this line or the next starts the block with '{'
                        if current_line_trimmed.contains('{') {
                            mod_block_start_line_idx = Some(next_line_idx);
                        } else if next_line_idx + 1 < lines.len()
                            && lines[next_line_idx + 1].trim().starts_with('{')
                        {
                            mod_block_start_line_idx = Some(next_line_idx + 1);
                        }
                        // If it's `mod foo;` or similar, mod_block_start_line_idx remains None, and we won't remove it.
                    }
                    break; // Found a significant line (mod or other code) or EOF
                }

                if let Some(block_start_idx) = mod_block_start_line_idx {
                    // Found 'mod <name> {' or 'mod <name>\n{'.
                    // Now count braces to find the end of this module.
                    let mut brace_level = 0;
                    let mut end_idx = block_start_idx; // Tracks the line index of the module's end.
                    let mut first_brace_found_in_module = false;

                    for k in block_start_idx..lines.len() {
                        end_idx = k;
                        for ch in lines[k].chars() {
                            if ch == '{' {
                                brace_level += 1;
                                first_brace_found_in_module = true;
                            } else if ch == '}' {
                                brace_level -= 1;
                            }
                        }
                        if first_brace_found_in_module && brace_level == 0 {
                            // Found the matching closing brace for the module.
                            break;
                        }
                    }

                    if first_brace_found_in_module && brace_level == 0 {
                        // Module block found and successfully parsed its extent.
                        // Skip all lines from the original `#[cfg(test)]` line up to `end_idx`.
                        i = end_idx + 1;
                        continue; // Continue to the next line in the outer loop, effectively skipping the test module.
                    }
                    // If parsing failed (e.g., EOF before closing brace, or no '{' found where expected),
                    // fall through to keep the `#[cfg(test)]` line and subsequent lines as they are.
                }
            }

            // If not part of a test module to be skipped (or parsing the module failed), add the current line.
            new_content.push_str(line);
            new_content.push('\n');
            i += 1;
        }

        // Adjust trailing newline to match original content:
        // If original content was not empty and didn't end with a newline,
        // but new_content (due to line-by-line push('\n')) does, remove the extra newline.
        if !content.is_empty() && !content.ends_with('\n') && new_content.ends_with('\n') {
            new_content.pop();
        } else if content.is_empty() && new_content.ends_with('\n') {
            // If original was empty, new_content might be just "\n". Clear it.
            new_content.clear();
        }

        new_content
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use filetime::{FileTime, set_file_mtime};
    use std::fs::{self, File};
    use std::io::{self, Write};
    #[cfg(unix)]
    use std::os::unix::fs::PermissionsExt;
    use std::path::{Path, PathBuf};
    use std::sync::atomic::Ordering;
    use std::time::{Duration, SystemTime};
    use tempfile::{TempDir, tempdir};

    // --- Test Helpers ---

    /// Create a temporary directory and sets up a specific file structure inside it.
    fn setup_test_directory(structure: &[(&str, Option<&[u8]>)]) -> io::Result<(TempDir, PathBuf)> {
        let dir = tempdir()?;
        let root = dir.path().to_path_buf();
        // Ensure root dir exists explicitly for canonicalization
        fs::create_dir_all(&root)?;

        for (path_str, content_opt) in structure {
            let full_path = root.join(path_str);
            if let Some(parent) = full_path.parent() {
                if !parent.exists() {
                    // Only create if it doesn't exist
                    fs::create_dir_all(parent)?;
                }
            }

            if let Some(content) = content_opt {
                // It's a file
                let mut file = File::create(&full_path)?;
                file.write_all(content)?;
            } else if path_str.ends_with('/') || path_str.is_empty() {
                if !full_path.exists() {
                    fs::create_dir_all(&full_path)?;
                }
            } else {
                File::create(&full_path)?;
            }
        }

        let canonical_root = fs::canonicalize(&root)?;
        Ok((dir, canonical_root))
    }

    /// Helper to create default scanner options
    fn default_opts() -> FsScannerOpts {
        FsScannerOpts {
            num_processor_threads: Some(2),
            num_walker_threads: Some(1),
            channel_capacity: 100,
            follow_symlinks: false,
            skip_content: true,
            no_gitignore: true,
            no_git_global: true,
            no_git_exclude: true,
            max_depth: None,
            max_file_size_for_content: None,
            text_detection_buffer_size: 512,
            text_detection_ratio: 0.85,
            modified_after: None,
            modified_before: None,
            permissions_filter: None,
            custom_ignore_path: None,
            ignore_patterns: Vec::new(),
            include_patterns: Vec::new(),
            ..Default::default()
        }
    }

    // Helper to get permission
    fn get_perms(meta: &fs::Metadata) -> u32 {
        meta.permissions().mode() & 0o170777
    }

    /// Recursive helper to assert tree structure. Timestamps are ignored by default.
    fn assert_nodes_equal(
        actual: &FsNode,
        expected: &FsNode,
        ignore_timestamps: bool,
        check_size: bool,
    ) {
        assert_eq!(
            actual.name(),
            expected.name(),
            "Name mismatch for path '{}'",
            actual.path().display()
        );
        assert_eq!(
            actual.path(),
            expected.path(),
            "Path mismatch for name '{}'",
            actual.name()
        );
        assert_eq!(
            actual.metadata().file_type,
            expected.metadata().file_type,
            "FileType mismatch for '{}'",
            actual.name()
        );

        // Optionally ignore timestamps as they can be tricky
        if !ignore_timestamps {
            // Allow for slight differences in timestamps due to system clock precision
            let allowed_diff = Duration::from_secs(2); // Allow 2 second difference
            let actual_time = actual.metadata().modified;
            let expected_time = expected.metadata().modified;
            if actual_time < expected_time {
                assert!(
                    expected_time
                        .duration_since(actual_time)
                        .unwrap_or_default()
                        <= allowed_diff,
                    "Timestamp mismatch for '{}': actual {:?} < expected {:?} beyond tolerance",
                    actual.name(),
                    actual_time,
                    expected_time
                );
            } else {
                assert!(
                    actual_time
                        .duration_since(expected_time)
                        .unwrap_or_default()
                        <= allowed_diff,
                    "Timestamp mismatch for '{}': actual {:?} > expected {:?} beyond tolerance",
                    actual.name(),
                    actual_time,
                    expected_time
                );
            }
        }

        // Compare permissions only on Unix where they are more meaningful in tests
        #[cfg(unix)]
        assert_eq!(
            actual.metadata().permissions & 0o777,
            expected.metadata().permissions & 0o777,
            "Permission mismatch (rwx) for '{}'",
            actual.name()
        );

        // Optionally compare size (useful for files, sometimes dirs/links)
        if check_size {
            assert_eq!(
                actual.metadata().size,
                expected.metadata().size,
                "Size mismatch for '{}'",
                actual.name()
            );
        }

        match (actual, expected) {
            (FsNode::Directory(act_dir), FsNode::Directory(exp_dir)) => {
                assert_eq!(
                    act_dir.contents.len(),
                    exp_dir.contents.len(),
                    "Mismatch in content count for dir '{}'",
                    act_dir.name
                );
                for (act_child, exp_child) in act_dir.contents.iter().zip(exp_dir.contents.iter()) {
                    let check_child_size = matches!(exp_child, FsNode::File(_) | FsNode::Binary(_));
                    assert_nodes_equal(act_child, exp_child, ignore_timestamps, check_child_size);
                }
            }
            (FsNode::File(act_file), FsNode::File(exp_file)) => {
                // Size already checked if check_size was true
                assert_eq!(
                    act_file.content, exp_file.content,
                    "File content mismatch for '{}'",
                    act_file.name
                );
                assert_eq!(
                    act_file.lines, exp_file.lines,
                    "File lines mismatch for '{}'",
                    act_file.name
                );
                assert_eq!(
                    act_file.chars, exp_file.chars,
                    "File chars mismatch for '{}'",
                    act_file.name
                );
            }
            (FsNode::Binary(act_bin), FsNode::Binary(exp_bin)) => {
                let _ = (act_bin, exp_bin);
            }
            (FsNode::Symlink(act_link), FsNode::Symlink(exp_link)) => {
                assert_eq!(
                    act_link.target, exp_link.target,
                    "Symlink target mismatch for '{}'",
                    act_link.name
                );
                assert_eq!(
                    act_link.target_is_dir, exp_link.target_is_dir,
                    "Symlink target_is_dir mismatch for '{}'",
                    act_link.name
                );
                assert_eq!(
                    act_link.target_exists, exp_link.target_exists,
                    "Symlink target_exists mismatch for '{}'",
                    act_link.name
                );
            }
            (a, b) => panic!(
                "Mismatched node variants even though types matched?: actual {a:#?}, expected {b:#?}",
            ),
        }
    }

    // Helper to get the expected root name from a PathBuf
    fn get_expected_root_name(root_path: &Path) -> String {
        root_path
            .file_name()
            .map_or_else(|| "/".to_string(), |n| n.to_string_lossy().to_string())
    }

    // --- Test Modules ---

    mod basic_scan {
        use super::*;
        use test_log::test;

        #[test]
        fn scan_empty_directory() {
            let (_temp_dir, root) = setup_test_directory(&[("", None)]).unwrap(); // Ensure root exists
            let opts = default_opts();
            let scanner = FsScanner::new(&root, opts, None).unwrap();
            let output = scanner.scan().unwrap();

            // Expected Root Node (metadata will be fetched during construction)
            let root_meta_fs = fs::metadata(&root).unwrap();
            let expected_root_meta = FsMetadata {
                size: root_meta_fs.len(),
                modified: root_meta_fs.modified().unwrap(),
                permissions: get_perms(&root_meta_fs),
                file_type: FsFileType::Directory,
            };
            let expected_root = FsDirectoryNode {
                name: get_expected_root_name(&root),
                path: root.clone(), // Expect absolute path for the root node
                metadata: expected_root_meta,
                contents: vec![],
            };

            // Check root node properties (ignore timestamp by default, don't check size for dir)
            assert_nodes_equal(
                &FsNode::Directory(output.root),
                &FsNode::Directory(expected_root.clone()),
                true,
                false,
            );

            // Assert Stats
            assert_eq!(
                output.stats.directories_processed.load(Ordering::Relaxed),
                0,
                "Stats: directories_processed"
            ); // Root isn't "processed" by walker/processor loop
            assert_eq!(
                output.stats.total_entries_processed(),
                0,
                "Stats: files_processed"
            );
            assert_eq!(
                output.stats.symlinks_processed.load(Ordering::Relaxed),
                0,
                "Stats: symlinks_processed"
            );
            assert_eq!(
                output.stats.total_entries_processed(),
                0,
                "Stats: total_entries_processed"
            );
            assert_eq!(
                output.stats.total_entries_skipped(),
                0,
                "Stats: total_entries_skipped"
            );
            // The root directory's size *is not* counted in total_bytes by the processor loop
            assert_eq!(
                output.stats.total_bytes.load(Ordering::Relaxed),
                0,
                "Stats: total_bytes"
            );
        }

        #[test]
        fn scan_single_file() {
            let file_content = b"hello";
            let (_temp_dir, root) =
                setup_test_directory(&[("file.txt", Some(file_content))]).unwrap();
            let opts = FsScannerOpts::default();
            let scanner = FsScanner::new(&root, opts, None).unwrap();
            let output = scanner.scan().unwrap();

            // Expected Tree
            let file_path = PathBuf::from("file.txt");
            let file_meta_fs = fs::metadata(root.join(&file_path)).unwrap();
            let expected_file = FsNode::File(FsFileNode {
                // Default is Binary if content skipped
                name: "file.txt".to_string(),
                path: file_path.clone(),
                metadata: FsMetadata {
                    size: file_content.len() as u64,
                    modified: file_meta_fs.modified().unwrap(),
                    permissions: get_perms(&file_meta_fs),
                    file_type: FsFileType::TextFile,
                },
                content: Some(String::from_utf8_lossy(file_content).into_owned()),
                chars: Some(file_content.len()),
                lines: Some(1),
                token_count: Some(0),
            });
            let root_meta_fs = fs::metadata(&root).unwrap();
            let expected_root_meta = FsMetadata {
                size: root_meta_fs.len(),
                modified: root_meta_fs.modified().unwrap(),
                permissions: get_perms(&root_meta_fs),
                file_type: FsFileType::Directory,
            };
            let expected_root = FsDirectoryNode {
                name: get_expected_root_name(&root),
                path: root.clone(), // Expect absolute path for the root node
                metadata: expected_root_meta,
                contents: vec![expected_file],
            };

            // Check root node properties recursively (ignore timestamp, check file size)
            assert_nodes_equal(
                &FsNode::Directory(output.root),
                &FsNode::Directory(expected_root.clone()),
                true,
                true,
            );

            // Assert Stats
            assert_eq!(
                output.stats.directories_processed(),
                0,
                "Stats: directories_processed"
            );
            assert_eq!(
                output.stats.binary_files_processed(),
                0,
                "Stats: binary_files_processed"
            );
            assert_eq!(
                output.stats.text_files_processed(),
                1,
                "Stats: text_files_processed"
            );
            assert_eq!(
                output.stats.symlinks_processed(),
                0,
                "Stats: symlinks_processed"
            );
            assert_eq!(
                output.stats.total_entries_processed(),
                1,
                "Stats: total_entries_processed"
            );
            assert_eq!(
                output.stats.total_bytes(),
                file_content.len() as u64,
                "Stats: total_bytes"
            );
        }

        #[test]
        fn scan_nested_structure() {
            let (_temp_dir, root) = setup_test_directory(&[
                ("file1.txt", Some(b"one")),
                ("subdir/", None), // Create directory explicitly
                ("subdir/file2.bin", Some(&[0x01, 0x02, 0x00])), // Add null byte for binary check
                ("subdir/nested/", None),
                ("subdir/nested/file3.txt", Some(b"three\nlines")),
            ])
            .unwrap();
            let mut opts = default_opts();
            opts.skip_content = false; // Read content to test type heuristic
            let scanner = FsScanner::new(&root, opts, None).unwrap();
            let output = scanner.scan().unwrap();

            // --- Expected Tree (Build bottom-up for clarity) ---
            let file1_path = PathBuf::from("file1.txt");
            let file1_meta_fs = fs::metadata(root.join(&file1_path)).unwrap();
            let expected_file1 = FsNode::File(FsFileNode {
                name: "file1.txt".to_string(),
                path: file1_path.clone(),
                metadata: FsMetadata {
                    size: 3,
                    modified: file1_meta_fs.modified().unwrap(),
                    permissions: get_perms(&file1_meta_fs),
                    file_type: FsFileType::TextFile,
                },
                content: Some("one".to_string()),
                lines: Some(1),
                chars: Some(3),
                token_count: Some(0),
            });

            let file2_path = PathBuf::from("subdir/file2.bin");
            let file2_meta_fs = fs::metadata(root.join(&file2_path)).unwrap();
            let expected_file2 = FsNode::Binary(FsBinaryNode {
                // Should be binary due to null byte
                name: "file2.bin".to_string(),
                path: file2_path.clone(),
                metadata: FsMetadata {
                    size: 3,
                    modified: file2_meta_fs.modified().unwrap(),
                    permissions: get_perms(&file2_meta_fs),
                    file_type: FsFileType::BinaryFile,
                },
            });

            let file3_path = PathBuf::from("subdir/nested/file3.txt");
            let file3_meta_fs = fs::metadata(root.join(&file3_path)).unwrap();
            let expected_file3 = FsNode::File(FsFileNode {
                name: "file3.txt".to_string(),
                path: file3_path.clone(),
                metadata: FsMetadata {
                    size: 11,
                    modified: file3_meta_fs.modified().unwrap(),
                    permissions: get_perms(&file3_meta_fs),
                    file_type: FsFileType::TextFile,
                },
                content: Some("three\nlines".to_string()),
                lines: Some(2),
                chars: Some(11),
                token_count: Some(0),
            });

            let nested_path = PathBuf::from("subdir/nested");
            let nested_meta_fs = fs::metadata(root.join(&nested_path)).unwrap();
            let expected_nested = FsNode::Directory(FsDirectoryNode {
                name: "nested".to_string(),
                path: nested_path.clone(),
                metadata: FsMetadata {
                    size: nested_meta_fs.len(),
                    modified: nested_meta_fs.modified().unwrap(),
                    permissions: get_perms(&nested_meta_fs),
                    file_type: FsFileType::Directory,
                },
                contents: vec![expected_file3],
            });

            let subdir_path = PathBuf::from("subdir");
            let subdir_meta_fs = fs::metadata(root.join(&subdir_path)).unwrap();
            let expected_subdir = FsNode::Directory(FsDirectoryNode {
                name: "subdir".to_string(),
                path: subdir_path.clone(),
                metadata: FsMetadata {
                    size: subdir_meta_fs.len(),
                    modified: subdir_meta_fs.modified().unwrap(),
                    permissions: get_perms(&subdir_meta_fs),
                    file_type: FsFileType::Directory,
                },
                contents: vec![expected_file2, expected_nested], // Sorted: file2.bin, nested
            });

            let root_meta_fs = fs::metadata(&root).unwrap();
            let expected_root = FsDirectoryNode {
                name: get_expected_root_name(&root),
                path: root.clone(), // Expect absolute path for the root node
                metadata: FsMetadata {
                    size: root_meta_fs.len(),
                    modified: root_meta_fs.modified().unwrap(),
                    permissions: get_perms(&root_meta_fs),
                    file_type: FsFileType::Directory,
                },
                contents: vec![expected_file1, expected_subdir], // Sorted: file1.txt, subdir
            };

            // --- Assertions ---
            // Check root node properties recursively (ignore timestamp, check file/binary sizes)
            assert_nodes_equal(
                &FsNode::Directory(output.root),
                &FsNode::Directory(expected_root.clone()),
                true,
                true,
            );

            // Stats
            assert_eq!(
                output.stats.directories_processed.load(Ordering::Relaxed),
                2,
                "Stats: directories_processed"
            ); // subdir, nested
            assert_eq!(
                output.stats.text_files_processed.load(Ordering::Relaxed),
                2,
                "Stats: text_files_processed"
            ); // file1, file3
            assert_eq!(
                output.stats.binary_files_processed.load(Ordering::Relaxed),
                1,
                "Stats: binary_files_processed"
            ); // file2
            assert_eq!(
                output.stats.total_entries_processed(),
                5,
                "Stats: total_entries_processed"
            );
            // Sum of file sizes + directory node metadata sizes (which might be non-zero on some platforms)
            let expected_total_bytes = 3 + 3 + 11 + nested_meta_fs.len() + subdir_meta_fs.len();
            assert_eq!(
                output.stats.total_bytes.load(Ordering::Relaxed),
                expected_total_bytes,
                "Stats: total_bytes"
            );
            assert_eq!(
                output.stats.total_lines.load(Ordering::Relaxed),
                1 + 2,
                "Stats: total_lines"
            );
            assert_eq!(
                output.stats.total_chars.load(Ordering::Relaxed),
                3 + 11,
                "Stats: total_chars"
            );
        }

        #[test]
        fn scan_with_max_depth() {
            let (_temp_dir, root) = setup_test_directory(&[
                ("file1.txt", Some(b"one")),
                ("subdir/", None),
                ("subdir/file2.bin", Some(&[0x01])), // Should not be processed
                ("subdir/nested/", None),            // Should not be processed
                ("subdir/nested/file3.txt", Some(b"three")), // Should not be processed
            ])
            .unwrap();
            let output = FsScanner::new(
                &root,
                FsScannerOpts {
                    max_depth: Some(1),
                    ..FsScannerOpts::default()
                },
                None,
            )
            .unwrap()
            .scan()
            .unwrap();

            // --- Expected Tree (Only depth 1) ---
            let file1_path = PathBuf::from("file1.txt");
            let file1_meta_fs = fs::metadata(root.join(&file1_path)).unwrap();
            let expected_file1 = FsNode::File(FsFileNode {
                name: "file1.txt".into(),
                path: file1_path,
                content: Some("one".into()),
                chars: Some(3),
                lines: Some(1),
                token_count: Some(0),
                metadata: FsMetadata {
                    size: 3,
                    modified: file1_meta_fs.modified().unwrap(),
                    permissions: get_perms(&file1_meta_fs),
                    file_type: FsFileType::TextFile,
                },
            });

            let subdir_path = PathBuf::from("subdir");
            let subdir_meta_fs = fs::metadata(root.join(&subdir_path)).unwrap();
            let expected_subdir = FsNode::Directory(FsDirectoryNode {
                name: "subdir".to_string(),
                path: subdir_path,
                metadata: FsMetadata {
                    size: subdir_meta_fs.len(),
                    modified: subdir_meta_fs.modified().unwrap(),
                    permissions: get_perms(&subdir_meta_fs),
                    file_type: FsFileType::Directory,
                },
                contents: vec![], // Contents not explored due to max_depth
            });

            let root_meta_fs = fs::metadata(&root).unwrap();
            let expected_root = FsDirectoryNode {
                name: get_expected_root_name(&root),
                path: root.clone(), // Expect absolute path for the root node
                metadata: FsMetadata {
                    size: root_meta_fs.len(),
                    modified: root_meta_fs.modified().unwrap(),
                    permissions: get_perms(&root_meta_fs),
                    file_type: FsFileType::Directory,
                },
                contents: vec![expected_file1, expected_subdir], // Sorted
            };

            // Check recursively (ignore timestamp, check file sizes)
            assert_nodes_equal(
                &FsNode::Directory(output.root),
                &FsNode::Directory(expected_root.clone()),
                true,
                true,
            );

            // Stats: Only entries at depth 1 are processed
            assert_eq!(
                output.stats.directories_processed(),
                1,
                "Stats: directories_processed"
            ); // subdir
            assert_eq!(
                output.stats.binary_files_processed(),
                0,
                "Stats: binary_files_processed"
            ); // file1.txt
            assert_eq!(
                output.stats.text_files_processed(),
                1,
                "Stats: text_files_processed"
            );
            assert_eq!(
                output.stats.total_entries_processed(),
                2,
                "Stats: total_entries_processed"
            );
            // Bytes = size of file1 + size of subdir dir node metadata
            assert_eq!(
                output.stats.total_bytes.load(Ordering::Relaxed),
                3 + subdir_meta_fs.len(),
                "Stats: total_bytes"
            );
        }
    }

    mod content_reading {
        use super::*;
        use test_log::test;

        #[test]
        fn read_text_file_content() {
            let content = "Line 1\nLine 2\nAnother Line\n"; // Ends with newline
            let (_temp_dir, root) =
                setup_test_directory(&[("test.txt", Some(content.as_bytes()))]).unwrap();
            let mut opts = default_opts();
            opts.skip_content = false;
            let scanner = FsScanner::new(&root, opts, None).unwrap();
            let output = scanner.scan().unwrap();

            assert_eq!(output.root.contents.len(), 1);
            let node = output.root.contents.first().expect("Should have one node"); // Use .first()
            match node {
                FsNode::File(file_node) => {
                    assert_eq!(
                        file_node.metadata.file_type,
                        FsFileType::TextFile,
                        "File Type"
                    );
                    assert_eq!(file_node.content.as_deref(), Some(content), "Content");
                    assert_eq!(file_node.lines, Some(3), "Lines");
                    assert_eq!(file_node.chars, Some(content.chars().count()), "Chars");
                    assert_eq!(output.stats.total_lines(), 3, "Stats Lines");
                    assert_eq!(
                        output.stats.total_chars.load(Ordering::Relaxed),
                        content.chars().count(),
                        "Stats Chars"
                    );
                }
                _ => panic!(
                    "Expected FsFileNode, got {:?}",
                    std::any::type_name_of_val(&node)
                ),
            }
            assert_eq!(
                output.stats.text_files_processed.load(Ordering::Relaxed),
                1,
                "Stats Text Files"
            );
            assert_eq!(
                output.stats.binary_files_processed.load(Ordering::Relaxed),
                0,
                "Stats Binary Files"
            );
            assert_eq!(
                output.stats.total_bytes.load(Ordering::Relaxed),
                content.len() as u64,
                "Stats Total Bytes"
            );
        }

        #[test]
        fn read_text_file_no_final_newline() {
            let content = "Line 1\nLine 2"; // No final newline
            let (_temp_dir, root) =
                setup_test_directory(&[("test.txt", Some(content.as_bytes()))]).unwrap();
            let mut opts = default_opts();
            opts.skip_content = false;
            let scanner = FsScanner::new(&root, opts, None).unwrap();
            let output = scanner.scan().unwrap();

            assert_eq!(output.root.contents.len(), 1);
            let node = output.root.contents.first().expect("Should have one node");
            match node {
                FsNode::File(file_node) => {
                    assert_eq!(
                        file_node.metadata.file_type,
                        FsFileType::TextFile,
                        "File Type"
                    );
                    assert_eq!(file_node.content.as_deref(), Some(content), "Content");
                    assert_eq!(file_node.lines, Some(2), "Lines"); // Correctly counts the last line
                    assert_eq!(file_node.chars, Some(content.chars().count()), "Chars");
                    assert_eq!(
                        output.stats.total_lines.load(Ordering::Relaxed),
                        2,
                        "Stats Lines"
                    );
                    assert_eq!(
                        output.stats.total_chars.load(Ordering::Relaxed),
                        content.chars().count(),
                        "Stats Chars"
                    );
                }
                _ => panic!(
                    "Expected FsFileNode, got {:?}",
                    std::any::type_name_of_val(&node)
                ),
            }
            assert_eq!(
                output.stats.text_files_processed.load(Ordering::Relaxed),
                1,
                "Stats Text Files"
            );
            assert_eq!(
                output.stats.binary_files_processed.load(Ordering::Relaxed),
                0,
                "Stats Binary Files"
            );
            assert_eq!(
                output.stats.total_bytes.load(Ordering::Relaxed),
                content.len() as u64,
                "Stats Total Bytes"
            );
        }

        #[test]
        fn identify_binary_file_null_byte() {
            let content = b"Hello\0World"; // Contains null byte
            let (_temp_dir, root) = setup_test_directory(&[("binary.dat", Some(content))]).unwrap();
            let mut opts = default_opts();
            opts.skip_content = false; // Heuristic runs even if we don't store content
            let scanner = FsScanner::new(&root, opts, None).unwrap();
            let output = scanner.scan().unwrap();

            assert_eq!(output.root.contents.len(), 1);
            let node = output.root.contents.first().expect("Should have one node"); // Use .first()
            match node {
                FsNode::Binary(bin_node) => {
                    assert_eq!(
                        bin_node.metadata.file_type,
                        FsFileType::BinaryFile,
                        "File Type"
                    );
                    assert_eq!(
                        output.stats.binary_files_processed.load(Ordering::Relaxed),
                        1,
                        "Stats Binary Files"
                    );
                }
                _ => panic!(
                    "Expected FsBinaryNode due to null byte, got {:?}",
                    std::any::type_name_of_val(&node)
                ),
            }
            assert_eq!(
                output.stats.text_files_processed.load(Ordering::Relaxed),
                0,
                "Stats Text Files"
            );
            assert_eq!(
                output.stats.total_bytes.load(Ordering::Relaxed),
                content.len() as u64,
                "Stats Total Bytes"
            );
        }

        #[test]
        fn identify_binary_file_low_ratio() {
            // More control/non-printable chars than printable ones (excluding whitespace)
            let content = b"abc\x01\x02\x03\x04\x05\x06\x07\x08def\x09\x0b\x0c"; // \x0a (\n) is whitespace
            let (_temp_dir, root) =
                setup_test_directory(&[("control.dat", Some(content))]).unwrap();
            let mut opts = default_opts();
            opts.skip_content = false;
            opts.text_detection_ratio = 0.7; // Set ratio for test predictability (6 printable / 15 total = 0.4)
            let scanner = FsScanner::new(&root, opts, None).unwrap();
            let output = scanner.scan().unwrap();

            assert_eq!(output.root.contents.len(), 1);
            let node = output.root.contents.first().expect("Should have one node");
            match node {
                FsNode::Binary(bin_node) => {
                    assert_eq!(
                        bin_node.metadata.file_type,
                        FsFileType::BinaryFile,
                        "File Type"
                    );
                    assert_eq!(
                        output.stats.binary_files_processed.load(Ordering::Relaxed),
                        1,
                        "Stats Binary Files"
                    );
                }
                _ => panic!(
                    "Expected FsBinaryNode due to low ratio, got {:?}",
                    std::any::type_name_of_val(&node)
                ),
            }
            assert_eq!(
                output.stats.text_files_processed.load(Ordering::Relaxed),
                0,
                "Stats Text Files"
            );
            assert_eq!(
                output.stats.total_bytes.load(Ordering::Relaxed),
                content.len() as u64,
                "Stats Total Bytes"
            );
        }

        #[test]
        fn skip_content_for_large_file_due_to_heuristic_size_limit() {
            // --- Setup ---
            const LINE_CONTENT: &str = "- List Item\n"; // 12 bytes
            const NUM_LINES: usize = 2 * 1024; // 2048 lines
            let expected_size: u64 = (LINE_CONTENT.len() * NUM_LINES) as u64; // 24576 bytes
            let max_size_for_content = expected_size - 1; // Set limit just below actual size

            // Create the large file content
            let large_content: String = LINE_CONTENT.repeat(NUM_LINES);

            // Setup test directory
            let (_temp_dir, root) =
                setup_test_directory(&[("large.txt", Some(large_content.as_bytes()))]).unwrap();

            // Configure scanner options
            let opts = FsScannerOpts {
                // Set the content size limit. This also affects the heuristic check.
                max_file_size_for_content: Some(max_size_for_content),
                skip_content: false, // Ensure we *would* read content if not for the limit
                ..Default::default()
            };

            // --- Act ---
            let scanner = FsScanner::new(&root, opts, None).unwrap();
            let output = scanner.scan().unwrap();

            // --- Assert Node Properties ---
            assert_eq!(
                output.root.contents.len(),
                1,
                "Expected exactly one node in the root directory"
            );
            let node = output.root.contents.first().expect("Should have one node");

            // Assert that the node is a Binary node because the heuristic preemptively classifies
            // files larger than `max_file_size_for_content` as binary.
            if let FsNode::Binary(bin_node) = node {
                assert_eq!(bin_node.name, "large.txt", "Node name");
                assert_eq!(bin_node.path, PathBuf::from("large.txt"), "Node path");
                assert_eq!(
                    bin_node.metadata.file_type,
                    FsFileType::BinaryFile,
                    "File type should be Binary due to heuristic size check"
                );
                assert_eq!(
                    bin_node.metadata.size, expected_size,
                    "Metadata size should match calculated size"
                );
            } else {
                panic!(
                    "Expected FsBinaryNode due to heuristic size limit, got: {:?}",
                    node
                );
            }

            // --- Assert Stats ---
            let stats = &output.stats;
            assert_eq!(
                stats.binary_files_processed.load(Ordering::Relaxed),
                1,
                "Stats: Binary files processed"
            );
            assert_eq!(
                stats.text_files_processed.load(Ordering::Relaxed),
                0,
                "Stats: Text files processed (should be 0)"
            );
            assert_eq!(
                stats.files_skipped_large_content.load(Ordering::Relaxed),
                0,
                "Stats: Files skipped large content (should be 0 as it was classified Binary first)"
            );
            assert_eq!(
                stats.total_entries_processed(),
                1,
                "Stats: Total entries processed"
            );
            assert_eq!(
                stats.total_bytes(),
                expected_size,
                "Stats: Total bytes processed"
            );
            assert_eq!(
                stats.total_lines(),
                0,
                "Stats: Total lines (content skipped)"
            );
            assert_eq!(
                stats.total_chars(),
                0,
                "Stats: Total chars (content skipped)"
            );
        }

        #[test]
        fn classify_large_file_as_binary_preemptively_by_heuristic() {
            // Heuristic checks max_file_size_for_content *before* reading buffer
            let large_content_size = 1024 * 2; // 2 KiB
            let large_content: Vec<u8> = std::iter::repeat(b'a').take(large_content_size).collect();
            let (_temp_dir, root) =
                setup_test_directory(&[("large_bin.dat", Some(&large_content))]).unwrap();
            let mut opts = default_opts();
            opts.skip_content = true; // Content skipped anyway by main process
            // Set max_file_size_for_content to trigger the pre-emptive binary classification in the heuristic
            opts.max_file_size_for_content = Some(large_content_size as u64 - 1);
            opts.text_detection_buffer_size = 1024; // Ensure buffer size itself wouldn't classify as binary if read

            let scanner = FsScanner::new(&root, opts, None).unwrap();
            let output = scanner.scan().unwrap();

            assert_eq!(output.root.contents.len(), 1);
            let node = output.root.contents.first().expect("Should have one node");
            match node {
                FsNode::Binary(bin_node) => {
                    // Classified as Binary preemptively by heuristic due to size limit check
                    assert_eq!(
                        bin_node.metadata.file_type,
                        FsFileType::BinaryFile,
                        "File Type"
                    );
                    assert_eq!(
                        bin_node.metadata.size, large_content_size as u64,
                        "File Size"
                    );
                    assert_eq!(
                        output.stats.binary_files_processed.load(Ordering::Relaxed),
                        1,
                        "Stats Binary Files"
                    );
                    assert_eq!(
                        output.stats.text_files_processed.load(Ordering::Relaxed),
                        0,
                        "Stats Text Files"
                    );
                    // Content wasn't skipped due to limit *during text processing*, heuristic decided Binary first
                    assert_eq!(
                        output
                            .stats
                            .files_skipped_large_content
                            .load(Ordering::Relaxed),
                        0,
                        "Stats Skipped Large Content"
                    );
                }
                _ => panic!(
                    "Expected FsBinaryNode due to preemptive size classification in heuristic, got {:?}",
                    std::any::type_name_of_val(&node)
                ),
            }
            assert_eq!(
                output.stats.total_bytes.load(Ordering::Relaxed),
                large_content_size as u64,
                "Stats Total Bytes"
            );
        }

        #[test]
        fn read_empty_file_content() {
            let (_temp_dir, root) = setup_test_directory(&[("empty.txt", Some(b""))]).unwrap();
            let mut opts = default_opts();
            opts.skip_content = false;
            let scanner = FsScanner::new(&root, opts, None).unwrap();
            let output = scanner.scan().unwrap();

            assert_eq!(output.root.contents.len(), 1);
            let node = output.root.contents.first().expect("Should have one node");
            match node {
                FsNode::File(file_node) => {
                    assert_eq!(
                        file_node.metadata.file_type,
                        FsFileType::TextFile,
                        "File Type"
                    );
                    assert_eq!(file_node.content.as_deref(), Some(""), "Content");
                    assert_eq!(file_node.lines, Some(0), "Lines"); // Code logic gives 0 lines for empty file content
                    assert_eq!(file_node.chars, Some(0), "Chars");
                    assert_eq!(
                        output.stats.total_lines.load(Ordering::Relaxed),
                        0,
                        "Stats Lines"
                    );
                    assert_eq!(
                        output.stats.total_chars.load(Ordering::Relaxed),
                        0,
                        "Stats Chars"
                    );
                }
                _ => panic!(
                    "Expected FsFileNode for empty file, got {:?}",
                    std::any::type_name_of_val(&node)
                ),
            }
            assert_eq!(
                output.stats.text_files_processed.load(Ordering::Relaxed),
                1,
                "Stats Text Files"
            );
            assert_eq!(
                output.stats.total_bytes.load(Ordering::Relaxed),
                0,
                "Stats Total Bytes"
            );
        }

        #[test]
        fn handle_invalid_utf8_file() {
            let invalid_utf8: &[u8] = &[0x80, 0x81, 0x82, 0x83]; // Invalid UTF-8 sequence
            let (_temp_dir, root) =
                setup_test_directory(&[("bad_utf8.txt", Some(invalid_utf8))]).unwrap();
            let mut opts = default_opts();
            opts.skip_content = false; // Attempt to read

            let scanner = FsScanner::new(&root, opts, None).unwrap();
            let output = scanner.scan().unwrap(); // Should not error out, but log/skip content

            assert_eq!(output.root.contents.len(), 1);
            let node = output.root.contents.first().expect("Should have one node"); // Use .first()

            match node {
                FsNode::File(file_node) => {
                    // Heuristic might classify as Text if buffer check passes ratio before hitting invalid bytes,
                    // but content reading *should* fail and result in None.
                    assert_eq!(
                        file_node.metadata.file_type,
                        FsFileType::TextFile,
                        "File Type"
                    ); // Heuristic likely passed buffer check
                    assert!(
                        file_node.content.is_none(),
                        "Content should be None due to UTF-8 read error"
                    );
                    assert!(
                        file_node.lines.is_none(),
                        "Lines should be None due to read error"
                    );
                    assert!(
                        file_node.chars.is_none(),
                        "Chars should be None due to read error"
                    );
                    assert_eq!(
                        file_node.metadata.size,
                        invalid_utf8.len() as u64,
                        "File Size"
                    );

                    assert_eq!(
                        output.stats.text_files_processed.load(Ordering::Relaxed),
                        1,
                        "Stats Text Files Processed (Attempted)"
                    );
                    // Should be skipped due to read error specifically
                    assert_eq!(
                        output
                            .stats
                            .files_skipped_read_error
                            .load(Ordering::Relaxed),
                        1,
                        "Stats Skipped Read Error"
                    );
                    // Should also count as a general processing error skip
                    assert_eq!(
                        output.stats.entries_skipped_error.load(Ordering::Relaxed),
                        1,
                        "Stats Entries Skipped Error"
                    );
                    assert_eq!(
                        output.stats.total_lines.load(Ordering::Relaxed),
                        0,
                        "Stats Total Lines"
                    );
                    assert_eq!(
                        output.stats.total_chars.load(Ordering::Relaxed),
                        0,
                        "Stats Total Chars"
                    );
                }
                FsNode::Binary(bin_node) => {
                    // Alternative: If the heuristic *did* catch the invalid UTF8 early
                    assert_eq!(
                        bin_node.metadata.file_type,
                        FsFileType::BinaryFile,
                        "File Type"
                    );
                    assert_eq!(
                        bin_node.metadata.size,
                        invalid_utf8.len() as u64,
                        "File Size"
                    );
                    assert_eq!(
                        output.stats.binary_files_processed.load(Ordering::Relaxed),
                        1,
                        "Stats Binary Files"
                    );
                    assert_eq!(
                        output
                            .stats
                            .files_skipped_read_error
                            .load(Ordering::Relaxed),
                        0,
                        "Stats Skipped Read Error"
                    );
                    assert_eq!(
                        output.stats.entries_skipped_error.load(Ordering::Relaxed),
                        0,
                        "Stats Entries Skipped Error"
                    ); // Not skipped if treated as binary
                }
                _ => panic!(
                    "Expected FsFileNode (content read failed) or FsBinaryNode (heuristic failed), got {:?}",
                    std::any::type_name_of_val(&node)
                ),
            }
            assert_eq!(
                output.stats.total_bytes.load(Ordering::Relaxed),
                invalid_utf8.len() as u64,
                "Stats Total Bytes"
            );
        }
    }

    mod filtering {
        use super::*;
        use std::thread;
        use test_log::test;

        #[test]
        fn filter_gitignore() {
            let (_temp_dir, root) = setup_test_directory(&[
                (
                    ".gitignore",
                    Some(b"ignored.txt\n*.tmp\nlogs/\n\n#comment\n!important.tmp\n"),
                ),
                ("keep.txt", Some(b"keep")),
                ("ignored.txt", Some(b"ignore")),  // Ignored by name
                ("data.tmp", Some(b"temp")),       // Ignored by *.tmp
                ("important.tmp", Some(b"vital")), // Explicitly NOT ignored by !important.tmp
                ("logs/", None),                   // Ignored dir
                ("logs/app.log", Some(b"logging")), // Ignored because parent dir ignored
            ])
            .unwrap();

            // Hypothesis: Create a dummy .git dir to see if it enables standard filters correctly.
            fs::create_dir(root.join(".git")).expect("Failed to create dummy .git directory");

            let mut opts = default_opts();
            opts.no_gitignore = false; // Enable gitignore processing
            opts.skip_content = true; // Keep it simple

            let scanner = FsScanner::new(&root, opts, None).unwrap();
            let output = scanner.scan().unwrap();

            // --- Expected Tree ---
            let keep_path = PathBuf::from("keep.txt");
            let keep_meta_fs = fs::metadata(root.join(&keep_path)).unwrap();
            let expected_keep = FsNode::File(FsFileNode {
                // Change to FileNode
                name: "keep.txt".to_string(),
                path: keep_path,
                metadata: FsMetadata {
                    size: 4,
                    modified: keep_meta_fs.modified().unwrap(),
                    permissions: get_perms(&keep_meta_fs),
                    file_type: FsFileType::TextFile, // Expect TextFile
                },
                content: None, // Content is skipped
                lines: None,   // Lines not counted
                chars: None,   // Chars not counted
                token_count: Some(0),
            });

            let important_path = PathBuf::from("important.tmp");
            let important_meta_fs = fs::metadata(root.join(&important_path)).unwrap();
            let expected_important = FsNode::File(FsFileNode {
                // Change to FileNode
                name: "important.tmp".to_string(),
                path: important_path,
                metadata: FsMetadata {
                    size: 5,
                    modified: important_meta_fs.modified().unwrap(),
                    permissions: get_perms(&important_meta_fs),
                    file_type: FsFileType::TextFile, // Expect TextFile
                },
                content: None, // Content is skipped
                lines: None,   // Lines not counted
                chars: None,   // Chars not counted
                token_count: Some(0),
            });

            let root_meta_fs = fs::metadata(&root).unwrap();
            let expected_root = FsDirectoryNode {
                name: get_expected_root_name(&root),
                path: root.clone(),
                metadata: FsMetadata {
                    size: root_meta_fs.len(),
                    modified: root_meta_fs.modified().unwrap(),
                    permissions: get_perms(&root_meta_fs),
                    file_type: FsFileType::Directory,
                },
                contents: vec![expected_important, expected_keep],
            };

            // Check recursively (ignore timestamp, check file sizes)
            assert_nodes_equal(
                &FsNode::Directory(output.root),
                &FsNode::Directory(expected_root.clone()),
                true,
                true,
            );

            // Stats: Check processed counts
            assert_eq!(output.stats.text_files_processed(), 2, "Stats Text Files");
            assert_eq!(
                output.stats.total_entries_processed(),
                2,
                "Stats Total Entries Processed"
            );
        }

        #[test]
        fn filter_custom_ignore_include() {
            let (_temp_dir, root) = setup_test_directory(&[
                ("image.jpg", Some(&[0xFF, 0xD8])),
                ("document.txt", Some(b"text")), // Should be ignored
                ("archive.zip", Some(&[0x50, 0x4B])),
                ("important.txt", Some(b"vital")), // Should be included
                ("report.pdf", Some(b"%PDF")),     // Not .txt, should be included
            ])
            .unwrap();
            let opts = FsScannerOpts {
                no_gitignore: true,                         // Ensure only our patterns apply
                ignore_patterns: vec!["*.txt".to_string()], // Ignore all txt files...
                include_patterns: vec!["important.txt".to_string()], // ...except this one
                ..Default::default()
            };

            let scanner = FsScanner::new(&root, opts, None).unwrap();
            let output = scanner.scan().unwrap();

            let important_path = PathBuf::from("important.txt");
            let important_meta_fs = fs::metadata(root.join(&important_path)).unwrap();
            let expected_important = FsNode::File(FsFileNode {
                // Expect FsFileNode
                name: "important.txt".to_string(),
                path: important_path.clone(), // Clone path
                metadata: FsMetadata {
                    size: 5,
                    modified: important_meta_fs.modified().unwrap(),
                    permissions: get_perms(&important_meta_fs),
                    file_type: FsFileType::TextFile, // Correctly expect TextFile
                },
                content: Some("vital".to_string()), // Expect content
                lines: Some(1),                     // Expect lines
                chars: Some(5),                     // Expect chars
                token_count: Some(0),
            });

            let root_meta_fs = fs::metadata(&root).unwrap();
            let expected_root = FsDirectoryNode {
                name: get_expected_root_name(&root),
                path: root.clone(),
                metadata: FsMetadata {
                    size: root_meta_fs.len(),
                    modified: root_meta_fs.modified().unwrap(),
                    permissions: get_perms(&root_meta_fs),
                    file_type: FsFileType::Directory,
                },
                contents: vec![expected_important],
            };

            // Check recursively (ignore timestamp, check file sizes)
            assert_nodes_equal(
                &FsNode::Directory(output.root),
                &FsNode::Directory(expected_root.clone()),
                true,
                true,
            );

            // Stats
            assert_eq!(
                output.stats.text_files_processed.load(Ordering::Relaxed),
                1, // important.txt is Text
                "Stats Text Files"
            );
            assert_eq!(
                output.stats.binary_files_processed.load(Ordering::Relaxed),
                0, // No binary files processed
                "Stats Binary Files"
            );
            assert_eq!(
                output.stats.total_entries_processed(),
                1, // Only important.txt
                "Stats Total Entries Processed"
            );
            assert_eq!(
                output.stats.total_bytes.load(Ordering::Relaxed),
                5, // Size of important.txt
                "Stats Total Bytes"
            );
            assert_eq!(
                output.stats.total_lines.load(Ordering::Relaxed),
                1, // Lines in important.txt
                "Stats Total Lines"
            );
            assert_eq!(
                output.stats.total_chars.load(Ordering::Relaxed),
                5, // Chars in important.txt
                "Stats Total Chars"
            );
            // Check skipped stats if possible (ignore crate details might affect exact counts)
            // The other 4 files (image, doc, archive, report) were likely skipped by the walker due to overrides.
            // The stats currently don't have a specific counter for walker skips due to overrides.
            // assert_eq!(output.stats.entries_skipped_ignored.load(Ordering::Relaxed), 4); // Approximate expectation
        }

        #[test]
        fn filter_modified_date() {
            let (_temp_dir, root) = setup_test_directory(&[
                ("old.txt", Some(b"ancient")),
                ("recent.txt", Some(b"new")),
                ("middle.txt", Some(b"mid")),
            ])
            .unwrap();

            // Use UTC for consistency across machines/timezones if possible
            let now = SystemTime::now();
            let two_hours_ago = now.checked_sub(Duration::from_secs(7200)).unwrap();
            let four_hours_ago = now.checked_sub(Duration::from_secs(14400)).unwrap();
            let one_hour_ago = now.checked_sub(Duration::from_secs(3600)).unwrap();
            let three_hours_ago = now.checked_sub(Duration::from_secs(10800)).unwrap();

            // Set modification times using filetime crate
            set_file_mtime(
                root.join("old.txt"),
                FileTime::from_system_time(four_hours_ago),
            )
            .unwrap();
            set_file_mtime(
                root.join("recent.txt"),
                FileTime::from_system_time(one_hour_ago),
            )
            .unwrap();
            set_file_mtime(
                root.join("middle.txt"),
                FileTime::from_system_time(three_hours_ago),
            )
            .unwrap();

            // Allow FS time to settle, especially on slower systems or VMs
            thread::sleep(Duration::from_millis(100));

            // --- Test Case 1: Modified After ---
            let mut opts_after = default_opts();
            opts_after.skip_content = true;
            opts_after.modified_after = Some(two_hours_ago); // Keep files modified *after* 2 hours ago (i.e., within the last 2 hours)

            println!(
                "Scanning 'after' {}...",
                two_hours_ago
                    .duration_since(SystemTime::UNIX_EPOCH)
                    .unwrap()
                    .as_secs()
            );
            let scanner_after = FsScanner::new(&root, opts_after, None).unwrap();
            let output_after = scanner_after.scan().unwrap();
            println!(
                "'After' scan done. Found {} entries.",
                output_after.root.contents.len()
            );

            assert_eq!(
                output_after.root.contents.len(),
                1,
                "'After' filter: Expected 1 entry"
            );
            // Expect FileNode because heuristic identifies "new" as text even if content skipped
            if let Some(FsNode::File(node)) = output_after.root.contents.first() {
                assert_eq!(
                    node.name, "recent.txt",
                    "'After' filter: Expected 'recent.txt'"
                );
                // Optionally assert file_type if needed, though the pattern match confirms it
                assert_eq!(node.metadata.file_type, FsFileType::TextFile);
            } else {
                panic!(
                    "'After' filter: Expected a File node, got {:?}", // Updated panic message
                    output_after.root.contents.first()
                );
            }
            assert_eq!(
                output_after.stats.total_entries_processed(),
                1,
                "'After' filter: Stats Processed"
            );
            assert_eq!(
                output_after
                    .stats
                    .entries_skipped_date
                    .load(Ordering::Relaxed),
                2,
                "'After' filter: Stats Skipped Date"
            ); // old.txt, middle.txt

            // --- Test Case 2: Modified Before ---
            thread::sleep(Duration::from_millis(50)); // Small delay between tests
            let mut opts_before = default_opts();
            opts_before.skip_content = true;
            opts_before.modified_before = Some(two_hours_ago); // Keep files modified *before* 2 hours ago

            println!(
                "Scanning 'before' {}...",
                two_hours_ago
                    .duration_since(SystemTime::UNIX_EPOCH)
                    .unwrap()
                    .as_secs()
            );
            let scanner_before = FsScanner::new(&root, opts_before, None).unwrap();
            let output_before = scanner_before.scan().unwrap();
            println!(
                "'Before' scan done. Found {} entries.",
                output_before.root.contents.len()
            );

            assert_eq!(
                output_before.root.contents.len(),
                2,
                "'Before' filter: Expected 2 entries"
            );
            let names_before: Vec<_> = output_before
                .root
                .contents
                .iter()
                .map(|n| n.name())
                .collect();
            assert!(
                names_before.contains(&"old.txt"), // Pass &&str by passing &"..."
                "'Before' filter: Missing 'old.txt'"
            );
            assert!(
                names_before.contains(&"middle.txt"), // Pass &&str by passing &"..."
                "'Before' filter: Missing 'middle.txt'"
            );
            assert_eq!(
                output_before.stats.total_entries_processed(),
                2,
                "'Before' filter: Stats Processed"
            );
            assert_eq!(
                output_before
                    .stats
                    .entries_skipped_date
                    .load(Ordering::Relaxed),
                1,
                "'Before' filter: Stats Skipped Date"
            ); // recent.txt

            // --- Test Case 3: Modified Between ---
            thread::sleep(Duration::from_millis(50));
            let mut opts_between = default_opts();
            opts_between.skip_content = true;
            opts_between.modified_after = Some(four_hours_ago); // Keep files modified *after* 4 hours ago
            opts_between.modified_before = Some(two_hours_ago); // AND *before* 2 hours ago

            println!(
                "Scanning 'between' {} and {}...",
                four_hours_ago
                    .duration_since(SystemTime::UNIX_EPOCH)
                    .unwrap()
                    .as_secs(),
                two_hours_ago
                    .duration_since(SystemTime::UNIX_EPOCH)
                    .unwrap()
                    .as_secs()
            );
            let scanner_between = FsScanner::new(&root, opts_between, None).unwrap();
            let output_between = scanner_between.scan().unwrap();
            println!(
                "'Between' scan done. Found {} entries.",
                output_between.root.contents.len()
            );

            assert_eq!(
                output_between.root.contents.len(),
                2, // Expect both old.txt and middle.txt
                "'Between' filter: Expected 2 entries"
            );
            // Check that both expected files are present and are File nodes
            let names_between: Vec<_> = output_between
                .root
                .contents
                .iter()
                .map(|n| {
                    assert!(
                        matches!(n, FsNode::File(_)),
                        "Expected File node, got {:?}",
                        n
                    );
                    n.name()
                })
                .collect();
            assert!(
                names_between.contains(&"old.txt"),
                "'Between' filter: Missing 'old.txt'"
            );
            assert!(
                names_between.contains(&"middle.txt"),
                "'Between' filter: Missing 'middle.txt'"
            );

            assert_eq!(
                output_between.stats.total_entries_processed(),
                2, // Correct processed count
                "'Between' filter: Stats Processed"
            );
            assert_eq!(
                output_between
                    .stats
                    .entries_skipped_date
                    .load(Ordering::Relaxed),
                1, // Only recent.txt should be skipped by date
                "'Between' filter: Stats Skipped Date"
            ); // recent.txt
        }

        #[test]
        #[cfg(unix)] // Permissions filtering is Unix-specific in the code
        fn filter_permissions() {
            let (_temp_dir, root) = setup_test_directory(&[
                ("readonly.txt", Some(b"ro")),
                ("writable.txt", Some(b"rw")),
                ("executable.sh", Some(b"exec")),
            ])
            .unwrap();

            // Set permissions (use standard octal notation)
            fs::set_permissions(root.join("readonly.txt"), fs::Permissions::from_mode(0o444))
                .unwrap(); // Read-only for all
            fs::set_permissions(root.join("writable.txt"), fs::Permissions::from_mode(0o666))
                .unwrap(); // Read-write for all
            fs::set_permissions(
                root.join("executable.sh"),
                fs::Permissions::from_mode(0o755),
            )
            .unwrap(); // rwxr-xr-x

            // --- Test Case 1: Filter for exact 0o666 ---
            let mut opts_rw = default_opts();
            opts_rw.permissions_filter = Some(0o666);

            let scanner_rw = FsScanner::new(&root, opts_rw, None).unwrap();
            let output_rw = scanner_rw.scan().unwrap();

            assert_eq!(
                output_rw.root.contents.len(),
                1,
                "Perm filter 0o666: Expected 1 entry"
            );
            if let Some(FsNode::File(node)) = output_rw.root.contents.first() {
                // Use .first()
                assert_eq!(
                    node.name, "writable.txt",
                    "Perm filter 0o666: Expected 'writable.txt'"
                );
            } else {
                panic!(
                    "Perm filter 0o666: Expected a File node, got {:?}",
                    output_rw.root.contents.first()
                );
            }
            assert_eq!(
                output_rw.stats.total_entries_processed(),
                1,
                "Perm filter 0o666: Stats Processed"
            );
            assert_eq!(
                output_rw
                    .stats
                    .entries_skipped_permission
                    .load(Ordering::Relaxed),
                2,
                "Perm filter 0o666: Stats Skipped Perm"
            ); // readonly, executable

            // --- Test Case 2: Filter for exact 0o755 ---
            let mut opts_exec = default_opts();
            opts_exec.permissions_filter = Some(0o755);

            let scanner_exec = FsScanner::new(&root, opts_exec, None).unwrap();
            let output_exec = scanner_exec.scan().unwrap();

            assert_eq!(
                output_exec.root.contents.len(),
                1,
                "Perm filter 0o755: Expected 1 entry"
            );
            if let Some(FsNode::File(node)) = output_exec.root.contents.first() {
                // Use .first()
                assert_eq!(
                    node.name, "executable.sh",
                    "Perm filter 0o755: Expected 'executable.sh'"
                );
            } else {
                panic!(
                    "Perm filter 0o755: Expected a File node, got {:?}",
                    output_exec.root.contents.first()
                );
            }
            assert_eq!(
                output_exec.stats.total_entries_processed(),
                1,
                "Perm filter 0o755: Stats Processed"
            );
            assert_eq!(
                output_exec
                    .stats
                    .entries_skipped_permission
                    .load(Ordering::Relaxed),
                2,
                "Perm filter 0o755: Stats Skipped Perm"
            ); // readonly, writable

            // --- Test Case 3: Filter for non-matching permissions ---
            let mut opts_nomatch = default_opts();
            opts_nomatch.permissions_filter = Some(0o777); // No file has these exact perms

            let scanner_nomatch = FsScanner::new(&root, opts_nomatch, None).unwrap();
            let output_nomatch = scanner_nomatch.scan().unwrap();

            assert_eq!(
                output_nomatch.root.contents.len(),
                0,
                "Perm filter 0o777: Expected 0 entries"
            );
            assert_eq!(
                output_nomatch.stats.total_entries_processed(),
                0,
                "Perm filter 0o777: Stats Processed"
            );
            assert_eq!(
                output_nomatch
                    .stats
                    .entries_skipped_permission
                    .load(Ordering::Relaxed),
                3,
                "Perm filter 0o777: Stats Skipped Perm"
            );
        }
    }

    mod error_handling {
        use super::*;
        use test_log::test;

        #[test]
        fn scan_invalid_root_path() {
            let non_existent_path = PathBuf::from("./surely_this_does_not_exist_42");
            let opts = default_opts();
            let result = FsScanner::new(&non_existent_path, opts, None);

            assert!(result.is_err());
            match result.err().unwrap() {
                // Should fail on canonicalize
                ScannerError::CanonicalizeError { path, .. } => {
                    assert_eq!(path, non_existent_path);
                }
                e => panic!("Expected CanonicalizeError, got {:?}", e),
            }
        }

        #[test]
        fn scan_root_is_file() {
            let (_temp_dir, root) =
                setup_test_directory(&[("root_file.txt", Some(b"i am a file"))]).unwrap();
            let file_path = root.join("root_file.txt");
            let opts = default_opts();
            let result = FsScanner::new(&file_path, opts, None); // Pass file path as root

            assert!(result.is_err());
            match result.err().unwrap() {
                // Should fail on the is_dir() check after canonicalization
                ScannerError::InvalidRoot(path) => {
                    // The path in the error might be canonicalized
                    assert_eq!(path, fs::canonicalize(&file_path).unwrap());
                }
                e => panic!("Expected InvalidRoot error, got {:?}", e),
            }
        }

        #[test]
        #[cfg(unix)] // Relies on setting restrictive permissions
        fn scan_permission_denied_on_subdir() {
            let (_temp_dir, root) = setup_test_directory(&[
                ("accessible.txt", Some(b"ok")),
                ("restricted_dir/", None),
                ("restricted_dir/secret.txt", Some(b"hidden")),
            ])
            .unwrap();

            let restricted_path = root.join("restricted_dir");

            // Make the directory inaccessible (no read/execute)
            fs::set_permissions(&restricted_path, fs::Permissions::from_mode(0o000)).unwrap();

            let opts = default_opts();
            let scanner = FsScanner::new(&root, opts, None).unwrap();
            let output = scanner.scan().unwrap(); // Should proceed but skip entering the restricted dir

            // Check that both the accessible file and the restricted directory node are present
            assert_eq!(
                output.root.contents.len(),
                2,
                "Expected accessible file and restricted dir node at root"
            );

            let mut found_accessible = false;
            let mut found_restricted_empty_dir = false;

            for node in &output.root.contents {
                match node {
                    FsNode::File(_) | FsNode::Binary(_) if node.name() == "accessible.txt" => {
                        found_accessible = true;
                    }
                    FsNode::Directory(d) if d.name == "restricted_dir" => {
                        assert!(
                            d.contents.is_empty(),
                            "Restricted directory node should have empty contents"
                        );
                        found_restricted_empty_dir = true;
                    }
                    _ => panic!("Unexpected node found in root: {:?}", node),
                }
            }

            assert!(found_accessible, "Accessible file node not found");
            assert!(
                found_restricted_empty_dir,
                "Restricted directory node not found or not empty"
            );

            // Check stats: two processed (file, dir node), one skipped (entering dir/reading file inside)
            assert_eq!(
                output.stats.total_entries_processed(),
                2, // accessible.txt and restricted_dir node itself
                "Stats: total_entries_processed"
            );
            // The skip happens in the *walker* when it tries to enter restricted_dir,
            // or potentially when the processor tries to read secret.txt (less likely path).
            assert_eq!(
                output.stats.entries_skipped_error.load(Ordering::Relaxed),
                1,
                "Stats: entries_skipped_error should be 1 (for walker error on restricted_dir)"
            );

            // Cleanup: Restore permissions so the temp dir can be deleted cleanly
            fs::set_permissions(&restricted_path, fs::Permissions::from_mode(0o755)).unwrap();
        }
    }
}