scour-secrets 0.19.0

use std::collections::HashMap;
use std::fs;
use std::io::{self, BufReader, BufWriter, Cursor, Read, Write};
use std::path::Path;
use std::sync::{Arc, Mutex};
use tracing::{info, warn};

use scour_secrets::secrets::{
    decrypt_secrets, encrypt_secrets, looks_encrypted, parse_secrets, serialize_secrets,
    SecretEntry, SecretsFormat,
};
use scour_secrets::{
    atomic_write, atomic_write_private, extract_context, extract_context_reader, ArchiveFilter,
    ArchiveFormat, ArchiveProcessor, ArchiveProgress, AtomicFileWriter, EntryCallback, FileReport,
    FileTypeProfile, LlmEntry, LogContextConfig, LogContextResult, MappingStore, Processor,
    ProcessorRegistry, ReportBuilder, ScanPattern, ScanStats, StreamScanner,
};
use zeroize::Zeroizing;

use crate::cli_args::Cli;
use crate::entropy::{
    entropy_histogram_bytes, entropy_scan_bytes, scanner_fallback, EntropyBuckets, EntropyConfig,
    NullSeekWriter, HISTOGRAM_THRESHOLDS,
};
use crate::input::{format_to_ext, is_structured_filename};
use crate::progress::{with_progress_scope, SharedProgressReporter};

/// Maximum output size buffered in memory when `--extract-context` is used
/// and the sanitized output is directed to stdout.
pub(crate) const MAX_CONTEXT_BUFFER_BYTES: u64 = 256 * 1024 * 1024; // 256 MiB

pub(crate) use scour_secrets::MIN_DISCOVERED_LITERAL_LEN;

/// Shared per-run collector for `--llm`: (label, sanitized_bytes) pairs in input order.
pub(crate) type LlmCollector = Arc<Mutex<Vec<LlmEntry>>>;

/// All per-run context needed to process a single file, stdin, or archive.
/// Every field is a reference or `Copy` type, so `FileProcessor<'a>` is `Copy`.
#[derive(Copy, Clone)]
pub(crate) struct FileProcessor<'a> {
    pub(crate) cli: &'a crate::cli_args::Cli,
    pub(crate) scanner: &'a Arc<StreamScanner>,
    pub(crate) registry: &'a Arc<ProcessorRegistry>,
    pub(crate) store: &'a Arc<MappingStore>,
    pub(crate) profiles: &'a [FileTypeProfile],
    pub(crate) report_builder: Option<&'a ReportBuilder>,
    pub(crate) progress: Option<&'a SharedProgressReporter>,
    pub(crate) llm_collector: Option<&'a LlmCollector>,
    pub(crate) entropy_configs: &'a Arc<Vec<crate::entropy::EntropyConfig>>,
    pub(crate) entropy_histogram_acc: Option<&'a Arc<Mutex<Vec<crate::entropy::EntropyBuckets>>>>,
    /// When `true`, a structured file's format-preserving scanner is built from
    /// the **entire** mapping store rather than only the values discovered while
    /// processing that file. This lets values found in *other* structured files
    /// (e.g. the same email or password in two configs) be redacted here too —
    /// including where they appear in comments or unstructured regions. Set on
    /// the output pass that runs after the discovery pre-pass has populated the
    /// store; the discovery pass itself uses the per-file delta.
    pub(crate) full_store_pass: bool,
}

use scour_secrets::entropy::merge_entropy_counts;

/// Run entropy scanning on `bytes` in-place, merging label counts into `stats`.
/// Returns the (potentially modified) bytes; does nothing when entropy configs
/// is empty, which is the common case.
fn apply_entropy_inplace(bytes: Vec<u8>, stats: &mut ScanStats, fp: FileProcessor<'_>) -> Vec<u8> {
    if fp.entropy_configs.is_empty() {
        return bytes;
    }
    let (out, lc) = entropy_scan_bytes(&bytes, fp.entropy_configs, fp.store);
    merge_entropy_counts(stats, lc);
    out
}

/// Complete a buffered scan: record the result to the report, run
/// context extraction, write or collect the output, and return
/// `had_matches`.
///
/// Called from every code path that already has the full sanitized
/// bytes in memory (structured path, `scan_plain_scanner` buffered
/// branches, etc.).
fn finalize_buffered_scan(
    output_bytes: &[u8],
    stats: &ScanStats,
    label: &str,
    method: &str,
    output_path: Option<&Path>,
    cli: &crate::cli_args::Cli,
    fp: FileProcessor<'_>,
) -> Result<bool, String> {
    let had_matches = stats.matches_found > 0;

    if let Some(rb) = fp.report_builder {
        rb.record_file(FileReport::from_scan_stats(
            label.to_string(),
            stats,
            method,
        ));
    }

    if cli.dry_run {
        tracing::info!(
            matches = stats.matches_found,
            replacements = stats.replacements_applied,
            "dry-run complete"
        );
        return Ok(had_matches);
    }

    maybe_extract_context(output_bytes, label, cli, fp.report_builder);
    write_or_collect(output_bytes, label, output_path, fp.llm_collector)?;
    Ok(had_matches)
}

fn accumulate_entropy_histogram(
    acc: &Arc<Mutex<Vec<EntropyBuckets>>>,
    buf: &[u8],
    configs: &[EntropyConfig],
) {
    let local = entropy_histogram_bytes(buf, configs);
    let mut guard = acc.lock().expect("entropy histogram lock");
    if guard.is_empty() {
        *guard = local;
    } else {
        for (dst, src) in guard.iter_mut().zip(local.iter()) {
            dst.merge(src);
        }
    }
}

pub(crate) fn print_entropy_histogram(buckets: &[EntropyBuckets]) {
    for b in buckets {
        let label_suffix = if b.label != "high_entropy_token" {
            format!(" [{}]", b.label)
        } else {
            String::new()
        };
        if b.total_candidates == 0 {
            eprintln!(
                "Entropy calibration{} — {} ({}–{} chars): no candidates found",
                label_suffix, b.charset_desc, b.min_length, b.max_length
            );
            continue;
        }
        eprintln!(
            "Entropy calibration{} — {} ({}–{} chars):",
            label_suffix, b.charset_desc, b.min_length, b.max_length
        );
        for (i, &thresh) in HISTOGRAM_THRESHOLDS.iter().enumerate() {
            let count = b.counts[i];
            let marker = if (thresh - b.configured_threshold).abs() < 1e-9 {
                "  ← threshold"
            } else {
                ""
            };
            eprintln!("  ≥{:.1} bits  {:>6}{}", thresh, count, marker);
        }
        if !HISTOGRAM_THRESHOLDS
            .iter()
            .any(|&t| (t - b.configured_threshold).abs() < 1e-9)
        {
            eprintln!(
                "  (configured threshold {:.2} bits falls between standard levels above)",
                b.configured_threshold
            );
        }
        eprintln!("  {} candidates examined", b.total_candidates);
    }
}

fn make_scan_callback(
    progress: Option<SharedProgressReporter>,
    label: impl Into<String>,
) -> impl FnMut(&scour_secrets::ScanProgress) {
    let label = label.into();
    move |scan_progress| {
        if let Some(reporter) = &progress {
            reporter
                .lock()
                .expect("progress reporter lock")
                .update_scan(&label, scan_progress);
        }
    }
}

fn scan_with_locations<R, W>(
    scanner: &StreamScanner,
    reader: R,
    writer: W,
    total_bytes: Option<u64>,
    progress_cb: impl FnMut(&scour_secrets::ScanProgress),
    max_locations: usize,
) -> Result<(ScanStats, Vec<scour_secrets::scanner::MatchLocation>, bool), String>
where
    R: std::io::Read,
    W: std::io::Write,
{
    let mut locations: Vec<scour_secrets::scanner::MatchLocation> = Vec::new();
    let mut truncated = false;
    let stats = scanner
        .scan_reader_with_callbacks(reader, writer, total_bytes, progress_cb, |loc| {
            if max_locations == 0 {
                return;
            }
            if locations.len() < max_locations {
                locations.push(loc);
            } else {
                truncated = true;
            }
        })
        .map_err(|e| format!("scanner error: {e}"))?;
    Ok((stats, locations, truncated))
}

/// Return `true` if the first 512 bytes look like binary.
fn looks_binary(data: &[u8]) -> bool {
    let sample = &data[..data.len().min(512)];
    if sample.contains(&0u8) {
        return true;
    }
    let non_text = sample
        .iter()
        .filter(|&&b| b < 0x20 && b != b'\n' && b != b'\r' && b != b'\t')
        .count();
    non_text as f64 / sample.len().max(1) as f64 > 0.10
}

fn build_log_context_config(cli: &Cli) -> LogContextConfig {
    let mut config = LogContextConfig::new()
        .with_context_lines(cli.context_lines)
        .with_max_matches(cli.max_context_matches)
        .case_sensitive(cli.context_case_sensitive);
    if !cli.context_keywords.is_empty() {
        config = if cli.context_keywords_replace {
            config.with_keywords(cli.context_keywords.iter().cloned())
        } else {
            config.with_extra_keywords(cli.context_keywords.iter().cloned())
        };
    }
    config
}

fn maybe_extract_context(
    bytes: &[u8],
    report_path: &str,
    cli: &Cli,
    report_builder: Option<&ReportBuilder>,
) {
    if !cli.extract_context {
        return;
    }
    let Some(rb) = report_builder else { return };
    let text = String::from_utf8_lossy(bytes);
    rb.set_file_log_context(
        report_path,
        extract_context(&text, &build_log_context_config(cli)),
    );
}

fn maybe_extract_context_reader(
    out_path: &Path,
    report_path: &str,
    cli: &Cli,
    report_builder: Option<&ReportBuilder>,
) {
    if !cli.extract_context {
        return;
    }
    let Some(rb) = report_builder else { return };
    let config = build_log_context_config(cli);
    let file = match fs::File::open(out_path) {
        Ok(f) => f,
        Err(e) => {
            warn!(error = %e, path = %out_path.display(), "--extract-context: failed to open output file for context scan");
            return;
        }
    };
    match extract_context_reader(BufReader::new(file), &config) {
        Ok(result) => rb.set_file_log_context(report_path, result),
        Err(e) => warn!(error = %e, "--extract-context: failed to read output for log context"),
    }
}

pub(crate) fn abs_label(path: &Path) -> String {
    fs::canonicalize(path)
        .unwrap_or_else(|_| std::env::current_dir().unwrap_or_default().join(path))
        .display()
        .to_string()
}

fn maybe_collect_for_llm(bytes: &[u8], label: &str, collector: Option<&LlmCollector>) {
    if let Some(c) = collector {
        if let Ok(mut guard) = c.lock() {
            guard.push((label.to_string(), bytes.to_vec()));
        }
    }
}

pub(crate) fn write_output(output_path: Option<&Path>, data: &[u8]) -> Result<(), String> {
    match output_path {
        Some(path) if path != Path::new("-") => {
            atomic_write(path, data)
                .map_err(|e| format!("failed to write {}: {e}", path.display()))?;
            info!(output = %path.display(), "output written");
        }
        _ => {
            let stdout = io::stdout();
            let mut lock = stdout.lock();
            lock.write_all(data)
                .map_err(|e| format!("failed to write to stdout: {e}"))?;
        }
    }
    Ok(())
}

fn write_or_collect(
    data: &[u8],
    label: &str,
    output_path: Option<&Path>,
    collector: Option<&LlmCollector>,
) -> Result<(), String> {
    if let Some(c) = collector {
        maybe_collect_for_llm(data, label, Some(c));
        Ok(())
    } else {
        write_output(output_path, data)
    }
}

fn file_size(path: &Path) -> Result<u64, String> {
    fs::metadata(path)
        .map(|metadata| metadata.len())
        .map_err(|e| format!("failed to stat {}: {e}", path.display()))
}

fn try_structured_processing(
    content: &[u8],
    filename: &str,
    registry: &Arc<ProcessorRegistry>,
    store: &Arc<MappingStore>,
    profiles: &[FileTypeProfile],
) -> Option<Result<Vec<u8>, String>> {
    let profile = profiles.iter().find(|p| p.matches_filename(filename))?;
    match registry.process(content, profile, store) {
        Ok(Some(result)) => Some(Ok(result)),
        Ok(None) => None,
        Err(e) => Some(Err(e.to_string())),
    }
}

/// Span-based structured processing: returns `Some(Ok(edited_bytes))` when a
/// profile matches *and* its processor supports byte-span edits (field values
/// replaced in place, format-preserving and leak-free); `None` when no profile
/// matches or the processor doesn't support edits (caller falls back to
/// [`try_structured_processing`]); `Some(Err)` on parse failure.
fn try_structured_edits(
    content: &[u8],
    filename: &str,
    registry: &Arc<ProcessorRegistry>,
    store: &Arc<MappingStore>,
    profiles: &[FileTypeProfile],
) -> Option<Result<(Vec<u8>, usize), String>> {
    let profile = profiles.iter().find(|p| p.matches_filename(filename))?;
    match registry.process_to_edits(content, profile, store) {
        Ok(Some(result)) => Some(Ok(result)),
        Ok(None) => None,
        Err(e) => Some(Err(e.to_string())),
    }
}

/// Compute the bytes the format-preserving scanner should run over for a
/// structured file, paired with the number of in-place field edits made (so the
/// caller can count profile-field redactions in the run summary). Returns:
///
/// - edit-applied bytes with the edit count, when the processor supports span edits;
/// - the original bytes with a zero count, when only the literal structured pass
///   applies (the store still gets populated and those values are counted later
///   by the scanner);
/// - `None` when no structured processing happened (caller uses the plain
///   scanner over the originals).
pub(crate) fn structured_base_bytes(
    input_bytes: &[u8],
    filename: &str,
    fp: &FileProcessor,
    strict: bool,
) -> Result<Option<(Vec<u8>, usize)>, String> {
    // Prefer span-based edit mode: exact, format-preserving, leak-free.
    match try_structured_edits(input_bytes, filename, fp.registry, fp.store, fp.profiles) {
        Some(Ok((edited, count))) => return Ok(Some((edited, count))),
        Some(Err(e)) if strict => return Err(format!("structured processing failed: {e}")),
        Some(Err(e)) => warn!(error = %e, "structured edit pass failed, trying literal pass"),
        None => {}
    }
    // Fall back to the literal structured pass (populates the store; the
    // format-preserving scanner then runs over the original bytes and counts the
    // re-matched values, so the edit count here is 0).
    match try_structured_processing(input_bytes, filename, fp.registry, fp.store, fp.profiles) {
        Some(Ok(_)) => Ok(Some((input_bytes.to_vec(), 0))),
        Some(Err(e)) if strict => Err(format!("structured processing failed: {e}")),
        Some(Err(e)) => {
            warn!(error = %e, "structured processing failed, falling back to scanner");
            Ok(None)
        }
        None => Ok(None),
    }
}

fn build_format_preserving_scanner(
    base_scanner: &Arc<StreamScanner>,
    store: &Arc<MappingStore>,
    snapshot: scour_secrets::store::StoreSnapshot,
) -> Result<StreamScanner, scour_secrets::error::SanitizeError> {
    let extra: Vec<ScanPattern> = store
        .iter_since(snapshot)
        .filter(|(_, orig, _)| orig.len() >= MIN_DISCOVERED_LITERAL_LEN)
        .filter_map(|(category, original, _)| {
            let s = original.as_str();
            // Label by category, never the value (labels appear in user-facing
            // report/findings/summary output, which must contain no secrets).
            let label = format!("field:{category}");
            match ScanPattern::from_literal(s, category, label) {
                Ok(pat) => Some(pat),
                Err(e) => {
                    warn!(error = %e, "could not compile field literal pattern");
                    None
                }
            }
        })
        .collect();

    base_scanner.for_structured_pass(extra)
}

pub(crate) fn load_profiles(path: &Path) -> Result<Vec<FileTypeProfile>, String> {
    let raw =
        fs::read(path).map_err(|e| format!("failed to read profile '{}': {e}", path.display()))?;
    let text = std::str::from_utf8(&raw)
        .map_err(|_| format!("profile '{}' is not valid UTF-8", path.display()))?;
    let ext = path.extension().and_then(|e| e.to_str()).unwrap_or("");
    let profiles: Vec<FileTypeProfile> = match ext {
        "json" => serde_json::from_str(text)
            .map_err(|e| format!("profile '{}': invalid JSON: {e}", path.display())),
        "yaml" | "yml" => serde_yaml_ng::from_str(text)
            .map_err(|e| format!("profile '{}': invalid YAML: {e}", path.display())),
        _ => serde_json::from_str(text)
            .or_else(|_| serde_yaml_ng::from_str(text))
            .map_err(|e| {
                format!(
                    "profile '{}': could not parse as JSON or YAML: {e}",
                    path.display()
                )
            }),
    }?;

    for (i, p) in profiles.iter().enumerate() {
        for pat in p.include.iter().chain(p.exclude.iter()) {
            glob::Pattern::new(pat).map_err(|e| {
                format!(
                    "profile '{}' entry {i}: invalid glob '{pat}': {e}",
                    path.display()
                )
            })?;
        }
    }

    Ok(profiles)
}

/// Whether a YAML document's first content line opens a flow sequence
/// (`[...]`). Appending block-sequence items to such a file would corrupt it,
/// so those files are re-serialized instead.
fn yaml_is_flow_sequence(raw: &[u8]) -> bool {
    raw.split(|&b| b == b'\n')
        .map(|line| {
            let start = line
                .iter()
                .position(|b| !b.is_ascii_whitespace())
                .unwrap_or(line.len());
            &line[start..]
        })
        .find(|l| !l.is_empty() && l[0] != b'#')
        .is_some_and(|l| l[0] == b'[')
}

/// Append store-discovered literal values to the secrets file at `path`,
/// preserving the file's on-disk form:
///
/// - **Encrypted file** (`was_encrypted`, requires `password`): decrypt →
///   parse → merge → serialize → re-encrypt (fresh salt + nonce) → atomic
///   write. Plaintext exists only in zeroizing memory; the file on disk is
///   never downgraded to plaintext.
/// - **Plaintext file**: a YAML file keeps its existing bytes verbatim —
///   comments included — and gets the new entries appended as sequence items;
///   JSON / TOML files are parsed and re-serialized in their own format
///   (extension-derived `format`, falling back to content detection, then
///   YAML for new files).
pub(crate) fn save_discovered_secrets(
    store: &Arc<MappingStore>,
    path: &Path,
    password: Option<&str>,
    was_encrypted: bool,
    format: Option<SecretsFormat>,
) -> std::result::Result<usize, String> {
    let mut new_entries: Vec<SecretEntry> = store
        .iter()
        // Never persist a literal the format-preserving scanner would itself
        // reject as too short (see MIN_DISCOVERED_LITERAL_LEN) — a 1–3 char
        // value written here poisons every future run with false positives.
        .filter(|(_, original, _)| original.len() >= MIN_DISCOVERED_LITERAL_LEN)
        .map(|(category, original, _)| {
            SecretEntry::new(original.to_string(), "literal", category.to_string())
                .with_label("discovered")
        })
        .collect();

    if new_entries.is_empty() {
        return Ok(0);
    }

    // Fail closed if the recorded flag and the bytes on disk disagree (e.g.
    // the file was swapped between load and write-back): never write plaintext
    // over a file that currently looks encrypted.
    let raw: Option<Zeroizing<Vec<u8>>> = if path.exists() {
        Some(Zeroizing::new(fs::read(path).map_err(|e| {
            format!("failed to read {}: {e}", path.display())
        })?))
    } else {
        None
    };
    let encrypted_now = raw.as_deref().is_some_and(|r| looks_encrypted(r));

    let (existing, effective_format): (Vec<SecretEntry>, SecretsFormat) = match &raw {
        Some(raw) if was_encrypted || encrypted_now => {
            let pw = password.ok_or_else(|| {
                format!(
                    "{} is encrypted but no password is available for write-back",
                    path.display()
                )
            })?;
            let plaintext = decrypt_secrets(raw, pw)
                .map_err(|e| format!("failed to decrypt {}: {e}", path.display()))?;
            let fmt = format.unwrap_or_else(|| SecretsFormat::detect(&plaintext));
            let entries = parse_secrets(&plaintext, Some(fmt))
                .map_err(|e| format!("failed to parse {}: {e}", path.display()))?;
            (entries, fmt)
        }
        Some(raw) => {
            let fmt = format.unwrap_or_else(|| SecretsFormat::detect(raw));
            let entries = parse_secrets(raw, Some(fmt))
                .map_err(|e| format!("failed to parse {}: {e}", path.display()))?;
            (entries, fmt)
        }
        None => (vec![], format.unwrap_or(SecretsFormat::Yaml)),
    };

    let existing_patterns: std::collections::HashSet<&str> =
        existing.iter().map(|e| e.pattern.as_str()).collect();

    new_entries.retain(|e| !existing_patterns.contains(e.pattern.as_str()));
    // The store can hold the same value under several categories (e.g. a
    // field rule and a scanner pattern both matched it). Keep one entry per
    // pattern — duplicates would inflate the file on every run. Sort+dedup
    // instead of a HashSet so no unzeroized copies of the values are made.
    new_entries.sort_by(|a, b| a.pattern.cmp(&b.pattern).then(a.category.cmp(&b.category)));
    new_entries.dedup_by(|a, b| a.pattern == b.pattern);
    let added = new_entries.len();

    if added == 0 {
        return Ok(0);
    }

    let output: Zeroizing<Vec<u8>> = if was_encrypted || encrypted_now {
        let mut all_entries: Vec<SecretEntry> = existing;
        all_entries.extend(new_entries);
        let serialized = Zeroizing::new(
            serialize_secrets(&all_entries, effective_format)
                .map_err(|e| format!("failed to serialize discovered secrets: {e}"))?,
        );
        let pw = password.expect("checked above: encrypted path requires a password");
        Zeroizing::new(
            encrypt_secrets(&serialized, pw)
                .map_err(|e| format!("failed to re-encrypt {}: {e}", path.display()))?,
        )
    } else if effective_format == SecretsFormat::Yaml
        && raw.as_deref().is_some_and(|r| !yaml_is_flow_sequence(r))
    {
        // Comment-preserving append: app bundles and hand-written secrets
        // files are comment-heavy documentation, and a re-serialize would
        // strip all of it. A YAML top-level block sequence accepts appended
        // items, so keep the existing bytes verbatim and add only the new
        // entries. (Flow-style `[...]` files take the re-serialize path.)
        let new_yaml = Zeroizing::new(
            serialize_secrets(&new_entries, SecretsFormat::Yaml)
                .map_err(|e| format!("failed to serialize discovered secrets: {e}"))?,
        );
        let mut buf = Zeroizing::new(raw.as_deref().map(|r| r.to_vec()).unwrap_or_default());
        if !buf.is_empty() && !buf.ends_with(b"\n") {
            buf.push(b'\n');
        }
        buf.extend_from_slice(&new_yaml);
        buf
    } else {
        let mut all_entries: Vec<SecretEntry> = existing;
        all_entries.extend(new_entries);
        Zeroizing::new(
            serialize_secrets(&all_entries, effective_format)
                .map_err(|e| format!("failed to serialize discovered secrets: {e}"))?,
        )
    };

    atomic_write_private(path, &output)
        .map_err(|e| format!("failed to write {}: {e}", path.display()))?;

    Ok(added)
}

fn record_archive_stats(rb: &ReportBuilder, stats: &scour_secrets::ArchiveStats) {
    for (path, method) in &stats.file_methods {
        if let Some(scan_stats) = stats.file_scan_stats.get(path) {
            rb.record_file(FileReport::from_scan_stats(
                path.clone(),
                scan_stats,
                method.clone(),
            ));
        } else {
            rb.record_file(FileReport::new(path.clone(), method.clone()));
        }
    }

    if stats.file_methods.is_empty() {
        let mut fr = FileReport::new(
            "(archive)",
            format!(
                "archive({} files, {} structured, {} scanner)",
                stats.files_processed, stats.structured_hits, stats.scanner_fallback
            ),
        );
        fr.bytes_processed = stats.total_input_bytes;
        fr.bytes_output = stats.total_output_bytes;
        rb.record_file(fr);
    }
}

fn print_archive_stats(output: &Path, stats: &scour_secrets::ArchiveStats) {
    info!(
        files = stats.files_processed,
        structured = stats.structured_hits,
        scanner = stats.scanner_fallback,
        output = %output.display(),
        "archive processing complete"
    );
}

mod archive;
mod file;
mod stdin;

#[cfg(test)]
mod tests {
    use super::*;
    use scour_secrets::{MappingStore, RandomGenerator};
    use std::sync::Arc;

    fn empty_store() -> Arc<MappingStore> {
        Arc::new(MappingStore::new(Arc::new(RandomGenerator::new()), None))
    }

    fn store_with_entry(original: &str) -> Arc<MappingStore> {
        let store = empty_store();
        store
            .get_or_insert(&scour_secrets::Category::AuthToken, original)
            .unwrap();
        store
    }

    // ── save_discovered_secrets ──────────────────────────────────────────────

    #[test]
    fn save_discovered_secrets_empty_store_returns_zero() {
        let dir = tempfile::tempdir().unwrap();
        let path = dir.path().join("secrets.yaml");
        let n = save_discovered_secrets(&empty_store(), &path, None, false, None).unwrap();
        assert_eq!(n, 0);
        assert!(
            !path.exists(),
            "no file should be created when store is empty"
        );
    }

    #[test]
    fn save_discovered_secrets_skips_short_values() {
        // A 1–3 char discovered value (e.g. a structured field that held "v")
        // must never be persisted: as a global literal it would corrupt every
        // future run. Nothing shorter than MIN_DISCOVERED_LITERAL_LEN is kept.
        let dir = tempfile::tempdir().unwrap();
        let path = dir.path().join("secrets.yaml");
        let store = empty_store();
        for v in ["v", "id", "abc"] {
            store
                .get_or_insert(&scour_secrets::Category::AuthToken, v)
                .unwrap();
        }

        let n = save_discovered_secrets(&store, &path, None, false, None).unwrap();
        assert_eq!(n, 0, "sub-threshold values must not be persisted");
        assert!(
            !path.exists(),
            "no file should be written when every value is too short"
        );
    }

    #[test]
    fn save_discovered_secrets_dedups_same_value_across_categories() {
        // The same value can land in the store under two categories (e.g. an
        // instance id matched by both a field rule and a scanner regex). Only
        // one literal entry may be persisted, or the file grows duplicate
        // patterns on every run.
        let dir = tempfile::tempdir().unwrap();
        let path = dir.path().join("secrets.yaml");
        let store = empty_store();
        store
            .get_or_insert(&scour_secrets::Category::AuthToken, "up7gpa36")
            .unwrap();
        store
            .get_or_insert(
                &scour_secrets::Category::Custom("instance_id".into()),
                "up7gpa36",
            )
            .unwrap();

        let n = save_discovered_secrets(&store, &path, None, false, None).unwrap();
        assert_eq!(n, 1, "one entry per pattern regardless of category");
        let content = fs::read_to_string(&path).unwrap();
        assert_eq!(
            content.matches("up7gpa36").count(),
            1,
            "value must appear exactly once in the file"
        );
    }

    #[test]
    fn save_discovered_secrets_writes_new_entries() {
        let dir = tempfile::tempdir().unwrap();
        let path = dir.path().join("secrets.yaml");
        let store = store_with_entry("abc123");

        let n = save_discovered_secrets(&store, &path, None, false, None).unwrap();
        assert_eq!(n, 1);
        let content = fs::read_to_string(&path).unwrap();
        assert!(
            content.contains("abc123"),
            "written file should contain the literal value"
        );
        assert!(
            content.contains("literal"),
            "entry kind should be 'literal'"
        );
    }

    #[test]
    fn save_discovered_secrets_skips_existing_patterns() {
        let dir = tempfile::tempdir().unwrap();
        let path = dir.path().join("secrets.yaml");

        // First write.
        let store = store_with_entry("abc123");
        save_discovered_secrets(&store, &path, None, false, None).unwrap();

        // Second write with the same value — should add 0 new entries.
        let n = save_discovered_secrets(&store, &path, None, false, None).unwrap();
        assert_eq!(n, 0);

        // File should still have exactly one entry.
        let content = fs::read_to_string(&path).unwrap();
        assert_eq!(content.matches("abc123").count(), 1);
    }

    #[test]
    fn save_discovered_secrets_adds_to_existing_file() {
        let dir = tempfile::tempdir().unwrap();
        let path = dir.path().join("secrets.yaml");

        let store1 = store_with_entry("first_secret");
        save_discovered_secrets(&store1, &path, None, false, None).unwrap();

        let store2 = store_with_entry("second_secret");
        let n = save_discovered_secrets(&store2, &path, None, false, None).unwrap();
        assert_eq!(n, 1);

        let content = fs::read_to_string(&path).unwrap();
        assert!(content.contains("first_secret"));
        assert!(content.contains("second_secret"));
    }

    #[test]
    fn save_discovered_secrets_preserves_yaml_comments() {
        // The write-back must not strip comments: app bundle secrets files
        // are comment-heavy documentation, and the handoff rewrites them on
        // every --app run. New entries are appended; existing bytes stay.
        let dir = tempfile::tempdir().unwrap();
        let path = dir.path().join("secrets.yaml");
        let original = "# Header comment — must survive the write-back\n\
                        - pattern: existing_secret\n\
                        \x20 kind: literal\n\
                        \x20 category: auth_token\n\
                        \x20 label: existing # inline comment\n";
        fs::write(&path, original).unwrap();

        let store = store_with_entry("freshly_discovered");
        let n = save_discovered_secrets(&store, &path, None, false, None).unwrap();
        assert_eq!(n, 1);

        let content = fs::read_to_string(&path).unwrap();
        assert!(
            content.starts_with(original),
            "existing bytes (comments included) must be untouched: {content}"
        );
        assert!(content.contains("freshly_discovered"));
        // And the appended result must still parse as a secrets file.
        let entries = parse_secrets(content.as_bytes(), None).unwrap();
        assert_eq!(entries.len(), 2);
    }

    #[test]
    fn save_discovered_secrets_reserializes_flow_yaml() {
        // A flow-style `[...]` document can't take appended block items;
        // it falls back to the re-serialize path.
        let dir = tempfile::tempdir().unwrap();
        let path = dir.path().join("secrets.yaml");
        fs::write(&path, "[]\n").unwrap();

        let store = store_with_entry("freshly_discovered");
        let n = save_discovered_secrets(&store, &path, None, false, None).unwrap();
        assert_eq!(n, 1);

        let content = fs::read_to_string(&path).unwrap();
        let entries = parse_secrets(content.as_bytes(), None).unwrap();
        assert_eq!(entries.len(), 1);
        assert_eq!(entries[0].pattern, "freshly_discovered");
    }

    #[test]
    fn save_discovered_secrets_errors_on_malformed_existing_file() {
        let dir = tempfile::tempdir().unwrap();
        let path = dir.path().join("secrets.yaml");
        fs::write(&path, b"this: is: not: valid: yaml: ][[[").unwrap();

        let store = store_with_entry("abc123");
        let result = save_discovered_secrets(&store, &path, None, false, None);
        assert!(
            result.is_err(),
            "malformed YAML should return an error, not silently lose data"
        );
    }

    #[test]
    fn save_discovered_secrets_encrypted_roundtrip() {
        let dir = tempfile::tempdir().unwrap();
        let path = dir.path().join("secrets.yaml.enc");
        let pw = "writeback-pw";

        // Seed an encrypted file with one existing entry.
        let initial = b"- pattern: existing_secret\n  kind: literal\n";
        fs::write(&path, encrypt_secrets(initial, pw).unwrap()).unwrap();

        let store = store_with_entry("discovered_secret");
        let n = save_discovered_secrets(&store, &path, Some(pw), true, None).unwrap();
        assert_eq!(n, 1);

        // File must still be encrypted — never downgraded to plaintext.
        let raw = fs::read(&path).unwrap();
        assert!(
            looks_encrypted(&raw),
            "write-back must keep the file encrypted"
        );
        assert!(
            !raw.windows(b"discovered_secret".len())
                .any(|w| w == b"discovered_secret"),
            "ciphertext must not contain the plaintext value"
        );

        // Same password decrypts; both old and new entries survive the merge.
        let plaintext = decrypt_secrets(&raw, pw).unwrap();
        let entries = parse_secrets(&plaintext, None).unwrap();
        let patterns: Vec<&str> = entries.iter().map(|e| e.pattern.as_str()).collect();
        assert!(patterns.contains(&"existing_secret"));
        assert!(patterns.contains(&"discovered_secret"));
    }

    #[test]
    fn save_discovered_secrets_encrypted_wrong_password_fails_unchanged() {
        let dir = tempfile::tempdir().unwrap();
        let path = dir.path().join("secrets.yaml.enc");
        fs::write(
            &path,
            encrypt_secrets(b"- pattern: existing\n  kind: literal\n", "right").unwrap(),
        )
        .unwrap();
        let before = fs::read(&path).unwrap();

        let store = store_with_entry("discovered");
        let result = save_discovered_secrets(&store, &path, Some("wrong"), true, None);
        assert!(result.is_err(), "wrong password must fail the write-back");
        assert_eq!(fs::read(&path).unwrap(), before, "file must be unchanged");
    }

    #[test]
    fn save_discovered_secrets_encrypted_without_password_fails_closed() {
        // Flag says plaintext but the bytes on disk are encrypted (file swapped
        // between load and write-back): must refuse rather than corrupt.
        let dir = tempfile::tempdir().unwrap();
        let path = dir.path().join("secrets.yaml");
        fs::write(&path, encrypt_secrets(b"- pattern: x\n", "pw").unwrap()).unwrap();
        let before = fs::read(&path).unwrap();

        let store = store_with_entry("discovered");
        let result = save_discovered_secrets(&store, &path, None, false, None);
        assert!(
            result.is_err(),
            "encrypted-looking file without password must fail"
        );
        assert_eq!(fs::read(&path).unwrap(), before);
    }

    #[test]
    fn save_discovered_secrets_preserves_json_format() {
        let dir = tempfile::tempdir().unwrap();
        let path = dir.path().join("secrets.json");
        fs::write(&path, br#"[{"pattern":"existing","kind":"literal"}]"#).unwrap();

        let store = store_with_entry("discovered");
        let n =
            save_discovered_secrets(&store, &path, None, false, Some(SecretsFormat::Json)).unwrap();
        assert_eq!(n, 1);

        // Must reparse as JSON — not YAML-overwritten.
        let raw = fs::read(&path).unwrap();
        let entries = parse_secrets(&raw, Some(SecretsFormat::Json))
            .expect("write-back must keep the file valid JSON");
        assert_eq!(entries.len(), 2);
    }

    #[test]
    fn save_discovered_secrets_preserves_toml_format() {
        let dir = tempfile::tempdir().unwrap();
        let path = dir.path().join("secrets.toml");
        fs::write(
            &path,
            b"[[secrets]]\npattern = \"existing\"\nkind = \"literal\"\n",
        )
        .unwrap();

        let store = store_with_entry("discovered");
        let n =
            save_discovered_secrets(&store, &path, None, false, Some(SecretsFormat::Toml)).unwrap();
        assert_eq!(n, 1);

        let raw = fs::read(&path).unwrap();
        let entries = parse_secrets(&raw, Some(SecretsFormat::Toml))
            .expect("write-back must keep the file valid TOML");
        assert_eq!(entries.len(), 2);
    }

    // ── looks_binary ─────────────────────────────────────────────────────────

    #[test]
    fn looks_binary_detects_null_byte() {
        assert!(looks_binary(b"hello\x00world"));
    }

    #[test]
    fn looks_binary_detects_high_control_char_ratio() {
        // Build a buffer where >10% of bytes are non-text control chars.
        let mut data = vec![0x01u8; 60]; // SOH — control char, not \n/\r/\t
        data.extend_from_slice(&[b'a'; 40]);
        assert!(looks_binary(&data), "60% control chars should look binary");
    }

    #[test]
    fn looks_binary_passes_plain_text() {
        let text = b"key = \"some value\"\nfoo = bar\npath = /tmp/file.txt\n";
        assert!(!looks_binary(text));
    }

    #[test]
    fn looks_binary_passes_text_with_tabs_and_cr() {
        let text = b"col1\tcol2\r\nval1\tval2\r\n";
        assert!(
            !looks_binary(text),
            "tabs and CR should not count as binary"
        );
    }

    #[test]
    fn looks_binary_samples_only_first_512_bytes() {
        // Put NUL after position 512 — should not be detected.
        let mut data = vec![b'a'; 600];
        data[513] = 0x00;
        assert!(
            !looks_binary(&data),
            "NUL after byte 512 should not trigger binary detection"
        );
    }

    // ── merge_entropy_counts ─────────────────────────────────────────────────

    #[test]
    fn merge_entropy_counts_adds_to_stats() {
        use scour_secrets::ScanStats;
        let mut stats = ScanStats::default();
        let mut counts = HashMap::new();
        counts.insert("high_entropy_token".to_string(), 3u64);
        counts.insert("other_label".to_string(), 2u64);
        merge_entropy_counts(&mut stats, counts);
        assert_eq!(stats.matches_found, 5);
        assert_eq!(stats.replacements_applied, 5);
        assert_eq!(stats.pattern_counts["high_entropy_token"], 3);
        assert_eq!(stats.pattern_counts["other_label"], 2);
    }

    #[test]
    fn merge_entropy_counts_empty_map_is_noop() {
        use scour_secrets::ScanStats;
        let mut stats = ScanStats::default();
        merge_entropy_counts(&mut stats, HashMap::new());
        assert_eq!(stats.matches_found, 0);
        assert_eq!(stats.replacements_applied, 0);
        assert!(stats.pattern_counts.is_empty());
    }
}