keyhog_core/spec/

load.rs

//! Detector loading pipeline: read TOML files and run the quality gate.

#![allow(clippy::result_large_err)] // SpecError carries a 128-byte toml::de::Error; boxing it would be a breaking API change.

use std::path::{Path, PathBuf};

use rayon::prelude::*;
use thiserror::Error;

use super::{validate_detector, DetectorFile, DetectorSpec, QualityIssue};
pub use crate::detector_file_io::{read_detector_toml_file, DETECTOR_TOML_FILE_BYTES};

/// Errors returned while loading or validating detector specifications.
#[derive(Debug, Error)]
#[allow(clippy::result_large_err)] // SpecError variants include 128-byte toml::de::Error; boxing would be a breaking API change.
pub enum SpecError {
    #[error(
        "failed to read detector path {path}: {source}. Fix: check the detector path exists and that the file is readable TOML"
    )]
    ReadFile {
        path: String,
        source: std::io::Error,
    },
    #[error(
        "invalid TOML in detector {path}: {source}. Fix: repair the TOML syntax in the detector file"
    )]
    InvalidToml {
        path: PathBuf,
        source: toml::de::Error,
    },
    #[error(
        "{failed_count} of {total} embedded detector(s) failed to parse, the binary \
         baked in a CORRUPT detector set, so its recall is silently degraded. This is \
         a build/source bug, not a runtime condition: the embedded corpus is compiled \
         in and cannot have been edited at runtime. Offending detector(s):\n{detail}\n\
         Fix: repair the named TOML(s) under `detectors/` (the toml error names the \
         line/column) and rebuild keyhog so build.rs re-embeds a valid set."
    )]
    EmbeddedCorpusCorrupt {
        failed_count: usize,
        total: usize,
        detail: String,
    },
    #[error(
        "{failed_count} of {total} detector file(s) from {dir} failed to load, \
         pass the quality gate, or exist at all, that is a partial detector \
         corpus, so keyhog is refusing to scan without a complete detector \
         corpus (a partial corpus silently drops recall). \
         Offending detector(s):\n{detail}\nFix: repair the named TOML file(s) \
         or add at least one valid `*.toml` detector spec, then rerun the scan."
    )]
    DetectorCorpusRejected {
        dir: String,
        failed_count: usize,
        total: usize,
        detail: String,
    },
}

/// Load all detector specs from a directory of TOML files.
/// Runs the quality gate on each detector and fails closed if any detector
/// cannot be read, parsed, or accepted by the gate.
///
/// # Examples
///
/// ```rust,no_run
/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
/// use keyhog_core::load_detectors;
/// use std::path::Path;
///
/// let detectors = load_detectors(Path::new("detectors"))?;
/// assert!(!detectors.is_empty());
/// # Ok(()) }
/// ```
pub fn load_detectors(dir: &Path) -> Result<Vec<DetectorSpec>, SpecError> {
    load_detectors_with_gate(dir, true)
}

/// Load detectors with optional quality gate enforcement.
/// When `enforce_gate` is `true`, detector read/parse/quality errors reject
/// the entire corpus instead of returning a partial detector set.
///
/// # Examples
///
/// ```ignore
/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
/// // Crate-internal hook for tests and CLI detector-cache owner code.
/// use keyhog_core::spec::load::load_detectors_with_gate;
/// use std::path::Path;
///
/// let _detectors = load_detectors_with_gate(Path::new("detectors"), true)?;
/// # Ok(()) }
/// ```
pub(crate) fn load_detectors_with_gate(
    dir: &Path,
    enforce_gate: bool,
) -> Result<Vec<DetectorSpec>, SpecError> {
    let toml_paths = discover_detector_tomls(dir, enforce_gate)?;
    let parsed = parse_detector_files(&toml_paths);
    assemble_detector_load(dir, enforce_gate, toml_paths.len(), parsed)
}

fn discover_detector_tomls(dir: &Path, enforce_gate: bool) -> Result<Vec<PathBuf>, SpecError> {
    let entries = std::fs::read_dir(dir).map_err(|e| SpecError::ReadFile {
        path: dir.display().to_string(),
        source: e,
    })?;
    let mut toml_paths = Vec::new();
    for entry in entries {
        let entry = entry.map_err(|e| SpecError::ReadFile {
            path: format!("directory entry under {}", dir.display()),
            source: e,
        })?;
        let path = entry.path();
        if path.extension().is_some_and(|ext| ext == "toml") {
            toml_paths.push(path);
        }
    }

    if enforce_gate && toml_paths.is_empty() {
        return Err(SpecError::DetectorCorpusRejected {
            dir: dir.display().to_string(),
            failed_count: 0,
            total: 0,
            detail:
                "  - no detector TOML files found; add at least one valid `*.toml` detector spec"
                    .to_string(),
        });
    }
    Ok(toml_paths)
}

fn parse_detector_files(toml_paths: &[PathBuf]) -> Vec<ReadDetectorOutcome> {
    toml_paths
        .par_iter()
        .map(|path| read_detector_file(path))
        .collect()
}

fn assemble_detector_load(
    dir: &Path,
    enforce_gate: bool,
    total: usize,
    parsed: Vec<ReadDetectorOutcome>,
) -> Result<Vec<DetectorSpec>, SpecError> {
    let mut load_state = DetectorLoadState::default();
    let mut detectors = Vec::with_capacity(parsed.len());

    for outcome in parsed {
        match outcome {
            ReadDetectorOutcome::Loaded { path, spec } => {
                if should_reject_detector(
                    &spec,
                    &path,
                    enforce_gate,
                    &mut load_state.gate_rejected,
                    &mut load_state.gate_errors,
                    &mut load_state.total_warnings,
                ) {
                    continue;
                }
                detectors.push(*spec);
            }
            ReadDetectorOutcome::Skipped { message } => {
                load_state.skipped += 1;
                load_state.load_errors.push(message);
            }
        }
    }

    // Sort before the duplicate-id scan so identical ids are adjacent and one
    // linear pass finds them. A detector id is a unique key, it selects the
    // checksum validator, suppression rules, and finding attribution, so two
    // detectors sharing an id silently shadow each other (the loser's
    // patterns/companions never fire). Law 10: surface it, don't let it pass.
    // Folded into the SAME gate as other corpus-integrity failures (fail closed
    // under the gate, logged otherwise) rather than a bespoke rejection path.
    detectors.sort_by(|a, b| a.id.cmp(&b.id));
    let mut duplicate_ids: Vec<&str> = detectors
        .windows(2)
        .filter(|w| w[0].id == w[1].id)
        .map(|w| w[0].id.as_str())
        .collect();
    duplicate_ids.dedup();
    if !duplicate_ids.is_empty() {
        load_state.gate_rejected += duplicate_ids.len();
        for id in duplicate_ids {
            load_state.gate_errors.push(format!(
                "duplicate detector id `{id}` (a later spec would shadow the earlier)"
            ));
        }
    }

    log_load_summary(&load_state);
    if enforce_gate && load_state.has_failures() {
        return Err(load_state.into_rejected_error(dir, total));
    }
    Ok(detectors)
}

#[derive(Default)]
struct DetectorLoadState {
    skipped: usize,
    load_errors: Vec<String>,
    gate_rejected: usize,
    gate_errors: Vec<String>,
    total_warnings: usize,
}

impl DetectorLoadState {
    fn has_failures(&self) -> bool {
        self.skipped > 0 || self.gate_rejected > 0
    }

    fn into_rejected_error(self, dir: &Path, total: usize) -> SpecError {
        let mut details = self.load_errors;
        details.extend(self.gate_errors);
        let detail = details
            .into_iter()
            .map(|line| format!("  - {line}"))
            .collect::<Vec<_>>()
            .join("\n");
        SpecError::DetectorCorpusRejected {
            dir: dir.display().to_string(),
            failed_count: self.skipped + self.gate_rejected,
            total,
            detail,
        }
    }
}

fn log_load_summary(state: &DetectorLoadState) {
    if state.skipped > 0 {
        // Aggregate into ONE actionable line instead of one warn! per file.
        // An "unknown field" parse error means the detector TOML declares a
        // field this binary's schema does not know, i.e. the corpus is newer
        // than the binary; the fix is `keyhog update`, not editing the file.
        let version_skew = state
            .load_errors
            .iter()
            .filter(|error| error.contains("unknown field"))
            .count();
        let examples = state
            .load_errors
            .iter()
            .take(3)
            .map(String::as_str)
            .collect::<Vec<_>>()
            .join(" | ");
        if version_skew > 0 {
            tracing::warn!(
                "skipped {} detector file(s); {} declare a field this keyhog version \
                 does not understand (the detector corpus is newer than the binary) - \
                 run `keyhog update`. Examples: {examples}",
                state.skipped,
                version_skew
            );
        } else {
            tracing::warn!(
                "skipped {} malformed/unreadable detector file(s) - run \
                 `keyhog detectors --detectors <DIR>` or -vv for the full list. \
                 Examples: {examples}",
                state.skipped
            );
        }
    }
    if state.gate_rejected > 0 {
        // Law 10: quality-gate rejections are not silent. The per-detector
        // causes are logged at warn! below; the aggregate is surfaced at
        // the default level so operators see why the detector set would have
        // been smaller than expected.
        tracing::warn!(
            "quality gate rejected {} detectors (see per-detector warnings above)",
            state.gate_rejected
        );
    }
    if state.total_warnings > 0 {
        // Advisory (non-rejecting) quality warnings describe detector-AUTHORING
        // nits on the already-validated, shipped detector set (e.g. "companion
        // regex is a pure character class; ALLOWED because within_lines <= 5").
        // They are build-time/authoring feedback, not an operator signal: the
        // bundled detectors passed the gate, so re-announcing their advisories
        // on every user command that loads detectors (`explain`, `detectors`,
        // a custom `--detectors` dir) is noise that drowns out the real
        // rejections above. Keep them at debug! (visible with `-vv` /
        // RUST_LOG=keyhog=debug for authors); errors and gate REJECTIONS stay
        // loud above (Law 10).
        tracing::debug!("quality gate: {} advisory warnings", state.total_warnings);
    }
}

enum ReadDetectorOutcome {
    Loaded {
        path: PathBuf,
        spec: Box<DetectorSpec>,
    },
    Skipped {
        message: String,
    },
}

fn read_detector_file(path: &Path) -> ReadDetectorOutcome {
    let contents = match read_detector_toml_file(path) {
        Ok(contents) => contents,
        Err(error) => {
            // LAW10: reporting-only; per-file detail stays at debug! (visible
            // with -vv), while `log_load_summary` warns and gated loads reject.
            // One warn! per skipped file floods stderr on a version-skewed or
            // partly-broken corpus (dozens of near-identical lines before any
            // finding), which is the opposite of actionable.
            let message = format!("failed to read {}: {}", path.display(), error);
            tracing::debug!(
                detector_path = %path.display(),
                error = %error,
                "skipping detector - unreadable file" // LAW10: aggregate warning surfaces the skipped file count and examples; gated loads reject
            );
            return ReadDetectorOutcome::Skipped { message };
        }
    };

    match toml::from_str::<DetectorFile>(&contents) {
        Ok(file) => ReadDetectorOutcome::Loaded {
            path: path.to_path_buf(),
            spec: Box::new(file.detector),
        },
        Err(error) => {
            // LAW10: reporting-only; per-file TOML detail stays at debug!,
            // while the summary warns with examples and gated loads reject the
            // incomplete corpus.
            let message = format!("failed to parse {}: {}", path.display(), error);
            tracing::debug!(
                detector_path = %path.display(),
                error = %error,
                "skipping detector - TOML parse failed" // LAW10: aggregate warning surfaces the skipped file count and examples; gated loads reject
            );
            ReadDetectorOutcome::Skipped { message }
        }
    }
}

fn should_reject_detector(
    spec: &DetectorSpec,
    path: &Path,
    enforce_gate: bool,
    gate_rejected: &mut usize,
    gate_errors: &mut Vec<String>,
    total_warnings: &mut usize,
) -> bool {
    let mut has_errors = false;
    let mut detector_errors = Vec::new();
    for issue in validate_detector(spec) {
        match issue {
            QualityIssue::Warning(warning) => {
                // Advisory only - the detector still loads and scans. This is
                // authoring feedback (see the aggregate at debug! in
                // `log_load_summary`), so keep it at debug! to stay out of
                // user-facing command output; errors below stay loud (Law 10).
                tracing::debug!(detector_path = %path.display(), "quality: {} - {}", spec.id, warning);
                *total_warnings += 1;
            }
            QualityIssue::Error(error) => {
                // Law 10: a detector that fails the quality gate must not be
                // silently loaded. The warning names the detector and the
                // issue so the author can fix it; when enforce_gate is true
                // the detector is rejected below.
                tracing::warn!(
                    detector_path = %path.display(),
                    "detector quality error: {}: {}",
                    spec.id,
                    error
                );
                detector_errors.push(format!("{}: {}: {}", path.display(), spec.id, error));
                has_errors = true;
            }
        }
    }

    if has_errors && enforce_gate {
        *gate_rejected += 1;
        gate_errors.extend(detector_errors);
        return true;
    }

    false
}

/// Load a set of detectors from a TOML string.
///
/// This is primarily used for dynamic detector injection and tests that need
/// an in-memory detector corpus.
pub(crate) fn load_detectors_from_str(toml_str: &str) -> Result<Vec<DetectorSpec>, SpecError> {
    let file: DetectorFile = toml::from_str(toml_str).map_err(|e| SpecError::InvalidToml {
        path: PathBuf::from("<string>"),
        source: e,
    })?;
    Ok(vec![file.detector])
}