keyhog_core/spec/
load.rs

1//! Detector loading pipeline: read TOML files and run the quality gate.
2
3#![allow(clippy::result_large_err)] // SpecError carries a 128-byte toml::de::Error; boxing it would be a breaking API change.
4
5use std::path::{Path, PathBuf};
6
7use rayon::prelude::*;
8use thiserror::Error;
9
10use super::{
11    migrate_legacy_success_policies, validate_detector, DetectorCorpusManifest, DetectorFile,
12    DetectorSpec, QualityIssue, DETECTOR_CORPUS_MANIFEST_FILE,
13    DETECTOR_CORPUS_MAX_FORWARD_SCHEMA_VERSION, DETECTOR_CORPUS_MIN_SCHEMA_VERSION,
14    DETECTOR_CORPUS_SCHEMA_VERSION,
15};
16pub use crate::detector_file_io::{read_detector_toml_file, DETECTOR_TOML_FILE_BYTES};
17
18/// Errors returned while loading or validating detector specifications.
19#[derive(Debug, Error)]
20#[allow(clippy::result_large_err)] // SpecError variants include 128-byte toml::de::Error; boxing would be a breaking API change.
21pub enum SpecError {
22    #[error(
23        "failed to read detector path {path}: {source}. Fix: check the detector path exists and that the file is readable TOML"
24    )]
25    ReadFile {
26        path: String,
27        source: std::io::Error,
28    },
29    #[error(
30        "invalid TOML in detector {path}: {source}. Fix: repair the TOML syntax in the detector file"
31    )]
32    InvalidToml {
33        path: PathBuf,
34        source: toml::de::Error,
35    },
36    #[error(
37        "invalid detector corpus manifest {path}: {source}. Fix: set `schema_version` \
38         to an integer supported by this keyhog binary and remove misspelled manifest fields"
39    )]
40    InvalidCorpusManifest {
41        path: PathBuf,
42        source: toml::de::Error,
43    },
44    #[error(
45        "unsupported detector corpus schema {found} declared by {path}; this binary \
46         supports schema {current} and bounded forward compatibility through schema \
47         {max_forward}. Fix: use a compatible detector corpus or update keyhog"
48    )]
49    UnsupportedCorpusSchema {
50        path: PathBuf,
51        found: u32,
52        current: u32,
53        max_forward: u32,
54    },
55    #[error(
56        "detector corpus {dir} declares supported forward schema {declared_schema}, \
57         while this binary owns schema {supported_schema}; {skipped_count} of {total} \
58         detector file(s) use fields this binary cannot interpret. Keyhog refuses to \
59         scan under newer parsing semantics or with a partial corpus because either \
60         would invalidate corpus identity and could silently drop recall. \
61         Compatibility detail:\n{detail}\nFix: update keyhog to load the complete \
62         detector corpus"
63    )]
64    ForwardIncompatibleCorpus {
65        dir: String,
66        declared_schema: u32,
67        supported_schema: u32,
68        skipped_count: usize,
69        total: usize,
70        detail: String,
71    },
72    #[error(
73        "{failed_count} of {total} embedded detector(s) failed to parse, the binary \
74         baked in a CORRUPT detector set, so its recall is silently degraded. This is \
75         a build/source bug, not a runtime condition: the embedded corpus is compiled \
76         in and cannot have been edited at runtime. Offending detector(s):\n{detail}\n\
77         Fix: repair the named TOML(s) under `detectors/` (the toml error names the \
78         line/column) and rebuild keyhog so build.rs re-embeds a valid set."
79    )]
80    EmbeddedCorpusCorrupt {
81        failed_count: usize,
82        total: usize,
83        detail: String,
84    },
85    #[error(
86        "{failed_count} of {total} detector file(s) from {dir} failed to load, \
87         pass the quality gate, or exist at all, that is a partial detector \
88         corpus, so keyhog is refusing to scan without a complete detector \
89         corpus (a partial corpus silently drops recall). \
90         Offending detector(s):\n{detail}\nFix: repair the named TOML file(s) \
91         or add at least one valid `*.toml` detector spec, then rerun the scan."
92    )]
93    DetectorCorpusRejected {
94        dir: String,
95        failed_count: usize,
96        total: usize,
97        detail: String,
98    },
99}
100
101/// A validated detector corpus paired with the schema identity that selected
102/// its normalization rules.
103///
104/// `schema_version` is the normalized directory identity: a missing manifest
105/// is schema 1, while an explicit manifest contributes its declared version.
106/// Keeping it beside `specs` prevents a caller from hashing normalized legacy
107/// specs as though they had been authored under the current schema.
108#[derive(Debug)]
109pub struct LoadedDetectorCorpus {
110    /// Fully parsed and validated detector specifications.
111    pub specs: Vec<DetectorSpec>,
112    /// Effective detector corpus schema version.
113    pub schema_version: u32,
114}
115
116impl LoadedDetectorCorpus {
117    /// Compute the schema-bound identity of these exact normalized specs.
118    pub fn compute_digest(&self) -> Result<[u8; 32], serde_json::Error> {
119        crate::compute_detector_corpus_digest_for_schema(&self.specs, self.schema_version)
120    }
121}
122
123/// Load all detector specs from a directory of TOML files.
124/// Runs the quality gate on each detector and fails closed if any detector
125/// cannot be read, parsed, or accepted by the gate.
126///
127/// # Examples
128///
129/// ```rust,no_run
130/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
131/// use keyhog_core::load_detectors;
132/// use std::path::Path;
133///
134/// let detectors = load_detectors(Path::new("detectors"))?;
135/// assert!(!detectors.is_empty());
136/// # Ok(()) }
137/// ```
138pub fn load_detectors(dir: &Path) -> Result<Vec<DetectorSpec>, SpecError> {
139    Ok(load_detector_corpus(dir)?.specs)
140}
141
142/// Load all detector specs together with their normalized corpus schema
143/// identity.
144pub fn load_detector_corpus(dir: &Path) -> Result<LoadedDetectorCorpus, SpecError> {
145    load_detector_corpus_with_gate(dir, true)
146}
147
148/// Load detectors with optional quality gate enforcement.
149/// When `enforce_gate` is `true`, detector read/parse/quality errors reject
150/// the entire corpus instead of returning a partial detector set.
151///
152/// # Examples
153///
154/// ```ignore
155/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
156/// // Crate-internal hook for tests and CLI detector-cache owner code.
157/// use keyhog_core::spec::load::load_detectors_with_gate;
158/// use std::path::Path;
159///
160/// let _detectors = load_detectors_with_gate(Path::new("detectors"), true)?;
161/// # Ok(()) }
162/// ```
163#[derive(Clone, Copy)]
164struct CorpusCompatibility {
165    schema_version: u32,
166    permits_forward_unknown_fields: bool,
167}
168
169pub(crate) fn load_detectors_with_gate(
170    dir: &Path,
171    enforce_gate: bool,
172) -> Result<Vec<DetectorSpec>, SpecError> {
173    Ok(load_detector_corpus_with_gate(dir, enforce_gate)?.specs)
174}
175
176fn load_detector_corpus_with_gate(
177    dir: &Path,
178    enforce_gate: bool,
179) -> Result<LoadedDetectorCorpus, SpecError> {
180    let compatibility = read_corpus_compatibility(dir)?;
181    let toml_paths = discover_detector_tomls(dir, enforce_gate)?;
182    let parsed = parse_detector_files(&toml_paths, compatibility);
183    let specs = assemble_detector_load(dir, enforce_gate, compatibility, toml_paths.len(), parsed)?;
184    Ok(LoadedDetectorCorpus {
185        specs,
186        schema_version: compatibility.schema_version,
187    })
188}
189
190fn read_corpus_compatibility(dir: &Path) -> Result<CorpusCompatibility, SpecError> {
191    let path = dir.join(DETECTOR_CORPUS_MANIFEST_FILE);
192    let contents = match std::fs::read_to_string(&path) {
193        Ok(contents) => contents,
194        Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
195            return Ok(CorpusCompatibility {
196                schema_version: DETECTOR_CORPUS_MIN_SCHEMA_VERSION,
197                permits_forward_unknown_fields: false,
198            });
199        }
200        Err(source) => {
201            return Err(SpecError::ReadFile {
202                path: path.display().to_string(),
203                source,
204            });
205        }
206    };
207    let manifest: DetectorCorpusManifest =
208        toml::from_str(&contents).map_err(|source| SpecError::InvalidCorpusManifest {
209            path: path.clone(),
210            source,
211        })?;
212    if !(DETECTOR_CORPUS_MIN_SCHEMA_VERSION..=DETECTOR_CORPUS_MAX_FORWARD_SCHEMA_VERSION)
213        .contains(&manifest.schema_version)
214    {
215        return Err(SpecError::UnsupportedCorpusSchema {
216            path,
217            found: manifest.schema_version,
218            current: DETECTOR_CORPUS_SCHEMA_VERSION,
219            max_forward: DETECTOR_CORPUS_MAX_FORWARD_SCHEMA_VERSION,
220        });
221    }
222    Ok(CorpusCompatibility {
223        schema_version: manifest.schema_version,
224        permits_forward_unknown_fields: manifest.schema_version > DETECTOR_CORPUS_SCHEMA_VERSION,
225    })
226}
227
228fn discover_detector_tomls(dir: &Path, enforce_gate: bool) -> Result<Vec<PathBuf>, SpecError> {
229    let entries = std::fs::read_dir(dir).map_err(|e| SpecError::ReadFile {
230        path: dir.display().to_string(),
231        source: e,
232    })?;
233    let mut toml_paths = Vec::new();
234    for entry in entries {
235        let entry = entry.map_err(|e| SpecError::ReadFile {
236            path: format!("directory entry under {}", dir.display()),
237            source: e,
238        })?;
239        let path = entry.path();
240        if path.extension().is_some_and(|ext| ext == "toml")
241            && path
242                .file_name()
243                .is_none_or(|name| name != DETECTOR_CORPUS_MANIFEST_FILE)
244        {
245            toml_paths.push(path);
246        }
247    }
248
249    if enforce_gate && toml_paths.is_empty() {
250        return Err(SpecError::DetectorCorpusRejected {
251            dir: dir.display().to_string(),
252            failed_count: 0,
253            total: 0,
254            detail:
255                "  - no detector TOML files found; add at least one valid `*.toml` detector spec"
256                    .to_string(),
257        });
258    }
259    Ok(toml_paths)
260}
261
262fn parse_detector_files(
263    toml_paths: &[PathBuf],
264    compatibility: CorpusCompatibility,
265) -> Vec<ReadDetectorOutcome> {
266    toml_paths
267        .par_iter()
268        .map(|path| read_detector_file(path, compatibility))
269        .collect()
270}
271
272fn assemble_detector_load(
273    dir: &Path,
274    enforce_gate: bool,
275    compatibility: CorpusCompatibility,
276    total: usize,
277    parsed: Vec<ReadDetectorOutcome>,
278) -> Result<Vec<DetectorSpec>, SpecError> {
279    let mut load_state = DetectorLoadState::default();
280    let mut detectors = Vec::with_capacity(parsed.len());
281
282    for outcome in parsed {
283        match outcome {
284            ReadDetectorOutcome::Loaded {
285                path,
286                spec,
287                legacy_migrations,
288            } => {
289                load_state.legacy_migrations += legacy_migrations;
290                if should_reject_detector(
291                    &spec,
292                    &path,
293                    enforce_gate,
294                    &mut load_state.gate_rejected,
295                    &mut load_state.gate_errors,
296                    &mut load_state.total_warnings,
297                ) {
298                    continue;
299                }
300                detectors.push(*spec);
301            }
302            ReadDetectorOutcome::ForwardSkipped { message } => {
303                load_state.forward_skipped += 1;
304                load_state.forward_errors.push(message);
305            }
306            ReadDetectorOutcome::Skipped { message } => {
307                load_state.skipped += 1;
308                load_state.load_errors.push(message);
309            }
310        }
311    }
312
313    // Sort before the duplicate-id scan so identical ids are adjacent and one
314    // linear pass finds them. A detector id is a unique key, it selects the
315    // checksum validator, suppression rules, and finding attribution, so two
316    // detectors sharing an id silently shadow each other (the loser's
317    // patterns/companions never fire). Law 10: surface it, don't let it pass.
318    // Folded into the SAME gate as other corpus-integrity failures (fail closed
319    // under the gate, logged otherwise) rather than a bespoke rejection path.
320    detectors.sort_by(|a, b| a.id.cmp(&b.id));
321    let mut duplicate_ids: Vec<&str> = detectors
322        .windows(2)
323        .filter(|w| w[0].id == w[1].id)
324        .map(|w| w[0].id.as_str())
325        .collect();
326    duplicate_ids.dedup();
327    if !duplicate_ids.is_empty() {
328        load_state.gate_rejected += duplicate_ids.len();
329        for id in duplicate_ids {
330            load_state.gate_errors.push(format!(
331                "duplicate detector id `{id}` (a later spec would shadow the earlier)"
332            ));
333        }
334    }
335
336    log_load_summary(&load_state);
337    if enforce_gate && compatibility.permits_forward_unknown_fields {
338        return Err(load_state.into_forward_error(dir, total, compatibility.schema_version));
339    }
340    if enforce_gate && load_state.has_failures() {
341        return Err(load_state.into_rejected_error(dir, total));
342    }
343    Ok(detectors)
344}
345
346#[derive(Default)]
347struct DetectorLoadState {
348    skipped: usize,
349    load_errors: Vec<String>,
350    forward_skipped: usize,
351    forward_errors: Vec<String>,
352    legacy_migrations: usize,
353    gate_rejected: usize,
354    gate_errors: Vec<String>,
355    total_warnings: usize,
356}
357
358impl DetectorLoadState {
359    fn has_failures(&self) -> bool {
360        self.skipped > 0 || self.forward_skipped > 0 || self.gate_rejected > 0
361    }
362
363    fn into_rejected_error(self, dir: &Path, total: usize) -> SpecError {
364        let mut details = self.load_errors;
365        details.extend(self.gate_errors);
366        let detail = details
367            .into_iter()
368            .map(|line| format!("  - {line}"))
369            .collect::<Vec<_>>()
370            .join("\n");
371        SpecError::DetectorCorpusRejected {
372            dir: dir.display().to_string(),
373            failed_count: self.skipped + self.gate_rejected,
374            total,
375            detail,
376        }
377    }
378    fn into_forward_error(self, dir: &Path, total: usize, declared_schema: u32) -> SpecError {
379        let detail = if self.forward_errors.is_empty() {
380            format!(
381                "  - {} declares schema {}; schema metadata is part of effective \
382                 corpus identity and cannot be interpreted as schema {}",
383                dir.join(DETECTOR_CORPUS_MANIFEST_FILE).display(),
384                declared_schema,
385                DETECTOR_CORPUS_SCHEMA_VERSION
386            )
387        } else {
388            self.forward_errors
389                .into_iter()
390                .map(|line| format!("  - {line}"))
391                .collect::<Vec<_>>()
392                .join("\n")
393        };
394        SpecError::ForwardIncompatibleCorpus {
395            dir: dir.display().to_string(),
396            declared_schema,
397            supported_schema: DETECTOR_CORPUS_SCHEMA_VERSION,
398            skipped_count: self.forward_skipped,
399            total,
400            detail,
401        }
402    }
403}
404
405fn log_load_summary(state: &DetectorLoadState) {
406    if state.skipped > 0 {
407        // Aggregate into ONE actionable line instead of one warn! per file.
408        // Unknown fields reaching this strict path are same-version typos or
409        // undeclared version skew; a declared bounded-forward corpus is handled
410        // separately below.
411        let version_skew = state
412            .load_errors
413            .iter()
414            .filter(|error| error.contains("unknown field"))
415            .count();
416        let examples = state
417            .load_errors
418            .iter()
419            .take(3)
420            .map(String::as_str)
421            .collect::<Vec<_>>()
422            .join(" | ");
423        if version_skew > 0 {
424            tracing::warn!(
425                "skipped {} detector file(s); {} contain unknown fields while the corpus \
426                 is using the current/legacy strict schema. Fix field typos, or add a \
427                 supported newer `{}` declaration when the fields are intentional. \
428                 Examples: {examples}",
429                state.skipped,
430                version_skew,
431                DETECTOR_CORPUS_MANIFEST_FILE
432            );
433        } else {
434            tracing::warn!(
435                "skipped {} malformed/unreadable detector file(s) - run \
436                 `keyhog detectors --detectors <DIR>` or -vv for the full list. \
437                 Examples: {examples}",
438                state.skipped
439            );
440        }
441    }
442    if state.forward_skipped > 0 {
443        let examples = state
444            .forward_errors
445            .iter()
446            .take(3)
447            .map(String::as_str)
448            .collect::<Vec<_>>()
449            .join(" | ");
450        tracing::warn!(
451            "detector corpus declared a supported forward schema; skipped {} detector \
452             file(s) that use newer fields rather than silently dropping those fields. \
453             Update keyhog for full recall. Examples: {examples}",
454            state.forward_skipped
455        );
456    }
457    if state.legacy_migrations > 0 {
458        tracing::warn!(
459            "migrated {} legacy schema-{} verifier success contract(s) to \
460             status_with_error_backstop; add an explicit success policy and a \
461             schema-{} corpus manifest",
462            state.legacy_migrations,
463            DETECTOR_CORPUS_MIN_SCHEMA_VERSION,
464            DETECTOR_CORPUS_SCHEMA_VERSION
465        );
466    }
467    if state.gate_rejected > 0 {
468        // Law 10: quality-gate rejections are not silent. The per-detector
469        // causes are logged at warn! below; the aggregate is surfaced at
470        // the default level so operators see why the detector set would have
471        // been smaller than expected.
472        tracing::warn!(
473            "quality gate rejected {} detectors (see per-detector warnings above)",
474            state.gate_rejected
475        );
476    }
477    if state.total_warnings > 0 {
478        // Advisory (non-rejecting) quality warnings describe detector-AUTHORING
479        // nits on the already-validated, shipped detector set (e.g. "companion
480        // regex is a pure character class; ALLOWED because within_lines <= 5").
481        // They are build-time/authoring feedback, not an operator signal: the
482        // bundled detectors passed the gate, so re-announcing their advisories
483        // on every user command that loads detectors (`explain`, `detectors`,
484        // a custom `--detectors` dir) is noise that drowns out the real
485        // rejections above. Keep them at debug! (visible with `-vv` /
486        // RUST_LOG=keyhog=debug for authors); errors and gate REJECTIONS stay
487        // loud above (Law 10).
488        tracing::debug!("quality gate: {} advisory warnings", state.total_warnings);
489    }
490}
491
492enum ReadDetectorOutcome {
493    Loaded {
494        path: PathBuf,
495        spec: Box<DetectorSpec>,
496        legacy_migrations: usize,
497    },
498    ForwardSkipped {
499        message: String,
500    },
501    Skipped {
502        message: String,
503    },
504}
505
506fn read_detector_file(path: &Path, compatibility: CorpusCompatibility) -> ReadDetectorOutcome {
507    let contents = match read_detector_toml_file(path) {
508        Ok(contents) => contents,
509        Err(error) => {
510            // LAW10: reporting-only; per-file detail stays at debug! (visible
511            // with -vv), while `log_load_summary` warns and gated loads reject.
512            // One warn! per skipped file floods stderr on a version-skewed or
513            // partly-broken corpus (dozens of near-identical lines before any
514            // finding), which is the opposite of actionable.
515            let message = format!("failed to read {}: {}", path.display(), error);
516            tracing::debug!(
517                detector_path = %path.display(),
518                error = %error,
519                "skipping detector - unreadable file" // LAW10: aggregate warning surfaces the skipped file count and examples; gated loads reject
520            );
521            return ReadDetectorOutcome::Skipped { message };
522        }
523    };
524
525    match toml::from_str::<DetectorFile>(&contents) {
526        Ok(mut file) => {
527            let legacy_migrations =
528                if compatibility.schema_version == DETECTOR_CORPUS_MIN_SCHEMA_VERSION {
529                    migrate_legacy_success_policies(&mut file.detector)
530                } else {
531                    0
532                };
533            ReadDetectorOutcome::Loaded {
534                path: path.to_path_buf(),
535                spec: Box::new(file.detector),
536                legacy_migrations,
537            }
538        }
539        Err(error) => {
540            let unknown_field = error.to_string().contains("unknown field");
541            if compatibility.permits_forward_unknown_fields && unknown_field {
542                let message = format!(
543                    "skipped {} under declared detector corpus schema {} because it uses \
544                     a field unknown to schema {}: {}. Fix: update keyhog to load this detector",
545                    path.display(),
546                    compatibility.schema_version,
547                    DETECTOR_CORPUS_SCHEMA_VERSION,
548                    error
549                );
550                tracing::warn!(
551                    detector_path = %path.display(),
552                    declared_schema = compatibility.schema_version,
553                    supported_schema = DETECTOR_CORPUS_SCHEMA_VERSION,
554                    error = %error,
555                    "skipping forward-schema detector without dropping unknown fields"
556                );
557                return ReadDetectorOutcome::ForwardSkipped { message };
558            }
559            let message = format!(
560                "failed to parse {} under detector corpus schema {}: {}. Fix: correct \
561                 misspelled or invalid detector fields; only a corpus manifest declaring \
562                 a supported newer schema permits an unknown future field",
563                path.display(),
564                compatibility.schema_version,
565                error
566            );
567            // LAW10: the default-level aggregate warning surfaces every skip,
568            // and gated loads return DetectorCorpusRejected with this detail.
569            tracing::debug!(
570                detector_path = %path.display(),
571                schema_version = compatibility.schema_version,
572                error = %error,
573                "skipping detector - TOML parse failed"
574            );
575            ReadDetectorOutcome::Skipped { message }
576        }
577    }
578}
579
580fn should_reject_detector(
581    spec: &DetectorSpec,
582    path: &Path,
583    enforce_gate: bool,
584    gate_rejected: &mut usize,
585    gate_errors: &mut Vec<String>,
586    total_warnings: &mut usize,
587) -> bool {
588    let mut has_errors = false;
589    let mut detector_errors = Vec::new();
590    for issue in validate_detector(spec) {
591        match issue {
592            QualityIssue::Warning(warning) => {
593                // Advisory only - the detector still loads and scans. This is
594                // authoring feedback (see the aggregate at debug! in
595                // `log_load_summary`), so keep it at debug! to stay out of
596                // user-facing command output; errors below stay loud (Law 10).
597                tracing::debug!(detector_path = %path.display(), "quality: {} - {}", spec.id, warning);
598                *total_warnings += 1;
599            }
600            QualityIssue::Error(error) => {
601                // Law 10: a detector that fails the quality gate must not be
602                // silently loaded. The warning names the detector and the
603                // issue so the author can fix it; when enforce_gate is true
604                // the detector is rejected below.
605                tracing::warn!(
606                    detector_path = %path.display(),
607                    "detector quality error: {}: {}",
608                    spec.id,
609                    error
610                );
611                detector_errors.push(format!("{}: {}: {}", path.display(), spec.id, error));
612                has_errors = true;
613            }
614        }
615    }
616
617    if has_errors && enforce_gate {
618        *gate_rejected += 1;
619        gate_errors.extend(detector_errors);
620        return true;
621    }
622
623    false
624}
625
626/// Load a set of detectors from a TOML string.
627///
628/// This is primarily used for dynamic detector injection and tests that need
629/// an in-memory detector corpus.
630pub(crate) fn load_detectors_from_str(toml_str: &str) -> Result<Vec<DetectorSpec>, SpecError> {
631    let file: DetectorFile = toml::from_str(toml_str).map_err(|e| SpecError::InvalidToml {
632        path: PathBuf::from("<string>"),
633        source: e,
634    })?;
635    Ok(vec![file.detector])
636}
keyhog_core/spec/load.rs

keyhog_core/spec/
load.rs
