1#![allow(clippy::result_large_err)] use std::path::{Path, PathBuf};
6
7use rayon::prelude::*;
8use thiserror::Error;
9
10use super::{
11 migrate_legacy_success_policies, validate_detector, DetectorCorpusManifest, DetectorFile,
12 DetectorSpec, QualityIssue, DETECTOR_CORPUS_MANIFEST_FILE,
13 DETECTOR_CORPUS_MAX_FORWARD_SCHEMA_VERSION, DETECTOR_CORPUS_MIN_SCHEMA_VERSION,
14 DETECTOR_CORPUS_SCHEMA_VERSION,
15};
16pub use crate::detector_file_io::{read_detector_toml_file, DETECTOR_TOML_FILE_BYTES};
17
18#[derive(Debug, Error)]
20#[allow(clippy::result_large_err)] pub enum SpecError {
22 #[error(
23 "failed to read detector path {path}: {source}. Fix: check the detector path exists and that the file is readable TOML"
24 )]
25 ReadFile {
27 path: String,
29 source: std::io::Error,
31 },
32 #[error(
33 "invalid TOML in detector {path}: {source}. Fix: repair the TOML syntax in the detector file"
34 )]
35 InvalidToml {
37 path: PathBuf,
39 source: toml::de::Error,
41 },
42 #[error(
43 "invalid detector corpus manifest {path}: {source}. Fix: set `schema_version` \
44 to an integer supported by this keyhog binary and remove misspelled manifest fields"
45 )]
46 InvalidCorpusManifest {
48 path: PathBuf,
50 source: toml::de::Error,
52 },
53 #[error(
54 "unsupported detector corpus schema {found} declared by {path}; this binary \
55 supports schema {current} and bounded forward compatibility through schema \
56 {max_forward}. Fix: use a compatible detector corpus or update keyhog"
57 )]
58 UnsupportedCorpusSchema {
60 path: PathBuf,
62 found: u32,
64 current: u32,
66 max_forward: u32,
68 },
69 #[error(
70 "detector corpus {dir} declares supported forward schema {declared_schema}, \
71 while this binary owns schema {supported_schema}; {skipped_count} of {total} \
72 detector file(s) use fields this binary cannot interpret. Keyhog refuses to \
73 scan under newer parsing semantics or with a partial corpus because either \
74 would invalidate corpus identity and could silently drop recall. \
75 Compatibility detail:\n{detail}\nFix: update keyhog to load the complete \
76 detector corpus"
77 )]
78 ForwardIncompatibleCorpus {
80 dir: String,
82 declared_schema: u32,
84 supported_schema: u32,
86 skipped_count: usize,
88 total: usize,
90 detail: String,
92 },
93 #[error(
94 "{failed_count} of {total} embedded detector(s) failed to parse, the binary \
95 baked in a CORRUPT detector set, so its recall is silently degraded. This is \
96 a build/source bug, not a runtime condition: the embedded corpus is compiled \
97 in and cannot have been edited at runtime. Offending detector(s):\n{detail}\n\
98 Fix: repair the named TOML(s) under `detectors/` (the toml error names the \
99 line/column) and rebuild keyhog so build.rs re-embeds a valid set."
100 )]
101 EmbeddedCorpusCorrupt {
103 failed_count: usize,
105 total: usize,
107 detail: String,
109 },
110 #[error(
111 "{failed_count} of {total} detector file(s) from {dir} failed to load, \
112 pass the quality gate, or exist at all, that is a partial detector \
113 corpus, so keyhog is refusing to scan without a complete detector \
114 corpus (a partial corpus silently drops recall). \
115 Offending detector(s):\n{detail}\nFix: repair the named TOML file(s) \
116 or add at least one valid `*.toml` detector spec, then rerun the scan."
117 )]
118 DetectorCorpusRejected {
120 dir: String,
122 failed_count: usize,
124 total: usize,
126 detail: String,
128 },
129}
130
131#[derive(Debug)]
139pub struct LoadedDetectorCorpus {
140 pub specs: Vec<DetectorSpec>,
142 pub schema_version: u32,
144}
145
146impl LoadedDetectorCorpus {
147 pub fn compute_digest(&self) -> Result<[u8; 32], serde_json::Error> {
149 crate::compute_detector_corpus_digest_for_schema(&self.specs, self.schema_version)
150 }
151}
152
153pub fn load_detectors(dir: &Path) -> Result<Vec<DetectorSpec>, SpecError> {
169 Ok(load_detector_corpus(dir)?.specs)
170}
171
172pub fn load_detector_corpus(dir: &Path) -> Result<LoadedDetectorCorpus, SpecError> {
175 load_detector_corpus_with_gate(dir, true)
176}
177
178#[derive(Clone, Copy)]
194struct CorpusCompatibility {
195 schema_version: u32,
196 permits_forward_unknown_fields: bool,
197}
198
199pub(crate) fn load_detectors_with_gate(
200 dir: &Path,
201 enforce_gate: bool,
202) -> Result<Vec<DetectorSpec>, SpecError> {
203 Ok(load_detector_corpus_with_gate(dir, enforce_gate)?.specs)
204}
205
206fn load_detector_corpus_with_gate(
207 dir: &Path,
208 enforce_gate: bool,
209) -> Result<LoadedDetectorCorpus, SpecError> {
210 let compatibility = read_corpus_compatibility(dir)?;
211 let toml_paths = discover_detector_tomls(dir, enforce_gate)?;
212 let parsed = parse_detector_files(&toml_paths, compatibility);
213 let specs = assemble_detector_load(dir, enforce_gate, compatibility, toml_paths.len(), parsed)?;
214 Ok(LoadedDetectorCorpus {
215 specs,
216 schema_version: compatibility.schema_version,
217 })
218}
219
220fn read_corpus_compatibility(dir: &Path) -> Result<CorpusCompatibility, SpecError> {
221 let path = dir.join(DETECTOR_CORPUS_MANIFEST_FILE);
222 let contents = match std::fs::read_to_string(&path) {
223 Ok(contents) => contents,
224 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
225 return Ok(CorpusCompatibility {
226 schema_version: DETECTOR_CORPUS_MIN_SCHEMA_VERSION,
227 permits_forward_unknown_fields: false,
228 });
229 }
230 Err(source) => {
231 return Err(SpecError::ReadFile {
232 path: path.display().to_string(),
233 source,
234 });
235 }
236 };
237 let manifest: DetectorCorpusManifest =
238 toml::from_str(&contents).map_err(|source| SpecError::InvalidCorpusManifest {
239 path: path.clone(),
240 source,
241 })?;
242 if !(DETECTOR_CORPUS_MIN_SCHEMA_VERSION..=DETECTOR_CORPUS_MAX_FORWARD_SCHEMA_VERSION)
243 .contains(&manifest.schema_version)
244 {
245 return Err(SpecError::UnsupportedCorpusSchema {
246 path,
247 found: manifest.schema_version,
248 current: DETECTOR_CORPUS_SCHEMA_VERSION,
249 max_forward: DETECTOR_CORPUS_MAX_FORWARD_SCHEMA_VERSION,
250 });
251 }
252 Ok(CorpusCompatibility {
253 schema_version: manifest.schema_version,
254 permits_forward_unknown_fields: manifest.schema_version > DETECTOR_CORPUS_SCHEMA_VERSION,
255 })
256}
257
258fn discover_detector_tomls(dir: &Path, enforce_gate: bool) -> Result<Vec<PathBuf>, SpecError> {
259 let entries = std::fs::read_dir(dir).map_err(|e| SpecError::ReadFile {
260 path: dir.display().to_string(),
261 source: e,
262 })?;
263 let mut toml_paths = Vec::new();
264 for entry in entries {
265 let entry = entry.map_err(|e| SpecError::ReadFile {
266 path: format!("directory entry under {}", dir.display()),
267 source: e,
268 })?;
269 let path = entry.path();
270 if path.extension().is_some_and(|ext| ext == "toml")
271 && path
272 .file_name()
273 .is_none_or(|name| name != DETECTOR_CORPUS_MANIFEST_FILE)
274 {
275 toml_paths.push(path);
276 }
277 }
278
279 if enforce_gate && toml_paths.is_empty() {
280 return Err(SpecError::DetectorCorpusRejected {
281 dir: dir.display().to_string(),
282 failed_count: 0,
283 total: 0,
284 detail:
285 " - no detector TOML files found; add at least one valid `*.toml` detector spec"
286 .to_string(),
287 });
288 }
289 Ok(toml_paths)
290}
291
292fn parse_detector_files(
293 toml_paths: &[PathBuf],
294 compatibility: CorpusCompatibility,
295) -> Vec<ReadDetectorOutcome> {
296 toml_paths
297 .par_iter()
298 .map(|path| read_detector_file(path, compatibility))
299 .collect()
300}
301
302fn assemble_detector_load(
303 dir: &Path,
304 enforce_gate: bool,
305 compatibility: CorpusCompatibility,
306 total: usize,
307 parsed: Vec<ReadDetectorOutcome>,
308) -> Result<Vec<DetectorSpec>, SpecError> {
309 let mut load_state = DetectorLoadState::default();
310 let mut detectors = Vec::with_capacity(parsed.len());
311
312 for outcome in parsed {
313 match outcome {
314 ReadDetectorOutcome::Loaded {
315 path,
316 spec,
317 legacy_migrations,
318 } => {
319 load_state.legacy_migrations += legacy_migrations;
320 if should_reject_detector(
321 &spec,
322 &path,
323 enforce_gate,
324 &mut load_state.gate_rejected,
325 &mut load_state.gate_errors,
326 &mut load_state.total_warnings,
327 ) {
328 continue;
329 }
330 detectors.push(*spec);
331 }
332 ReadDetectorOutcome::ForwardSkipped { message } => {
333 load_state.forward_skipped += 1;
334 load_state.forward_errors.push(message);
335 }
336 ReadDetectorOutcome::Skipped { message } => {
337 load_state.skipped += 1;
338 load_state.load_errors.push(message);
339 }
340 }
341 }
342
343 detectors.sort_by(|a, b| a.id.cmp(&b.id));
351 let mut duplicate_ids: Vec<&str> = detectors
352 .windows(2)
353 .filter(|w| w[0].id == w[1].id)
354 .map(|w| w[0].id.as_str())
355 .collect();
356 duplicate_ids.dedup();
357 if !duplicate_ids.is_empty() {
358 load_state.gate_rejected += duplicate_ids.len();
359 for id in duplicate_ids {
360 load_state.gate_errors.push(format!(
361 "duplicate detector id `{id}` (a later spec would shadow the earlier)"
362 ));
363 }
364 }
365
366 log_load_summary(&load_state);
367 if enforce_gate && compatibility.permits_forward_unknown_fields {
368 return Err(load_state.into_forward_error(dir, total, compatibility.schema_version));
369 }
370 if enforce_gate && load_state.has_failures() {
371 return Err(load_state.into_rejected_error(dir, total));
372 }
373 Ok(detectors)
374}
375
376#[derive(Default)]
377struct DetectorLoadState {
378 skipped: usize,
379 load_errors: Vec<String>,
380 forward_skipped: usize,
381 forward_errors: Vec<String>,
382 legacy_migrations: usize,
383 gate_rejected: usize,
384 gate_errors: Vec<String>,
385 total_warnings: usize,
386}
387
388impl DetectorLoadState {
389 fn has_failures(&self) -> bool {
390 self.skipped > 0 || self.forward_skipped > 0 || self.gate_rejected > 0
391 }
392
393 fn into_rejected_error(self, dir: &Path, total: usize) -> SpecError {
394 let mut details = self.load_errors;
395 details.extend(self.gate_errors);
396 let detail = details
397 .into_iter()
398 .map(|line| format!(" - {line}"))
399 .collect::<Vec<_>>()
400 .join("\n");
401 SpecError::DetectorCorpusRejected {
402 dir: dir.display().to_string(),
403 failed_count: self.skipped + self.gate_rejected,
404 total,
405 detail,
406 }
407 }
408 fn into_forward_error(self, dir: &Path, total: usize, declared_schema: u32) -> SpecError {
409 let detail = if self.forward_errors.is_empty() {
410 format!(
411 " - {} declares schema {}; schema metadata is part of effective \
412 corpus identity and cannot be interpreted as schema {}",
413 dir.join(DETECTOR_CORPUS_MANIFEST_FILE).display(),
414 declared_schema,
415 DETECTOR_CORPUS_SCHEMA_VERSION
416 )
417 } else {
418 self.forward_errors
419 .into_iter()
420 .map(|line| format!(" - {line}"))
421 .collect::<Vec<_>>()
422 .join("\n")
423 };
424 SpecError::ForwardIncompatibleCorpus {
425 dir: dir.display().to_string(),
426 declared_schema,
427 supported_schema: DETECTOR_CORPUS_SCHEMA_VERSION,
428 skipped_count: self.forward_skipped,
429 total,
430 detail,
431 }
432 }
433}
434
435fn log_load_summary(state: &DetectorLoadState) {
436 if state.skipped > 0 {
437 let version_skew = state
442 .load_errors
443 .iter()
444 .filter(|error| error.contains("unknown field"))
445 .count();
446 let examples = state
447 .load_errors
448 .iter()
449 .take(3)
450 .map(String::as_str)
451 .collect::<Vec<_>>()
452 .join(" | ");
453 if version_skew > 0 {
454 tracing::warn!(
455 "skipped {} detector file(s); {} contain unknown fields while the corpus \
456 is using the current/legacy strict schema. Fix field typos, or add a \
457 supported newer `{}` declaration when the fields are intentional. \
458 Examples: {examples}",
459 state.skipped,
460 version_skew,
461 DETECTOR_CORPUS_MANIFEST_FILE
462 );
463 } else {
464 tracing::warn!(
465 "skipped {} malformed/unreadable detector file(s) - run \
466 `keyhog detectors --detectors <DIR>` or -vv for the full list. \
467 Examples: {examples}",
468 state.skipped
469 );
470 }
471 }
472 if state.forward_skipped > 0 {
473 let examples = state
474 .forward_errors
475 .iter()
476 .take(3)
477 .map(String::as_str)
478 .collect::<Vec<_>>()
479 .join(" | ");
480 tracing::warn!(
481 "detector corpus declared a supported forward schema; skipped {} detector \
482 file(s) that use newer fields rather than silently dropping those fields. \
483 Update keyhog for full recall. Examples: {examples}",
484 state.forward_skipped
485 );
486 }
487 if state.legacy_migrations > 0 {
488 tracing::warn!(
489 "migrated {} legacy schema-{} verifier success contract(s) to \
490 status_with_error_backstop; add an explicit success policy and a \
491 schema-{} corpus manifest",
492 state.legacy_migrations,
493 DETECTOR_CORPUS_MIN_SCHEMA_VERSION,
494 DETECTOR_CORPUS_SCHEMA_VERSION
495 );
496 }
497 if state.gate_rejected > 0 {
498 tracing::warn!(
503 "quality gate rejected {} detectors (see per-detector warnings above)",
504 state.gate_rejected
505 );
506 }
507 if state.total_warnings > 0 {
508 tracing::debug!("quality gate: {} advisory warnings", state.total_warnings);
519 }
520}
521
522enum ReadDetectorOutcome {
523 Loaded {
524 path: PathBuf,
525 spec: Box<DetectorSpec>,
526 legacy_migrations: usize,
527 },
528 ForwardSkipped {
529 message: String,
530 },
531 Skipped {
532 message: String,
533 },
534}
535
536fn read_detector_file(path: &Path, compatibility: CorpusCompatibility) -> ReadDetectorOutcome {
537 let contents = match read_detector_toml_file(path) {
538 Ok(contents) => contents,
539 Err(error) => {
540 let message = format!("failed to read {}: {}", path.display(), error);
546 tracing::debug!(
547 detector_path = %path.display(),
548 error = %error,
549 "skipping detector - unreadable file" );
551 return ReadDetectorOutcome::Skipped { message };
552 }
553 };
554
555 match toml::from_str::<DetectorFile>(&contents) {
556 Ok(mut file) => {
557 let legacy_migrations =
558 if compatibility.schema_version == DETECTOR_CORPUS_MIN_SCHEMA_VERSION {
559 migrate_legacy_success_policies(&mut file.detector)
560 } else {
561 0
562 };
563 ReadDetectorOutcome::Loaded {
564 path: path.to_path_buf(),
565 spec: Box::new(file.detector),
566 legacy_migrations,
567 }
568 }
569 Err(error) => {
570 let unknown_field = error.to_string().contains("unknown field");
571 if compatibility.permits_forward_unknown_fields && unknown_field {
572 let message = format!(
573 "skipped {} under declared detector corpus schema {} because it uses \
574 a field unknown to schema {}: {}. Fix: update keyhog to load this detector",
575 path.display(),
576 compatibility.schema_version,
577 DETECTOR_CORPUS_SCHEMA_VERSION,
578 error
579 );
580 tracing::warn!(
581 detector_path = %path.display(),
582 declared_schema = compatibility.schema_version,
583 supported_schema = DETECTOR_CORPUS_SCHEMA_VERSION,
584 error = %error,
585 "skipping forward-schema detector without dropping unknown fields"
586 );
587 return ReadDetectorOutcome::ForwardSkipped { message };
588 }
589 let message = format!(
590 "failed to parse {} under detector corpus schema {}: {}. Fix: correct \
591 misspelled or invalid detector fields; only a corpus manifest declaring \
592 a supported newer schema permits an unknown future field",
593 path.display(),
594 compatibility.schema_version,
595 error
596 );
597 tracing::debug!(
600 detector_path = %path.display(),
601 schema_version = compatibility.schema_version,
602 error = %error,
603 "skipping detector - TOML parse failed"
604 );
605 ReadDetectorOutcome::Skipped { message }
606 }
607 }
608}
609
610fn should_reject_detector(
611 spec: &DetectorSpec,
612 path: &Path,
613 enforce_gate: bool,
614 gate_rejected: &mut usize,
615 gate_errors: &mut Vec<String>,
616 total_warnings: &mut usize,
617) -> bool {
618 let mut has_errors = false;
619 let mut detector_errors = Vec::new();
620 for issue in validate_detector(spec) {
621 match issue {
622 QualityIssue::Warning(warning) => {
623 tracing::debug!(detector_path = %path.display(), "quality: {} - {}", spec.id, warning);
628 *total_warnings += 1;
629 }
630 QualityIssue::Error(error) => {
631 tracing::warn!(
636 detector_path = %path.display(),
637 "detector quality error: {}: {}",
638 spec.id,
639 error
640 );
641 detector_errors.push(format!("{}: {}: {}", path.display(), spec.id, error));
642 has_errors = true;
643 }
644 }
645 }
646
647 if has_errors && enforce_gate {
648 *gate_rejected += 1;
649 gate_errors.extend(detector_errors);
650 return true;
651 }
652
653 false
654}
655
656pub(crate) fn load_detectors_from_str(toml_str: &str) -> Result<Vec<DetectorSpec>, SpecError> {
661 let file: DetectorFile = toml::from_str(toml_str).map_err(|e| SpecError::InvalidToml {
662 path: PathBuf::from("<string>"),
663 source: e,
664 })?;
665 Ok(vec![file.detector])
666}