1#![allow(clippy::result_large_err)] use std::path::{Path, PathBuf};
6
7use rayon::prelude::*;
8use thiserror::Error;
9
10use super::{
11 migrate_legacy_success_policies, validate_detector_for_corpus_schema, DetectorCorpusManifest,
12 DetectorFile, DetectorSpec, QualityIssue, DETECTOR_CORPUS_MANIFEST_FILE,
13 DETECTOR_CORPUS_MAX_FORWARD_SCHEMA_VERSION, DETECTOR_CORPUS_MIN_SCHEMA_VERSION,
14 DETECTOR_CORPUS_SCHEMA_VERSION, HARD_NEGATIVE_TEST_EVIDENCE_SCHEMA_VERSION,
15};
16pub use crate::detector_file_io::{read_detector_toml_file, DETECTOR_TOML_FILE_BYTES};
17
18#[derive(Debug, Error)]
20#[allow(clippy::result_large_err)] pub enum SpecError {
22 #[error(
23 "failed to read detector path {path}: {source}. Fix: check the detector path exists and that the file is readable TOML"
24 )]
25 ReadFile {
27 path: String,
29 source: std::io::Error,
31 },
32 #[error(
33 "invalid TOML in detector {path}: {source}. Fix: repair the TOML syntax in the detector file"
34 )]
35 InvalidToml {
37 path: PathBuf,
39 source: toml::de::Error,
41 },
42 #[error(
43 "invalid detector corpus manifest {path}: {source}. Fix: set `schema_version` \
44 to an integer supported by this keyhog binary and remove misspelled manifest fields"
45 )]
46 InvalidCorpusManifest {
48 path: PathBuf,
50 source: toml::de::Error,
52 },
53 #[error(
54 "unsupported detector corpus schema {found} declared by {path}; this binary \
55 supports schema {current} and bounded forward compatibility through schema \
56 {max_forward}. Fix: use a compatible detector corpus or update keyhog"
57 )]
58 UnsupportedCorpusSchema {
60 path: PathBuf,
62 found: u32,
64 current: u32,
66 max_forward: u32,
68 },
69 #[error(
70 "detector corpus {dir} declares supported forward schema {declared_schema}, \
71 while this binary owns schema {supported_schema}; {skipped_count} of {total} \
72 detector file(s) use fields this binary cannot interpret. Keyhog refuses to \
73 scan under newer parsing semantics or with a partial corpus because either \
74 would invalidate corpus identity and could silently drop recall. \
75 Compatibility detail:\n{detail}\nFix: update keyhog to load the complete \
76 detector corpus"
77 )]
78 ForwardIncompatibleCorpus {
80 dir: String,
82 declared_schema: u32,
84 supported_schema: u32,
86 skipped_count: usize,
88 total: usize,
90 detail: String,
92 },
93 #[error(
94 "{failed_count} of {total} embedded detector(s) failed to parse, the binary \
95 baked in a CORRUPT detector set, so its recall is silently degraded. This is \
96 a build/source bug, not a runtime condition: the embedded corpus is compiled \
97 in and cannot have been edited at runtime. Offending detector(s):\n{detail}\n\
98 Fix: repair the named TOML(s) under `detectors/` (the toml error names the \
99 line/column) and rebuild keyhog so build.rs re-embeds a valid set."
100 )]
101 EmbeddedCorpusCorrupt {
103 failed_count: usize,
105 total: usize,
107 detail: String,
109 },
110 #[error(
111 "{failed_count} of {total} detector file(s) from {dir} failed to load, \
112 pass the quality gate, or exist at all, that is a partial detector \
113 corpus, so keyhog is refusing to scan without a complete detector \
114 corpus (a partial corpus silently drops recall). \
115 Offending detector(s):\n{detail}\nFix: repair the named TOML file(s) \
116 or add at least one valid `*.toml` detector spec, then rerun the scan."
117 )]
118 DetectorCorpusRejected {
120 dir: String,
122 failed_count: usize,
124 total: usize,
126 detail: String,
128 },
129}
130
131#[derive(Debug)]
139pub struct LoadedDetectorCorpus {
140 pub specs: Vec<DetectorSpec>,
142 pub schema_version: u32,
144}
145
146impl LoadedDetectorCorpus {
147 pub fn compute_digest(&self) -> Result<[u8; 32], serde_json::Error> {
149 crate::compute_detector_corpus_digest_for_schema(&self.specs, self.schema_version)
150 }
151}
152
153pub fn load_detectors(dir: &Path) -> Result<Vec<DetectorSpec>, SpecError> {
169 Ok(load_detector_corpus(dir)?.specs)
170}
171
172pub fn load_detector_corpus(dir: &Path) -> Result<LoadedDetectorCorpus, SpecError> {
175 load_detector_corpus_with_gate(dir, true)
176}
177
178#[derive(Clone, Copy)]
179struct CorpusCompatibility {
180 schema_version: u32,
181 permits_forward_unknown_fields: bool,
182}
183
184pub fn load_detectors_with_gate(
202 dir: &Path,
203 enforce_gate: bool,
204) -> Result<Vec<DetectorSpec>, SpecError> {
205 Ok(load_detector_corpus_with_gate(dir, enforce_gate)?.specs)
206}
207
208fn load_detector_corpus_with_gate(
209 dir: &Path,
210 enforce_gate: bool,
211) -> Result<LoadedDetectorCorpus, SpecError> {
212 let compatibility = read_corpus_compatibility(dir)?;
213 let toml_paths = discover_detector_tomls(dir, enforce_gate)?;
214 let parsed = parse_detector_files(&toml_paths, compatibility);
215 let specs = assemble_detector_load(dir, enforce_gate, compatibility, toml_paths.len(), parsed)?;
216 Ok(LoadedDetectorCorpus {
217 specs,
218 schema_version: compatibility.schema_version,
219 })
220}
221
222fn read_corpus_compatibility(dir: &Path) -> Result<CorpusCompatibility, SpecError> {
223 let path = dir.join(DETECTOR_CORPUS_MANIFEST_FILE);
224 let contents = match std::fs::read_to_string(&path) {
225 Ok(contents) => contents,
226 Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
227 return Ok(CorpusCompatibility {
228 schema_version: DETECTOR_CORPUS_MIN_SCHEMA_VERSION,
229 permits_forward_unknown_fields: false,
230 });
231 }
232 Err(source) => {
233 return Err(SpecError::ReadFile {
234 path: path.display().to_string(),
235 source,
236 });
237 }
238 };
239 let manifest: DetectorCorpusManifest =
240 toml::from_str(&contents).map_err(|source| SpecError::InvalidCorpusManifest {
241 path: path.clone(),
242 source,
243 })?;
244 if !(DETECTOR_CORPUS_MIN_SCHEMA_VERSION..=DETECTOR_CORPUS_MAX_FORWARD_SCHEMA_VERSION)
245 .contains(&manifest.schema_version)
246 {
247 return Err(SpecError::UnsupportedCorpusSchema {
248 path,
249 found: manifest.schema_version,
250 current: DETECTOR_CORPUS_SCHEMA_VERSION,
251 max_forward: DETECTOR_CORPUS_MAX_FORWARD_SCHEMA_VERSION,
252 });
253 }
254 Ok(CorpusCompatibility {
255 schema_version: manifest.schema_version,
256 permits_forward_unknown_fields: manifest.schema_version > DETECTOR_CORPUS_SCHEMA_VERSION,
257 })
258}
259
260fn discover_detector_tomls(dir: &Path, enforce_gate: bool) -> Result<Vec<PathBuf>, SpecError> {
261 let entries = std::fs::read_dir(dir).map_err(|e| SpecError::ReadFile {
262 path: dir.display().to_string(),
263 source: e,
264 })?;
265 let mut toml_paths = Vec::new();
266 for entry in entries {
267 let entry = entry.map_err(|e| SpecError::ReadFile {
268 path: format!("directory entry under {}", dir.display()),
269 source: e,
270 })?;
271 let path = entry.path();
272 if path.extension().is_some_and(|ext| ext == "toml")
273 && path
274 .file_name()
275 .is_none_or(|name| name != DETECTOR_CORPUS_MANIFEST_FILE)
276 {
277 toml_paths.push(path);
278 }
279 }
280
281 if enforce_gate && toml_paths.is_empty() {
282 return Err(SpecError::DetectorCorpusRejected {
283 dir: dir.display().to_string(),
284 failed_count: 0,
285 total: 0,
286 detail:
287 " - no detector TOML files found; add at least one valid `*.toml` detector spec"
288 .to_string(),
289 });
290 }
291 Ok(toml_paths)
292}
293
294fn parse_detector_files(
295 toml_paths: &[PathBuf],
296 compatibility: CorpusCompatibility,
297) -> Vec<ReadDetectorOutcome> {
298 toml_paths
299 .par_iter()
300 .map(|path| read_detector_file(path, compatibility))
301 .collect()
302}
303
304fn assemble_detector_load(
305 dir: &Path,
306 enforce_gate: bool,
307 compatibility: CorpusCompatibility,
308 total: usize,
309 parsed: Vec<ReadDetectorOutcome>,
310) -> Result<Vec<DetectorSpec>, SpecError> {
311 let mut load_state = DetectorLoadState::default();
312 let mut detectors = Vec::with_capacity(parsed.len());
313
314 for outcome in parsed {
315 match outcome {
316 ReadDetectorOutcome::Loaded {
317 path,
318 spec,
319 legacy_migrations,
320 } => {
321 load_state.legacy_migrations += legacy_migrations;
322 if should_reject_detector(
323 &spec,
324 &path,
325 enforce_gate,
326 compatibility.schema_version,
327 &mut load_state.gate_rejected,
328 &mut load_state.gate_errors,
329 &mut load_state.total_warnings,
330 ) {
331 continue;
332 }
333 detectors.push(*spec);
334 }
335 ReadDetectorOutcome::ForwardSkipped { message } => {
336 load_state.forward_skipped += 1;
337 load_state.forward_errors.push(message);
338 }
339 ReadDetectorOutcome::Skipped { message } => {
340 load_state.skipped += 1;
341 load_state.load_errors.push(message);
342 }
343 }
344 }
345
346 detectors.sort_by(|a, b| a.id.cmp(&b.id));
354 let mut duplicate_ids: Vec<&str> = detectors
355 .windows(2)
356 .filter(|w| w[0].id == w[1].id)
357 .map(|w| w[0].id.as_str())
358 .collect();
359 duplicate_ids.dedup();
360 if !duplicate_ids.is_empty() {
361 load_state.gate_rejected += duplicate_ids.len();
362 for id in duplicate_ids {
363 load_state.gate_errors.push(format!(
364 "duplicate detector id `{id}` (a later spec would shadow the earlier)"
365 ));
366 }
367 }
368
369 log_load_summary(&load_state);
370 if enforce_gate && compatibility.permits_forward_unknown_fields {
371 return Err(load_state.into_forward_error(dir, total, compatibility.schema_version));
372 }
373 if enforce_gate && load_state.has_failures() {
374 return Err(load_state.into_rejected_error(dir, total));
375 }
376 Ok(detectors)
377}
378
379#[derive(Default)]
380struct DetectorLoadState {
381 skipped: usize,
382 load_errors: Vec<String>,
383 forward_skipped: usize,
384 forward_errors: Vec<String>,
385 legacy_migrations: usize,
386 gate_rejected: usize,
387 gate_errors: Vec<String>,
388 total_warnings: usize,
389}
390
391impl DetectorLoadState {
392 fn has_failures(&self) -> bool {
393 self.skipped > 0 || self.forward_skipped > 0 || self.gate_rejected > 0
394 }
395
396 fn into_rejected_error(self, dir: &Path, total: usize) -> SpecError {
397 let mut details = self.load_errors;
398 details.extend(self.gate_errors);
399 let detail = details
400 .into_iter()
401 .map(|line| format!(" - {line}"))
402 .collect::<Vec<_>>()
403 .join("\n");
404 SpecError::DetectorCorpusRejected {
405 dir: dir.display().to_string(),
406 failed_count: self.skipped + self.gate_rejected,
407 total,
408 detail,
409 }
410 }
411 fn into_forward_error(self, dir: &Path, total: usize, declared_schema: u32) -> SpecError {
412 let detail = if self.forward_errors.is_empty() {
413 format!(
414 " - {} declares schema {}; schema metadata is part of effective \
415 corpus identity and cannot be interpreted as schema {}",
416 dir.join(DETECTOR_CORPUS_MANIFEST_FILE).display(),
417 declared_schema,
418 DETECTOR_CORPUS_SCHEMA_VERSION
419 )
420 } else {
421 self.forward_errors
422 .into_iter()
423 .map(|line| format!(" - {line}"))
424 .collect::<Vec<_>>()
425 .join("\n")
426 };
427 SpecError::ForwardIncompatibleCorpus {
428 dir: dir.display().to_string(),
429 declared_schema,
430 supported_schema: DETECTOR_CORPUS_SCHEMA_VERSION,
431 skipped_count: self.forward_skipped,
432 total,
433 detail,
434 }
435 }
436}
437
438fn log_load_summary(state: &DetectorLoadState) {
439 if state.skipped > 0 {
440 let version_skew = state
445 .load_errors
446 .iter()
447 .filter(|error| error.contains("unknown field"))
448 .count();
449 let examples = state
450 .load_errors
451 .iter()
452 .take(3)
453 .map(String::as_str)
454 .collect::<Vec<_>>()
455 .join(" | ");
456 if version_skew > 0 {
457 tracing::warn!(
458 "skipped {} detector file(s); {} contain unknown fields while the corpus \
459 is using the current/legacy strict schema. Fix field typos, or add a \
460 supported newer `{}` declaration when the fields are intentional. \
461 Examples: {examples}",
462 state.skipped,
463 version_skew,
464 DETECTOR_CORPUS_MANIFEST_FILE
465 );
466 } else {
467 tracing::warn!(
468 "skipped {} malformed/unreadable detector file(s) - run \
469 `keyhog detectors --detectors <DIR>` or -vv for the full list. \
470 Examples: {examples}",
471 state.skipped
472 );
473 }
474 }
475 if state.forward_skipped > 0 {
476 let examples = state
477 .forward_errors
478 .iter()
479 .take(3)
480 .map(String::as_str)
481 .collect::<Vec<_>>()
482 .join(" | ");
483 tracing::warn!(
484 "detector corpus declared a supported forward schema; skipped {} detector \
485 file(s) that use newer fields rather than silently dropping those fields. \
486 Update keyhog for full recall. Examples: {examples}",
487 state.forward_skipped
488 );
489 }
490 if state.legacy_migrations > 0 {
491 tracing::warn!(
492 "migrated {} legacy schema-{} verifier success contract(s) to \
493 status_with_error_backstop; add an explicit success policy and a \
494 schema-{} corpus manifest",
495 state.legacy_migrations,
496 DETECTOR_CORPUS_MIN_SCHEMA_VERSION,
497 DETECTOR_CORPUS_SCHEMA_VERSION
498 );
499 }
500 if state.gate_rejected > 0 {
501 tracing::warn!(
506 "quality gate rejected {} detectors (see per-detector warnings above)",
507 state.gate_rejected
508 );
509 }
510 if state.total_warnings > 0 {
511 tracing::debug!("quality gate: {} advisory warnings", state.total_warnings);
522 }
523}
524
525enum ReadDetectorOutcome {
526 Loaded {
527 path: PathBuf,
528 spec: Box<DetectorSpec>,
529 legacy_migrations: usize,
530 },
531 ForwardSkipped {
532 message: String,
533 },
534 Skipped {
535 message: String,
536 },
537}
538
539const SEMANTIC_POLICY_SCHEMA_VERSION: u32 = 4;
540
541#[derive(Default)]
542struct DeclaredSchemaFields {
543 semantic_policy: bool,
544 hard_negative_test_evidence: bool,
545}
546
547fn declared_schema_fields(contents: &str) -> Result<DeclaredSchemaFields, toml::de::Error> {
548 let document = toml::from_str::<toml::Value>(contents)?;
549 let Some(detector) = document.get("detector").and_then(toml::Value::as_table) else {
550 return Ok(DeclaredSchemaFields::default());
551 };
552 let semantic_policy = [
553 "capture_role",
554 "anchor_role",
555 "allowed_source_roles",
556 "required_evidence",
557 ]
558 .iter()
559 .any(|field| detector.contains_key(*field));
560 let hard_negative_test_evidence = detector
561 .get("tests")
562 .and_then(toml::Value::as_array)
563 .is_some_and(|tests| {
564 tests.iter().any(|test| {
565 test.as_table().is_some_and(|table| {
566 ["pattern_index", "negative_class"]
567 .iter()
568 .any(|field| table.contains_key(*field))
569 })
570 })
571 });
572 Ok(DeclaredSchemaFields {
573 semantic_policy,
574 hard_negative_test_evidence,
575 })
576}
577
578fn read_detector_file(path: &Path, compatibility: CorpusCompatibility) -> ReadDetectorOutcome {
579 let contents = match read_detector_toml_file(path) {
580 Ok(contents) => contents,
581 Err(error) => {
582 let message = format!("failed to read {}: {}", path.display(), error);
588 tracing::debug!(
589 detector_path = %path.display(),
590 error = %error,
591 "skipping detector - unreadable file" );
593 return ReadDetectorOutcome::Skipped { message };
594 }
595 };
596 let declared_fields = if compatibility.schema_version
597 < HARD_NEGATIVE_TEST_EVIDENCE_SCHEMA_VERSION
598 {
599 match declared_schema_fields(&contents) {
600 Ok(fields) => fields,
601 Err(error) => {
602 return ReadDetectorOutcome::Skipped {
603 message: format!(
604 "failed to parse {} under detector corpus schema {}: {}. Fix: correct \
605 misspelled or invalid detector fields; only a corpus manifest declaring \
606 a supported newer schema permits an unknown future field",
607 path.display(),
608 compatibility.schema_version,
609 error
610 ),
611 };
612 }
613 }
614 } else {
615 DeclaredSchemaFields::default()
616 };
617 if compatibility.schema_version < SEMANTIC_POLICY_SCHEMA_VERSION
618 && declared_fields.semantic_policy
619 {
620 return ReadDetectorOutcome::Skipped {
621 message: format!(
622 "{} declares semantic policy fields that require corpus schema {SEMANTIC_POLICY_SCHEMA_VERSION}; corpus.toml declares schema {}",
623 path.display(),
624 compatibility.schema_version
625 ),
626 };
627 }
628 if declared_fields.hard_negative_test_evidence {
629 return ReadDetectorOutcome::Skipped {
630 message: format!(
631 "{} declares hard-negative test evidence fields that require corpus schema {HARD_NEGATIVE_TEST_EVIDENCE_SCHEMA_VERSION}; corpus.toml declares schema {}",
632 path.display(),
633 compatibility.schema_version
634 ),
635 };
636 }
637
638 match toml::from_str::<DetectorFile>(&contents) {
639 Ok(mut file) => {
640 let legacy_migrations =
641 if compatibility.schema_version == DETECTOR_CORPUS_MIN_SCHEMA_VERSION {
642 migrate_legacy_success_policies(&mut file.detector)
643 } else {
644 0
645 };
646 ReadDetectorOutcome::Loaded {
647 path: path.to_path_buf(),
648 spec: Box::new(file.detector),
649 legacy_migrations,
650 }
651 }
652 Err(error) => {
653 let unknown_field = error.to_string().contains("unknown field");
654 if compatibility.permits_forward_unknown_fields && unknown_field {
655 let message = format!(
656 "skipped {} under declared detector corpus schema {} because it uses \
657 a field unknown to schema {}: {}. Fix: update keyhog to load this detector",
658 path.display(),
659 compatibility.schema_version,
660 DETECTOR_CORPUS_SCHEMA_VERSION,
661 error
662 );
663 tracing::warn!(
664 detector_path = %path.display(),
665 declared_schema = compatibility.schema_version,
666 supported_schema = DETECTOR_CORPUS_SCHEMA_VERSION,
667 error = %error,
668 "skipping forward-schema detector without dropping unknown fields"
669 );
670 return ReadDetectorOutcome::ForwardSkipped { message };
671 }
672 let message = format!(
673 "failed to parse {} under detector corpus schema {}: {}. Fix: correct \
674 misspelled or invalid detector fields; only a corpus manifest declaring \
675 a supported newer schema permits an unknown future field",
676 path.display(),
677 compatibility.schema_version,
678 error
679 );
680 tracing::debug!(
683 detector_path = %path.display(),
684 schema_version = compatibility.schema_version,
685 error = %error,
686 "skipping detector - TOML parse failed"
687 );
688 ReadDetectorOutcome::Skipped { message }
689 }
690 }
691}
692
693fn should_reject_detector(
694 spec: &DetectorSpec,
695 path: &Path,
696 enforce_gate: bool,
697 corpus_schema_version: u32,
698 gate_rejected: &mut usize,
699 gate_errors: &mut Vec<String>,
700 total_warnings: &mut usize,
701) -> bool {
702 let mut has_errors = false;
703 let mut detector_errors = Vec::new();
704 for issue in validate_detector_for_corpus_schema(spec, corpus_schema_version) {
705 match issue {
706 QualityIssue::Warning(warning) => {
707 tracing::debug!(detector_path = %path.display(), "quality: {} - {}", spec.id, warning);
712 *total_warnings += 1;
713 }
714 QualityIssue::Error(error) => {
715 tracing::warn!(
720 detector_path = %path.display(),
721 "detector quality error: {}: {}",
722 spec.id,
723 error
724 );
725 detector_errors.push(format!("{}: {}: {}", path.display(), spec.id, error));
726 has_errors = true;
727 }
728 }
729 }
730
731 if has_errors && enforce_gate {
732 *gate_rejected += 1;
733 gate_errors.extend(detector_errors);
734 return true;
735 }
736
737 false
738}
739
740pub(crate) fn load_detectors_from_str(toml_str: &str) -> Result<Vec<DetectorSpec>, SpecError> {
745 let file: DetectorFile = toml::from_str(toml_str).map_err(|e| SpecError::InvalidToml {
746 path: PathBuf::from("<string>"),
747 source: e,
748 })?;
749 Ok(vec![file.detector])
750}