mod allowlist;
pub(crate) use allowlist::load_rule_suppressor;
mod dispatch;
pub(crate) use dispatch::{
automatic_backend_recovery_allowed, canonical_source_classes,
record_completed_backend_recovery, scan_selected_batch, AutorouteMeasurementReceipt,
BackendRecoveryPlan, COALESCED_CHUNK_SCAN_CEILING_BYTES, COALESCED_CHUNK_SCAN_CEILING_MB,
};
mod postprocess;
pub(crate) mod reporting;
mod run;
mod streaming;
mod workflow_state;
pub(crate) use workflow_state::{merkle_skipped_unchanged, record_merkle_skipped_unchanged};
use crate::args::ScanArgs;
use crate::orchestrator_config::{
auto_discover_detectors, autoroute_config_digest, backend_override_cli_value,
configure_threads, gpu_runtime_policy_from_args, load_effective_detector_corpus,
parse_backend_override, resolve_scan_config, resolved_scan_config_for_scanner,
validate_detector_mode_selection, validate_explicit_detector_path, DetectorCorpusProvenance,
LoadedDetectorCorpus, ResolvedEngineRuntimeSettings, ResolvedScanConfig,
};
use crate::style;
use anyhow::{Context, Result};
use keyhog_core::{Chunk, DetectorSpec, MerkleLoadStatus, RawMatch, Source};
use keyhog_scanner::{CompiledScanner, GpuInitPolicy};
#[cfg(feature = "git")]
use std::path::PathBuf;
use std::sync::Arc;
fn collect_detector_signatures(detectors: &[DetectorSpec]) -> std::collections::HashSet<Arc<str>> {
detectors
.iter()
.flat_map(|detector| {
detector
.patterns
.iter()
.map(|pattern| Arc::from(pattern.regex.as_str()))
})
.chain(detectors.iter().flat_map(|detector| {
detector
.companions
.iter()
.map(|companion| Arc::from(companion.regex.as_str()))
}))
.collect()
}
fn filter_disabled_detectors(
detectors: &mut Vec<DetectorSpec>,
disabled_detectors: &std::collections::HashSet<String>,
) -> usize {
let known_ids: std::collections::HashSet<String> = detectors
.iter()
.map(|detector| detector.id.clone())
.collect();
let mut removed: std::collections::HashSet<String> = disabled_detectors
.intersection(&known_ids)
.cloned()
.collect();
if removed.is_empty() {
return 0;
}
let mut required_by: std::collections::HashMap<String, Vec<String>> =
std::collections::HashMap::new();
for detector in detectors.iter() {
for relation in &detector.detector_relations {
if relation.kind == keyhog_core::DetectorRelationKind::Requires
&& known_ids.contains(&relation.detector_id)
{
required_by
.entry(relation.detector_id.clone())
.or_default()
.push(detector.id.clone());
}
}
}
let mut queue: std::collections::VecDeque<String> = removed.iter().cloned().collect();
while let Some(target_id) = queue.pop_front() {
if let Some(dependents) = required_by.get(&target_id) {
for dependent_id in dependents {
if removed.insert(dependent_id.clone()) {
queue.push_back(dependent_id.clone());
}
}
}
}
let before = detectors.len();
detectors.retain(|detector| !removed.contains(&detector.id));
for detector in detectors.iter_mut() {
detector
.detector_relations
.retain(|relation| !removed.contains(&relation.detector_id));
}
before - detectors.len()
}
const LOW_RAM_HOST_THRESHOLD_MB: u64 = 4096;
const LOW_RAM_MAX_MATCHES_PER_CHUNK: usize = 500;
const LOW_RAM_MAX_DECODE_BYTES: usize = 256 * 1024;
#[derive(Debug)]
pub(crate) struct GpuUnavailableError {
diagnostic: String,
}
impl GpuUnavailableError {
fn new(diagnostic: impl Into<String>) -> Self {
Self {
diagnostic: diagnostic.into(),
}
}
}
impl std::fmt::Display for GpuUnavailableError {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str(&self.diagnostic)
}
}
impl std::error::Error for GpuUnavailableError {}
const DAEMON_GPU_REMEDIATION: &str =
"Run `keyhog backend --self-test` and repair the GPU driver/runtime, or start the daemon with `--backend simd` or `--backend cpu`.";
fn daemon_gpu_failure(diagnostic: impl std::fmt::Display) -> anyhow::Error {
GpuUnavailableError::new(format!("{diagnostic} {DAEMON_GPU_REMEDIATION}")).into()
}
pub(crate) fn daemon_gpu_preflight_failure(diagnostic: String) -> anyhow::Error {
let diagnostic = diagnostic.trim().trim_end_matches('.');
daemon_gpu_failure(format_args!(
"daemon start: required GPU preflight failed: {diagnostic}."
))
}
pub(crate) fn daemon_compile_failure(error: &keyhog_scanner::ScanError) -> anyhow::Error {
match error {
keyhog_scanner::ScanError::Gpu(diagnostic) => daemon_gpu_failure(format_args!(
"daemon GPU initialization failed while compiling the scanner: {diagnostic}."
)),
_ => anyhow::anyhow!("daemon: compiling scanner from detector specs: {error}"),
}
}
fn apply_host_runtime_limits(
effective_config: &mut ResolvedScanConfig,
hw: &keyhog_scanner::HardwareCaps,
) {
effective_config.min_confidence = effective_config.scanner.min_confidence;
effective_config.ml_enabled = effective_config.scanner.ml_enabled;
let Some(mem_mb) = hw.total_memory_mb else {
return;
};
if mem_mb >= LOW_RAM_HOST_THRESHOLD_MB {
return;
}
let prev_matches = effective_config.scanner.max_matches_per_chunk;
let prev_decode = effective_config.scanner.max_decode_bytes;
let new_matches = prev_matches.min(LOW_RAM_MAX_MATCHES_PER_CHUNK);
let new_decode = prev_decode.min(LOW_RAM_MAX_DECODE_BYTES);
effective_config.scanner.max_matches_per_chunk = new_matches;
effective_config.scanner.max_decode_bytes = new_decode;
if new_matches != prev_matches || new_decode != prev_decode {
static LOW_RAM_CAP_WARNED: std::sync::OnceLock<()> = std::sync::OnceLock::new();
if LOW_RAM_CAP_WARNED.set(()).is_ok() {
eprintln!(
"keyhog: low-RAM host ({mem_mb} MiB < {LOW_RAM_HOST_THRESHOLD_MB}): capping scan limits to avoid OOM: max_decode_bytes {prev_decode} -> {new_decode}, max_matches_per_chunk {prev_matches} -> {new_matches}. Reduce scan scope or use a host with more memory; run `keyhog config --effective` to inspect the configured limits."
);
}
}
}
fn persistent_runtime_requires_gpu(
surface: &str,
backend_override: Option<keyhog_scanner::ScanBackend>,
gpu_required_by_route: bool,
) -> Result<bool> {
match backend_override {
None => Ok(gpu_required_by_route),
Some(backend) if backend.is_gpu() && gpu_required_by_route => Ok(true),
Some(backend) if backend.is_gpu() => Err(persistent_runtime_gpu_failure(
surface,
format!(
"{surface} --backend {} cannot be honored: this build and host have no eligible physical GPU path.",
backend.label()
),
)),
Some(keyhog_scanner::ScanBackend::SimdCpu | keyhog_scanner::ScanBackend::CpuFallback) => {
Ok(false)
}
Some(unknown) => anyhow::bail!(
"{surface} backend {unknown:?} is not supported by this KeyHog build; choose auto, gpu-cuda, gpu-wgpu, simd, or cpu"
),
}
}
fn persistent_runtime_gpu_failure(
surface: &str,
diagnostic: impl std::fmt::Display,
) -> anyhow::Error {
if surface == "daemon" {
daemon_gpu_failure(diagnostic)
} else {
GpuUnavailableError::new(format!(
"{diagnostic} Run `keyhog backend --self-test` and repair the GPU driver/runtime, or run `keyhog {surface} --backend simd` or `keyhog {surface} --backend cpu`."
))
.into()
}
}
fn validate_persistent_gpu_initialization(
surface: &str,
gpu_required: bool,
gpu_ready: bool,
) -> Result<()> {
if gpu_required && !gpu_ready {
return Err(persistent_runtime_gpu_failure(
surface,
format_args!("{surface} GPU initialization failed: the detected physical GPU is unavailable or incompatible with the compiled scanner, driver, or runtime; refusing to announce readiness."),
));
}
Ok(())
}
#[cfg(test)]
fn validate_persistent_gpu_warmup(
surface: &str,
gpu_required: bool,
degrade_before: u64,
degrade_after: u64,
) -> Result<()> {
if gpu_required && degrade_after != degrade_before {
return Err(persistent_runtime_gpu_failure(
surface,
format_args!("{surface} GPU warmup degraded before readiness; refusing to apply persistent warm autoroute evidence."),
));
}
Ok(())
}
#[cfg(test)]
fn daemon_requires_gpu(
backend_override: Option<keyhog_scanner::ScanBackend>,
gpu_required_by_route: bool,
) -> Result<bool> {
persistent_runtime_requires_gpu("daemon", backend_override, gpu_required_by_route)
}
#[cfg(test)]
fn validate_daemon_gpu_initialization(gpu_required: bool, gpu_ready: bool) -> Result<()> {
validate_persistent_gpu_initialization("daemon", gpu_required, gpu_ready)
}
#[cfg(test)]
fn validate_daemon_gpu_warmup(
gpu_required: bool,
degrade_before: u64,
degrade_after: u64,
) -> Result<()> {
validate_persistent_gpu_warmup("daemon", gpu_required, degrade_before, degrade_after)
}
pub(crate) use postprocess::render_credential;
#[cfg(unix)]
pub(crate) use postprocess::{
dedup_for_report, skipped_findings_from_deduped, suppresses_allowlist_match,
suppresses_test_fixture,
};
#[doc(hidden)]
pub(crate) use dispatch::backend_requires_coalesced_batch_pipeline_for_test;
#[doc(hidden)]
pub(crate) use run::scan_exit_code;
#[cfg(unix)]
pub(crate) use run::render_daemon_request_profile;
#[doc(hidden)]
pub(crate) use reporting::{
fmt_secs, render_progress_bar, render_reporting_ticker_line, render_severity_line,
render_ticker_line, render_verification_line, render_verification_ticker_line,
verification_breakdown, TickerGuard,
};
pub(crate) use dispatch::{
autoroute_engine_identity, autoroute_executable_identity, autoroute_gpu_artifact_identity,
CachedBackendRouter,
};
pub(crate) use dispatch::{
bind_autoroute_cache_to_execution_packs, inspect_autoroute_cache,
load_execution_pack_generation_binding, AutorouteReadiness, StagedAutorouteCache,
};
pub(crate) use streaming::{scan_streaming_source, StreamingSourceEvent};
fn resolved_default_autoroute_config() -> ResolvedScanConfig {
let mut resolved = resolved_scan_config_for_scanner(keyhog_scanner::ScannerConfig::default());
resolved.threads = Some(crate::orchestrator_config::keyhog_worker_threads());
resolved
}
pub(crate) fn autoroute_default_config_identity() -> String {
format!(
"{:016x}",
autoroute_config_digest(&resolved_default_autoroute_config())
)
}
fn router_gpu_participates(
backend_override: Option<keyhog_scanner::ScanBackend>,
runtime_policy: keyhog_scanner::gpu::GpuRuntimePolicy,
) -> bool {
backend_override.map_or_else(
|| runtime_policy != keyhog_scanner::gpu::GpuRuntimePolicy::Disabled,
keyhog_scanner::ScanBackend::is_gpu,
)
}
fn select_router_hardware<T>(
gpu_participates: bool,
probe_gpu: impl FnOnce() -> T,
probe_host: impl FnOnce() -> T,
) -> T {
if gpu_participates {
probe_gpu()
} else {
probe_host()
}
}
fn probe_router_hardware(gpu_participates: bool) -> keyhog_scanner::hw_probe::HardwareCaps {
select_router_hardware(
gpu_participates,
|| keyhog_scanner::hw_probe::probe_hardware().clone(),
keyhog_scanner::hw_probe::probe_host_hardware,
)
}
pub(crate) fn probe_route_hardware(
backend_override: Option<keyhog_scanner::ScanBackend>,
runtime_policy: keyhog_scanner::gpu::GpuRuntimePolicy,
) -> keyhog_scanner::hw_probe::HardwareCaps {
probe_router_hardware(router_gpu_participates(backend_override, runtime_policy))
}
pub(crate) fn cached_autoroute_router_for_default_config(
scanner: &CompiledScanner,
detectors: &[DetectorSpec],
backend_override: Option<keyhog_scanner::ScanBackend>,
) -> CachedBackendRouter {
let rules_digest = keyhog_core::hex_encode(&keyhog_core::compute_spec_hash(detectors));
let resolved = resolved_default_autoroute_config();
let gpu_participates = router_gpu_participates(backend_override, resolved.gpu_runtime_policy);
cached_autoroute_router(
scanner,
rules_digest,
autoroute_config_digest(&resolved),
gpu_participates,
crate::autoroute_cache_path::resolve_autoroute_cache_path(None),
)
}
fn cached_autoroute_router(
scanner: &CompiledScanner,
rules_digest: String,
config_digest: u64,
gpu_participates: bool,
autoroute_cache_path: Result<Option<std::path::PathBuf>, String>,
) -> CachedBackendRouter {
let hw_caps = probe_router_hardware(gpu_participates);
let pattern_count = scanner.runtime_status().pattern_count;
CachedBackendRouter::new(
hw_caps,
pattern_count,
rules_digest,
config_digest,
gpu_participates,
autoroute_cache_path,
scanner,
)
}
pub(crate) struct DefaultScanFilter {
signatures: std::collections::HashSet<Arc<str>>,
disabled_detectors: std::collections::HashSet<String>,
detector_min_confidence: std::collections::HashMap<String, f64>,
test_fixture_suppressions: crate::test_fixture_suppressions::TestFixtureSuppressions,
no_suppress_test_fixtures: bool,
min_confidence: f64,
min_severity: Option<keyhog_core::Severity>,
allowlist: keyhog_core::Allowlist,
}
impl DefaultScanFilter {
pub(crate) fn for_guard(detectors: &[DetectorSpec]) -> Self {
let signatures = collect_detector_signatures(detectors);
let disabled_detectors = std::collections::HashSet::new();
let detector_min_confidence = compose_detector_min_confidence(
&mut detectors.to_vec(),
std::collections::HashMap::new(),
);
Self {
signatures,
disabled_detectors,
detector_min_confidence,
test_fixture_suppressions:
crate::test_fixture_suppressions::TestFixtureSuppressions::bundled(),
no_suppress_test_fixtures: false,
min_confidence: 0.0,
min_severity: None,
allowlist: keyhog_core::Allowlist::default(),
}
}
pub(crate) fn finalize_count(
&self,
scanner: &CompiledScanner,
matches: Vec<RawMatch>,
) -> Option<usize> {
let filter = postprocess::MatchFilter {
scanner,
signatures: &self.signatures,
disabled_detectors: &self.disabled_detectors,
test_fixture_suppressions: &self.test_fixture_suppressions,
no_suppress_test_fixtures: self.no_suppress_test_fixtures,
detector_min_confidence: &self.detector_min_confidence,
min_confidence: self.min_confidence,
min_severity: self.min_severity,
};
match postprocess::filter_and_resolve_matches(&filter, matches, &self.allowlist) {
Ok(finalized) => Some(finalized.len()),
Err(e) => {
tracing::warn!("guard: match finalization failed: {}", e);
None
}
}
}
}
fn compose_detector_min_confidence(
detectors: &mut [DetectorSpec],
mut floors: std::collections::HashMap<String, f64>,
) -> std::collections::HashMap<String, f64> {
for detector in detectors.iter() {
if let Some(floor) = detector.min_confidence {
floors.entry(detector.id.clone()).or_insert(floor);
}
}
for detector in detectors {
if let Some(floor) = floors.get(&detector.id) {
detector.min_confidence = Some(*floor);
}
}
floors
}
pub(crate) struct DefaultScanRuntime {
scanner: Arc<CompiledScanner>,
router: CachedBackendRouter,
detector_count: usize,
worker_threads: usize,
backend_override: Option<keyhog_scanner::ScanBackend>,
recover_automatic_backend_faults: bool,
filter: Option<DefaultScanFilter>,
}
impl DefaultScanRuntime {
pub(crate) fn new(
scanner: Arc<CompiledScanner>,
detectors: &[DetectorSpec],
backend_override: Option<keyhog_scanner::ScanBackend>,
) -> Self {
let router =
cached_autoroute_router_for_default_config(&scanner, detectors, backend_override);
Self::new_with_router(scanner, detectors, router).with_backend_override(backend_override)
}
fn new_with_router(
scanner: Arc<CompiledScanner>,
detectors: &[DetectorSpec],
router: CachedBackendRouter,
) -> Self {
Self {
scanner,
router,
detector_count: detectors.len(),
worker_threads: rayon::current_num_threads(),
backend_override: None,
recover_automatic_backend_faults: automatic_backend_recovery_allowed(
None,
false,
keyhog_scanner::gpu::gpu_runtime_policy(),
),
filter: None,
}
}
pub(crate) fn with_filter(mut self, filter: DefaultScanFilter) -> Self {
self.filter = Some(filter);
self
}
pub(crate) fn filter_and_resolve(&self, matches: Vec<RawMatch>) -> Result<Vec<RawMatch>> {
let Some(f) = self.filter.as_ref() else {
anyhow::bail!(
"internal: DefaultScanRuntime has no resolved suppression filter; \
setup_default_scan_runtime must install one before filtering matches"
);
};
let filter = postprocess::MatchFilter {
scanner: &self.scanner,
signatures: &f.signatures,
disabled_detectors: &f.disabled_detectors,
test_fixture_suppressions: &f.test_fixture_suppressions,
no_suppress_test_fixtures: f.no_suppress_test_fixtures,
detector_min_confidence: &f.detector_min_confidence,
min_confidence: f.min_confidence,
min_severity: f.min_severity,
};
postprocess::filter_and_resolve_matches(&filter, matches, &f.allowlist)
}
pub(crate) fn with_backend_override(
mut self,
backend: Option<keyhog_scanner::ScanBackend>,
) -> Self {
self.backend_override = backend;
self.recover_automatic_backend_faults = automatic_backend_recovery_allowed(
backend,
false,
keyhog_scanner::gpu::gpu_runtime_policy(),
);
self
}
pub(crate) fn detector_count(&self) -> usize {
self.detector_count
}
pub(crate) fn worker_threads(&self) -> usize {
self.worker_threads
}
pub(crate) fn warm(&self) {
self.scanner.warm();
}
fn validate_explicit_backend(&self, subcommand_name: &str) -> Result<()> {
match self.backend_override {
Some(backend) if backend.is_gpu() => {
let eligible = self
.scanner
.gpu_backend_candidates()
.iter()
.any(|candidate| candidate.backend == backend && candidate.is_eligible())
&& self.scanner.warm_backend(backend);
if !eligible {
return Err(GpuUnavailableError::new(format!(
"{subcommand_name} --backend {} cannot be honored on this host/build; its GPU driver path is not ready. Run `keyhog backend --self-test`, use `--backend simd`, or use `--backend cpu`",
backend.label()
))
.into());
}
}
Some(keyhog_scanner::ScanBackend::SimdCpu) => {
self.scanner.initialize_simd_backend().map_err(|error| {
anyhow::anyhow!(
"{subcommand_name} --backend simd cannot be honored because Hyperscan initialization failed: {error}. Run `keyhog backend --self-test` or choose --backend cpu"
)
})?;
}
Some(keyhog_scanner::ScanBackend::CpuFallback) | None => {}
Some(backend) => {
anyhow::bail!(
"{subcommand_name} cannot validate the requested backend `{}` in this build",
backend.label()
);
}
}
Ok(())
}
pub(crate) fn prepare_persistent_daemon(
self,
backend_override: Option<keyhog_scanner::ScanBackend>,
) -> Result<Self> {
self.prepare_persistent_runtime(backend_override, "daemon")
}
pub(crate) fn prepare_persistent_watch(
self,
backend_override: Option<keyhog_scanner::ScanBackend>,
) -> Result<Self> {
self.prepare_persistent_runtime(backend_override, "watch")
}
fn prepare_persistent_runtime(
mut self,
backend_override: Option<keyhog_scanner::ScanBackend>,
surface: &'static str,
) -> Result<Self> {
self.scanner.warm();
let gpu_candidates = self.scanner.gpu_backend_candidates();
let required_routes = match backend_override {
Some(backend) => vec![backend],
None => self
.router
.persistent_routes()
.map_err(anyhow::Error::from)?,
};
let simd_required = required_routes.contains(&keyhog_scanner::ScanBackend::SimdCpu);
if simd_required {
self.scanner
.initialize_simd_backend()
.map_err(|error| {
anyhow::anyhow!(
"{surface} requires SIMD but Hyperscan initialization failed: {error}. Run `keyhog backend --self-test` or choose --backend cpu"
)
})?;
}
let gpu_routes: Vec<_> = required_routes
.iter()
.copied()
.filter(|backend| backend.is_gpu())
.collect();
let requested_gpu_is_eligible = match backend_override {
Some(backend) if backend.is_gpu() => gpu_candidates
.iter()
.any(|candidate| candidate.backend == backend && candidate.is_eligible()),
_ => !gpu_routes.is_empty(),
};
let gpu_must_be_ready =
persistent_runtime_requires_gpu(surface, backend_override, requested_gpu_is_eligible)?;
let gpu_ready = gpu_must_be_ready
&& !gpu_routes.is_empty()
&& gpu_routes
.iter()
.all(|backend| self.scanner.warm_backend(*backend));
validate_persistent_gpu_initialization(surface, gpu_must_be_ready, gpu_ready)?;
if gpu_must_be_ready {
let warmup = keyhog_core::Chunk {
data: format!("keyhog {surface} accelerator warmup\n").into(),
metadata: keyhog_core::ChunkMetadata {
source_type: format!("{surface}-warmup").into(),
..Default::default()
},
};
self.scanner.clear_fragment_cache();
for backend in &gpu_routes {
self.scanner
.scan_chunks_with_backend(std::slice::from_ref(&warmup), *backend)?;
}
self.scanner.clear_fragment_cache();
}
tracing::info!(
simd_initialized = self.scanner.simd_backend_initialized(),
gpu_ready,
selected_gpu_routes = ?gpu_routes,
gpu_must_be_ready,
surface,
"persistent runtime backends initialized"
);
self.router = self.router.for_persistent_runtime();
Ok(self)
}
pub(crate) fn scan_chunk(&self, chunk: &Chunk) -> Result<Vec<RawMatch>> {
if chunk.data.is_empty() {
return Ok(Vec::new());
}
let selection = self.router.choose_with_plan(
self.scanner.as_ref(),
self.backend_override,
std::slice::from_ref(chunk),
)?;
let backend = selection.backend;
let batch = std::slice::from_ref(chunk);
let outcome = scan_selected_batch(
self.scanner.as_ref(),
batch,
backend,
#[cfg(feature = "gpu")]
selection.ordered_gpu.as_deref(),
selection.phase1_plan.as_ref(),
selection.execution_route,
selection
.recovery_plan
.filter(|_| self.recover_automatic_backend_faults),
)
.with_context(|| {
format!(
"selected backend {} failed during single-chunk dispatch",
backend.label()
)
})?;
dispatch::record_profiled_batch_route(
batch,
self.backend_override
.map_or("auto", keyhog_scanner::ScanBackend::label),
&selection,
&outcome,
);
if let Some(recovery) = outcome.recovery.as_ref() {
self.router
.quarantine_recovered_route(&selection, recovery)?;
}
Ok(outcome.per_chunk.into_iter().flatten().collect())
}
pub(crate) fn clear_fragment_cache(&self) {
self.scanner.clear_fragment_cache();
}
pub(crate) fn into_parts(self) -> (Arc<CompiledScanner>, CachedBackendRouter) {
(self.scanner, self.router)
}
}
fn default_runtime_gpu_init_policy(
backend_override: Option<keyhog_scanner::ScanBackend>,
) -> GpuInitPolicy {
backend_override.map_or(
GpuInitPolicy::FromRuntimePolicy,
GpuInitPolicy::SelectedBackend,
)
}
pub(crate) fn compile_default_scan_runtime(
detectors: Vec<DetectorSpec>,
backend_override: Option<keyhog_scanner::ScanBackend>,
map_compile_error: impl FnOnce(&keyhog_scanner::ScanError) -> anyhow::Error,
) -> Result<DefaultScanRuntime> {
let gpu_policy = default_runtime_gpu_init_policy(backend_override);
let detectors: Arc<[DetectorSpec]> = detectors.into();
let scanner = Arc::new(
CompiledScanner::compile_shared_with_gpu_policy_and_tuning(
Arc::clone(&detectors),
gpu_policy,
&keyhog_scanner::ScannerTuningConfig::default(),
)
.map_err(|error| map_compile_error(&error))?,
);
Ok(DefaultScanRuntime::new(
scanner,
&detectors,
backend_override,
))
}
pub(crate) fn setup_default_scan_runtime(
detectors_path: &std::path::Path,
detectors_cli_explicit: bool,
cache_dir: Option<std::path::PathBuf>,
threads: Option<usize>,
backend_override: Option<keyhog_scanner::ScanBackend>,
subcommand_name: &'static str,
warm: bool,
filter_root: Option<&std::path::Path>,
) -> Result<DefaultScanRuntime> {
setup_default_scan_runtime_with_rayon_policy(
detectors_path,
detectors_cli_explicit,
cache_dir,
threads,
backend_override,
subcommand_name,
warm,
filter_root,
RayonSetupPolicy::RequireKeyHogOwned,
)
}
#[derive(Clone, Copy)]
enum RayonSetupPolicy {
RequireKeyHogOwned,
#[cfg(test)]
ReuseTestHarnessPool,
}
#[cfg(test)]
pub(crate) fn setup_default_scan_runtime_for_test(
detectors_path: &std::path::Path,
detectors_cli_explicit: bool,
cache_dir: Option<std::path::PathBuf>,
threads: Option<usize>,
backend_override: Option<keyhog_scanner::ScanBackend>,
subcommand_name: &'static str,
warm: bool,
filter_root: Option<&std::path::Path>,
) -> Result<DefaultScanRuntime> {
setup_default_scan_runtime_with_rayon_policy(
detectors_path,
detectors_cli_explicit,
cache_dir,
threads,
backend_override,
subcommand_name,
warm,
filter_root,
RayonSetupPolicy::ReuseTestHarnessPool,
)
}
fn setup_default_scan_runtime_with_rayon_policy(
detectors_path: &std::path::Path,
detectors_cli_explicit: bool,
cache_dir: Option<std::path::PathBuf>,
threads: Option<usize>,
backend_override: Option<keyhog_scanner::ScanBackend>,
subcommand_name: &'static str,
warm: bool,
filter_root: Option<&std::path::Path>,
rayon_policy: RayonSetupPolicy,
) -> Result<DefaultScanRuntime> {
use clap::Parser;
crate::runtime_preflight::validate_scan_runtime_config()?;
let mut synthetic = ScanArgs::try_parse_from(["keyhog-scan"]).context(
"internal: constructing default ScanArgs for watch/scan-system config resolution",
)?;
synthetic.detectors = detectors_path.to_path_buf();
synthetic.detectors_cli_explicit = detectors_cli_explicit;
synthetic.cache_dir = cache_dir;
synthetic.threads = threads;
synthetic.backend = backend_override.map(|backend| backend_override_cli_value(backend).into());
synthetic.path = filter_root.map(std::path::Path::to_path_buf);
let mut effective_config = resolve_scan_config(&mut synthetic)?;
let requested_detector_mode = synthetic.detectors_mode.map(Into::into);
validate_detector_mode_selection(synthetic.detectors_cli_explicit, requested_detector_mode)?;
validate_explicit_detector_path(&synthetic.detectors, synthetic.detectors_cli_explicit)?;
let detectors_path = auto_discover_detectors(&synthetic.detectors)?;
let detectors_path_for_compile = detectors_path.clone();
ResolvedEngineRuntimeSettings::from(&effective_config).apply();
let hw = keyhog_scanner::hw_probe::probe_hardware();
let worker_threads = match rayon_policy {
RayonSetupPolicy::RequireKeyHogOwned => {
configure_threads(effective_config.threads, hw.physical_cores)?
}
#[cfg(test)]
RayonSetupPolicy::ReuseTestHarnessPool => {
let current = rayon::current_num_threads();
if let Some(requested) = effective_config.threads {
if requested != current {
anyhow::bail!(
"test harness Rayon pool has {current} threads, but the isolated runtime requested {requested}"
);
}
}
current
}
};
effective_config.threads = Some(worker_threads);
apply_host_runtime_limits(&mut effective_config, &hw);
keyhog_scanner::gpu::require_gpu_preflight().map_err(|diagnostic| {
GpuUnavailableError::new(format!(
"cannot start `{subcommand_name}` with the resolved GPU policy: {diagnostic}"
))
})?;
let mut detectors = load_effective_detector_corpus(
&detectors_path,
requested_detector_mode,
!synthetic.lockdown,
)
.context("loading effective detector corpus")?
.detectors;
let disabled_detectors = effective_config.disabled_detectors.clone();
if !disabled_detectors.is_empty() {
let before = detectors.len();
filter_disabled_detectors(&mut detectors, &disabled_detectors);
if detectors.is_empty() && before > 0 {
anyhow::bail!(
"all {before} loaded detector(s) were disabled by .keyhog.toml \
[detector.<id>] enabled = false. Leave at least one detector enabled to run \
`{subcommand_name}`, or remove the config."
);
}
}
let rules_digest = keyhog_core::hex_encode(&keyhog_core::compute_spec_hash(&detectors));
let mut detector_min_confidence = compose_detector_min_confidence(
&mut detectors,
effective_config.detector_min_confidence.clone(),
);
if synthetic.precision {
let floor = effective_config.scanner.min_confidence;
for detector_floor in detector_min_confidence.values_mut() {
*detector_floor = detector_floor.max(floor);
}
detector_min_confidence =
compose_detector_min_confidence(&mut detectors, detector_min_confidence);
}
let gpu_init_policy = gpu_init_policy_for_args(
&synthetic,
effective_config.autoroute_cache_path.as_deref(),
effective_config.autoroute_gpu,
effective_config.autoroute_calibration,
);
let scanner = Arc::new(
CompiledScanner::compile_with_gpu_policy_and_tuning(
detectors.clone(),
gpu_init_policy,
&effective_config.scanner_tuning,
)
.map_err(|error| {
crate::orchestrator_config::detector_compile_failed(
subcommand_name,
&detectors_path_for_compile,
&error,
)
})?
.with_config(effective_config.engine_scanner_config()),
);
let gpu_participates = router_gpu_participates(
effective_config.backend_override,
effective_config.gpu_runtime_policy,
);
let router = cached_autoroute_router(
&scanner,
rules_digest,
autoroute_config_digest(&effective_config),
gpu_participates,
Ok(effective_config.autoroute_cache_path.clone()),
);
let mut scan_runtime = DefaultScanRuntime::new_with_router(scanner, &detectors, router)
.with_backend_override(effective_config.backend_override);
scan_runtime.validate_explicit_backend(subcommand_name)?;
if let Some(root) = filter_root {
let signatures = collect_detector_signatures(&detectors);
let allowlist = allowlist::load_allowlist(Some(root), &effective_config.allowlist)?;
let test_fixture_suppressions = if effective_config.report.no_suppress_test_fixtures {
crate::test_fixture_suppressions::TestFixtureSuppressions::empty()
} else {
crate::test_fixture_suppressions::TestFixtureSuppressions::bundled()
};
scan_runtime = scan_runtime.with_filter(DefaultScanFilter {
signatures,
disabled_detectors,
detector_min_confidence,
test_fixture_suppressions,
no_suppress_test_fixtures: effective_config.report.no_suppress_test_fixtures,
min_confidence: effective_config.min_confidence,
min_severity: effective_config
.report
.severity
.as_ref()
.map(|s| s.to_severity()),
allowlist,
});
}
drop(detectors);
run::release_allocator_arenas_after_construction();
if warm {
scan_runtime.warm();
}
Ok(scan_runtime)
}
#[doc(hidden)]
pub(crate) fn router_gpu_participates_for_test(
backend_override: Option<keyhog_scanner::ScanBackend>,
runtime_policy: keyhog_scanner::gpu::GpuRuntimePolicy,
) -> bool {
router_gpu_participates(backend_override, runtime_policy)
}
#[doc(hidden)]
pub(crate) fn router_uses_gpu_probe_for_test(gpu_participates: bool) -> bool {
select_router_hardware(gpu_participates, || true, || false)
}
#[doc(hidden)]
pub(crate) fn gpu_init_policy_for_args_for_test(args: &ScanArgs) -> GpuInitPolicy {
gpu_init_policy_for_args(
args,
None,
args.autoroute_gpu && !args.no_autoroute_gpu,
args.autoroute_calibrate,
)
}
#[doc(hidden)]
pub(crate) fn gpu_init_policy_for_resolved_autoroute_for_test(
args: &ScanArgs,
autoroute_cache_path: Option<&std::path::Path>,
autoroute_gpu: bool,
autoroute_calibration: bool,
) -> GpuInitPolicy {
gpu_init_policy_for_args(
args,
autoroute_cache_path,
autoroute_gpu,
autoroute_calibration,
)
}
#[doc(hidden)]
pub(crate) fn explicit_backend_override(
raw: Option<&str>,
) -> Result<Option<keyhog_scanner::ScanBackend>> {
parse_backend_override(raw)
}
#[doc(hidden)]
pub(crate) fn allowlist_root_for_test(path: &std::path::Path) -> std::path::PathBuf {
allowlist::allowlist_root(path)
}
#[doc(hidden)]
pub(crate) fn scanner_panic_notice_for_test(panicked: bool) -> Option<String> {
reporting::scanner_panic_notice(panicked)
}
#[doc(hidden)]
pub(crate) fn resolve_scan_exit_for_test(
has_new_entries: bool,
incremental_cache_failed: bool,
source_coverage_incomplete: bool,
) -> u8 {
run::resolve_scan_exit(run::ScanOutcome {
has_new_entries,
incremental_cache_failed,
source_coverage_incomplete,
..run::ScanOutcome::default()
})
}
fn execution_pack_policy_for_args(
args: &ScanArgs,
) -> keyhog_scanner::execution_pack::ExecutionPackPolicy {
use keyhog_scanner::execution_pack::ExecutionPackPolicy;
if args.fast {
ExecutionPackPolicy::Fast
} else if args.deep {
ExecutionPackPolicy::Deep
} else if args.precision {
ExecutionPackPolicy::Precision
} else {
ExecutionPackPolicy::Default
}
}
pub(crate) struct ScanOrchestrator {
pub(crate) args: ScanArgs,
pub(crate) detector_count: usize,
#[cfg(feature = "verify")]
pub(crate) verifier_detectors: Option<Arc<[DetectorSpec]>>,
pub(crate) detector_spec_hash: [u8; 32],
pub(crate) detector_rules_digest: String,
pub(crate) detector_corpus_digest: String,
pub(crate) detector_corpus_provenance: DetectorCorpusProvenance,
pub(crate) scanner: Arc<CompiledScanner>,
pub(crate) signatures: std::collections::HashSet<Arc<str>>,
pub(crate) test_fixture_suppressions: crate::test_fixture_suppressions::TestFixtureSuppressions,
pub(crate) disabled_detectors: std::collections::HashSet<String>,
pub(crate) detector_min_confidence: std::collections::HashMap<String, f64>,
pub(crate) effective_config: ResolvedScanConfig,
autoroute_measurement_observer: Option<dispatch::AutorouteMeasurementObserver>,
#[cfg(test)]
scanner_dispatch_starts: Arc<std::sync::atomic::AtomicUsize>,
early_profile_session: Option<keyhog_profile::Session>,
early_profile_build: Option<std::thread::JoinHandle<keyhog_profile::BuildIdentityV2>>,
}
impl ScanOrchestrator {
#[inline]
fn record_scanner_dispatch_start(&self) {
#[cfg(test)]
self.scanner_dispatch_starts
.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
}
pub(crate) fn new(mut args: ScanArgs) -> Result<Self> {
let early_profile_session = if args.profile || args.profile_out.is_some() {
let identity = keyhog_profile::RunIdentity::new(
env!("CARGO_PKG_VERSION"),
"pending-detector-corpus",
"pending-config",
"pending-source",
"orchestrator-construction",
"pending-backend-policy",
);
let session = keyhog_profile::Session::start(identity).map_err(anyhow::Error::new)?;
crate::set_operator_profile_active(true);
Some(session)
} else {
None
};
let early_profile_build = early_profile_session
.as_ref()
.map(|_| std::thread::spawn(run::profiler_build_identity));
keyhog_scanner::gpu::set_gpu_runtime_policy(gpu_runtime_policy_from_args(&args));
let positional_stdin = args
.input
.iter()
.any(|path| path == std::path::Path::new("-"));
if positional_stdin && args.input.len() > 1 {
anyhow::bail!(
"stdin shorthand `-` cannot be combined with other positional scan roots; use either `keyhog scan -` or scan the filesystem roots without `-`"
);
}
if positional_stdin || matches!(args.path.as_deref().and_then(|p| p.to_str()), Some("-")) {
args.stdin = true;
args.input.clear();
args.path = None;
}
if args.path.is_none() {
args.path = args.input.first().cloned();
}
#[cfg(feature = "git")]
if args.git_staged && args.path.is_none() {
args.path = Some(PathBuf::from("."));
}
if !args.stdin {
for root in args.scan_roots() {
crate::path_validation::validate_cli_path_arg(&root, "scan path")?;
}
}
let mut effective_config = resolve_scan_config(&mut args)?;
ResolvedEngineRuntimeSettings::from(&effective_config).apply();
let disabled_detectors = effective_config.disabled_detectors.clone();
let mut detector_min_confidence = effective_config.detector_min_confidence.clone();
if effective_config.require_lockdown && !args.lockdown {
anyhow::bail!(
".keyhog.toml sets [lockdown] require = true, but --lockdown was not passed. \
Re-run with --lockdown to enforce the configured hardening, or remove the \
requirement from .keyhog.toml."
);
}
let hw = {
let _profile_span = keyhog_profile::span(keyhog_profile::Stage::BackendAcquire);
keyhog_scanner::hw_probe::probe_hardware()
};
let worker_threads = configure_threads(args.threads, hw.physical_cores)?;
args.threads = Some(worker_threads);
effective_config.threads = Some(worker_threads);
let (requested_detector_mode, detectors_path) = {
let _profile_span = keyhog_profile::span(keyhog_profile::Stage::DetectorValidate);
let requested_detector_mode = args.detectors_mode.map(Into::into);
validate_detector_mode_selection(args.detectors_cli_explicit, requested_detector_mode)?;
validate_explicit_detector_path(&args.detectors, args.detectors_cli_explicit)?;
let detectors_path = auto_discover_detectors(&args.detectors)?;
(requested_detector_mode, detectors_path)
};
let resolved_config_digest =
crate::orchestrator_config::matcher_resolved_config_digest(&effective_config);
let runtime_identity = keyhog_scanner::hw_probe::hyperscan_runtime_identity();
let gpu_init_policy = {
let _profile_span = keyhog_profile::span(keyhog_profile::Stage::ExecutionPackSelect);
gpu_init_policy_for_args(
&args,
effective_config.autoroute_cache_path.as_deref(),
effective_config.autoroute_gpu,
effective_config.autoroute_calibration,
)
};
let (mut loaded_corpus, detector_execution_pack) = {
let _profile_span = keyhog_profile::span(keyhog_profile::Stage::DetectorLoad);
if !detectors_path.exists() && requested_detector_mode.is_none() {
let policy = execution_pack_policy_for_args(&args);
let execution_pack_directory =
crate::execution_pack_install::installed_execution_pack_directory()
.context("resolving the installed execution-pack directory")?;
let installed = match effective_config.backend_override {
Some(backend) => {
let pack_backend =
keyhog_scanner::execution_pack::ExecutionPackBackend::from_scan_backend(
backend,
)
.context(
"the selected scan backend has no execution-pack identity",
)?;
crate::execution_pack_install::
load_installed_detector_execution_pack_for_backend(
policy,
pack_backend,
)
}
None => crate::execution_pack_install::
load_installed_preferred_detector_execution_pack(policy),
};
match installed {
Ok(pack) => (None, Some(pack)),
Err(error) if !execution_pack_directory.exists() => {
tracing::warn!(
error = %error,
"no installed execution-pack generation; parsing embedded detectors"
);
let embedded = || -> anyhow::Result<LoadedDetectorCorpus> {
load_effective_detector_corpus(
&detectors_path,
requested_detector_mode,
!args.lockdown,
)
.context("loading effective detector corpus")
};
(Some(embedded()?), None)
}
Err(error) => {
return Err(error).context(
"loading authenticated detector execution pack; run a verified install or self-update",
);
}
}
} else {
(
Some(
load_effective_detector_corpus(
&detectors_path,
requested_detector_mode,
!args.lockdown,
)
.context("loading effective detector corpus")?,
),
None,
)
}
};
#[cfg(feature = "verify")]
let verifier_enabled = effective_config.report.verify;
#[cfg(not(feature = "verify"))]
let verifier_enabled = false;
let schemas_required = !disabled_detectors.is_empty() || verifier_enabled;
if loaded_corpus.is_none() && schemas_required {
let pack = detector_execution_pack.as_ref().context(
"installed detector schemas are required, but the authenticated execution pack was not retained",
)?;
let ir_bytes = pack
.section(keyhog_scanner::execution_pack::ExecutionPackSectionKind::DetectorIr)
.context("installed execution pack has no detector IR section")?;
let ir = keyhog_scanner::execution_pack::CanonicalDetectorExecutionIr::decode_runtime(
ir_bytes,
)
.map_err(anyhow::Error::msg)?;
if ir.digest() != pack.identity().detector_digest {
anyhow::bail!(
"installed detector IR identity does not match its authenticated pack"
);
}
let embedded_count = ir.detectors().len();
loaded_corpus = Some(LoadedDetectorCorpus {
detectors: ir.into_detectors(),
schema_version: keyhog_core::DETECTOR_CORPUS_SCHEMA_VERSION,
provenance: DetectorCorpusProvenance {
mode: "embedded",
source: format!("authenticated execution pack {}", pack.path().display()),
embedded_count,
custom_count: 0,
},
});
}
let direct_pack_hydration = loaded_corpus.is_none();
let (detector_corpus_schema_version, mut detector_corpus_provenance, mut detectors) =
match loaded_corpus {
Some(loaded) => (loaded.schema_version, loaded.provenance, loaded.detectors),
None => {
let pack = detector_execution_pack.as_ref().context(
"direct scanner hydration requires a retained authenticated execution pack",
)?;
(
keyhog_core::DETECTOR_CORPUS_SCHEMA_VERSION,
DetectorCorpusProvenance {
mode: "embedded",
source: format!(
"authenticated execution pack {}",
pack.path().display()
),
embedded_count: 0,
custom_count: 0,
},
Vec::new(),
)
}
};
let detector_validation_span =
keyhog_profile::span(keyhog_profile::Stage::DetectorValidate);
if !disabled_detectors.is_empty() {
let before = detectors.len();
let dropped = filter_disabled_detectors(&mut detectors, &disabled_detectors);
if dropped > 0 {
if detectors.is_empty() {
let mut disabled_ids: Vec<&str> =
disabled_detectors.iter().map(String::as_str).collect();
disabled_ids.sort_unstable();
let listed = if disabled_ids.len() <= 16 {
disabled_ids.join(", ")
} else {
format!(
"{} ... ({} total)",
disabled_ids[..16].join(", "),
disabled_ids.len()
)
};
anyhow::bail!(
"all {before} loaded detector(s) were disabled by .keyhog.toml \
[detector.<id>] enabled = false ({listed}). Fix: leave at least \
one detector enabled, remove the config, or use .keyhogignore for \
specific finding suppressions. Refusing to scan with no detectors \
loaded."
);
}
tracing::info!(
target: "keyhog::config",
dropped,
"disabled detectors via .keyhog.toml [detector.<id>] enabled = false"
);
} else {
let palette = style::for_stderr();
eprintln!(
"{} .keyhog.toml disables detector id(s) {disabled_detectors:?}, but none matched the loaded corpus. \
Detector ids come from `keyhog detectors`; accelerated slots use the same canonical TOML id.",
style::warn("WARN", &palette)
);
}
}
let mut detector_rules_digest =
keyhog_core::hex_encode(&keyhog_core::compute_spec_hash(&detectors));
let mut detector_corpus_digest = keyhog_core::hex_encode(
&keyhog_core::compute_detector_corpus_digest_for_schema(
&detectors,
detector_corpus_schema_version,
)
.context("serializing effective detector corpus identity")?,
);
apply_host_runtime_limits(&mut effective_config, &hw);
detector_min_confidence =
compose_detector_min_confidence(&mut detectors, detector_min_confidence);
if args.precision {
let floor = effective_config.scanner.min_confidence;
for v in detector_min_confidence.values_mut() {
*v = v.max(floor);
}
}
detector_min_confidence =
compose_detector_min_confidence(&mut detectors, detector_min_confidence);
let mut detector_spec_hash = keyhog_core::compute_spec_hash(&detectors);
drop(detector_validation_span);
let mut detector_count = detectors.len();
let mut signatures = collect_detector_signatures(&detectors);
let detectors: Option<Arc<[DetectorSpec]>> =
(!direct_pack_hydration).then(|| detectors.into());
let scanner = {
let _pack_span = keyhog_profile::span(keyhog_profile::Stage::ExecutionPackMap);
let compiled = if disabled_detectors.is_empty() {
match detector_execution_pack.as_ref() {
Some(pack) => {
CompiledScanner::compile_from_execution_pack_with_gpu_policy_and_tuning(
pack,
gpu_init_policy,
&effective_config.scanner_tuning,
)
}
None => {
let detectors = detectors.as_ref().context(
"embedded/debug scanner construction requires detector schemas",
)?;
keyhog_scanner::compile_shared_with_matcher_artifact_cache(
Arc::clone(detectors),
gpu_init_policy,
&effective_config.scanner_tuning,
resolved_config_digest,
None,
runtime_identity.as_deref(),
)
.map(|(scanner, outcome)| {
tracing::debug!(
target: "keyhog::matcher_artifact_cache",
outcome = outcome.as_str(),
"matcher artifact cache outcome"
);
scanner
})
}
}
} else {
let detectors = detectors
.as_ref()
.context("disabled-detector scanner construction requires detector schemas")?;
keyhog_scanner::compile_shared_with_matcher_artifact_cache(
Arc::clone(detectors),
gpu_init_policy,
&effective_config.scanner_tuning,
resolved_config_digest,
None,
runtime_identity.as_deref(),
)
.map(|(scanner, outcome)| {
tracing::debug!(
target: "keyhog::matcher_artifact_cache",
outcome = outcome.as_str(),
"matcher artifact cache outcome"
);
scanner
})
};
Arc::new(
compiled
.with_context(|| {
format!("materializing scanner for {detector_count} detectors")
})?
.with_config(effective_config.engine_scanner_config()),
)
};
if direct_pack_hydration {
detector_count = scanner.detector_count();
signatures = scanner.detector_signature_sources();
for (id, floor) in scanner.declared_detector_min_confidence() {
detector_min_confidence
.entry(id.to_owned())
.or_insert(floor);
}
if args.precision {
let floor = effective_config.scanner.min_confidence;
for value in detector_min_confidence.values_mut() {
*value = value.max(floor);
}
}
let runtime = scanner.runtime_status();
detector_rules_digest = keyhog_core::hex_encode(&runtime.compiled_plan_digest);
let pack = detector_execution_pack.as_ref().context(
"direct scanner hydration requires a retained authenticated execution pack",
)?;
detector_corpus_digest = keyhog_core::hex_encode(&pack.identity().detector_digest);
detector_corpus_provenance.embedded_count = detector_count;
let mut hasher = blake3::Hasher::new();
hasher.update(b"keyhog-effective-installed-detector-spec-v1\0");
hasher.update(&runtime.compiled_plan_digest);
hasher.update(&pack.identity().detector_digest);
hasher.update(&effective_config.scanner.min_confidence.to_le_bytes());
let mut floors: Vec<_> = detector_min_confidence.iter().collect();
floors.sort_unstable_by(|left, right| left.0.cmp(right.0));
for (id, floor) in floors {
hasher.update(&(id.len() as u64).to_le_bytes());
hasher.update(id.as_bytes());
hasher.update(&floor.to_le_bytes());
}
detector_spec_hash = *hasher.finalize().as_bytes();
}
#[cfg(feature = "verify")]
let verifier_detectors = if verifier_enabled { detectors } else { None };
#[cfg(not(feature = "verify"))]
drop(detectors);
run::release_allocator_arenas_after_construction();
let test_fixture_suppressions = if args.no_suppress_test_fixtures {
crate::test_fixture_suppressions::TestFixtureSuppressions::empty()
} else {
crate::test_fixture_suppressions::TestFixtureSuppressions::bundled()
};
Ok(Self {
args,
detector_count,
#[cfg(feature = "verify")]
verifier_detectors,
detector_spec_hash,
detector_rules_digest,
detector_corpus_digest,
detector_corpus_provenance,
scanner,
signatures,
test_fixture_suppressions,
disabled_detectors,
detector_min_confidence,
effective_config,
autoroute_measurement_observer: None,
#[cfg(test)]
scanner_dispatch_starts: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
early_profile_session,
early_profile_build,
})
}
pub(crate) fn scanner(&self) -> &CompiledScanner {
self.scanner.as_ref()
}
pub(crate) fn prepare_autoroute_calibration_gpu_artifact(&self) -> Result<()> {
self.scanner
.prepare_autoroute_calibration_gpu_artifact()
.map_err(|error| anyhow::anyhow!(error))
}
pub(crate) fn observe_autoroute_calibration_measurements(
&mut self,
observer: dispatch::AutorouteMeasurementObserver,
) -> Result<()> {
if !self.effective_config.autoroute_calibration {
anyhow::bail!("measured-route observation requires an autoroute calibration runtime");
}
self.autoroute_measurement_observer = Some(observer);
Ok(())
}
pub(crate) fn args(&self) -> &ScanArgs {
&self.args
}
pub(crate) fn incremental_cache_path(&self) -> Result<Option<std::path::PathBuf>> {
if !self.args.incremental {
return Ok(None);
}
if self.args.lockdown {
tracing::warn!("lockdown mode: --incremental disabled (cache writes refused)");
eprintln!(
"warning: --incremental disabled because --lockdown forbids cache reads/writes; scanning without the incremental cache"
);
return Ok(None);
}
match self.configured_incremental_cache_path() {
Some(path) => Ok(Some(path)),
None => anyhow::bail!(
"--incremental was requested, but no default cache directory is available. \
Fix: set XDG_CACHE_HOME or HOME, or pass --incremental-cache <PATH>."
),
}
}
pub(crate) fn lockdown_persistence_cache_paths(&self) -> Vec<std::path::PathBuf> {
if !(self.args.incremental || self.args.incremental_cache.is_some()) {
return Vec::new();
}
self.configured_incremental_cache_path()
.into_iter()
.collect()
}
fn configured_incremental_cache_path(&self) -> Option<std::path::PathBuf> {
self.args
.incremental_cache
.clone()
.or_else(keyhog_core::merkle_default_cache_path)
}
pub(crate) fn build_merkle_index(
&self,
path: Option<&std::path::Path>,
) -> (
Option<Arc<keyhog_core::MerkleIndex>>,
Option<keyhog_core::MerkleLoadStatus>,
) {
let Some(path) = path else {
return (None, None);
};
let report =
keyhog_core::MerkleIndex::load_with_spec_report(path, &self.detector_spec_hash);
if let Some(warning) = incremental_cache_warning(report.status()) {
eprintln!("{warning}");
}
let status = report.status().clone();
let idx = report.into_index();
tracing::info!("incremental scan: loaded merkle index");
(Some(Arc::new(idx)), Some(status))
}
#[doc(hidden)]
pub(crate) fn scan_sources_for_test(
&self,
sources: Vec<Box<dyn Source>>,
show_progress: bool,
merkle: Option<Arc<keyhog_core::MerkleIndex>>,
) -> Result<Vec<RawMatch>> {
self.scan_sources(sources, show_progress, merkle, None)
}
#[doc(hidden)]
pub(crate) fn from_parts_for_test(
args: ScanArgs,
detectors: Vec<DetectorSpec>,
scanner: Arc<CompiledScanner>,
signatures: std::collections::HashSet<Arc<str>>,
test_fixture_suppressions: crate::test_fixture_suppressions::TestFixtureSuppressions,
) -> Self {
let batch_pipeline = args.batch_pipeline && !args.no_batch_pipeline;
let threads = args.threads;
let reader_threads = args.reader_threads;
let fused_batch = args
.fused_batch
.unwrap_or(crate::orchestrator_config::FUSED_BATCH_DEFAULT); let fused_depth = args.fused_depth;
let detector_spec_hash = keyhog_core::compute_spec_hash(&detectors);
let detector_rules_digest = keyhog_core::hex_encode(&detector_spec_hash);
let detector_corpus_digest = detector_rules_digest.clone();
let detector_corpus_provenance = DetectorCorpusProvenance {
mode: "provided",
source: "library/test constructor".to_string(),
embedded_count: 0,
custom_count: detectors.len(),
};
let detector_count = detectors.len();
#[cfg(feature = "verify")]
let verifier_detectors = args.verify.then(|| detectors.into());
#[cfg(not(feature = "verify"))]
drop(detectors);
Self {
args,
detector_count,
#[cfg(feature = "verify")]
verifier_detectors,
detector_spec_hash,
detector_rules_digest,
detector_corpus_provenance,
detector_corpus_digest,
scanner,
signatures,
test_fixture_suppressions,
disabled_detectors: std::collections::HashSet::new(),
detector_min_confidence: std::collections::HashMap::new(),
autoroute_measurement_observer: None,
#[cfg(test)]
scanner_dispatch_starts: Arc::new(std::sync::atomic::AtomicUsize::new(0)),
early_profile_session: None,
early_profile_build: None,
effective_config: ResolvedScanConfig {
backend_override: Some(keyhog_scanner::ScanBackend::CpuFallback),
batch_pipeline,
threads,
reader_threads,
fused_batch,
fused_depth,
gpu_runtime_policy: keyhog_scanner::gpu::GpuRuntimePolicy::Auto,
autoroute_gpu: false,
autoroute_calibration: false,
scanner: keyhog_scanner::ScannerConfig::default(),
min_confidence: keyhog_scanner::ScannerConfig::default().min_confidence,
ml_enabled: keyhog_scanner::ScannerConfig::default().ml_enabled,
detector_min_confidence: std::collections::HashMap::new(),
disabled_detectors: std::collections::HashSet::new(),
require_lockdown: false,
regex_dfa_limit: None,
gpu_batch_input_limit: None,
max_file_size: None,
#[cfg(feature = "git")]
max_commits: crate::orchestrator_config::MAX_COMMITS_DEFAULT,
no_default_excludes: false,
exclude_paths: Vec::new(),
incremental: false,
incremental_cache_path: None,
hyperscan_cache_dir: None,
autoroute_cache_path: None,
matcher_cache_path: None,
calibration_cache_path: None,
calibration_entry_count: 0,
calibration_digest: 0,
aws_canary_accounts: Vec::new(),
scanner_tuning: keyhog_scanner::ScannerTuningConfig::default(),
allowlist: crate::orchestrator_config::ResolvedAllowlistConfig {
file: None,
require_reason: false,
require_approved_by: false,
max_expires_days: None,
},
source_limits: keyhog_sources::SourceLimits::default(),
report: crate::orchestrator_config::ResolvedReportPolicy {
format: crate::args::OutputFormat::Text,
severity: None,
dedup: crate::args::CliDedupScope::Credential,
verify: false,
lockdown: false,
show_secrets: false,
no_suppress_test_fixtures: false,
hide_client_safe: false,
},
verify: crate::orchestrator_config::ResolvedVerifyPolicy::disabled(),
},
}
}
}
pub(crate) fn incremental_cache_warning(status: &MerkleLoadStatus) -> Option<String> {
match status {
MerkleLoadStatus::Missing { .. } | MerkleLoadStatus::Loaded { .. } => None,
MerkleLoadStatus::ReadFailed { path, error } => Some(format!(
"warning: incremental cache {} could not be read: {error}; starting from an empty cache and rewriting it after this scan",
path.display()
)),
MerkleLoadStatus::ParseFailed { path, error } => Some(format!(
"warning: incremental cache {} could not be parsed: {error}; starting from an empty cache and rewriting it after this scan",
path.display()
)),
MerkleLoadStatus::SchemaMismatch {
path,
version,
expected,
} => Some(format!(
"warning: incremental cache {} uses schema version {version}, expected {expected}; starting from an empty cache and rewriting it after this scan",
path.display()
)),
MerkleLoadStatus::SpecChanged { path } => Some(format!(
"warning: incremental cache {} was built for a different detector/config identity; starting from an empty cache and rewriting it after this scan",
path.display()
)),
MerkleLoadStatus::InvalidEntryHash {
path,
entry_path,
hash,
} => Some(format!(
"warning: incremental cache {} has an invalid hash for entry {} ({hash}); starting from an empty cache and rewriting it after this scan",
path.display(),
entry_path
)),
}
}
fn gpu_init_policy_for_args(
args: &ScanArgs,
autoroute_cache_path: Option<&std::path::Path>,
autoroute_gpu: bool,
autoroute_calibration: bool,
) -> GpuInitPolicy {
if let Some(policy) = backend_name_gpu_policy(args.backend.as_deref()) {
return policy;
}
if args.no_gpu && !args.require_gpu {
return GpuInitPolicy::ForceDisabled;
}
if autoroute_calibration && autoroute_gpu {
return GpuInitPolicy::FromRuntimePolicy;
}
if filesystem_auto_scan_cannot_route_gpu(args) && !args.require_gpu {
if autoroute_cache_path.is_some_and(std::path::Path::exists) {
return GpuInitPolicy::FromRuntimePolicy;
}
return GpuInitPolicy::ForceDisabled;
}
GpuInitPolicy::FromRuntimePolicy
}
fn backend_name_gpu_policy(name: Option<&str>) -> Option<GpuInitPolicy> {
let name = name?.trim();
if name.eq_ignore_ascii_case("auto") {
return None;
}
keyhog_scanner::hw_probe::parse_backend_str(name).map(backend_gpu_policy)
}
fn backend_gpu_policy(backend: keyhog_scanner::ScanBackend) -> GpuInitPolicy {
GpuInitPolicy::SelectedBackend(backend)
}
fn filesystem_auto_scan_cannot_route_gpu(args: &ScanArgs) -> bool {
if args.batch_pipeline && !args.no_batch_pipeline {
return false;
}
if args.path.is_none() {
return false;
}
if args.stdin {
return false;
}
#[cfg(feature = "binary")]
if args.binary {
return false;
}
#[cfg(feature = "git")]
if args.git_blobs.is_some() || args.git_diff.is_some() || args.git_history.is_some() {
return false;
}
#[cfg(feature = "github")]
if args.github_org.is_some() {
return false;
}
#[cfg(feature = "gitlab")]
if args.gitlab_group.is_some() {
return false;
}
#[cfg(feature = "bitbucket")]
if args.bitbucket_workspace.is_some() {
return false;
}
#[cfg(feature = "s3")]
if args.s3_bucket.is_some() {
return false;
}
#[cfg(feature = "gcs")]
if args.gcs_bucket.is_some() {
return false;
}
#[cfg(feature = "azure")]
if args.azure_container_url.is_some() {
return false;
}
#[cfg(feature = "docker")]
if args.docker_image.is_some() {
return false;
}
#[cfg(feature = "web")]
if args.url.is_some() {
return false;
}
if args
.source
.as_ref()
.is_some_and(|sources| !sources.is_empty())
{
return false;
}
true
}
#[cfg(test)]
mod tests;