use keyhog_core::{
CompanionMap, CredentialHash, MatchLocation, RawMatch, SensitiveString, Severity,
};
use keyhog_scanner::telemetry::{DogfoodEvent, StaticRecoveryStatus};
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
pub(crate) const WIRE_VERSION: u32 = 14;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub(crate) struct WarmBackendIdentity {
pub engine: String,
pub gpu_artifact: Option<String>,
pub binary_sha256: String,
pub detector_rules_digest: String,
pub config_digest: String,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub(crate) struct WarmBackendStatus {
pub ready: bool,
pub daemon_generation: String,
pub identity: WarmBackendIdentity,
pub required_backends: Vec<String>,
pub initialized_backends: Vec<String>,
pub reason: Option<String>,
pub repair_command: Option<String>,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub(crate) struct MassScanStats {
pub batches: u64,
pub chunks: u64,
pub bytes: u64,
pub gpu_batches: u64,
pub gpu_chunks: u64,
pub gpu_bytes: u64,
pub duration_ms: u64,
}
impl MassScanStats {
pub(crate) fn gpu_is_primary(self) -> bool {
self.bytes > 0 && self.gpu_bytes > self.bytes.saturating_sub(self.gpu_bytes)
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub(crate) struct ProfileStageMeasurement {
pub stage: String,
pub calls: u64,
pub elapsed_ns: u64,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub(crate) struct RequestProfile {
pub request_id: String,
pub wall_time_ns: u64,
pub stages: Vec<ProfileStageMeasurement>,
pub dropped_span_events: u64,
pub dropped_point_events: u64,
pub dropped_annotations: u64,
pub sampled_out_events: u64,
}
pub(crate) const MAX_FRAME_BYTES: u32 = 64 * 1024 * 1024;
pub(crate) const MASS_BATCH_BYTES: usize = 8 * 1024 * 1024;
pub(crate) const MASS_BATCH_CHUNKS: usize = 1024;
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "op", rename_all = "snake_case")]
pub(crate) enum Request {
Hello,
ScanText {
path: Option<String>,
text: String,
dogfood: bool,
profile: bool,
},
ScanPath {
path: String,
working_dir: Option<String>,
dogfood: bool,
profile: bool,
},
MassBegin { dogfood: bool, profile: bool },
MassBatch {
#[serde(with = "protected_chunks")]
chunks: Vec<keyhog_core::Chunk>,
},
MassFilesystemBegin {
root: String,
max_file_size: u64,
ignore_paths: Vec<String>,
respect_default_excludes: bool,
reader_threads: Option<usize>,
incremental_cache: Option<String>,
},
MassFilesystemDrain,
MassEnd,
Health,
Shutdown,
GuardCommitBegin {
repo_path: String,
index_fingerprint: String,
hash_algorithm: String,
entries: Vec<GuardWireManifestEntry>,
},
GuardCommitBlob {
transaction_id: u64,
blob_oid: String,
object_size: u64,
#[serde(with = "protected_chunks")]
payload: Vec<keyhog_core::Chunk>,
},
GuardCommitFinish {
transaction_id: u64,
client_objects_streamed: u64,
client_bytes_streamed: u64,
},
GuardAdd {
root: String,
mode: String,
},
GuardRemove {
root: String,
},
GuardStatus {
root: String,
},
GuardReconcile {
root: String,
},
GuardList,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub(crate) enum Response {
Hello {
wire_version: u32,
keyhog_version: String,
git_hash: String,
detector_rules_digest: String,
backend_policy: String,
detector_count: usize,
uptime_secs: u64,
warm_backend: WarmBackendStatus,
mass_service: bool,
mass_gpu_primary_required: bool,
},
ScanResults {
path: Option<String>,
#[serde(with = "protected_raw_matches")]
matches: Vec<RawMatch>,
engine_example_suppressions: u64,
dogfood_events: Vec<DogfoodEvent>,
static_recovery_rejections: BTreeMap<String, u64>,
static_recovery_status: StaticRecoveryStatus,
dogfood_detail_events_dropped: u64,
source_coverage_gaps: SourceCoverageGaps,
backend_recovery: RequiredOption<BackendRecoveryStatus>,
profile: RequiredOption<RequestProfile>,
},
MassReady,
MassFilesystemReady,
MassFilesystemComplete {
source_coverage_gaps: SourceCoverageGaps,
skipped_unchanged: usize,
},
MassFilesystemIncrementalError { message: String },
MassComplete { stats: MassScanStats },
Health {
uptime_secs: u64,
scans_served: u64,
active_scans: u32,
detector_count: usize,
backend_recoveries: u64,
last_backend_fault: Option<BackendRecoveryStatus>,
guard_roots_registered: u64,
guard_roots_current: u64,
guard_roots_blocked: u64,
guard_roots_degraded: u64,
guard_active_transactions: u64,
warm_backend: WarmBackendStatus,
},
Error { message: String },
Shutdown,
GuardCommitPlan {
transaction_id: u64,
clean_hits: Vec<String>,
required_blob_oids: Vec<String>,
max_blob_bytes: u64,
},
GuardCommitBlobAck {
transaction_id: u64,
blob_oid: String,
bytes_scanned: u64,
findings_count: u64,
},
GuardCommitReceipt {
objects_requested: u64,
objects_hit: u64,
objects_scanned: u64,
objects_skipped: u64,
bytes_requested: u64,
bytes_hit: u64,
bytes_scanned: u64,
findings_count: u64,
coverage_gaps: u64,
terminal_state: String,
terminal_sequence: u64,
},
GuardAdded {
root: String,
state: String,
terminal_sequence: u64,
},
GuardRemoved,
GuardStatusResult {
root: String,
mode: String,
state: String,
terminal_sequence: u64,
accepted_event_sequence: u64,
completed_event_sequence: u64,
pending_events: u64,
files_scanned: u64,
bytes_scanned: u64,
attestation_hits: u64,
attestation_misses: u64,
findings_count: u64,
coverage_gaps: u64,
initial_reconciliation_time: Option<u64>,
last_reconciliation_time: Option<u64>,
scanner_residency: String,
backend_route_label: String,
build_identity_short: String,
detector_digest_short: String,
suppression_digest_short: String,
config_digest_short: String,
autoroute_evidence_status: String,
store_schema_version: u32,
store_path: String,
repair_command: String,
},
GuardReconcileStarted {
root: String,
},
GuardListResult {
roots: Vec<GuardListEntry>,
},
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub(crate) struct GuardWireManifestEntry {
pub path: String,
pub kind: String,
pub object_oid: String,
pub object_size: u64,
pub raw_mode: u32,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub(crate) struct GuardListEntry {
pub root: String,
pub mode: String,
pub state: String,
pub terminal_sequence: u64,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub(crate) struct BackendRecoveryStatus {
pub failed_backend: String,
pub recovery_backend: String,
pub recovered_ranges: Vec<RecoveredInputRangeStatus>,
pub recovered_chunks: usize,
pub recovered_bytes: u64,
pub reason: String,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum RequiredOption<T> {
None,
Some(T),
}
#[cfg(test)]
impl<T> RequiredOption<T> {
pub(crate) fn is_none(&self) -> bool {
matches!(self, RequiredOption::None)
}
pub(crate) fn expect(self, msg: &str) -> T {
match self {
RequiredOption::Some(v) => v,
RequiredOption::None => panic!("{msg}"),
}
}
}
impl<T> From<Option<T>> for RequiredOption<T> {
fn from(opt: Option<T>) -> Self {
opt.map_or(RequiredOption::None, RequiredOption::Some)
}
}
impl<T> From<RequiredOption<T>> for Option<T> {
fn from(req: RequiredOption<T>) -> Self {
match req {
RequiredOption::None => None,
RequiredOption::Some(v) => Some(v),
}
}
}
impl<T: Serialize> Serialize for RequiredOption<T> {
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
match self {
RequiredOption::None => serializer.serialize_none(),
RequiredOption::Some(v) => v.serialize(serializer),
}
}
}
impl<'de, T: Deserialize<'de>> Deserialize<'de> for RequiredOption<T> {
fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
struct RequiredOptionVisitor<T> {
marker: std::marker::PhantomData<T>,
}
impl<'de, T: Deserialize<'de>> serde::de::Visitor<'de> for RequiredOptionVisitor<T> {
type Value = RequiredOption<T>;
fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
f.write_str("a required optional value")
}
fn visit_none<E>(self) -> Result<Self::Value, E> {
Ok(RequiredOption::None)
}
fn visit_unit<E>(self) -> Result<Self::Value, E> {
Ok(RequiredOption::None)
}
fn visit_map<M>(self, map: M) -> Result<Self::Value, M::Error>
where
M: serde::de::MapAccess<'de>,
{
let de = serde::de::value::MapAccessDeserializer::new(map);
T::deserialize(de).map(RequiredOption::Some)
}
fn visit_seq<S>(self, seq: S) -> Result<Self::Value, S::Error>
where
S: serde::de::SeqAccess<'de>,
{
let de = serde::de::value::SeqAccessDeserializer::new(seq);
T::deserialize(de).map(RequiredOption::Some)
}
}
deserializer.deserialize_any(RequiredOptionVisitor {
marker: std::marker::PhantomData,
})
}
}
impl<T> Default for RequiredOption<T> {
fn default() -> Self {
RequiredOption::None
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub(crate) struct RecoveredInputRangeStatus {
pub chunk_index: usize,
pub byte_start: usize,
pub byte_end: usize,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
pub(crate) struct SourceCoverageGaps {
pub over_max_size: usize,
pub binary: usize,
pub unreadable: usize,
pub git_object_unreadable: usize,
pub archive_truncated: usize,
pub binary_section_name_unresolved: usize,
pub source_truncated: usize,
pub structured_source_parse_failures: usize,
pub archive_duplicate_scan_unavailable: usize,
pub git_lfs_pointer: usize,
pub source_failed: usize,
}
impl SourceCoverageGaps {
pub(crate) fn total(self) -> usize {
self.over_max_size
+ self.binary
+ self.unreadable
+ self.git_object_unreadable
+ self.archive_truncated
+ self.binary_section_name_unresolved
+ self.source_truncated
+ self.structured_source_parse_failures
+ self.archive_duplicate_scan_unavailable
+ self.git_lfs_pointer
+ self.source_failed
}
pub(crate) fn fail_class_total(self) -> usize {
self.unreadable
+ self.git_object_unreadable
+ self.archive_truncated
+ self.binary_section_name_unresolved
+ self.source_truncated
+ self.structured_source_parse_failures
+ self.archive_duplicate_scan_unavailable
+ self.git_lfs_pointer
+ self.source_failed
}
pub(crate) fn is_empty(self) -> bool {
self.total() == 0
}
#[cfg(test)]
pub(crate) fn fail_class_empty(self) -> bool {
self.fail_class_total() == 0
}
}
mod protected_chunks {
use keyhog_core::{Chunk, ChunkMetadata, SensitiveString};
use serde::ser::SerializeSeq;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
#[derive(Serialize)]
struct ChunkRef<'a> {
#[serde(serialize_with = "serialize_sensitive")]
data: &'a SensitiveString,
metadata: &'a ChunkMetadata,
}
#[derive(Deserialize)]
struct ChunkOwned {
data: String,
metadata: ChunkMetadata,
}
pub(super) fn serialize<S>(chunks: &[Chunk], serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let mut sequence = serializer.serialize_seq(Some(chunks.len()))?;
for chunk in chunks {
sequence.serialize_element(&ChunkRef {
data: &chunk.data,
metadata: &chunk.metadata,
})?;
}
sequence.end()
}
pub(super) fn deserialize<'de, D>(deserializer: D) -> Result<Vec<Chunk>, D::Error>
where
D: Deserializer<'de>,
{
Vec::<ChunkOwned>::deserialize(deserializer).map(|chunks| {
chunks
.into_iter()
.map(|chunk| Chunk {
data: chunk.data.into(),
metadata: chunk.metadata,
})
.collect()
})
}
fn serialize_sensitive<S>(value: &&SensitiveString, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
serializer.serialize_str(value.as_str())
}
}
mod protected_raw_matches {
use super::{CompanionMap, CredentialHash, MatchLocation, RawMatch, SensitiveString, Severity};
use serde::ser::{SerializeMap, SerializeSeq};
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use std::collections::HashMap;
use std::sync::Arc;
#[derive(Serialize)]
struct DaemonRawMatchRef<'a> {
detector_id: &'a str,
detector_name: &'a str,
service: &'a str,
severity: Severity,
#[serde(serialize_with = "serialize_sensitive")]
credential: &'a SensitiveString,
credential_hash: CredentialHash,
#[serde(serialize_with = "serialize_companions")]
companions: &'a CompanionMap,
location: &'a MatchLocation,
#[serde(skip_serializing_if = "Option::is_none")]
entropy: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
confidence: Option<f64>,
}
#[derive(Deserialize)]
struct DaemonRawMatchOwned {
detector_id: String,
detector_name: String,
service: String,
severity: Severity,
#[serde(deserialize_with = "deserialize_sensitive")]
credential: SensitiveString,
credential_hash: CredentialHash,
companions: HashMap<String, String>,
location: MatchLocation,
entropy: Option<f64>,
confidence: Option<f64>,
}
impl From<DaemonRawMatchOwned> for RawMatch {
fn from(wire: DaemonRawMatchOwned) -> Self {
Self {
detector_id: Arc::from(wire.detector_id),
detector_name: Arc::from(wire.detector_name),
service: Arc::from(wire.service),
severity: wire.severity,
credential: wire.credential,
credential_hash: wire.credential_hash,
companions: wire
.companions
.into_iter()
.map(|(name, value)| (Arc::from(name), value))
.collect(),
location: wire.location,
entropy: wire.entropy,
confidence: wire.confidence,
}
}
}
pub(super) fn serialize<S>(matches: &[RawMatch], serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let mut sequence = serializer.serialize_seq(Some(matches.len()))?;
for raw_match in matches {
sequence.serialize_element(&DaemonRawMatchRef {
detector_id: raw_match.detector_id.as_ref(),
detector_name: raw_match.detector_name.as_ref(),
service: raw_match.service.as_ref(),
severity: raw_match.severity,
credential: &raw_match.credential,
credential_hash: raw_match.credential_hash,
companions: &raw_match.companions,
location: &raw_match.location,
entropy: raw_match.entropy,
confidence: raw_match.confidence,
})?;
}
sequence.end()
}
pub(super) fn deserialize<'de, D>(deserializer: D) -> Result<Vec<RawMatch>, D::Error>
where
D: Deserializer<'de>,
{
Vec::<DaemonRawMatchOwned>::deserialize(deserializer)
.map(|matches| matches.into_iter().map(RawMatch::from).collect())
}
fn serialize_sensitive<S>(
credential: &&SensitiveString, serializer: S,
) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
serializer.serialize_str(credential.as_str())
}
fn deserialize_sensitive<'de, D>(deserializer: D) -> Result<SensitiveString, D::Error>
where
D: Deserializer<'de>,
{
String::deserialize(deserializer).map(SensitiveString::from)
}
fn serialize_companions<S>(companions: &&CompanionMap, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let mut map = serializer.serialize_map(Some(companions.len()))?;
for (name, value) in companions.iter() {
map.serialize_entry(name.as_ref(), value)?;
}
map.end()
}
}
pub(crate) fn request_kind(request: &Request) -> &'static str {
match request {
Request::Hello => "Hello",
Request::ScanText { .. } => "ScanText",
Request::ScanPath { .. } => "ScanPath",
Request::MassBegin { .. } => "MassBegin",
Request::MassBatch { .. } => "MassBatch",
Request::MassFilesystemBegin { .. } => "MassFilesystemBegin",
Request::MassFilesystemDrain => "MassFilesystemDrain",
Request::MassEnd => "MassEnd",
Request::Health => "Health",
Request::Shutdown => "Shutdown",
Request::GuardCommitBegin { .. } => "GuardCommitBegin",
Request::GuardCommitBlob { .. } => "GuardCommitBlob",
Request::GuardCommitFinish { .. } => "GuardCommitFinish",
Request::GuardAdd { .. } => "GuardAdd",
Request::GuardRemove { .. } => "GuardRemove",
Request::GuardStatus { .. } => "GuardStatus",
Request::GuardReconcile { .. } => "GuardReconcile",
Request::GuardList => "GuardList",
}
}
pub(crate) fn response_kind(response: &Response) -> &'static str {
match response {
Response::Hello { .. } => "Hello",
Response::Health { .. } => "Health",
Response::ScanResults { .. } => "ScanResults",
Response::MassReady => "MassReady",
Response::MassFilesystemReady => "MassFilesystemReady",
Response::MassFilesystemComplete { .. } => "MassFilesystemComplete",
Response::MassFilesystemIncrementalError { .. } => "MassFilesystemIncrementalError",
Response::MassComplete { .. } => "MassComplete",
Response::Shutdown => "Shutdown",
Response::Error { .. } => "Error",
Response::GuardCommitPlan { .. } => "GuardCommitPlan",
Response::GuardCommitBlobAck { .. } => "GuardCommitBlobAck",
Response::GuardCommitReceipt { .. } => "GuardCommitReceipt",
Response::GuardAdded { .. } => "GuardAdded",
Response::GuardRemoved => "GuardRemoved",
Response::GuardStatusResult { .. } => "GuardStatusResult",
Response::GuardReconcileStarted { .. } => "GuardReconcileStarted",
Response::GuardListResult { .. } => "GuardListResult",
}
}