1use crate::collector::CollectorCapability;
2use crate::metrics::{MacroStageId, MetricId, METRICS};
3use serde::{Deserialize, Serialize};
4use std::sync::atomic::{AtomicU64, Ordering};
5use std::time::{SystemTime, UNIX_EPOCH};
6
7pub const PROFILE_SCHEMA: &str = "keyhog-profile-v1";
9
10const fn legacy_component_version() -> u16 {
11 1
12}
13
14pub const STAGE_MEASUREMENT_VERSION: u16 = 1;
15pub const RUN_IDENTITY_VERSION: u16 = 1;
16pub const STATE_TRANSITION_VERSION: u16 = 1;
17pub const RESOURCE_SAMPLE_VERSION: u16 = 1;
18pub const RESOURCE_SNAPSHOT_VERSION: u16 = 2;
19pub const STATE_MEASUREMENT_VERSION: u16 = 1;
20pub const RESOURCE_USAGE_VERSION: u16 = 1;
21pub const RUN_PROFILE_VERSION: u16 = 3;
22pub const WORKLOAD_MEASUREMENTS_VERSION: u16 = 1;
23
24static RUN_SEQUENCE: AtomicU64 = AtomicU64::new(0);
25
26#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)]
28#[serde(rename_all = "kebab-case")]
29#[repr(usize)]
30pub enum Stage {
31 SourceAcquire = 0,
32 SourceWalk,
33 SourceRead,
34 Preprocess,
35 Phase1Triggers,
36 BackendDispatch,
37 HotPatterns,
38 ConfirmedPatterns,
39 Phase2Prefilter,
40 Phase2KeywordAc,
41 Phase2SharedAc,
42 Phase2AnchoredVerify,
43 Phase2WholeChunk,
44 GenericDetection,
45 Entropy,
46 MachineLearning,
47 Decode,
48 Suppression,
49 LiveVerification,
50 Reporting,
51 SourceQueueWait,
52 IncrementalLookup,
53 BackendSelect,
54 ResultMerge,
55 ScannerQueueWait,
56 AutorouteCalibration,
58 BoundaryScan,
62 DetectorLoad,
64 DetectorValidate,
66 ExecutionPackSelect,
68 ExecutionPackMap,
70 BackendAcquire,
72 BackendInit,
74 Teardown,
76 ScanPipeline,
78 ScannerCompile,
80}
81
82impl Stage {
83 pub const ALL: [Self; 36] = [
85 Self::SourceAcquire,
86 Self::SourceWalk,
87 Self::SourceRead,
88 Self::Preprocess,
89 Self::Phase1Triggers,
90 Self::BackendDispatch,
91 Self::HotPatterns,
92 Self::ConfirmedPatterns,
93 Self::Phase2Prefilter,
94 Self::Phase2KeywordAc,
95 Self::Phase2SharedAc,
96 Self::Phase2AnchoredVerify,
97 Self::Phase2WholeChunk,
98 Self::GenericDetection,
99 Self::Entropy,
100 Self::MachineLearning,
101 Self::Decode,
102 Self::Suppression,
103 Self::LiveVerification,
104 Self::Reporting,
105 Self::SourceQueueWait,
106 Self::IncrementalLookup,
107 Self::BackendSelect,
108 Self::ResultMerge,
109 Self::ScannerQueueWait,
110 Self::AutorouteCalibration,
111 Self::BoundaryScan,
112 Self::DetectorLoad,
113 Self::DetectorValidate,
114 Self::ExecutionPackSelect,
115 Self::ExecutionPackMap,
116 Self::BackendAcquire,
117 Self::BackendInit,
118 Self::Teardown,
119 Self::ScanPipeline,
120 Self::ScannerCompile,
121 ];
122
123 #[inline]
124 pub(crate) const fn index(self) -> usize {
125 self as usize
126 }
127
128 pub const fn metric_id(self) -> MetricId {
133 match self {
134 Self::SourceAcquire => MetricId::SourceAcquire,
135 Self::SourceWalk => MetricId::SourceWalk,
136 Self::SourceRead => MetricId::SourceRead,
137 Self::Preprocess => MetricId::Preprocess,
138 Self::Phase1Triggers => MetricId::Phase1Triggers,
139 Self::BackendDispatch => MetricId::BackendDispatch,
140 Self::HotPatterns => MetricId::HotPatterns,
141 Self::ConfirmedPatterns => MetricId::ConfirmedPatterns,
142 Self::Phase2Prefilter => MetricId::Phase2Prefilter,
143 Self::Phase2KeywordAc => MetricId::Phase2KeywordAc,
144 Self::Phase2SharedAc => MetricId::Phase2SharedAc,
145 Self::Phase2AnchoredVerify => MetricId::Phase2AnchoredVerify,
146 Self::Phase2WholeChunk => MetricId::Phase2WholeChunk,
147 Self::GenericDetection => MetricId::GenericDetection,
148 Self::Entropy => MetricId::Entropy,
149 Self::MachineLearning => MetricId::MachineLearning,
150 Self::Decode => MetricId::Decode,
151 Self::Suppression => MetricId::Suppression,
152 Self::LiveVerification => MetricId::LiveVerification,
153 Self::Reporting => MetricId::Reporting,
154 Self::SourceQueueWait => MetricId::SourceQueueWait,
155 Self::IncrementalLookup => MetricId::IncrementalLookup,
156 Self::BackendSelect => MetricId::BackendSelect,
157 Self::ResultMerge => MetricId::ResultMerge,
158 Self::ScannerQueueWait => MetricId::ScannerQueueWait,
159 Self::AutorouteCalibration => MetricId::AutorouteCalibration,
160 Self::BoundaryScan => MetricId::BoundaryScan,
161 Self::DetectorLoad => MetricId::DetectorLoad,
162 Self::DetectorValidate => MetricId::DetectorValidate,
163 Self::ExecutionPackSelect => MetricId::ExecutionPackSelect,
164 Self::ExecutionPackMap => MetricId::ExecutionPackMap,
165 Self::BackendAcquire => MetricId::BackendAcquire,
166 Self::BackendInit => MetricId::BackendInit,
167 Self::Teardown => MetricId::Teardown,
168 Self::ScanPipeline => MetricId::ScanPipeline,
169 Self::ScannerCompile => MetricId::ScannerCompile,
170 }
171 }
172
173 #[inline]
175 pub const fn is_container(self) -> bool {
176 matches!(self, Self::ScanPipeline)
177 }
178
179 pub const fn as_str(self) -> &'static str {
181 METRICS[self.metric_id() as usize].name
182 }
183
184 pub const fn macro_stage_id(self) -> MacroStageId {
186 match self {
187 Self::SourceAcquire | Self::SourceWalk | Self::SourceRead | Self::SourceQueueWait => {
188 MacroStageId::Acquire
189 }
190 Self::Preprocess
191 | Self::Phase1Triggers
192 | Self::BackendDispatch
193 | Self::HotPatterns
194 | Self::ConfirmedPatterns
195 | Self::Phase2Prefilter
196 | Self::Phase2KeywordAc
197 | Self::Phase2SharedAc
198 | Self::Phase2AnchoredVerify
199 | Self::Phase2WholeChunk
200 | Self::GenericDetection
201 | Self::Entropy
202 | Self::MachineLearning
203 | Self::Decode
204 | Self::IncrementalLookup
205 | Self::BackendSelect
206 | Self::ScannerQueueWait
207 | Self::AutorouteCalibration
208 | Self::BoundaryScan
209 | Self::DetectorLoad
210 | Self::DetectorValidate
211 | Self::ExecutionPackSelect
212 | Self::ExecutionPackMap
213 | Self::BackendAcquire
214 | Self::BackendInit
215 | Self::Teardown
216 | Self::ScanPipeline
217 | Self::ScannerCompile => MacroStageId::Scan,
218 Self::Suppression | Self::ResultMerge => MacroStageId::Resolve,
219 Self::LiveVerification => MacroStageId::Verify,
220 Self::Reporting => MacroStageId::Report,
221 }
222 }
223}
224
225#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
227pub struct StageMeasurement {
228 #[serde(default = "legacy_component_version")]
229 pub version: u16,
230 pub stage: Stage,
231 pub elapsed_ns: u64,
232 pub calls: u64,
233 pub attributed_ns: u64,
234}
235
236#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]
238#[serde(rename_all = "kebab-case")]
239pub enum CacheState {
240 #[default]
241 Unknown,
242 Disabled,
243 Cold,
244 Warm,
245}
246
247#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]
249#[serde(rename_all = "kebab-case")]
250pub enum DaemonState {
251 #[default]
252 Off,
253 Client,
254 Worker,
255 Mass,
256}
257
258#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)]
260#[serde(rename_all = "kebab-case")]
261pub enum RunState {
262 Created,
263 Acquiring,
264 Scanning,
265 Resolving,
266 Verifying,
267 Reporting,
268 Completed,
269 Failed,
270}
271
272#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
274pub struct RunIdentity {
275 #[serde(default = "legacy_component_version")]
276 pub version: u16,
277 pub run_id: String,
278 pub binary_version: String,
279 pub detector_digest: String,
280 pub config_digest: String,
281 pub source_kind: String,
282 pub workload_class: String,
283 pub backend_requested: String,
284 pub backend_selected: Option<String>,
285 pub cache_state: CacheState,
286 pub daemon_state: DaemonState,
287 pub scanner_threads: usize,
288 pub reader_threads: Option<usize>,
289 pub logical_cpus: usize,
290}
291
292impl RunIdentity {
293 pub fn new(
295 binary_version: impl Into<String>,
296 detector_digest: impl Into<String>,
297 config_digest: impl Into<String>,
298 source_kind: impl Into<String>,
299 workload_class: impl Into<String>,
300 backend_requested: impl Into<String>,
301 ) -> Self {
302 let sequence = RUN_SEQUENCE.fetch_add(1, Ordering::Relaxed);
303 let unix_ns = SystemTime::now()
304 .duration_since(UNIX_EPOCH)
305 .unwrap_or_default()
306 .as_nanos();
307 Self {
308 version: RUN_IDENTITY_VERSION,
309 run_id: format!("{}-{unix_ns}-{sequence}", std::process::id()),
310 binary_version: binary_version.into(),
311 detector_digest: detector_digest.into(),
312 config_digest: config_digest.into(),
313 source_kind: source_kind.into(),
314 workload_class: workload_class.into(),
315 backend_requested: backend_requested.into(),
316 backend_selected: None,
317 cache_state: CacheState::Unknown,
318 daemon_state: DaemonState::Off,
319 scanner_threads: 0,
320 reader_threads: None,
321 logical_cpus: crate::host_parallelism::logical_cpu_count(),
322 }
323 }
324}
325
326#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
328pub struct StateTransition {
329 #[serde(default = "legacy_component_version")]
330 pub version: u16,
331 pub state: RunState,
332 pub elapsed_ns: u64,
333}
334
335#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
337pub struct ResourceSample {
338 #[serde(default = "legacy_component_version")]
339 pub version: u16,
340 pub state: RunState,
341 pub elapsed_ns: u64,
342 pub snapshot: ResourceSnapshot,
343}
344
345#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)]
347pub struct ResourceSnapshot {
348 #[serde(default = "legacy_component_version")]
349 pub version: u16,
350 pub cpu_time_ms: Option<u64>,
351 pub resident_bytes: Option<u64>,
352 pub virtual_bytes: Option<u64>,
353 pub thread_count: Option<u64>,
354 #[serde(default)]
356 pub resident_high_water_bytes: Option<u64>,
357 #[serde(default)]
359 pub swap_bytes: Option<u64>,
360}
361
362impl Default for ResourceSnapshot {
363 fn default() -> Self {
364 Self {
365 version: RESOURCE_SNAPSHOT_VERSION,
366 cpu_time_ms: None,
367 resident_bytes: None,
368 virtual_bytes: None,
369 thread_count: None,
370 resident_high_water_bytes: None,
371 swap_bytes: None,
372 }
373 }
374}
375
376#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
378pub struct StateMeasurement {
379 #[serde(default = "legacy_component_version")]
380 pub version: u16,
381 pub state: RunState,
382 pub elapsed_ns: u64,
383 pub cpu_time_ms: Option<u64>,
384 pub aggregate_cpu_milli_percent: Option<u64>,
385 pub resident_start_bytes: Option<u64>,
386 pub resident_end_bytes: Option<u64>,
387 pub threads_start: Option<u64>,
388 pub threads_end: Option<u64>,
389}
390
391#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
393pub struct ResourceUsage {
394 #[serde(default = "legacy_component_version")]
395 pub version: u16,
396 pub start: ResourceSnapshot,
397 pub finish: ResourceSnapshot,
398 pub max_observed_resident_bytes: Option<u64>,
399 pub max_observed_threads: Option<u64>,
400 pub aggregate_cpu_percent: Option<f64>,
401}
402
403impl Default for ResourceUsage {
404 fn default() -> Self {
405 Self {
406 version: RESOURCE_USAGE_VERSION,
407 start: ResourceSnapshot::default(),
408 finish: ResourceSnapshot::default(),
409 max_observed_resident_bytes: None,
410 max_observed_threads: None,
411 aggregate_cpu_percent: None,
412 }
413 }
414}
415
416#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
418pub struct WorkloadMeasurements {
419 #[serde(default = "legacy_component_version")]
420 pub version: u16,
421 pub container_bytes: Option<u64>,
422 pub expanded_payload_bytes: Option<u64>,
423 pub derived_decoder_bytes: Option<u64>,
424 pub backend_dispatched_bytes: Option<u64>,
425}
426
427impl Default for WorkloadMeasurements {
428 fn default() -> Self {
429 Self {
430 version: WORKLOAD_MEASUREMENTS_VERSION,
431 container_bytes: None,
432 expanded_payload_bytes: None,
433 derived_decoder_bytes: None,
434 backend_dispatched_bytes: None,
435 }
436 }
437}
438
439impl WorkloadMeasurements {
440 pub(crate) fn measured(derived_decoder_bytes: u64, backend_dispatched_bytes: u64) -> Self {
441 Self {
442 version: WORKLOAD_MEASUREMENTS_VERSION,
443 container_bytes: None,
444 expanded_payload_bytes: None,
445 derived_decoder_bytes: Some(derived_decoder_bytes),
446 backend_dispatched_bytes: Some(backend_dispatched_bytes),
447 }
448 }
449}
450
451#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
453pub struct RunProfile {
454 #[serde(default = "legacy_component_version")]
455 pub version: u16,
456 pub schema: String,
457 pub identity: RunIdentity,
458 pub status: RunState,
459 pub wall_time_ns: u64,
460 pub input_bytes: u64,
461 pub input_units: u64,
462 #[serde(default)]
463 pub workload: WorkloadMeasurements,
464 pub stages: Vec<StageMeasurement>,
465 pub transitions: Vec<StateTransition>,
466 #[serde(default)]
467 pub states: Vec<StateMeasurement>,
468 #[serde(default)]
469 pub collectors: Vec<CollectorCapability>,
470 pub resource_samples: Vec<ResourceSample>,
471 pub resources: ResourceUsage,
472 #[serde(default = "legacy_hardware_gap")]
474 pub hardware: crate::Evidence<crate::hardware::HardwareRunEvidenceV2>,
475 #[serde(default = "legacy_system_gap")]
477 pub system: crate::Evidence<crate::system::SystemRunEvidenceV2>,
478 #[serde(default)]
480 pub compile_surfaces: Vec<crate::schema_v2::CompileSurfaceRecordV2>,
481}
482
483fn legacy_system_gap() -> crate::Evidence<crate::system::SystemRunEvidenceV2> {
484 crate::Evidence::unavailable(crate::EvidenceGap::LegacyV1NotRecorded)
485}
486
487fn legacy_hardware_gap() -> crate::Evidence<crate::hardware::HardwareRunEvidenceV2> {
488 crate::Evidence::unavailable(crate::EvidenceGap::LegacyV1NotRecorded)
489}
490
491impl RunProfile {
492 pub fn to_json_pretty(&self) -> serde_json::Result<String> {
494 serde_json::to_string_pretty(self)
495 }
496
497 pub fn render_text(&self) -> String {
499 let reader_threads = self
500 .identity
501 .reader_threads
502 .map_or_else(|| "auto".to_owned(), |threads| threads.to_string());
503 let throughput_mib_s = if self.wall_time_ns == 0 {
504 0.0
505 } else {
506 self.input_bytes as f64 * 1_000_000_000.0 / self.wall_time_ns as f64 / (1024.0 * 1024.0)
507 };
508 let mut output = format!(
509 "KeyHog profile {}\n\
510 state={} source={} workload={} backend_requested={} backend_selected={} cache={} daemon={} wall_ms={:.3}\n\
511 version={} detector_digest={} config_digest={}\n\
512 input_bytes={} input_units={} throughput_mib_s={throughput_mib_s:.3} scanner_threads={} reader_threads={} logical_cpus={}\n",
513 self.identity.run_id,
514 state_name(self.status),
515 self.identity.source_kind,
516 self.identity.workload_class,
517 self.identity.backend_requested,
518 self.identity
519 .backend_selected
520 .as_deref()
521 .unwrap_or("unselected"),
522 cache_name(self.identity.cache_state),
523 daemon_name(self.identity.daemon_state),
524 self.wall_time_ns as f64 / 1_000_000.0,
525 self.identity.binary_version,
526 self.identity.detector_digest,
527 self.identity.config_digest,
528 self.input_bytes,
529 self.input_units,
530 self.identity.scanner_threads,
531 reader_threads,
532 self.identity.logical_cpus,
533 );
534 for state in &self.states {
535 output.push_str(&format!(
536 "macro {:<12} wall_ms={:.3}",
537 state_name(state.state),
538 state.elapsed_ns as f64 / 1_000_000.0,
539 ));
540 if let Some(cpu) = state.aggregate_cpu_milli_percent {
541 output.push_str(&format!(" cpu={:.1}%", cpu as f64 / 1_000.0));
542 } else {
543 output.push_str(" cpu=unavailable");
544 }
545 if let (Some(start), Some(finish)) =
546 (state.resident_start_bytes, state.resident_end_bytes)
547 {
548 output.push_str(&format!(" rss_bytes={start}->{finish}"));
549 }
550 if let (Some(start), Some(finish)) = (state.threads_start, state.threads_end) {
551 output.push_str(&format!(" threads={start}->{finish}"));
552 }
553 output.push('\n');
554 }
555 for stage in &self.stages {
556 let per_call_us = if stage.calls == 0 {
557 0.0
558 } else {
559 stage.elapsed_ns as f64 / stage.calls as f64 / 1_000.0
560 };
561 output.push_str(&format!(
562 " {:<24} {:>10.3} ms calls={} per_call_us={per_call_us:.3} attributed_ms={:.3}\n",
563 stage.stage.as_str(),
564 stage.elapsed_ns as f64 / 1_000_000.0,
565 stage.calls,
566 stage.attributed_ns as f64 / 1_000_000.0,
567 ));
568 }
569 if let Some(state) = self.states.iter().max_by_key(|state| state.elapsed_ns) {
570 output.push_str(&format!(
571 "bottleneck macro={} wall_ms={:.3}",
572 state_name(state.state),
573 state.elapsed_ns as f64 / 1_000_000.0,
574 ));
575 }
576 if let Some(stage) = self
577 .stages
578 .iter()
579 .filter(|stage| !stage.stage.is_container())
580 .max_by_key(|stage| stage.elapsed_ns)
581 {
582 output.push_str(&format!(
583 " summed_stage={} summed_ms={:.3}",
584 stage.stage.as_str(),
585 stage.elapsed_ns as f64 / 1_000_000.0,
586 ));
587 }
588 if !self.states.is_empty() || !self.stages.is_empty() {
589 output.push('\n');
590 }
591 if let Some(cpu) = self.resources.aggregate_cpu_percent {
592 output.push_str(&format!("resources aggregate_cpu={cpu:.1}%"));
593 } else {
594 output.push_str("resources aggregate_cpu=unavailable");
595 }
596 if let Some(rss) = self.resources.max_observed_resident_bytes {
597 output.push_str(&format!(" max_observed_rss_bytes={rss}"));
598 }
599 if let Some(threads) = self.resources.max_observed_threads {
600 output.push_str(&format!(" max_observed_threads={threads}"));
601 }
602 output.push('\n');
603 for capability in &self.collectors {
604 output.push_str(&format!(
605 "collector {} availability={}",
606 capability.collector.as_str(),
607 capability.availability.as_str(),
608 ));
609 if let Some(detail) = &capability.detail {
610 output.push_str(&format!(" detail={detail}"));
611 }
612 output.push('\n');
613 }
614 output
615 }
616
617 pub fn render_markdown(&self) -> String {
619 crate::CausalProfileV2::from_v1(self.clone()).render_markdown()
620 }
621}
622
623fn state_name(state: RunState) -> &'static str {
624 match state {
625 RunState::Created => "created",
626 RunState::Acquiring => "acquiring",
627 RunState::Scanning => "scanning",
628 RunState::Resolving => "resolving",
629 RunState::Verifying => "verifying",
630 RunState::Reporting => "reporting",
631 RunState::Completed => "completed",
632 RunState::Failed => "failed",
633 }
634}
635
636fn cache_name(state: CacheState) -> &'static str {
637 match state {
638 CacheState::Unknown => "unknown",
639 CacheState::Disabled => "disabled",
640 CacheState::Cold => "cold",
641 CacheState::Warm => "warm",
642 }
643}
644
645fn daemon_name(state: DaemonState) -> &'static str {
646 match state {
647 DaemonState::Off => "off",
648 DaemonState::Client => "client",
649 DaemonState::Worker => "worker",
650 DaemonState::Mass => "mass",
651 }
652}