Skip to main content

keyhog_profile/
schema_v2.rs

1use crate::{
2    CacheState, CollectorCapability, DaemonState, ResourceSample, ResourceUsage, RunProfile,
3    RunState, StageMeasurement, StateMeasurement, StateTransition,
4};
5use serde::{Deserialize, Serialize};
6
7pub const PROFILE_SCHEMA_V2: &str = "keyhog-profile";
8pub const PROFILE_SCHEMA_V2_MAJOR: u16 = 2;
9pub const PROFILE_SCHEMA_V2_MINOR: u16 = 8;
10pub const PROFILE_ENVELOPE_V2_VERSION: u16 = 1;
11pub const CAUSAL_PROFILE_V2_VERSION: u16 = 8;
12pub const CAUSAL_IDENTITY_V2_VERSION: u16 = 1;
13pub const EVENT_SCHEMA_VERSION: u16 = 7;
14pub const METRIC_REGISTRY_VERSION: u16 = 6;
15pub const EXPORTER_VERSION: u16 = 1;
16pub const STAGE_CONCURRENCY_V2_VERSION: u16 = 1;
17pub const WORKER_OCCUPANCY_V2_VERSION: u16 = 1;
18pub const CACHE_EFFECTIVENESS_V2_VERSION: u16 = 1;
19pub const INDEXED_COUNTER_V2_VERSION: u16 = 1;
20pub const RETRY_RECORD_V2_VERSION: u16 = 1;
21pub const OBSERVER_EFFECT_V2_VERSION: u16 = 1;
22pub const ESTIMATED_NS_PER_SPAN_EVENT: u64 = 35;
23/// Why a v2 evidence field has no measured value.
24#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)]
25#[serde(rename_all = "kebab-case")]
26pub enum EvidenceGap {
27    LegacyV1NotRecorded,
28    CollectorDisabled,
29    PermissionDenied,
30    Unsupported,
31    Unavailable,
32}
33
34/// A measured value or an explicit reason why no value exists.
35#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
36#[serde(tag = "status", rename_all = "kebab-case")]
37pub enum Evidence<T> {
38    Recorded { value: T },
39    Unavailable { reason: EvidenceGap },
40}
41
42impl<T> Evidence<T> {
43    pub fn recorded(value: T) -> Self {
44        Self::Recorded { value }
45    }
46
47    pub const fn unavailable(reason: EvidenceGap) -> Self {
48        Self::Unavailable { reason }
49    }
50
51    pub fn value(&self) -> Option<&T> {
52        match self {
53            Self::Recorded { value } => Some(value),
54            Self::Unavailable { .. } => None,
55        }
56    }
57}
58
59fn legacy_gap<T>() -> Evidence<T> {
60    Evidence::unavailable(EvidenceGap::LegacyV1NotRecorded)
61}
62
63/// Independent major and minor version for one schema family.
64#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)]
65pub struct SchemaVersionV2 {
66    pub version: u16,
67    pub major: u16,
68    pub minor: u16,
69}
70
71/// Producer identity for the code that emitted an artifact.
72#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
73pub struct ProducerIdentityV2 {
74    pub version: u16,
75    pub profile_crate_version: String,
76    pub exporter_version: u16,
77}
78
79/// Digest protecting the canonical profile artifact.
80#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
81pub struct ArtifactIntegrityV2 {
82    pub version: u16,
83    pub algorithm: String,
84    pub digest: String,
85}
86
87/// Self-describing envelope for a v2 causal profile.
88#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
89pub struct ProfileEnvelopeV2 {
90    pub version: u16,
91    pub schema: String,
92    pub schema_version: SchemaVersionV2,
93    pub event_schema_version: u16,
94    pub metric_registry_version: u16,
95    pub producer: ProducerIdentityV2,
96    pub integrity: Evidence<ArtifactIntegrityV2>,
97}
98
99/// Exact host and operating environment used by one run.
100#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
101pub struct HostIdentityV2 {
102    pub version: u16,
103    pub operating_system: Evidence<String>,
104    pub kernel_version: Evidence<String>,
105    pub architecture: Evidence<String>,
106    pub cpu_model: Evidence<String>,
107    pub logical_cpus: u32,
108    pub physical_cores: Evidence<u32>,
109    pub cpu_features_digest: Evidence<String>,
110    pub affinity_digest: Evidence<String>,
111    pub numa_digest: Evidence<String>,
112}
113
114/// Exact executable and toolchain identity used by one run.
115#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
116pub struct BuildIdentityV2 {
117    pub version: u16,
118    pub binary_version: String,
119    pub binary_digest: Evidence<String>,
120    pub source_revision: Evidence<String>,
121    pub build_profile: Evidence<String>,
122    pub target_triple: Evidence<String>,
123    pub feature_digest: Evidence<String>,
124    pub compiler_identity: Evidence<String>,
125    pub allocator_identity: Evidence<String>,
126    pub linked_backend_digest: Evidence<String>,
127}
128
129/// Detector corpus and compiled execution-plan identity.
130#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
131pub struct DetectorIdentityV2 {
132    pub version: u16,
133    pub corpus_digest: String,
134    pub compiled_plan_digest: Evidence<String>,
135    pub enabled_detector_digest: Evidence<String>,
136    pub backend_database_digest: Evidence<String>,
137    pub external_provenance_digest: Evidence<String>,
138}
139
140/// Canonical resolved configuration and policy identity.
141#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
142pub struct ConfigIdentityV2 {
143    pub version: u16,
144    pub resolved_config_digest: String,
145    pub policy_digest: Evidence<String>,
146    pub preset: Evidence<String>,
147    pub protection_state: Evidence<String>,
148}
149
150/// Safe source adapter and target identity.
151#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
152pub struct SourceIdentityV2 {
153    pub version: u16,
154    pub adapters: Vec<String>,
155    pub target_digest: Evidence<String>,
156    pub partition_digest: Evidence<String>,
157}
158
159/// Measured workload shape used to classify comparable runs.
160#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
161pub struct WorkloadIdentityV2 {
162    pub version: u16,
163    pub class: String,
164    pub raw_source_bytes: u64,
165    pub source_units: u64,
166    pub container_bytes: Evidence<u64>,
167    pub expanded_payload_bytes: Evidence<u64>,
168    pub derived_decoder_bytes: Evidence<u64>,
169    pub backend_dispatched_bytes: Evidence<u64>,
170    pub size_bucket: Evidence<String>,
171    pub fanout_bucket: Evidence<String>,
172}
173
174/// One actual route completed for a measured batch.
175#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
176pub struct BatchRouteV2 {
177    pub version: u16,
178    pub batch_sequence: u64,
179    pub workload_key_digest: String,
180    pub requested_backend: String,
181    pub selected_backend: String,
182    pub completed_backend: String,
183    pub recovered_from_backend: Evidence<String>,
184}
185
186/// Requested, selected, completed, and recovered backend identity.
187#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
188pub struct RouteIdentityV2 {
189    pub version: u16,
190    pub request_mode: String,
191    pub requested_backend: String,
192    pub selected_backend: Evidence<String>,
193    pub completed_backend: Evidence<String>,
194    pub autoroute_decision_digest: Evidence<String>,
195    pub batches: Vec<BatchRouteV2>,
196    /// Routes omitted after the retained-batch cap; defaults to 0 for older profiles.
197    #[serde(default)]
198    pub dropped_batches: u64,
199}
200
201impl RouteIdentityV2 {
202    /// Build aggregate route identity from the exact completed batch records.
203    pub fn from_recorded_batches(requested_backend: String, batches: Vec<BatchRouteV2>) -> Self {
204        Self::from_recorded_batches_with_drops(requested_backend, batches, 0)
205    }
206
207    /// Build aggregate route identity, including explicit drop accounting.
208    pub fn from_recorded_batches_with_drops(
209        requested_backend: String,
210        batches: Vec<BatchRouteV2>,
211        dropped_batches: u64,
212    ) -> Self {
213        let request_mode = if requested_backend == "auto" {
214            "autoroute"
215        } else {
216            "explicit"
217        };
218        Self {
219            version: 1,
220            request_mode: request_mode.to_owned(),
221            selected_backend: aggregate_backend(&batches, |batch| &batch.selected_backend),
222            completed_backend: aggregate_backend(&batches, |batch| &batch.completed_backend),
223            requested_backend,
224            autoroute_decision_digest: Evidence::unavailable(EvidenceGap::Unavailable),
225            batches,
226            dropped_batches,
227        }
228    }
229}
230
231fn aggregate_backend(
232    batches: &[BatchRouteV2],
233    backend: impl Fn(&BatchRouteV2) -> &str,
234) -> Evidence<String> {
235    let mut labels = batches.iter().map(backend);
236    let Some(first) = labels.next() else {
237        return Evidence::unavailable(EvidenceGap::Unavailable);
238    };
239    if labels.all(|label| label == first) {
240        Evidence::recorded(first.to_owned())
241    } else {
242        Evidence::recorded("mixed".to_owned())
243    }
244}
245
246/// Cache families whose preparation state changes run cost.
247#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)]
248#[serde(rename_all = "kebab-case")]
249pub enum CacheLayerKindV2 {
250    LegacyAggregate,
251    Detector,
252    Merkle,
253    Autoroute,
254    Verifier,
255    Daemon,
256    PageCache,
257    HyperscanShards,
258    MatcherArtifacts,
259    GpuPrograms,
260    LockFiles,
261}
262
263/// State and generation identity for one cache layer.
264#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
265pub struct CacheLayerV2 {
266    pub version: u16,
267    pub layer: CacheLayerKindV2,
268    pub state: CacheState,
269    pub generation: Evidence<String>,
270    pub digest: Evidence<String>,
271}
272
273/// Daemon mode and request linkage for one run.
274#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
275pub struct DaemonIdentityV2 {
276    pub version: u16,
277    pub state: DaemonState,
278    pub generation: Evidence<String>,
279    pub request_id: Evidence<String>,
280    pub parent_request_id: Evidence<String>,
281    pub ready_age_ns: Evidence<u64>,
282}
283
284/// Whether scanner coverage was complete, partial, or unknown.
285#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)]
286#[serde(rename_all = "kebab-case")]
287pub enum CoverageStateV2 {
288    Complete,
289    Partial,
290    Failed,
291    Cancelled,
292    Unknown,
293}
294
295/// Terminal outcome and result identity for one run.
296#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
297pub struct OutcomeIdentityV2 {
298    pub version: u16,
299    pub status: RunState,
300    pub coverage: CoverageStateV2,
301    pub error_count: Evidence<u64>,
302    pub exit_code: Evidence<i32>,
303    pub findings_digest: Evidence<String>,
304    pub report_digest: Evidence<String>,
305}
306
307impl OutcomeIdentityV2 {
308    /// Construct terminal outcome evidence recorded by the production caller.
309    pub fn recorded(
310        status: RunState,
311        coverage: CoverageStateV2,
312        error_count: u64,
313        exit_code: i32,
314        findings_digest: Evidence<String>,
315        report_digest: Evidence<String>,
316    ) -> Self {
317        Self {
318            version: 1,
319            status,
320            coverage,
321            error_count: Evidence::recorded(error_count),
322            exit_code: Evidence::recorded(exit_code),
323            findings_digest,
324            report_digest,
325        }
326    }
327}
328
329/// Comparison identity joining every timing-relevant dimension.
330#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
331pub struct CausalRunIdentityV2 {
332    pub version: u16,
333    pub run_id: String,
334    pub parent_run_id: Evidence<String>,
335    pub benchmark_pair_id: Evidence<String>,
336    pub repeat_group_id: Evidence<String>,
337    pub host: HostIdentityV2,
338    pub build: BuildIdentityV2,
339    pub detectors: DetectorIdentityV2,
340    pub config: ConfigIdentityV2,
341    pub source: SourceIdentityV2,
342    pub workload: WorkloadIdentityV2,
343    pub route: RouteIdentityV2,
344    pub caches: Vec<CacheLayerV2>,
345    pub daemon: DaemonIdentityV2,
346    pub outcome: OutcomeIdentityV2,
347    pub scanner_threads_requested: usize,
348    pub reader_threads_requested: Evidence<usize>,
349    pub reader_threads_resolved: Evidence<usize>,
350}
351
352/// Causal origin of the work measured by one span.
353#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]
354#[serde(rename_all = "kebab-case")]
355#[repr(u8)]
356pub enum WorkOrigin {
357    /// Work performed directly on the caller's own input.
358    #[default]
359    Root = 0,
360    /// Work performed inside an accepted decode-through input.
361    Decoded = 1,
362    /// Work performed on input derived from earlier pipeline output.
363    Derived = 2,
364    /// Work repeated after an earlier attempt failed or was recovered.
365    Retried = 3,
366}
367
368impl WorkOrigin {
369    /// Every work origin in stable wire order.
370    pub const ALL: [Self; 4] = [Self::Root, Self::Decoded, Self::Derived, Self::Retried];
371
372    /// Whether this origin counts as attributed (non-root) pipeline work.
373    pub const fn is_attributed_work(self) -> bool {
374        !matches!(self, Self::Root)
375    }
376}
377
378/// One nested or linked causal interval.
379#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
380pub struct SpanRecordV2 {
381    pub version: u16,
382    pub span_id: u64,
383    pub parent_span_id: Evidence<u64>,
384    pub metric_id: crate::MetricId,
385    pub start_ns: u64,
386    pub inclusive_ns: u64,
387    pub exclusive_ns: u64,
388    pub thread_id: u64,
389    pub task_id: Evidence<u64>,
390    #[serde(default = "legacy_gap")]
391    pub worker_id: Evidence<u64>,
392    #[serde(default)]
393    pub work_origin: WorkOrigin,
394    /// Raw cycle and instruction readings captured at span begin and end.
395    #[serde(default = "legacy_gap")]
396    pub hardware: Evidence<crate::hardware::SpanHardwareV2>,
397}
398
399/// One exact logarithmic bucket of a caller-recorded value distribution.
400#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
401pub struct DistributionBucketV2 {
402    pub version: u16,
403    pub lower_bound: u64,
404    pub upper_bound: u64,
405    pub count: u64,
406}
407
408/// Caller-recorded logarithmic distribution for one typed metric.
409#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
410pub struct MetricDistributionV2 {
411    pub version: u16,
412    pub metric_id: crate::MetricId,
413    pub call_count: u64,
414    pub minimum: u64,
415    pub maximum: u64,
416    pub buckets: Vec<DistributionBucketV2>,
417}
418
419/// One matched producer enqueue and consumer dequeue through a bounded queue.
420#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
421pub struct QueueLinkV2 {
422    pub version: u16,
423    pub queue: crate::QueueId,
424    pub sequence: u64,
425    pub producer_thread_id: u64,
426    pub producer_elapsed_ns: u64,
427    pub consumer_thread_id: u64,
428    pub consumer_elapsed_ns: u64,
429}
430
431/// Current depth and high-water mark for one bounded queue slot.
432#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
433pub struct QueueDepthV2 {
434    pub version: u16,
435    pub queue: crate::QueueId,
436    pub current: u64,
437    pub high_water: u64,
438    pub enqueues: u64,
439    pub dequeues: u64,
440}
441
442/// Per-worker load observed from one counter shard.
443#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
444pub struct WorkerLoadV2 {
445    pub version: u16,
446    pub worker_id: u64,
447    pub calls: u64,
448    pub elapsed_ns: u64,
449}
450
451/// Work-stealing imbalance evidence merged from every worker shard.
452#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
453pub struct WorkerImbalanceV2 {
454    pub version: u16,
455    pub worker_count: u64,
456    pub total_calls: u64,
457    pub total_elapsed_ns: u64,
458    /// Share of total calls handled by the busiest worker, in parts per million.
459    pub max_share_ppm: u64,
460    /// Upper-median per-worker share of total calls, in parts per million.
461    pub median_share_ppm: u64,
462    /// Share of registered workers that recorded zero calls, in parts per million.
463    pub idle_share_ppm: u64,
464    pub workers: Vec<WorkerLoadV2>,
465}
466
467/// Wall-clock occupancy of one micro-function across every worker.
468///
469/// `window_ns` is the span from the first start to the last end of any call to
470/// this micro-function, so `elapsed_ns / window_ns` is the average number of
471/// workers inside it while it was running. A stage whose concurrency is near
472/// one ran serially even when the pool was large.
473#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
474pub struct StageConcurrencyV2 {
475    pub version: u16,
476    pub metric_id: crate::MetricId,
477    pub macro_stage_id: crate::MacroStageId,
478    pub calls: u64,
479    /// Summed inclusive time across every call on every worker.
480    pub elapsed_ns: u64,
481    /// First start to last end across every worker, relative to session start.
482    pub window_ns: u64,
483    pub first_start_ns: u64,
484    pub last_end_ns: u64,
485    /// Workers that recorded at least one call.
486    pub worker_count: u64,
487    /// Largest single-worker contribution to `elapsed_ns`.
488    pub max_worker_elapsed_ns: u64,
489    /// `elapsed_ns / window_ns` in thousandths; 1000 means strictly serial.
490    pub concurrency_milli: u64,
491    /// Time inside calls the caller explicitly declared serial.
492    pub declared_serial_ns: u64,
493    pub declared_serial_calls: u64,
494    /// Bytes the caller attributed to this micro-function.
495    pub bytes: u64,
496}
497
498/// Busy, blocked, and idle time for one worker across the whole session.
499#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
500pub struct WorkerOccupancyRowV2 {
501    pub version: u16,
502    pub worker_id: u64,
503    /// Time inside outermost non-blocked spans; nested spans are not counted twice.
504    pub busy_ns: u64,
505    /// Time inside outermost blocked-wait spans.
506    pub blocked_ns: u64,
507    /// Outermost spans entered by this worker.
508    pub calls: u64,
509}
510
511/// Pool-wide busy versus idle accounting merged from every worker shard.
512#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
513pub struct WorkerOccupancyV2 {
514    pub version: u16,
515    /// Workers that registered a counter shard, busy or not.
516    pub worker_count: u64,
517    /// Workers that recorded at least one outermost span.
518    pub active_worker_count: u64,
519    pub busy_ns: u64,
520    pub blocked_ns: u64,
521    pub calls: u64,
522    pub busiest_busy_ns: u64,
523    pub median_busy_ns: u64,
524    pub workers: Vec<WorkerOccupancyRowV2>,
525}
526
527/// Hit and miss counts for one reuse cache.
528#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
529pub struct CacheEffectivenessV2 {
530    pub version: u16,
531    pub cache: crate::CacheId,
532    pub hits: u64,
533    pub misses: u64,
534    /// `hits / (hits + misses)` in parts per million.
535    pub hit_rate_ppm: u64,
536}
537
538/// Retry attempts recorded for one cause.
539///
540/// `attempts` counts every attempt, not every operation that was eventually
541/// retried, so a path that retries a thousand times reads as a thousand.
542#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
543pub struct RetryRecordV2 {
544    pub version: u16,
545    pub cause: crate::RetryCause,
546    pub attempts: u64,
547}
548
549/// One indexed counter family, summed per slot across every worker.
550///
551/// The caller owns the slot labels. `slots` is always
552/// [`crate::INDEXED_COUNTER_SLOTS`] long so two runs diff positionally.
553#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
554pub struct IndexedCounterRecordV2 {
555    pub version: u16,
556    pub counter: crate::IndexedCounterId,
557    pub slots: Vec<u64>,
558    /// Records addressed to a slot outside the fixed range, never folded in.
559    pub dropped_out_of_range: u64,
560}
561
562/// Blocked wait time attributed separately from runnable execution for one stage.
563#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
564pub struct BlockedWaitRecordV2 {
565    pub version: u16,
566    pub metric_id: crate::MetricId,
567    pub macro_stage_id: crate::MacroStageId,
568    pub calls: u64,
569    pub blocked_ns: u64,
570}
571pub const COMPILE_SURFACE_RECORD_V2_VERSION: u16 = 1;
572
573const fn legacy_component_version() -> u16 {
574    1
575}
576
577/// Compilation and load metrics recorded for one compile surface class.
578#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
579pub struct CompileSurfaceRecordV2 {
580    #[serde(default = "legacy_component_version")]
581    pub version: u16,
582    pub surface: crate::CompileSurfaceId,
583    pub name: String,
584    pub runtime_compiles: u64,
585    pub loads: u64,
586    pub install_compiles: u64,
587    pub update_compiles: u64,
588    pub developer_compiles: u64,
589}
590
591/// One exact logarithmic latency bucket.
592#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
593pub struct LatencyBucketV2 {
594    pub version: u16,
595    pub lower_bound_ns: u64,
596    pub upper_bound_ns: u64,
597    pub count: u64,
598}
599
600/// Allocation-free hot-path call latency distribution for one micro-function.
601#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
602pub struct LatencyDistributionV2 {
603    pub version: u16,
604    pub metric_id: crate::MetricId,
605    pub macro_stage_id: crate::MacroStageId,
606    pub call_count: u64,
607    pub minimum_ns: u64,
608    pub maximum_ns: u64,
609    /// Nearest-rank p50, represented by the retained logarithmic bucket's upper bound.
610    pub p50_ns: u64,
611    /// Nearest-rank p90, represented by the retained logarithmic bucket's upper bound.
612    pub p90_ns: u64,
613    /// Nearest-rank p95, represented by the retained logarithmic bucket's upper bound.
614    pub p95_ns: u64,
615    /// Nearest-rank p99, represented by the retained logarithmic bucket's upper bound.
616    pub p99_ns: u64,
617    pub buckets: Vec<LatencyBucketV2>,
618}
619
620/// Estimated profiler observer effect and instrumentation overhead (Row 108).
621#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
622pub struct ObserverEffectV2 {
623    pub version: u16,
624    /// Estimated total nanoseconds spent in profiler instrumentation.
625    pub estimated_overhead_ns: u64,
626    /// Estimated fraction of wall time attributed to profiler overhead, in parts per million.
627    pub overhead_share_ppm: u64,
628    /// Total instrumentation events (spans, counters, annotations) counted.
629    pub total_instrumentation_events: u64,
630    /// Measured or calibrated per-event instrumentation cost in nanoseconds.
631    pub estimated_ns_per_event: u64,
632    /// Per-stage observer effect breakdown.
633    pub stage_overhead: Vec<StageOverheadV2>,
634}
635
636impl ObserverEffectV2 {
637    /// Estimate profiler observer effect from recorded stage calls and wall time.
638    pub fn estimate(stages: &[StageMeasurement], wall_time_ns: u64) -> Self {
639        let mut stage_overhead = Vec::with_capacity(stages.len());
640        let mut total_events = 0_u64;
641
642        for stage in stages {
643            total_events = total_events.saturating_add(stage.calls);
644            let estimated_overhead_ns = stage.calls.saturating_mul(ESTIMATED_NS_PER_SPAN_EVENT);
645            let overhead_share_ppm = if stage.elapsed_ns > 0 {
646                (estimated_overhead_ns as u128 * 1_000_000 / stage.elapsed_ns as u128)
647                    .min(1_000_000) as u64
648            } else {
649                0
650            };
651            stage_overhead.push(StageOverheadV2 {
652                version: OBSERVER_EFFECT_V2_VERSION,
653                stage: stage.stage,
654                calls: stage.calls,
655                estimated_overhead_ns,
656                overhead_share_ppm,
657            });
658        }
659
660        let total_overhead_ns = total_events.saturating_mul(ESTIMATED_NS_PER_SPAN_EVENT);
661        let overhead_share_ppm = if wall_time_ns > 0 {
662            (total_overhead_ns as u128 * 1_000_000 / wall_time_ns as u128).min(1_000_000) as u64
663        } else {
664            0
665        };
666
667        Self {
668            version: OBSERVER_EFFECT_V2_VERSION,
669            estimated_overhead_ns: total_overhead_ns,
670            overhead_share_ppm,
671            total_instrumentation_events: total_events,
672            estimated_ns_per_event: ESTIMATED_NS_PER_SPAN_EVENT,
673            stage_overhead,
674        }
675    }
676}
677
678/// Estimated observer effect for one stage based on its event/call frequency.
679#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
680pub struct StageOverheadV2 {
681    pub version: u16,
682    pub stage: crate::Stage,
683    pub calls: u64,
684    pub estimated_overhead_ns: u64,
685    pub overhead_share_ppm: u64,
686}
687
688/// One typed counter or gauge materialized from fixed runtime storage.
689#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
690pub struct TypedMetricRecordV2 {
691    pub version: u16,
692    pub metric_id: crate::MetricId,
693    pub kind: crate::MetricKind,
694    pub value: u64,
695}
696
697/// One bounded instantaneous event with a typed numeric payload.
698#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
699pub struct PointEventV2 {
700    pub version: u16,
701    pub sequence: u64,
702    pub event_id: crate::EventId,
703    pub elapsed_ns: u64,
704    pub thread_id: u64,
705    pub value: u64,
706    #[serde(default = "legacy_gap")]
707    pub task_id: Evidence<u64>,
708    #[serde(default = "legacy_gap")]
709    pub worker_id: Evidence<u64>,
710}
711
712/// One bounded typed numeric annotation on the run timeline.
713#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
714pub struct AnnotationV2 {
715    pub version: u16,
716    pub sequence: u64,
717    pub annotation_id: crate::AnnotationId,
718    pub elapsed_ns: u64,
719    pub thread_id: u64,
720    pub value: u64,
721    #[serde(default = "legacy_gap")]
722    pub task_id: Evidence<u64>,
723    #[serde(default = "legacy_gap")]
724    pub worker_id: Evidence<u64>,
725}
726
727/// Bounded event stream with explicit availability and loss accounting.
728#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
729pub struct EventStreamV2 {
730    pub version: u16,
731    pub availability: Evidence<bool>,
732    pub dropped_events: u64,
733    #[serde(default)]
734    pub dropped_span_events: u64,
735    #[serde(default)]
736    pub dropped_point_events: u64,
737    #[serde(default)]
738    pub dropped_annotations: u64,
739    #[serde(default)]
740    pub sampled_out_events: u64,
741    pub spans: Vec<SpanRecordV2>,
742    #[serde(default)]
743    pub point_events: Vec<PointEventV2>,
744    #[serde(default)]
745    pub annotations: Vec<AnnotationV2>,
746}
747
748/// Versioned causal profile envelope and measurements.
749#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
750pub struct CausalProfileV2 {
751    pub version: u16,
752    pub envelope: ProfileEnvelopeV2,
753    pub identity: CausalRunIdentityV2,
754    pub status: RunState,
755    pub wall_time_ns: u64,
756    pub stages: Vec<StageMeasurement>,
757    pub transitions: Vec<StateTransition>,
758    pub states: Vec<StateMeasurement>,
759    pub collectors: Vec<CollectorCapability>,
760    pub resource_samples: Vec<ResourceSample>,
761    pub resources: ResourceUsage,
762    #[serde(default)]
763    pub typed_metrics: Vec<TypedMetricRecordV2>,
764    #[serde(default)]
765    pub latency_distributions: Vec<LatencyDistributionV2>,
766    /// Wall-clock occupancy per micro-function; empty on profiles before 2.8.
767    #[serde(default)]
768    pub stage_concurrency: Vec<StageConcurrencyV2>,
769    /// Per-worker busy and blocked time; absent on profiles before 2.8.
770    #[serde(default)]
771    pub worker_occupancy: Option<WorkerOccupancyV2>,
772    /// Queue depth high-water evidence; empty on profiles before 2.8.
773    #[serde(default)]
774    pub queue_depths: Vec<QueueDepthV2>,
775    /// Per-stage blocked wait time; empty on profiles before 2.8.
776    #[serde(default)]
777    pub blocked_waits: Vec<BlockedWaitRecordV2>,
778    /// Reuse-cache hit rates; empty on profiles before 2.8.
779    #[serde(default)]
780    pub caches: Vec<CacheEffectivenessV2>,
781    /// Indexed counter families; empty on profiles before 2.8.
782    #[serde(default)]
783    pub indexed_counters: Vec<IndexedCounterRecordV2>,
784    /// Retry attempts by cause; empty on profiles before 2.8.
785    #[serde(default)]
786    pub retries: Vec<RetryRecordV2>,
787    /// Compilation surface metrics by class and phase; empty on profiles before 2.9.
788    #[serde(default)]
789    pub compile_surfaces: Vec<CompileSurfaceRecordV2>,
790    /// Derived bottleneck analysis; absent on profiles before 2.8.
791    #[serde(default)]
792    pub insight: Option<crate::insight::RunInsightV2>,
793    /// Estimated profiler observer effect; absent on profiles before 2.8.
794    #[serde(default)]
795    pub observer_effect: Option<ObserverEffectV2>,
796    pub events: EventStreamV2,
797    /// CPU hardware evidence collected across the run.
798    #[serde(default = "legacy_gap")]
799    pub hardware: Evidence<crate::hardware::HardwareRunEvidenceV2>,
800    /// Memory, IO, and system evidence collected across the run.
801    #[serde(default = "legacy_gap")]
802    pub system: Evidence<crate::system::SystemRunEvidenceV2>,
803}
804
805impl CausalProfileV2 {
806    /// Migrate a v1 aggregate profile and capture build evidence available to this executable.
807    pub fn from_v1(profile: RunProfile) -> Self {
808        let build = BuildIdentityV2::capture_legacy(&profile.identity.binary_version);
809        Self::from_v1_with_build(profile, build)
810    }
811
812    /// Migrate a v1 aggregate profile with exact final-binary build identity.
813    pub fn from_v1_with_build(profile: RunProfile, build: BuildIdentityV2) -> Self {
814        let RunProfile {
815            identity,
816            status,
817            wall_time_ns,
818            input_bytes,
819            input_units,
820            workload,
821            stages,
822            transitions,
823            states,
824            collectors,
825            resource_samples,
826            resources,
827            hardware,
828            system,
829            compile_surfaces,
830            ..
831        } = profile;
832        let selected_backend = identity
833            .backend_selected
834            .clone()
835            .map_or_else(legacy_gap, Evidence::recorded);
836        let reader_threads_requested = identity
837            .reader_threads
838            .map_or_else(legacy_gap, Evidence::recorded);
839        let causal_identity = CausalRunIdentityV2 {
840            version: CAUSAL_IDENTITY_V2_VERSION,
841            run_id: identity.run_id,
842            parent_run_id: legacy_gap(),
843            benchmark_pair_id: legacy_gap(),
844            repeat_group_id: legacy_gap(),
845            host: HostIdentityV2::capture(),
846            build,
847            detectors: DetectorIdentityV2 {
848                version: 1,
849                corpus_digest: identity.detector_digest,
850                compiled_plan_digest: legacy_gap(),
851                enabled_detector_digest: legacy_gap(),
852                backend_database_digest: legacy_gap(),
853                external_provenance_digest: legacy_gap(),
854            },
855            config: ConfigIdentityV2 {
856                version: 1,
857                resolved_config_digest: identity.config_digest,
858                policy_digest: legacy_gap(),
859                preset: legacy_gap(),
860                protection_state: legacy_gap(),
861            },
862            source: SourceIdentityV2 {
863                version: 1,
864                adapters: vec![identity.source_kind],
865                target_digest: legacy_gap(),
866                partition_digest: legacy_gap(),
867            },
868            workload: WorkloadIdentityV2::capture(crate::WorkloadIdentityInput {
869                class: &identity.workload_class,
870                raw_source_bytes: input_bytes,
871                source_units: input_units,
872                container_bytes: workload.container_bytes,
873                expanded_payload_bytes: workload.expanded_payload_bytes,
874                derived_decoder_bytes: workload.derived_decoder_bytes,
875                backend_dispatched_bytes: workload.backend_dispatched_bytes,
876            }),
877            route: RouteIdentityV2 {
878                version: 1,
879                request_mode: "legacy-v1".to_owned(),
880                requested_backend: identity.backend_requested,
881                selected_backend,
882                completed_backend: legacy_gap(),
883                autoroute_decision_digest: legacy_gap(),
884                batches: Vec::new(),
885                dropped_batches: 0,
886            },
887            caches: vec![CacheLayerV2 {
888                version: 1,
889                layer: CacheLayerKindV2::LegacyAggregate,
890                state: identity.cache_state,
891                generation: legacy_gap(),
892                digest: legacy_gap(),
893            }],
894            daemon: DaemonIdentityV2 {
895                version: 1,
896                state: identity.daemon_state,
897                generation: legacy_gap(),
898                request_id: legacy_gap(),
899                parent_request_id: legacy_gap(),
900                ready_age_ns: legacy_gap(),
901            },
902            outcome: OutcomeIdentityV2 {
903                version: 1,
904                status,
905                coverage: CoverageStateV2::Unknown,
906                error_count: legacy_gap(),
907                exit_code: legacy_gap(),
908                findings_digest: legacy_gap(),
909                report_digest: legacy_gap(),
910            },
911            scanner_threads_requested: identity.scanner_threads,
912            reader_threads_requested,
913            reader_threads_resolved: legacy_gap(),
914        };
915        let observer_effect = Some(ObserverEffectV2::estimate(&stages, wall_time_ns));
916        Self {
917            version: CAUSAL_PROFILE_V2_VERSION,
918            envelope: ProfileEnvelopeV2 {
919                version: PROFILE_ENVELOPE_V2_VERSION,
920                schema: PROFILE_SCHEMA_V2.to_owned(),
921                schema_version: SchemaVersionV2 {
922                    version: 1,
923                    major: PROFILE_SCHEMA_V2_MAJOR,
924                    minor: PROFILE_SCHEMA_V2_MINOR,
925                },
926                event_schema_version: EVENT_SCHEMA_VERSION,
927                metric_registry_version: METRIC_REGISTRY_VERSION,
928                producer: ProducerIdentityV2 {
929                    version: 1,
930                    profile_crate_version: env!("CARGO_PKG_VERSION").to_owned(),
931                    exporter_version: EXPORTER_VERSION,
932                },
933                integrity: legacy_gap(),
934            },
935            identity: causal_identity,
936            status,
937            wall_time_ns,
938            stages,
939            transitions,
940            states,
941            collectors,
942            resource_samples,
943            resources,
944            typed_metrics: Vec::new(),
945            latency_distributions: Vec::new(),
946            stage_concurrency: Vec::new(),
947            worker_occupancy: None,
948            queue_depths: Vec::new(),
949            blocked_waits: Vec::new(),
950            caches: Vec::new(),
951            indexed_counters: Vec::new(),
952            retries: Vec::new(),
953            compile_surfaces: if compile_surfaces.is_empty() {
954                crate::compile_surface_reports()
955            } else {
956                compile_surfaces
957            },
958            insight: None,
959            observer_effect,
960            events: EventStreamV2 {
961                version: 3,
962                availability: legacy_gap(),
963                dropped_events: 0,
964                dropped_span_events: 0,
965                dropped_point_events: 0,
966                dropped_annotations: 0,
967                sampled_out_events: 0,
968                spans: Vec::new(),
969                point_events: Vec::new(),
970                annotations: Vec::new(),
971            },
972            hardware,
973            system,
974        }
975    }
976    /// Render a clean, tabular Markdown report for terminal and browser inspection (Row 108).
977    pub fn render_markdown(&self) -> String {
978        let mut out = String::with_capacity(4096);
979        let input_bytes = self.identity.workload.raw_source_bytes;
980        let input_units = self.identity.workload.source_units;
981        let throughput_mib_s = if self.wall_time_ns == 0 {
982            0.0
983        } else {
984            input_bytes as f64 * 1_000_000_000.0 / self.wall_time_ns as f64 / (1024.0 * 1024.0)
985        };
986
987        let backend_req = &self.identity.route.requested_backend;
988        let backend_sel = self
989            .identity
990            .route
991            .selected_backend
992            .value()
993            .map_or("unselected", |s| s.as_str());
994        let workload_class = &self.identity.workload.class;
995        let adapter = self
996            .identity
997            .source
998            .adapters
999            .first()
1000            .map_or("unknown", |s| s.as_str());
1001        let worker_count = self.worker_occupancy.as_ref().map_or(1, |w| w.worker_count);
1002
1003        out.push_str(&format!("# KeyHog Profile: {}\n\n", self.identity.run_id));
1004        out.push_str(&format!(
1005            "- **Status**: `{:?}`\n\
1006             - **Source**: `{adapter}`\n\
1007             - **Workload**: `{workload_class}`\n\
1008             - **Backend**: requested=`{backend_req}` selected=`{backend_sel}`\n\
1009             - **Wall Time**: {:.3} ms\n\
1010             - **Throughput**: {throughput_mib_s:.3} MiB/s\n\
1011             - **Input**: {input_bytes} bytes ({input_units} units)\n\
1012             - **Hardware**: logical_cpus={} workers={worker_count}\n\n",
1013            self.status,
1014            self.wall_time_ns as f64 / 1_000_000.0,
1015            self.identity.host.logical_cpus,
1016        ));
1017
1018        out.push_str("## Stages\n\n");
1019        out.push_str("| Stage | Calls | Elapsed (ms) | Self (ms) | Blocked (ms) | Concurrency | Queue Depth |\n");
1020        out.push_str("| :--- | ---: | ---: | ---: | ---: | ---: | :--- |\n");
1021
1022        for stage in &self.stages {
1023            let metric_id = crate::MetricId::from(stage.stage);
1024            let blocked_rec = self
1025                .blocked_waits
1026                .iter()
1027                .find(|bw| bw.metric_id == metric_id);
1028            let concurrency_rec = self
1029                .stage_concurrency
1030                .iter()
1031                .find(|sc| sc.metric_id == metric_id);
1032
1033            let matching_queue = match stage.stage {
1034                crate::Stage::SourceQueueWait | crate::Stage::ScannerQueueWait => {
1035                    Some(crate::QueueId::ScannerWork)
1036                }
1037                crate::Stage::BackendDispatch => Some(crate::QueueId::BackendBatch),
1038                crate::Stage::Decode => Some(crate::QueueId::DecoderWork),
1039                crate::Stage::LiveVerification => Some(crate::QueueId::LiveVerification),
1040                _ => None,
1041            };
1042
1043            let blocked_str = if let Some(bw) = blocked_rec {
1044                if bw.blocked_ns > 0 {
1045                    format!("{:.3}", bw.blocked_ns as f64 / 1_000_000.0)
1046                } else {
1047                    "-".to_string()
1048                }
1049            } else {
1050                "-".to_string()
1051            };
1052
1053            let concurrency_str = if let Some(sc) = concurrency_rec {
1054                if sc.concurrency_milli > 0 {
1055                    format!("{:.2}x", sc.concurrency_milli as f64 / 1000.0)
1056                } else {
1057                    "-".to_string()
1058                }
1059            } else {
1060                "-".to_string()
1061            };
1062
1063            let queue_depth_str = if let Some(qid) = matching_queue {
1064                if let Some(q) = self.queue_depths.iter().find(|qd| qd.queue == qid) {
1065                    format!("{}/{}", q.high_water, q.current)
1066                } else {
1067                    "-".to_string()
1068                }
1069            } else {
1070                "-".to_string()
1071            };
1072
1073            out.push_str(&format!(
1074                "| {} | {} | {:.3} | {:.3} | {} | {} | {} |\n",
1075                stage.stage.as_str(),
1076                stage.calls,
1077                stage.elapsed_ns as f64 / 1_000_000.0,
1078                stage.attributed_ns as f64 / 1_000_000.0,
1079                blocked_str,
1080                concurrency_str,
1081                queue_depth_str,
1082            ));
1083        }
1084        out.push('\n');
1085
1086        if !self.queue_depths.is_empty() {
1087            out.push_str("## Queue Depths\n\n");
1088            out.push_str("| Queue | Current | Peak (High Water) | Enqueues | Dequeues |\n");
1089            out.push_str("| :--- | ---: | ---: | ---: | ---: |\n");
1090            for q in &self.queue_depths {
1091                out.push_str(&format!(
1092                    "| {} | {} | {} | {} | {} |\n",
1093                    q.queue.as_str(),
1094                    q.current,
1095                    q.high_water,
1096                    q.enqueues,
1097                    q.dequeues,
1098                ));
1099            }
1100            out.push('\n');
1101        }
1102
1103        if !self.caches.is_empty() {
1104            out.push_str("## Caches\n\n");
1105            out.push_str("| Cache | Hits | Misses | Hit Rate |\n");
1106            out.push_str("| :--- | ---: | ---: | ---: |\n");
1107            for c in &self.caches {
1108                let hit_pct = c.hit_rate_ppm as f64 / 10_000.0;
1109                out.push_str(&format!(
1110                    "| {} | {} | {} | {:.2}% |\n",
1111                    c.cache.as_str(),
1112                    c.hits,
1113                    c.misses,
1114                    hit_pct,
1115                ));
1116            }
1117            out.push('\n');
1118        }
1119
1120        if let Some(worker_occ) = &self.worker_occupancy {
1121            out.push_str("## Worker Occupancy\n\n");
1122            out.push_str(&format!(
1123                "- **Registered Workers**: {}\n\
1124                 - **Active Workers**: {}\n\
1125                 - **Total Busy**: {:.3} ms\n\
1126                 - **Total Blocked**: {:.3} ms\n\
1127                 - **Busiest Worker**: {:.3} ms\n\n",
1128                worker_occ.worker_count,
1129                worker_occ.active_worker_count,
1130                worker_occ.busy_ns as f64 / 1_000_000.0,
1131                worker_occ.blocked_ns as f64 / 1_000_000.0,
1132                worker_occ.busiest_busy_ns as f64 / 1_000_000.0,
1133            ));
1134            if !worker_occ.workers.is_empty() {
1135                out.push_str("| Worker ID | Calls | Busy (ms) | Blocked (ms) |\n");
1136                out.push_str("| ---: | ---: | ---: | ---: |\n");
1137                for w in &worker_occ.workers {
1138                    out.push_str(&format!(
1139                        "| {} | {} | {:.3} | {:.3} |\n",
1140                        w.worker_id,
1141                        w.calls,
1142                        w.busy_ns as f64 / 1_000_000.0,
1143                        w.blocked_ns as f64 / 1_000_000.0,
1144                    ));
1145                }
1146                out.push('\n');
1147            }
1148        }
1149
1150        out
1151    }
1152}