1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
//! [`MetricsRecorderPort`] — in-process operational metrics sink.
//!
//! This port is deliberately distinct from [`StatisticsPort`]. That one
//! records a handful of durable business counters that may be backed by
//! Postgres, so it is `async` and fallible. This one is the sink for
//! rich, high-frequency operational metrics (latency histograms,
//! per-outcome counters) exported to Prometheus straight from process
//! memory.
//!
//! Recording an observation is a synchronous, lock-free, infallible
//! operation: it must never block a deliberation and must never fail it.
//! So the methods take `&self`, return nothing, and are not `async` —
//! instrumentation can never change a use case's control flow.
//!
//! The application layer depends only on this trait; the concrete metric
//! registry lives in an adapter. Use cases call `observe_*` / `record_*`
//! at the point where each measurement first becomes available.
//!
//! [`StatisticsPort`]: super::StatisticsPort
use crate::value_objects::{
CeremonyOutcome, DeliberationOutcome, Discrimination, DurationMs, LlmErrorKind, Score,
ScoringMode, Specialty, StepStatus, TokenUsage,
};
pub trait MetricsRecorderPort: Send + Sync {
/// Observe the end-to-end wall-clock duration of a deliberation that
/// ran to completion, regardless of its terminal outcome.
fn observe_deliberation_duration(&self, specialty: &Specialty, duration: DurationMs);
/// Record the terminal [`DeliberationOutcome`] of a deliberation —
/// the failure rate of the product surfaces here.
fn record_deliberation_outcome(&self, specialty: &Specialty, outcome: DeliberationOutcome);
/// Observe the score of the winning proposal. Recorded only on the
/// success path (a `NoValidProposal` outcome has no winner).
fn observe_winner_score(&self, specialty: &Specialty, score: Score);
/// Observe the latency of a single LLM-judge rating call, whether it
/// succeeded or failed. A call that times out reports a latency near
/// the judge's deadline — the leading signal that the judge is
/// approaching its timeout cliff.
fn observe_judge_latency(&self, model: &str, duration: DurationMs);
/// Observe a judge's `[0.0, 1.0]` verdict for one proposal — the
/// basis for score calibration and threshold tuning.
fn observe_judge_score(&self, model: &str, score: Score);
/// Record a failed LLM-judge call, classified by [`LlmErrorKind`].
/// A judge error fails the deliberation at the validating gate, and
/// the kind dictates the response (timeout vs rate-limit vs
/// credential rotation).
fn record_judge_error(&self, model: &str, kind: LlmErrorKind);
/// Record a failed proposing-agent call, partitioned by `provider`
/// (vllm / openai / anthropic) and [`LlmErrorKind`]. A 429 here is
/// backpressure into every deliberation that routes to the provider.
fn record_provider_error(&self, provider: &str, kind: LlmErrorKind);
/// Record token usage for one LLM-judge call. Combined with judge
/// discrimination, this is the cost side of the judge's ROI.
fn record_judge_tokens(&self, model: &str, usage: TokenUsage);
/// Record token usage for one proposing-agent call, by provider —
/// cost attribution and prompt-bloat detection.
fn record_provider_tokens(&self, provider: &str, usage: TokenUsage);
/// Observe the latency of one proposing-agent call, by `provider` and
/// `operation` (generate / critique / revise). Recorded on every path.
fn observe_provider_request(&self, provider: &str, operation: &str, duration: DurationMs);
/// Increment the in-flight gauge for `provider` at the start of a
/// call. With vLLM serialised (`max-num-seqs=1`), a sustained
/// in-flight depth above 1 is the leading indicator of the latency
/// cliff — concurrency that any other backend would absorb queues here.
fn inc_provider_in_flight(&self, provider: &str);
/// Decrement the in-flight gauge for `provider` when a call returns,
/// whether it succeeded or failed.
fn dec_provider_in_flight(&self, provider: &str);
/// Record whether the scoring policy re-ranked the winner of a
/// deliberation. The killer signal for the LLM judge's worth: a
/// `reranked` ratio near zero means the judge never changes the
/// outcome and is pure cost.
fn record_discrimination(&self, specialty: &Specialty, result: Discrimination);
/// Record the terminal outcome of a ceremony run, by definition name —
/// the completion-rate and failure-mode split per ceremony type.
fn record_ceremony_outcome(&self, ceremony: &str, outcome: CeremonyOutcome);
/// Observe the end-to-end duration of a ceremony run that reached a
/// terminal state.
fn observe_ceremony_duration(&self, ceremony: &str, duration: DurationMs);
/// Observe the duration of a single ceremony step — slow-step
/// isolation within a ceremony.
fn observe_ceremony_step_duration(&self, ceremony: &str, step: &str, duration: DurationMs);
/// Record that a ceremony step finished with `status` — per-step volume
/// and the failure split.
fn record_ceremony_step(&self, ceremony: &str, step: &str, status: StepStatus);
/// Record that no transition out of `from_state` was satisfiable — a
/// guard deadlock or a missing event, distinct from a normal pending
/// wait.
fn record_ceremony_transition_blocked(&self, ceremony: &str, from_state: &str);
/// Observe the latency of one NATS publish, by event `subject_kind`.
fn observe_nats_publish(&self, subject_kind: &str, duration: DurationMs);
/// Record a failed NATS publish, by `subject_kind` and `reason`
/// (serialize vs publish). Completion events drive downstream
/// orchestration, so a silent publish failure loses work.
fn record_nats_publish_error(&self, subject_kind: &str, reason: &str);
/// Set the gauge of Postgres connections currently checked out of the
/// pool. Sampled at scrape time; saturation here cascades into
/// readiness failures, so it is the leading pool-exhaustion signal.
fn set_postgres_pool_in_use(&self, connections: i64);
/// Record which branch a judge-aware scorer took for one proposal. A
/// spike in `uniform_fallback` while a judge is configured means the
/// judge silently is not scoring — an otherwise-undetectable misconfig.
fn record_scoring_mode(&self, mode: ScoringMode);
}
/// A [`MetricsRecorderPort`] that discards every observation.
///
/// The default sink wherever metrics are not the subject under test:
/// unit tests, benches, and any composition that does not export
/// Prometheus. Mirrors [`NullObserver`](super::NullObserver) for the
/// observer port.
#[derive(Debug, Clone, Copy, Default)]
pub struct NoopMetricsRecorder;
impl MetricsRecorderPort for NoopMetricsRecorder {
fn observe_deliberation_duration(&self, _specialty: &Specialty, _duration: DurationMs) {}
fn record_deliberation_outcome(&self, _specialty: &Specialty, _outcome: DeliberationOutcome) {}
fn observe_winner_score(&self, _specialty: &Specialty, _score: Score) {}
fn observe_judge_latency(&self, _model: &str, _duration: DurationMs) {}
fn observe_judge_score(&self, _model: &str, _score: Score) {}
fn record_judge_error(&self, _model: &str, _kind: LlmErrorKind) {}
fn record_provider_error(&self, _provider: &str, _kind: LlmErrorKind) {}
fn record_judge_tokens(&self, _model: &str, _usage: TokenUsage) {}
fn record_provider_tokens(&self, _provider: &str, _usage: TokenUsage) {}
fn observe_provider_request(&self, _provider: &str, _operation: &str, _duration: DurationMs) {}
fn inc_provider_in_flight(&self, _provider: &str) {}
fn dec_provider_in_flight(&self, _provider: &str) {}
fn record_discrimination(&self, _specialty: &Specialty, _result: Discrimination) {}
fn record_ceremony_outcome(&self, _ceremony: &str, _outcome: CeremonyOutcome) {}
fn observe_ceremony_duration(&self, _ceremony: &str, _duration: DurationMs) {}
fn observe_ceremony_step_duration(&self, _ceremony: &str, _step: &str, _duration: DurationMs) {}
fn record_ceremony_step(&self, _ceremony: &str, _step: &str, _status: StepStatus) {}
fn record_ceremony_transition_blocked(&self, _ceremony: &str, _from_state: &str) {}
fn observe_nats_publish(&self, _subject_kind: &str, _duration: DurationMs) {}
fn record_nats_publish_error(&self, _subject_kind: &str, _reason: &str) {}
fn set_postgres_pool_in_use(&self, _connections: i64) {}
fn record_scoring_mode(&self, _mode: ScoringMode) {}
}