1use agent_effects::{
59 EffectObserver, EffectRecord, EffectStatus, Observation, RuntimeEvent, Transition,
60};
61use opentelemetry::KeyValue;
62use opentelemetry::metrics::{Counter, Gauge, Histogram, Meter, UpDownCounter};
63
64#[derive(Clone, Debug)]
66pub struct OtelObserver {
67 started: Counter<u64>,
68 completed: Counter<u64>,
69 failed: Counter<u64>,
70 rejected: Counter<u64>,
71 unknown: Counter<u64>,
72 needs_intervention: Counter<u64>,
73 retries: Counter<u64>,
74 compensation_started: Counter<u64>,
75 compensation_completed: Counter<u64>,
76 compensation_failed: Counter<u64>,
77 pending_approval: UpDownCounter<i64>,
78 duration: Histogram<f64>,
79 attempt_duration: Histogram<f64>,
80 unknown_duration: Histogram<f64>,
81 awaiting_intervention: UpDownCounter<i64>,
82 attempts: Counter<u64>,
83 retry_delay: Histogram<f64>,
84 verification_started: Counter<u64>,
85 verification_outcomes: Counter<u64>,
86 approval_decisions: Counter<u64>,
87 approval_duration: Histogram<f64>,
88 operator_resolutions: Counter<u64>,
89 lease_expired: Counter<u64>,
90 lease_conflicts: Counter<u64>,
91 lease_renewal_failures: Counter<u64>,
92 recovery_passes: Counter<u64>,
93 recovery_duration: Histogram<f64>,
94 recovery_effects: Counter<u64>,
95 recovery_backlog: Gauge<u64>,
96 pruned: Counter<u64>,
97}
98
99impl OtelObserver {
100 pub fn global() -> Self {
103 Self::new(&opentelemetry::global::meter("agent-effects"))
104 }
105
106 #[allow(clippy::too_many_lines)]
109 pub fn new(meter: &Meter) -> Self {
110 let counter = |name: &'static str, description: &'static str| {
111 meter
112 .u64_counter(name)
113 .with_unit("{effect}")
114 .with_description(description)
115 .build()
116 };
117 let seconds = |name: &'static str, description: &'static str| {
118 meter
119 .f64_histogram(name)
120 .with_unit("s")
121 .with_description(description)
122 .build()
123 };
124 Self {
125 started: counter("agent_effects.started", "Effects recorded"),
126 completed: counter("agent_effects.completed", "Effects committed"),
127 failed: counter(
128 "agent_effects.failed",
129 "Effects that definitely did not apply",
130 ),
131 rejected: counter(
132 "agent_effects.rejected",
133 "Effects refused by a precondition or an approver",
134 ),
135 unknown: counter(
136 "agent_effects.unknown",
137 "Attempts whose outcome became unknown",
138 ),
139 needs_intervention: counter(
140 "agent_effects.needs_intervention",
141 "Effects escalated to an operator",
142 ),
143 retries: meter
144 .u64_counter("agent_effects.retry.count")
145 .with_unit("{retry}")
146 .with_description("Retries scheduled")
147 .build(),
148 compensation_started: counter(
149 "agent_effects.compensation.started",
150 "Compensations started",
151 ),
152 compensation_completed: counter(
153 "agent_effects.compensation.completed",
154 "Effects undone",
155 ),
156 compensation_failed: counter(
157 "agent_effects.compensation.failed",
158 "Compensations that failed for good",
159 ),
160 pending_approval: meter
161 .i64_up_down_counter("agent_effects.pending_approval")
162 .with_unit("{effect}")
163 .with_description("Effects awaiting approval (changes since this process started)")
164 .build(),
165 duration: seconds(
166 "agent_effects.duration",
167 "Time from recording an effect to its committing, failing or being rejected",
168 ),
169 attempt_duration: seconds("agent_effects.attempt.duration", "Duration of one attempt"),
170 unknown_duration: seconds(
171 "agent_effects.unknown.duration",
172 "Time an effect spent with an unknown outcome",
173 ),
174 awaiting_intervention: meter
175 .i64_up_down_counter("agent_effects.awaiting_intervention")
176 .with_unit("{effect}")
177 .with_description(
178 "Effects waiting for an operator (changes since this process started)",
179 )
180 .build(),
181 attempts: meter
182 .u64_counter("agent_effects.attempts")
183 .with_unit("{attempt}")
184 .with_description("Attempts started, first or retry")
185 .build(),
186 retry_delay: seconds(
187 "agent_effects.retry.delay",
188 "Time from scheduling a retry until it may start",
189 ),
190 verification_started: meter
191 .u64_counter("agent_effects.verification.started")
192 .with_unit("{verification}")
193 .with_description("Verifications started")
194 .build(),
195 verification_outcomes: meter
196 .u64_counter("agent_effects.verification.outcomes")
197 .with_unit("{verification}")
198 .with_description("How verifications ended")
199 .build(),
200 approval_decisions: meter
201 .u64_counter("agent_effects.approval.decisions")
202 .with_unit("{decision}")
203 .with_description("Approval decisions, by a provider or an operator")
204 .build(),
205 approval_duration: seconds(
206 "agent_effects.approval.duration",
207 "Time from requesting approval to the decision",
208 ),
209 operator_resolutions: meter
210 .u64_counter("agent_effects.operator.resolutions")
211 .with_unit("{decision}")
212 .with_description("Operator decisions on unresolved effects")
213 .build(),
214 lease_expired: counter(
215 "agent_effects.lease.expired",
216 "Attempts whose worker lost its lease mid-flight",
217 ),
218 lease_conflicts: meter
219 .u64_counter("agent_effects.lease.conflicts")
220 .with_unit("{conflict}")
221 .with_description("A worker found another holding an effect's lease")
222 .build(),
223 lease_renewal_failures: meter
224 .u64_counter("agent_effects.lease.renewal_failures")
225 .with_unit("{renewal}")
226 .with_description("Lease renewals that failed with a store error")
227 .build(),
228 recovery_passes: meter
229 .u64_counter("agent_effects.recovery.passes")
230 .with_unit("{pass}")
231 .with_description("Recovery passes")
232 .build(),
233 recovery_duration: seconds(
234 "agent_effects.recovery.duration",
235 "Duration of one recovery pass",
236 ),
237 recovery_effects: counter(
238 "agent_effects.recovery.effects",
239 "What recovery passes did, per effect",
240 ),
241 recovery_backlog: meter
242 .u64_gauge("agent_effects.recovery.backlog")
243 .with_unit("{effect}")
244 .with_description("What this worker's latest recovery pass found waiting")
245 .build(),
246 pruned: meter
247 .u64_counter("agent_effects.pruned")
248 .with_unit("{effect}")
249 .with_description("Settled records deleted by retention")
250 .build(),
251 }
252 }
253}
254
255fn tagged(attrs: &[KeyValue; 2], key: &'static str, value: &'static str) -> [KeyValue; 3] {
256 let [name, kind] = attrs.clone();
257 [name, kind, KeyValue::new(key, value)]
258}
259
260fn count(n: usize) -> u64 {
261 u64::try_from(n).unwrap_or(u64::MAX)
262}
263
264fn attributes(record: &EffectRecord) -> [KeyValue; 2] {
265 [
266 KeyValue::new("effect.name", record.key.name.to_string()),
267 KeyValue::new("effect.kind", record.kind.as_str()),
268 ]
269}
270
271impl OtelObserver {
272 fn on_decision(&self, o: &Observation<'_>, attrs: &[KeyValue; 2]) {
274 if o.from == EffectStatus::Verifying {
275 let outcome = match o.transition {
276 Transition::VerificationConfirmed => Some("confirmed"),
277 Transition::VerificationNotApplied | Transition::ScheduleRetry => {
278 Some("not_applied")
279 }
280 Transition::VerificationConflict => Some("conflict"),
281 Transition::OutcomeUnknown => Some("inconclusive"),
282 _ => None,
283 };
284 if let Some(outcome) = outcome {
285 self.verification_outcomes
286 .add(1, &tagged(attrs, "outcome", outcome));
287 }
288 }
289 if o.from == EffectStatus::AwaitingApproval {
290 let decision = match o.transition {
291 Transition::Approve => Some("approved"),
292 Transition::Deny => Some("denied"),
293 _ => None,
294 };
295 if let Some(decision) = decision {
296 let decided = tagged(attrs, "decision", decision);
297 self.approval_decisions.add(1, &decided);
298 self.approval_duration
299 .record(o.in_previous_status.as_secs_f64(), &decided);
300 }
301 }
302 }
303}
304
305impl EffectObserver for OtelObserver {
306 fn on_created(&self, record: &EffectRecord) {
307 self.started.add(1, &attributes(record));
308 }
309
310 fn on_transition(&self, o: &Observation<'_>) {
311 let attrs = attributes(o.record);
312 let settled = |counter: &Counter<u64>| {
313 counter.add(1, &attrs);
314 self.duration.record(o.since_created.as_secs_f64(), &attrs);
315 };
316 match o.to {
317 EffectStatus::Committed => settled(&self.completed),
318 EffectStatus::Failed => settled(&self.failed),
319 EffectStatus::Rejected => settled(&self.rejected),
320 EffectStatus::Unknown => self.unknown.add(1, &attrs),
321 EffectStatus::NeedsIntervention => self.needs_intervention.add(1, &attrs),
322 EffectStatus::Compensated => self.compensation_completed.add(1, &attrs),
323 EffectStatus::CompensationFailed => self.compensation_failed.add(1, &attrs),
324 _ => {}
325 }
326 let waits_for_operator = |s: EffectStatus| {
327 matches!(
328 s,
329 EffectStatus::NeedsIntervention | EffectStatus::CompensationFailed
330 )
331 };
332 if waits_for_operator(o.to) && !waits_for_operator(o.from) {
333 self.awaiting_intervention.add(1, &attrs);
334 } else if waits_for_operator(o.from) && !waits_for_operator(o.to) {
335 self.awaiting_intervention.add(-1, &attrs);
336 }
337 match o.transition {
338 Transition::ScheduleRetry => {
339 self.retries.add(1, &attrs);
340 if let Some(delay) = o
341 .record
342 .next_attempt_at
343 .and_then(|at| at.duration_since(o.record.updated_at).ok())
344 {
345 self.retry_delay.record(delay.as_secs_f64(), &attrs);
346 }
347 }
348 Transition::StartAttempt => {
349 let attempt = if o.record.attempt_count > 1 {
350 "retry"
351 } else {
352 "first"
353 };
354 self.attempts.add(1, &tagged(&attrs, "attempt", attempt));
355 }
356 Transition::StartVerification => self.verification_started.add(1, &attrs),
357 Transition::LeaseExpired => self.lease_expired.add(1, &attrs),
358 Transition::StartCompensation => self.compensation_started.add(1, &attrs),
359 Transition::RequestApproval => self.pending_approval.add(1, &attrs),
360 Transition::ResolvedApplied => self
361 .operator_resolutions
362 .add(1, &tagged(&attrs, "resolution", "applied")),
363 Transition::ResolvedNotApplied => self
364 .operator_resolutions
365 .add(1, &tagged(&attrs, "resolution", "not_applied")),
366 Transition::ResolvedRetry => self
367 .operator_resolutions
368 .add(1, &tagged(&attrs, "resolution", "retry")),
369 Transition::ResolvedCompensated => self
370 .operator_resolutions
371 .add(1, &tagged(&attrs, "resolution", "compensated")),
372 _ => {}
373 }
374 self.on_decision(o, &attrs);
375 match o.from {
376 EffectStatus::AwaitingApproval => self.pending_approval.add(-1, &attrs),
377 EffectStatus::Executing => {
378 let [name, kind] = attrs.clone();
379 let outcome = KeyValue::new("outcome", o.to.as_str());
380 self.attempt_duration
381 .record(o.in_previous_status.as_secs_f64(), &[name, kind, outcome]);
382 }
383 EffectStatus::Unknown => self
384 .unknown_duration
385 .record(o.in_previous_status.as_secs_f64(), &attrs),
386 _ => {}
387 }
388 }
389
390 fn on_event(&self, event: &RuntimeEvent<'_>) {
391 match event {
392 RuntimeEvent::LeaseConflict { name } => self
393 .lease_conflicts
394 .add(1, &[KeyValue::new("effect.name", name.to_string())]),
395 RuntimeEvent::LeaseRenewalFailed => self.lease_renewal_failures.add(1, &[]),
396 RuntimeEvent::RecoveryPass { report, duration } => {
397 let result = match report {
398 None => "failed",
399 Some(report) if report.stopped => "stopped",
400 Some(_) => "completed",
401 };
402 let result = [KeyValue::new("result", result)];
403 self.recovery_passes.add(1, &result);
404 self.recovery_duration
405 .record(duration.as_secs_f64(), &result);
406 let Some(report) = report else { return };
407 for (outcome, n) in [
408 ("resumed", report.resumed.len()),
409 ("marked_unknown", report.marked_unknown.len()),
410 ("skipped", report.skipped.len()),
411 ("unhandled", report.unhandled.len()),
412 ("resume_error", report.resume_errors.len()),
413 ] {
414 if n > 0 {
415 self.recovery_effects
416 .add(count(n), &[KeyValue::new("outcome", outcome)]);
417 }
418 }
419 let due =
420 report.resumed.len() + report.unhandled.len() + report.resume_errors.len();
421 for (state, n) in [
422 ("due", due),
423 ("not_due", report.not_due.len()),
424 ("awaiting_operator", report.waiting_for_operator.len()),
425 ] {
426 self.recovery_backlog
427 .record(count(n), &[KeyValue::new("state", state)]);
428 }
429 }
430 RuntimeEvent::Pruned { status, count } => self
431 .pruned
432 .add(*count, &[KeyValue::new("status", status.as_str())]),
433 _ => {}
434 }
435 }
436}