Skip to main content

agent_effects_otel/
lib.rs

1//! OpenTelemetry metrics for [`agent-effects`](https://crates.io/crates/agent-effects).
2//!
3//! ```no_run
4//! use agent_effects::Runtime;
5//! use agent_effects_memory::MemoryStore;
6//! use agent_effects_otel::OtelObserver;
7//!
8//! // Uses the global meter provider; set it up with your exporter (OTLP,
9//! // Prometheus, …) as usual.
10//! let runtime = Runtime::builder(MemoryStore::new())
11//!     .observer(OtelObserver::global())
12//!     .build();
13//! ```
14//!
15//! Instruments, each with attributes `effect.name` and `effect.kind` unless
16//! the table says otherwise:
17//!
18//! | Instrument | Kind | Counts / measures |
19//! |---|---|---|
20//! | `agent_effects.started` | counter | effects recorded |
21//! | `agent_effects.completed` | counter | effects committed |
22//! | `agent_effects.failed` | counter | effects that definitely did not apply |
23//! | `agent_effects.rejected` | counter | effects refused by a precondition or approver |
24//! | `agent_effects.unknown` | counter | outcomes that became unknown: **watch this one**. A spike means remote systems are answering ambiguously |
25//! | `agent_effects.needs_intervention` | counter | effects escalated to an operator |
26//! | `agent_effects.awaiting_intervention` | up/down counter | effects waiting for an operator: escalated or failed compensations (deltas since start) |
27//! | `agent_effects.attempts` | counter, `attempt` = `first` \| `retry` | attempts started |
28//! | `agent_effects.retry.count` | counter | retries scheduled |
29//! | `agent_effects.retry.delay` | histogram, s | time from scheduling a retry until it may start |
30//! | `agent_effects.verification.started` | counter | verifications started |
31//! | `agent_effects.verification.outcomes` | counter, `outcome` = `confirmed` \| `not_applied` \| `conflict` \| `inconclusive` | how verifications ended |
32//! | `agent_effects.compensation.started` / `.completed` / `.failed` | counter | compensations |
33//! | `agent_effects.pending_approval` | up/down counter | effects awaiting approval (deltas since start) |
34//! | `agent_effects.approval.decisions` | counter, `decision` = `approved` \| `denied` | approval decisions |
35//! | `agent_effects.approval.duration` | histogram, s, `decision` | approval requested → decided |
36//! | `agent_effects.operator.resolutions` | counter, `resolution` = `applied` \| `not_applied` \| `retry` \| `compensated` | operator decisions on unresolved effects |
37//! | `agent_effects.lease.expired` | counter | attempts whose worker lost its lease mid-flight |
38//! | `agent_effects.lease.conflicts` | counter, `effect.name` only | a worker found another holding an effect |
39//! | `agent_effects.lease.renewal_failures` | counter, no attributes | lease renewals that failed with a store error |
40//! | `agent_effects.duration` | histogram, s | recorded → committed, failed or rejected |
41//! | `agent_effects.attempt.duration` | histogram, s | one attempt, also tagged `outcome` (the status it ended in) |
42//! | `agent_effects.unknown.duration` | histogram, s | time spent unknown before being resolved |
43//! | `agent_effects.recovery.passes` | counter, `result` = `completed` \| `stopped` \| `failed` | recovery passes |
44//! | `agent_effects.recovery.duration` | histogram, s, `result` | one recovery pass |
45//! | `agent_effects.recovery.effects` | counter, `outcome` = `resumed` \| `marked_unknown` \| `skipped` \| `unhandled` \| `resume_error` | what passes did, per effect |
46//! | `agent_effects.recovery.backlog` | gauge, `state` = `due` \| `not_due` \| `awaiting_operator` | what this worker's latest pass found waiting |
47//! | `agent_effects.pruned` | counter, `status` | settled records deleted by retention |
48//!
49//! Recovery and pruning instruments carry no effect attributes.
50//!
51//! No attribute carries an effect id, a logical key, an input, an output, an
52//! actor or an error message: they are unbounded and may identify customers.
53//! Every attribute value comes from a small fixed set, apart from
54//! `effect.name`, which the application chooses. Traces need no extra crate: the runtime's `tracing` spans
55//! (`agent_effect.execute`, `agent_effect.compensate`) export through
56//! `tracing-opentelemetry` like any other.
57
58use agent_effects::{
59    EffectObserver, EffectRecord, EffectStatus, Observation, RuntimeEvent, Transition,
60};
61use opentelemetry::KeyValue;
62use opentelemetry::metrics::{Counter, Gauge, Histogram, Meter, UpDownCounter};
63
64/// An [`EffectObserver`] that records OpenTelemetry metrics.
65#[derive(Clone, Debug)]
66pub struct OtelObserver {
67    started: Counter<u64>,
68    completed: Counter<u64>,
69    failed: Counter<u64>,
70    rejected: Counter<u64>,
71    unknown: Counter<u64>,
72    needs_intervention: Counter<u64>,
73    retries: Counter<u64>,
74    compensation_started: Counter<u64>,
75    compensation_completed: Counter<u64>,
76    compensation_failed: Counter<u64>,
77    pending_approval: UpDownCounter<i64>,
78    duration: Histogram<f64>,
79    attempt_duration: Histogram<f64>,
80    unknown_duration: Histogram<f64>,
81    awaiting_intervention: UpDownCounter<i64>,
82    attempts: Counter<u64>,
83    retry_delay: Histogram<f64>,
84    verification_started: Counter<u64>,
85    verification_outcomes: Counter<u64>,
86    approval_decisions: Counter<u64>,
87    approval_duration: Histogram<f64>,
88    operator_resolutions: Counter<u64>,
89    lease_expired: Counter<u64>,
90    lease_conflicts: Counter<u64>,
91    lease_renewal_failures: Counter<u64>,
92    recovery_passes: Counter<u64>,
93    recovery_duration: Histogram<f64>,
94    recovery_effects: Counter<u64>,
95    recovery_backlog: Gauge<u64>,
96    pruned: Counter<u64>,
97}
98
99impl OtelObserver {
100    /// Instruments from the global meter provider, under the meter name
101    /// `agent-effects`.
102    pub fn global() -> Self {
103        Self::new(&opentelemetry::global::meter("agent-effects"))
104    }
105
106    /// Instruments from `meter`.
107    // One instrument per field: long, but flat.
108    #[allow(clippy::too_many_lines)]
109    pub fn new(meter: &Meter) -> Self {
110        let counter = |name: &'static str, description: &'static str| {
111            meter
112                .u64_counter(name)
113                .with_unit("{effect}")
114                .with_description(description)
115                .build()
116        };
117        let seconds = |name: &'static str, description: &'static str| {
118            meter
119                .f64_histogram(name)
120                .with_unit("s")
121                .with_description(description)
122                .build()
123        };
124        Self {
125            started: counter("agent_effects.started", "Effects recorded"),
126            completed: counter("agent_effects.completed", "Effects committed"),
127            failed: counter(
128                "agent_effects.failed",
129                "Effects that definitely did not apply",
130            ),
131            rejected: counter(
132                "agent_effects.rejected",
133                "Effects refused by a precondition or an approver",
134            ),
135            unknown: counter(
136                "agent_effects.unknown",
137                "Attempts whose outcome became unknown",
138            ),
139            needs_intervention: counter(
140                "agent_effects.needs_intervention",
141                "Effects escalated to an operator",
142            ),
143            retries: meter
144                .u64_counter("agent_effects.retry.count")
145                .with_unit("{retry}")
146                .with_description("Retries scheduled")
147                .build(),
148            compensation_started: counter(
149                "agent_effects.compensation.started",
150                "Compensations started",
151            ),
152            compensation_completed: counter(
153                "agent_effects.compensation.completed",
154                "Effects undone",
155            ),
156            compensation_failed: counter(
157                "agent_effects.compensation.failed",
158                "Compensations that failed for good",
159            ),
160            pending_approval: meter
161                .i64_up_down_counter("agent_effects.pending_approval")
162                .with_unit("{effect}")
163                .with_description("Effects awaiting approval (changes since this process started)")
164                .build(),
165            duration: seconds(
166                "agent_effects.duration",
167                "Time from recording an effect to its committing, failing or being rejected",
168            ),
169            attempt_duration: seconds("agent_effects.attempt.duration", "Duration of one attempt"),
170            unknown_duration: seconds(
171                "agent_effects.unknown.duration",
172                "Time an effect spent with an unknown outcome",
173            ),
174            awaiting_intervention: meter
175                .i64_up_down_counter("agent_effects.awaiting_intervention")
176                .with_unit("{effect}")
177                .with_description(
178                    "Effects waiting for an operator (changes since this process started)",
179                )
180                .build(),
181            attempts: meter
182                .u64_counter("agent_effects.attempts")
183                .with_unit("{attempt}")
184                .with_description("Attempts started, first or retry")
185                .build(),
186            retry_delay: seconds(
187                "agent_effects.retry.delay",
188                "Time from scheduling a retry until it may start",
189            ),
190            verification_started: meter
191                .u64_counter("agent_effects.verification.started")
192                .with_unit("{verification}")
193                .with_description("Verifications started")
194                .build(),
195            verification_outcomes: meter
196                .u64_counter("agent_effects.verification.outcomes")
197                .with_unit("{verification}")
198                .with_description("How verifications ended")
199                .build(),
200            approval_decisions: meter
201                .u64_counter("agent_effects.approval.decisions")
202                .with_unit("{decision}")
203                .with_description("Approval decisions, by a provider or an operator")
204                .build(),
205            approval_duration: seconds(
206                "agent_effects.approval.duration",
207                "Time from requesting approval to the decision",
208            ),
209            operator_resolutions: meter
210                .u64_counter("agent_effects.operator.resolutions")
211                .with_unit("{decision}")
212                .with_description("Operator decisions on unresolved effects")
213                .build(),
214            lease_expired: counter(
215                "agent_effects.lease.expired",
216                "Attempts whose worker lost its lease mid-flight",
217            ),
218            lease_conflicts: meter
219                .u64_counter("agent_effects.lease.conflicts")
220                .with_unit("{conflict}")
221                .with_description("A worker found another holding an effect's lease")
222                .build(),
223            lease_renewal_failures: meter
224                .u64_counter("agent_effects.lease.renewal_failures")
225                .with_unit("{renewal}")
226                .with_description("Lease renewals that failed with a store error")
227                .build(),
228            recovery_passes: meter
229                .u64_counter("agent_effects.recovery.passes")
230                .with_unit("{pass}")
231                .with_description("Recovery passes")
232                .build(),
233            recovery_duration: seconds(
234                "agent_effects.recovery.duration",
235                "Duration of one recovery pass",
236            ),
237            recovery_effects: counter(
238                "agent_effects.recovery.effects",
239                "What recovery passes did, per effect",
240            ),
241            recovery_backlog: meter
242                .u64_gauge("agent_effects.recovery.backlog")
243                .with_unit("{effect}")
244                .with_description("What this worker's latest recovery pass found waiting")
245                .build(),
246            pruned: meter
247                .u64_counter("agent_effects.pruned")
248                .with_unit("{effect}")
249                .with_description("Settled records deleted by retention")
250                .build(),
251        }
252    }
253}
254
255fn tagged(attrs: &[KeyValue; 2], key: &'static str, value: &'static str) -> [KeyValue; 3] {
256    let [name, kind] = attrs.clone();
257    [name, kind, KeyValue::new(key, value)]
258}
259
260fn count(n: usize) -> u64 {
261    u64::try_from(n).unwrap_or(u64::MAX)
262}
263
264fn attributes(record: &EffectRecord) -> [KeyValue; 2] {
265    [
266        KeyValue::new("effect.name", record.key.name.to_string()),
267        KeyValue::new("effect.kind", record.kind.as_str()),
268    ]
269}
270
271impl OtelObserver {
272    /// How a verification or an approval ended.
273    fn on_decision(&self, o: &Observation<'_>, attrs: &[KeyValue; 2]) {
274        if o.from == EffectStatus::Verifying {
275            let outcome = match o.transition {
276                Transition::VerificationConfirmed => Some("confirmed"),
277                Transition::VerificationNotApplied | Transition::ScheduleRetry => {
278                    Some("not_applied")
279                }
280                Transition::VerificationConflict => Some("conflict"),
281                Transition::OutcomeUnknown => Some("inconclusive"),
282                _ => None,
283            };
284            if let Some(outcome) = outcome {
285                self.verification_outcomes
286                    .add(1, &tagged(attrs, "outcome", outcome));
287            }
288        }
289        if o.from == EffectStatus::AwaitingApproval {
290            let decision = match o.transition {
291                Transition::Approve => Some("approved"),
292                Transition::Deny => Some("denied"),
293                _ => None,
294            };
295            if let Some(decision) = decision {
296                let decided = tagged(attrs, "decision", decision);
297                self.approval_decisions.add(1, &decided);
298                self.approval_duration
299                    .record(o.in_previous_status.as_secs_f64(), &decided);
300            }
301        }
302    }
303}
304
305impl EffectObserver for OtelObserver {
306    fn on_created(&self, record: &EffectRecord) {
307        self.started.add(1, &attributes(record));
308    }
309
310    fn on_transition(&self, o: &Observation<'_>) {
311        let attrs = attributes(o.record);
312        let settled = |counter: &Counter<u64>| {
313            counter.add(1, &attrs);
314            self.duration.record(o.since_created.as_secs_f64(), &attrs);
315        };
316        match o.to {
317            EffectStatus::Committed => settled(&self.completed),
318            EffectStatus::Failed => settled(&self.failed),
319            EffectStatus::Rejected => settled(&self.rejected),
320            EffectStatus::Unknown => self.unknown.add(1, &attrs),
321            EffectStatus::NeedsIntervention => self.needs_intervention.add(1, &attrs),
322            EffectStatus::Compensated => self.compensation_completed.add(1, &attrs),
323            EffectStatus::CompensationFailed => self.compensation_failed.add(1, &attrs),
324            _ => {}
325        }
326        let waits_for_operator = |s: EffectStatus| {
327            matches!(
328                s,
329                EffectStatus::NeedsIntervention | EffectStatus::CompensationFailed
330            )
331        };
332        if waits_for_operator(o.to) && !waits_for_operator(o.from) {
333            self.awaiting_intervention.add(1, &attrs);
334        } else if waits_for_operator(o.from) && !waits_for_operator(o.to) {
335            self.awaiting_intervention.add(-1, &attrs);
336        }
337        match o.transition {
338            Transition::ScheduleRetry => {
339                self.retries.add(1, &attrs);
340                if let Some(delay) = o
341                    .record
342                    .next_attempt_at
343                    .and_then(|at| at.duration_since(o.record.updated_at).ok())
344                {
345                    self.retry_delay.record(delay.as_secs_f64(), &attrs);
346                }
347            }
348            Transition::StartAttempt => {
349                let attempt = if o.record.attempt_count > 1 {
350                    "retry"
351                } else {
352                    "first"
353                };
354                self.attempts.add(1, &tagged(&attrs, "attempt", attempt));
355            }
356            Transition::StartVerification => self.verification_started.add(1, &attrs),
357            Transition::LeaseExpired => self.lease_expired.add(1, &attrs),
358            Transition::StartCompensation => self.compensation_started.add(1, &attrs),
359            Transition::RequestApproval => self.pending_approval.add(1, &attrs),
360            Transition::ResolvedApplied => self
361                .operator_resolutions
362                .add(1, &tagged(&attrs, "resolution", "applied")),
363            Transition::ResolvedNotApplied => self
364                .operator_resolutions
365                .add(1, &tagged(&attrs, "resolution", "not_applied")),
366            Transition::ResolvedRetry => self
367                .operator_resolutions
368                .add(1, &tagged(&attrs, "resolution", "retry")),
369            Transition::ResolvedCompensated => self
370                .operator_resolutions
371                .add(1, &tagged(&attrs, "resolution", "compensated")),
372            _ => {}
373        }
374        self.on_decision(o, &attrs);
375        match o.from {
376            EffectStatus::AwaitingApproval => self.pending_approval.add(-1, &attrs),
377            EffectStatus::Executing => {
378                let [name, kind] = attrs.clone();
379                let outcome = KeyValue::new("outcome", o.to.as_str());
380                self.attempt_duration
381                    .record(o.in_previous_status.as_secs_f64(), &[name, kind, outcome]);
382            }
383            EffectStatus::Unknown => self
384                .unknown_duration
385                .record(o.in_previous_status.as_secs_f64(), &attrs),
386            _ => {}
387        }
388    }
389
390    fn on_event(&self, event: &RuntimeEvent<'_>) {
391        match event {
392            RuntimeEvent::LeaseConflict { name } => self
393                .lease_conflicts
394                .add(1, &[KeyValue::new("effect.name", name.to_string())]),
395            RuntimeEvent::LeaseRenewalFailed => self.lease_renewal_failures.add(1, &[]),
396            RuntimeEvent::RecoveryPass { report, duration } => {
397                let result = match report {
398                    None => "failed",
399                    Some(report) if report.stopped => "stopped",
400                    Some(_) => "completed",
401                };
402                let result = [KeyValue::new("result", result)];
403                self.recovery_passes.add(1, &result);
404                self.recovery_duration
405                    .record(duration.as_secs_f64(), &result);
406                let Some(report) = report else { return };
407                for (outcome, n) in [
408                    ("resumed", report.resumed.len()),
409                    ("marked_unknown", report.marked_unknown.len()),
410                    ("skipped", report.skipped.len()),
411                    ("unhandled", report.unhandled.len()),
412                    ("resume_error", report.resume_errors.len()),
413                ] {
414                    if n > 0 {
415                        self.recovery_effects
416                            .add(count(n), &[KeyValue::new("outcome", outcome)]);
417                    }
418                }
419                let due =
420                    report.resumed.len() + report.unhandled.len() + report.resume_errors.len();
421                for (state, n) in [
422                    ("due", due),
423                    ("not_due", report.not_due.len()),
424                    ("awaiting_operator", report.waiting_for_operator.len()),
425                ] {
426                    self.recovery_backlog
427                        .record(count(n), &[KeyValue::new("state", state)]);
428                }
429            }
430            RuntimeEvent::Pruned { status, count } => self
431                .pruned
432                .add(*count, &[KeyValue::new("status", status.as_str())]),
433            _ => {}
434        }
435    }
436}