Skip to main content

harn_vm/triggers/
registry.rs

1use serde::{Deserialize, Serialize};
2use std::cell::RefCell;
3use std::collections::{BTreeMap, BTreeSet, VecDeque};
4use std::path::PathBuf;
5use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
6use std::sync::{Arc, Mutex};
7
8use time::OffsetDateTime;
9use uuid::Uuid;
10
11use crate::event_log::{active_event_log, AnyEventLog, EventLog, LogEvent, Topic};
12use crate::llm::trigger_predicate::TriggerPredicateBudget;
13use crate::secrets::{configured_default_chain, SecretProvider};
14use crate::triggers::test_util::clock;
15use crate::trust_graph::AutonomyTier;
16
17use super::aggregation::TriggerAggregationConfig;
18use super::dispatcher::TriggerRetryConfig;
19use super::flow_control::TriggerFlowControlConfig;
20use super::ProviderId;
21
22mod handler;
23mod id;
24
25pub use handler::{AgentScope, TargetExpr, TriggerHandlerSpec};
26pub use id::TriggerId;
27
28#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
29#[serde(rename_all = "snake_case")]
30pub enum TriggerState {
31    Registering,
32    Active,
33    Paused,
34    Draining,
35    Terminated,
36}
37
38impl TriggerState {
39    pub fn as_str(self) -> &'static str {
40        match self {
41            Self::Registering => "registering",
42            Self::Active => "active",
43            Self::Paused => "paused",
44            Self::Draining => "draining",
45            Self::Terminated => "terminated",
46        }
47    }
48}
49
50#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
51#[serde(rename_all = "snake_case")]
52pub enum TriggerBindingSource {
53    Manifest,
54    Dynamic,
55}
56
57impl TriggerBindingSource {
58    pub fn as_str(self) -> &'static str {
59        match self {
60            Self::Manifest => "manifest",
61            Self::Dynamic => "dynamic",
62        }
63    }
64}
65
66#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
67#[serde(rename_all = "snake_case")]
68pub enum TriggerBudgetExhaustionStrategy {
69    #[default]
70    False,
71    RetryLater,
72    Fail,
73    Warn,
74}
75
76impl TriggerBudgetExhaustionStrategy {
77    pub fn as_str(self) -> &'static str {
78        match self {
79            Self::False => "false",
80            Self::RetryLater => "retry_later",
81            Self::Fail => "fail",
82            Self::Warn => "warn",
83        }
84    }
85}
86
87#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
88pub struct OrchestratorBudgetConfig {
89    pub daily_cost_usd: Option<f64>,
90    pub hourly_cost_usd: Option<f64>,
91}
92
93#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
94pub struct OrchestratorBudgetSnapshot {
95    pub daily_cost_usd: Option<f64>,
96    pub hourly_cost_usd: Option<f64>,
97    pub cost_today_usd_micros: u64,
98    pub cost_hour_usd_micros: u64,
99    pub day_utc: i32,
100    pub hour_utc: i64,
101}
102
103#[derive(Debug)]
104struct OrchestratorBudgetState {
105    config: OrchestratorBudgetConfig,
106    day_utc: i32,
107    hour_utc: i64,
108    cost_today_usd_micros: u64,
109    cost_hour_usd_micros: u64,
110}
111
112impl Default for OrchestratorBudgetState {
113    fn default() -> Self {
114        Self {
115            config: OrchestratorBudgetConfig::default(),
116            day_utc: utc_day_key(),
117            hour_utc: utc_hour_key(),
118            cost_today_usd_micros: 0,
119            cost_hour_usd_micros: 0,
120        }
121    }
122}
123
124#[derive(Clone)]
125pub struct TriggerPredicateSpec {
126    pub raw: String,
127    pub callable: crate::value::VmCallable,
128}
129
130impl std::fmt::Debug for TriggerPredicateSpec {
131    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
132        f.debug_struct("TriggerPredicateSpec")
133            .field("raw", &self.raw)
134            .finish()
135    }
136}
137
138#[derive(Clone, Debug)]
139pub struct TriggerBindingSpec {
140    pub id: String,
141    pub source: TriggerBindingSource,
142    pub kind: String,
143    pub provider: ProviderId,
144    pub autonomy_tier: AutonomyTier,
145    pub handler: TriggerHandlerSpec,
146    pub dispatch_priority: super::worker_queue::WorkerQueuePriority,
147    pub when: Option<TriggerPredicateSpec>,
148    pub when_budget: Option<TriggerPredicateBudget>,
149    pub retry: TriggerRetryConfig,
150    pub match_events: Vec<String>,
151    pub dedupe_key: Option<String>,
152    pub dedupe_retention_days: u32,
153    pub filter: Option<String>,
154    pub daily_cost_usd: Option<f64>,
155    pub hourly_cost_usd: Option<f64>,
156    pub max_autonomous_decisions_per_hour: Option<u64>,
157    pub max_autonomous_decisions_per_day: Option<u64>,
158    pub on_budget_exhausted: TriggerBudgetExhaustionStrategy,
159    pub max_concurrent: Option<u32>,
160    pub flow_control: TriggerFlowControlConfig,
161    /// CH-04 (#1875): optional aggregation buffer. When set, the
162    /// dispatcher accumulates matching events into a per-(binding,
163    /// partition_key) buffer and dispatches the handler with a batched
164    /// event when `count` is reached or the `window` elapses.
165    pub aggregation: Option<TriggerAggregationConfig>,
166    pub manifest_path: Option<PathBuf>,
167    pub package_name: Option<String>,
168    pub definition_fingerprint: String,
169}
170
171#[derive(Debug)]
172pub struct TriggerMetrics {
173    pub received: AtomicU64,
174    pub dispatched: AtomicU64,
175    pub failed: AtomicU64,
176    pub dlq: AtomicU64,
177    pub last_received_ms: Mutex<Option<i64>>,
178    pub cost_total_usd_micros: AtomicU64,
179    pub cost_today_usd_micros: AtomicU64,
180    pub cost_hour_usd_micros: AtomicU64,
181    pub autonomous_decisions_total: AtomicU64,
182    pub autonomous_decisions_today: AtomicU64,
183    pub autonomous_decisions_hour: AtomicU64,
184}
185
186impl Default for TriggerMetrics {
187    fn default() -> Self {
188        Self {
189            received: AtomicU64::new(0),
190            dispatched: AtomicU64::new(0),
191            failed: AtomicU64::new(0),
192            dlq: AtomicU64::new(0),
193            last_received_ms: Mutex::new(None),
194            cost_total_usd_micros: AtomicU64::new(0),
195            cost_today_usd_micros: AtomicU64::new(0),
196            cost_hour_usd_micros: AtomicU64::new(0),
197            autonomous_decisions_total: AtomicU64::new(0),
198            autonomous_decisions_today: AtomicU64::new(0),
199            autonomous_decisions_hour: AtomicU64::new(0),
200        }
201    }
202}
203
204#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
205pub struct TriggerMetricsSnapshot {
206    pub received: u64,
207    pub dispatched: u64,
208    pub failed: u64,
209    pub dlq: u64,
210    pub in_flight: u64,
211    pub last_received_ms: Option<i64>,
212    pub cost_total_usd_micros: u64,
213    pub cost_today_usd_micros: u64,
214    pub cost_hour_usd_micros: u64,
215    pub autonomous_decisions_total: u64,
216    pub autonomous_decisions_today: u64,
217    pub autonomous_decisions_hour: u64,
218}
219
220pub struct TriggerBinding {
221    pub id: TriggerId,
222    pub version: u32,
223    pub source: TriggerBindingSource,
224    pub kind: String,
225    pub provider: ProviderId,
226    pub autonomy_tier: AutonomyTier,
227    pub handler: TriggerHandlerSpec,
228    pub dispatch_priority: super::worker_queue::WorkerQueuePriority,
229    pub when: Option<TriggerPredicateSpec>,
230    pub when_budget: Option<TriggerPredicateBudget>,
231    pub retry: TriggerRetryConfig,
232    pub match_events: Vec<String>,
233    pub dedupe_key: Option<String>,
234    pub dedupe_retention_days: u32,
235    pub filter: Option<String>,
236    pub daily_cost_usd: Option<f64>,
237    pub hourly_cost_usd: Option<f64>,
238    pub max_autonomous_decisions_per_hour: Option<u64>,
239    pub max_autonomous_decisions_per_day: Option<u64>,
240    pub on_budget_exhausted: TriggerBudgetExhaustionStrategy,
241    pub max_concurrent: Option<u32>,
242    pub flow_control: TriggerFlowControlConfig,
243    /// CH-04 (#1875): see `TriggerBindingSpec::aggregation`. Cloned at
244    /// registration so the dispatcher does not need to lock the registry
245    /// for every emit.
246    pub aggregation: Option<TriggerAggregationConfig>,
247    pub manifest_path: Option<PathBuf>,
248    pub package_name: Option<String>,
249    pub definition_fingerprint: String,
250    pub state: Mutex<TriggerState>,
251    pub metrics: TriggerMetrics,
252    pub in_flight: AtomicU64,
253    pub cancel_token: Arc<AtomicBool>,
254    pub predicate_state: Mutex<TriggerPredicateState>,
255}
256
257#[derive(Clone, Debug, Default)]
258pub struct TriggerPredicateState {
259    pub budget_day_utc: Option<i32>,
260    pub budget_hour_utc: Option<i64>,
261    pub consecutive_failures: u32,
262    pub breaker_open_until_ms: Option<i64>,
263    pub recent_cost_usd_micros: VecDeque<u64>,
264}
265
266impl std::fmt::Debug for TriggerBinding {
267    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
268        f.debug_struct("TriggerBinding")
269            .field("id", &self.id)
270            .field("version", &self.version)
271            .field("source", &self.source)
272            .field("kind", &self.kind)
273            .field("provider", &self.provider)
274            .field("handler_kind", &self.handler.kind())
275            .field("state", &self.state_snapshot())
276            .finish()
277    }
278}
279
280impl TriggerBinding {
281    pub fn snapshot(&self) -> TriggerBindingSnapshot {
282        TriggerBindingSnapshot {
283            id: self.id.as_str().to_string(),
284            version: self.version,
285            source: self.source,
286            kind: self.kind.clone(),
287            provider: self.provider.as_str().to_string(),
288            autonomy_tier: self.autonomy_tier,
289            handler_kind: self.handler.kind().to_string(),
290            state: self.state_snapshot(),
291            metrics: self.metrics_snapshot(),
292            daily_cost_usd: self.daily_cost_usd,
293            hourly_cost_usd: self.hourly_cost_usd,
294            max_autonomous_decisions_per_hour: self.max_autonomous_decisions_per_hour,
295            max_autonomous_decisions_per_day: self.max_autonomous_decisions_per_day,
296            on_budget_exhausted: self.on_budget_exhausted,
297        }
298    }
299
300    fn new(spec: TriggerBindingSpec, version: u32) -> Self {
301        Self {
302            id: TriggerId::new(spec.id),
303            version,
304            source: spec.source,
305            kind: spec.kind,
306            provider: spec.provider,
307            autonomy_tier: spec.autonomy_tier,
308            handler: spec.handler,
309            dispatch_priority: spec.dispatch_priority,
310            when: spec.when,
311            when_budget: spec.when_budget,
312            retry: spec.retry,
313            match_events: spec.match_events,
314            dedupe_key: spec.dedupe_key,
315            dedupe_retention_days: spec.dedupe_retention_days,
316            filter: spec.filter,
317            daily_cost_usd: spec.daily_cost_usd,
318            hourly_cost_usd: spec.hourly_cost_usd,
319            max_autonomous_decisions_per_hour: spec.max_autonomous_decisions_per_hour,
320            max_autonomous_decisions_per_day: spec.max_autonomous_decisions_per_day,
321            on_budget_exhausted: spec.on_budget_exhausted,
322            max_concurrent: spec.max_concurrent,
323            flow_control: spec.flow_control,
324            aggregation: spec.aggregation,
325            manifest_path: spec.manifest_path,
326            package_name: spec.package_name,
327            definition_fingerprint: spec.definition_fingerprint,
328            state: Mutex::new(TriggerState::Registering),
329            metrics: TriggerMetrics::default(),
330            in_flight: AtomicU64::new(0),
331            cancel_token: Arc::new(AtomicBool::new(false)),
332            predicate_state: Mutex::new(TriggerPredicateState::default()),
333        }
334    }
335
336    pub fn binding_key(&self) -> String {
337        format!("{}@v{}", self.id.as_str(), self.version)
338    }
339
340    pub fn state_snapshot(&self) -> TriggerState {
341        *self.state.lock().expect("trigger state poisoned")
342    }
343
344    pub fn metrics_snapshot(&self) -> TriggerMetricsSnapshot {
345        TriggerMetricsSnapshot {
346            received: self.metrics.received.load(Ordering::Relaxed),
347            dispatched: self.metrics.dispatched.load(Ordering::Relaxed),
348            failed: self.metrics.failed.load(Ordering::Relaxed),
349            dlq: self.metrics.dlq.load(Ordering::Relaxed),
350            in_flight: self.in_flight.load(Ordering::Relaxed),
351            last_received_ms: *self
352                .metrics
353                .last_received_ms
354                .lock()
355                .expect("trigger metrics poisoned"),
356            cost_total_usd_micros: self.metrics.cost_total_usd_micros.load(Ordering::Relaxed),
357            cost_today_usd_micros: self.metrics.cost_today_usd_micros.load(Ordering::Relaxed),
358            cost_hour_usd_micros: self.metrics.cost_hour_usd_micros.load(Ordering::Relaxed),
359            autonomous_decisions_total: self
360                .metrics
361                .autonomous_decisions_total
362                .load(Ordering::Relaxed),
363            autonomous_decisions_today: self
364                .metrics
365                .autonomous_decisions_today
366                .load(Ordering::Relaxed),
367            autonomous_decisions_hour: self
368                .metrics
369                .autonomous_decisions_hour
370                .load(Ordering::Relaxed),
371        }
372    }
373}
374
375#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
376pub struct TriggerBindingSnapshot {
377    pub id: String,
378    pub version: u32,
379    pub source: TriggerBindingSource,
380    pub kind: String,
381    pub provider: String,
382    pub autonomy_tier: AutonomyTier,
383    pub handler_kind: String,
384    pub state: TriggerState,
385    pub metrics: TriggerMetricsSnapshot,
386    pub daily_cost_usd: Option<f64>,
387    pub hourly_cost_usd: Option<f64>,
388    pub max_autonomous_decisions_per_hour: Option<u64>,
389    pub max_autonomous_decisions_per_day: Option<u64>,
390    pub on_budget_exhausted: TriggerBudgetExhaustionStrategy,
391}
392
393#[derive(Clone, Copy, Debug, PartialEq, Eq)]
394pub enum TriggerDispatchOutcome {
395    Dispatched,
396    Failed,
397    Dlq,
398}
399
400#[derive(Debug)]
401pub enum TriggerRegistryError {
402    DuplicateId(String),
403    InvalidSpec(String),
404    UnknownId(String),
405    UnknownBindingVersion { id: String, version: u32 },
406    EventLog(String),
407}
408
409impl std::fmt::Display for TriggerRegistryError {
410    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
411        match self {
412            Self::DuplicateId(id) => write!(f, "duplicate trigger id '{id}'"),
413            Self::InvalidSpec(message) | Self::EventLog(message) => f.write_str(message),
414            Self::UnknownId(id) => write!(f, "unknown trigger id '{id}'"),
415            Self::UnknownBindingVersion { id, version } => {
416                write!(f, "unknown trigger binding '{id}' version {version}")
417            }
418        }
419    }
420}
421
422impl std::error::Error for TriggerRegistryError {}
423
424#[derive(Default)]
425pub struct TriggerRegistry {
426    bindings: BTreeMap<String, Vec<Arc<TriggerBinding>>>,
427    by_provider: BTreeMap<String, BTreeSet<String>>,
428    event_log: Option<Arc<AnyEventLog>>,
429    secret_provider: Option<Arc<dyn SecretProvider>>,
430}
431
432thread_local! {
433    static TRIGGER_REGISTRY: RefCell<TriggerRegistry> = RefCell::new(TriggerRegistry::default());
434}
435
436thread_local! {
437    static ORCHESTRATOR_BUDGET: RefCell<OrchestratorBudgetState> =
438        RefCell::new(OrchestratorBudgetState::default());
439}
440
441const TERMINATED_VERSION_RETENTION_LIMIT: usize = 2;
442
443const TRIGGERS_LIFECYCLE_TOPIC: &str = "triggers.lifecycle";
444const PREDICATE_COST_WINDOW: usize = 100;
445
446#[derive(Clone, Debug, Deserialize)]
447struct LifecycleStateTransitionRecord {
448    id: String,
449    version: u32,
450    #[serde(default)]
451    definition_fingerprint: Option<String>,
452    to_state: TriggerState,
453}
454
455#[derive(Clone, Debug)]
456struct HistoricalLifecycleRecord {
457    occurred_at_ms: i64,
458    transition: LifecycleStateTransitionRecord,
459}
460
461#[derive(Clone, Copy, Debug, PartialEq, Eq)]
462pub struct RecordedTriggerBinding {
463    pub version: u32,
464    pub received_at: OffsetDateTime,
465}
466
467#[derive(Clone, Copy, Debug, Default)]
468struct HistoricalVersionLookup {
469    matching_version: Option<u32>,
470    max_version: Option<u32>,
471}
472
473pub fn clear_trigger_registry() {
474    TRIGGER_REGISTRY.with(|slot| {
475        *slot.borrow_mut() = TriggerRegistry::default();
476    });
477    clear_orchestrator_budget();
478    super::aggregation::clear_aggregation_state();
479}
480
481pub fn install_orchestrator_budget(config: OrchestratorBudgetConfig) {
482    ORCHESTRATOR_BUDGET.with(|slot| {
483        let mut state = slot.borrow_mut();
484        rollover_orchestrator_budget(&mut state);
485        state.config = config;
486    });
487}
488
489pub fn clear_orchestrator_budget() {
490    ORCHESTRATOR_BUDGET.with(|slot| {
491        *slot.borrow_mut() = OrchestratorBudgetState::default();
492    });
493}
494
495pub fn snapshot_orchestrator_budget() -> OrchestratorBudgetSnapshot {
496    ORCHESTRATOR_BUDGET.with(|slot| {
497        let mut state = slot.borrow_mut();
498        rollover_orchestrator_budget(&mut state);
499        OrchestratorBudgetSnapshot {
500            daily_cost_usd: state.config.daily_cost_usd,
501            hourly_cost_usd: state.config.hourly_cost_usd,
502            cost_today_usd_micros: state.cost_today_usd_micros,
503            cost_hour_usd_micros: state.cost_hour_usd_micros,
504            day_utc: state.day_utc,
505            hour_utc: state.hour_utc,
506        }
507    })
508}
509
510pub fn note_orchestrator_budget_cost(cost_usd_micros: u64) {
511    if cost_usd_micros == 0 {
512        return;
513    }
514    ORCHESTRATOR_BUDGET.with(|slot| {
515        let mut state = slot.borrow_mut();
516        rollover_orchestrator_budget(&mut state);
517        state.cost_today_usd_micros = state.cost_today_usd_micros.saturating_add(cost_usd_micros);
518        state.cost_hour_usd_micros = state.cost_hour_usd_micros.saturating_add(cost_usd_micros);
519    });
520}
521
522pub(crate) fn note_binding_budget_cost(binding: &TriggerBinding, cost_usd_micros: u64) {
523    if cost_usd_micros == 0 {
524        return;
525    }
526    reset_binding_budget_windows(binding);
527    binding
528        .metrics
529        .cost_total_usd_micros
530        .fetch_add(cost_usd_micros, Ordering::Relaxed);
531    binding
532        .metrics
533        .cost_today_usd_micros
534        .fetch_add(cost_usd_micros, Ordering::Relaxed);
535    binding
536        .metrics
537        .cost_hour_usd_micros
538        .fetch_add(cost_usd_micros, Ordering::Relaxed);
539}
540
541pub fn orchestrator_budget_would_exceed(expected_cost_usd_micros: u64) -> Option<&'static str> {
542    ORCHESTRATOR_BUDGET.with(|slot| {
543        let mut state = slot.borrow_mut();
544        rollover_orchestrator_budget(&mut state);
545        if state.config.hourly_cost_usd.is_some_and(|limit| {
546            micros_to_usd(
547                state
548                    .cost_hour_usd_micros
549                    .saturating_add(expected_cost_usd_micros),
550            ) > limit
551        }) {
552            return Some("orchestrator_hourly_budget_exceeded");
553        }
554        if state.config.daily_cost_usd.is_some_and(|limit| {
555            micros_to_usd(
556                state
557                    .cost_today_usd_micros
558                    .saturating_add(expected_cost_usd_micros),
559            ) > limit
560        }) {
561            return Some("orchestrator_daily_budget_exceeded");
562        }
563        None
564    })
565}
566
567pub fn reset_binding_budget_windows(binding: &TriggerBinding) {
568    let today = utc_day_key();
569    let hour = utc_hour_key();
570    let mut state = binding
571        .predicate_state
572        .lock()
573        .expect("trigger predicate state poisoned");
574    if state.budget_day_utc != Some(today) {
575        state.budget_day_utc = Some(today);
576        binding
577            .metrics
578            .cost_today_usd_micros
579            .store(0, Ordering::Relaxed);
580        binding
581            .metrics
582            .autonomous_decisions_today
583            .store(0, Ordering::Relaxed);
584    }
585    if state.budget_hour_utc != Some(hour) {
586        state.budget_hour_utc = Some(hour);
587        binding
588            .metrics
589            .cost_hour_usd_micros
590            .store(0, Ordering::Relaxed);
591        binding
592            .metrics
593            .autonomous_decisions_hour
594            .store(0, Ordering::Relaxed);
595    }
596}
597
598pub fn binding_budget_would_exceed(
599    binding: &TriggerBinding,
600    expected_cost_usd_micros: u64,
601) -> Option<&'static str> {
602    reset_binding_budget_windows(binding);
603    if binding.hourly_cost_usd.is_some_and(|limit| {
604        micros_to_usd(
605            binding
606                .metrics
607                .cost_hour_usd_micros
608                .load(Ordering::Relaxed)
609                .saturating_add(expected_cost_usd_micros),
610        ) > limit
611    }) {
612        return Some("hourly_budget_exceeded");
613    }
614    if binding.daily_cost_usd.is_some_and(|limit| {
615        micros_to_usd(
616            binding
617                .metrics
618                .cost_today_usd_micros
619                .load(Ordering::Relaxed)
620                .saturating_add(expected_cost_usd_micros),
621        ) > limit
622    }) {
623        return Some("daily_budget_exceeded");
624    }
625    None
626}
627
628pub fn binding_autonomy_budget_would_exceed(binding: &TriggerBinding) -> Option<&'static str> {
629    reset_binding_budget_windows(binding);
630    if binding
631        .max_autonomous_decisions_per_hour
632        .is_some_and(|limit| {
633            binding
634                .metrics
635                .autonomous_decisions_hour
636                .load(Ordering::Relaxed)
637                .saturating_add(1)
638                > limit
639        })
640    {
641        return Some("hourly_autonomy_budget_exceeded");
642    }
643    if binding
644        .max_autonomous_decisions_per_day
645        .is_some_and(|limit| {
646            binding
647                .metrics
648                .autonomous_decisions_today
649                .load(Ordering::Relaxed)
650                .saturating_add(1)
651                > limit
652        })
653    {
654        return Some("daily_autonomy_budget_exceeded");
655    }
656    None
657}
658
659pub fn note_autonomous_decision(binding: &TriggerBinding) {
660    reset_binding_budget_windows(binding);
661    binding
662        .metrics
663        .autonomous_decisions_total
664        .fetch_add(1, Ordering::Relaxed);
665    binding
666        .metrics
667        .autonomous_decisions_today
668        .fetch_add(1, Ordering::Relaxed);
669    binding
670        .metrics
671        .autonomous_decisions_hour
672        .fetch_add(1, Ordering::Relaxed);
673}
674
675pub fn expected_predicate_cost_usd_micros(binding: &TriggerBinding) -> u64 {
676    let state = binding
677        .predicate_state
678        .lock()
679        .expect("trigger predicate state poisoned");
680    if let Some(average) = average_cost_sample_micros(&state.recent_cost_usd_micros) {
681        return average;
682    }
683    binding
684        .when_budget
685        .as_ref()
686        .and_then(|budget| budget.max_cost_usd)
687        .map(usd_to_micros)
688        .unwrap_or_default()
689}
690
691pub fn record_predicate_cost_sample(binding: &TriggerBinding, cost_usd_micros: u64) {
692    let mut state = binding
693        .predicate_state
694        .lock()
695        .expect("trigger predicate state poisoned");
696    state.recent_cost_usd_micros.push_back(cost_usd_micros);
697    while state.recent_cost_usd_micros.len() > PREDICATE_COST_WINDOW {
698        state.recent_cost_usd_micros.pop_front();
699    }
700}
701
702fn average_cost_sample_micros(samples: &VecDeque<u64>) -> Option<u64> {
703    if samples.is_empty() {
704        return None;
705    }
706    let total: u128 = samples.iter().map(|sample| u128::from(*sample)).sum();
707    Some((total / samples.len() as u128) as u64)
708}
709
710pub fn usd_to_micros(value: f64) -> u64 {
711    if !value.is_finite() || value <= 0.0 {
712        return 0;
713    }
714    (value * 1_000_000.0).ceil() as u64
715}
716
717pub fn micros_to_usd(value: u64) -> f64 {
718    value as f64 / 1_000_000.0
719}
720
721fn rollover_orchestrator_budget(state: &mut OrchestratorBudgetState) {
722    let today = utc_day_key();
723    let hour = utc_hour_key();
724    if state.day_utc != today {
725        state.day_utc = today;
726        state.cost_today_usd_micros = 0;
727    }
728    if state.hour_utc != hour {
729        state.hour_utc = hour;
730        state.cost_hour_usd_micros = 0;
731    }
732}
733
734fn utc_day_key() -> i32 {
735    (clock::now_utc().date()
736        - time::Date::from_calendar_date(1970, time::Month::January, 1).expect("valid epoch date"))
737    .whole_days() as i32
738}
739
740fn utc_hour_key() -> i64 {
741    clock::now_utc().unix_timestamp() / 3_600
742}
743
744pub fn snapshot_trigger_bindings() -> Vec<TriggerBindingSnapshot> {
745    TRIGGER_REGISTRY.with(|slot| {
746        let registry = slot.borrow();
747        let mut snapshots = Vec::new();
748        for bindings in registry.bindings.values() {
749            for binding in bindings {
750                snapshots.push(binding.snapshot());
751            }
752        }
753        snapshots.sort_by(|left, right| {
754            left.id
755                .cmp(&right.id)
756                .then(left.version.cmp(&right.version))
757                .then(left.state.as_str().cmp(right.state.as_str()))
758        });
759        snapshots
760    })
761}
762
763#[allow(clippy::arc_with_non_send_sync)]
764pub fn resolve_trigger_binding_as_of(
765    id: &str,
766    as_of: OffsetDateTime,
767) -> Result<Arc<TriggerBinding>, TriggerRegistryError> {
768    let version = binding_version_as_of(id, as_of)?;
769    resolve_trigger_binding_version(id, version)
770}
771
772#[allow(clippy::arc_with_non_send_sync)]
773pub fn resolve_live_or_as_of(
774    id: &str,
775    recorded: RecordedTriggerBinding,
776) -> Result<Arc<TriggerBinding>, TriggerRegistryError> {
777    match resolve_live_trigger_binding(id, Some(recorded.version)) {
778        Ok(binding) => Ok(binding),
779        Err(TriggerRegistryError::UnknownBindingVersion { .. }) => {
780            let binding = resolve_trigger_binding_as_of(id, recorded.received_at)?;
781            let mut metadata = BTreeMap::new();
782            metadata.insert("trigger_id".to_string(), serde_json::json!(id));
783            metadata.insert(
784                "recorded_version".to_string(),
785                serde_json::json!(recorded.version),
786            );
787            metadata.insert(
788                "received_at".to_string(),
789                serde_json::json!(recorded
790                    .received_at
791                    .format(&time::format_description::well_known::Rfc3339)
792                    .unwrap_or_else(|_| recorded.received_at.to_string())),
793            );
794            metadata.insert(
795                "resolved_version".to_string(),
796                serde_json::json!(binding.version),
797            );
798            crate::events::log_warn_meta(
799                "replay.binding_version_gc_fallback",
800                "trigger replay fell back to lifecycle history after binding version GC",
801                metadata,
802            );
803            Ok(binding)
804        }
805        Err(error) => Err(error),
806    }
807}
808
809pub fn binding_version_as_of(id: &str, as_of: OffsetDateTime) -> Result<u32, TriggerRegistryError> {
810    TRIGGER_REGISTRY.with(|slot| {
811        let registry = slot.borrow();
812        registry.binding_version_as_of(id, as_of)
813    })
814}
815
816#[allow(clippy::arc_with_non_send_sync)]
817fn resolve_trigger_binding_version(
818    id: &str,
819    version: u32,
820) -> Result<Arc<TriggerBinding>, TriggerRegistryError> {
821    TRIGGER_REGISTRY.with(|slot| {
822        let registry = slot.borrow();
823        registry
824            .binding(id, version)
825            .ok_or_else(|| TriggerRegistryError::UnknownBindingVersion {
826                id: id.to_string(),
827                version,
828            })
829    })
830}
831
832#[allow(clippy::arc_with_non_send_sync)]
833pub fn resolve_live_trigger_binding(
834    id: &str,
835    version: Option<u32>,
836) -> Result<Arc<TriggerBinding>, TriggerRegistryError> {
837    TRIGGER_REGISTRY.with(|slot| {
838        let registry = slot.borrow();
839        if let Some(version) = version {
840            let binding = registry.binding(id, version).ok_or_else(|| {
841                TriggerRegistryError::UnknownBindingVersion {
842                    id: id.to_string(),
843                    version,
844                }
845            })?;
846            if binding.state_snapshot() == TriggerState::Terminated {
847                return Err(TriggerRegistryError::UnknownBindingVersion {
848                    id: id.to_string(),
849                    version,
850                });
851            }
852            return Ok(binding);
853        }
854
855        registry
856            .live_bindings_any_source(id)
857            .into_iter()
858            .max_by_key(|binding| binding.version)
859            .ok_or_else(|| TriggerRegistryError::UnknownId(id.to_string()))
860    })
861}
862
863/// Snapshot active channel-source trigger bindings whose `channel:` selector
864/// in `match_events[0]` matches the supplied (scope, scope_id, name) tuple.
865///
866/// Used by [`crate::channels`] fan-out (CH-02 / #1872). Returns bindings
867/// sorted by id then version so dispatch ordering is deterministic.
868pub(crate) fn channel_bindings_matching(
869    scope: &str,
870    scope_id: &str,
871    name: &str,
872) -> Vec<Arc<TriggerBinding>> {
873    TRIGGER_REGISTRY.with(|slot| {
874        let registry = slot.borrow();
875        let Some(binding_ids) = registry.by_provider.get("channel") else {
876            return Vec::new();
877        };
878        let mut bindings = Vec::new();
879        for id in binding_ids {
880            let Some(versions) = registry.bindings.get(id) else {
881                continue;
882            };
883            for binding in versions {
884                if binding.state_snapshot() != TriggerState::Active {
885                    continue;
886                }
887                let Some(selector_raw) = binding.match_events.first() else {
888                    continue;
889                };
890                let Ok(selector) = crate::channels::ChannelSelector::parse(selector_raw) else {
891                    continue;
892                };
893                // For tenant-default ("Current") selectors we resolve "current"
894                // to the scope_id of the emit, since the trigger and the
895                // emitter share the same registry / runtime context.
896                if !selector.matches(scope, scope_id, name, scope_id) {
897                    continue;
898                }
899                bindings.push(binding.clone());
900            }
901        }
902        bindings.sort_by(|left, right| {
903            left.id
904                .as_str()
905                .cmp(right.id.as_str())
906                .then(left.version.cmp(&right.version))
907        });
908        bindings
909    })
910}
911
912pub(crate) fn matching_bindings(event: &super::TriggerEvent) -> Vec<Arc<TriggerBinding>> {
913    TRIGGER_REGISTRY.with(|slot| {
914        let registry = slot.borrow();
915        let Some(binding_ids) = registry.by_provider.get(event.provider.as_str()) else {
916            return Vec::new();
917        };
918
919        let mut bindings = Vec::new();
920        for id in binding_ids {
921            let Some(versions) = registry.bindings.get(id) else {
922                continue;
923            };
924            for binding in versions {
925                if binding.state_snapshot() != TriggerState::Active {
926                    continue;
927                }
928                if !binding.match_events.is_empty()
929                    && !binding
930                        .match_events
931                        .iter()
932                        .any(|kind| trigger_event_kind_matches(event, kind))
933                {
934                    continue;
935                }
936                bindings.push(binding.clone());
937            }
938        }
939
940        bindings.sort_by(|left, right| {
941            left.id
942                .as_str()
943                .cmp(right.id.as_str())
944                .then(left.version.cmp(&right.version))
945        });
946        bindings
947    })
948}
949
950fn trigger_event_kind_matches(event: &super::TriggerEvent, expected: &str) -> bool {
951    if expected == event.kind {
952        return true;
953    }
954    expected
955        .strip_prefix(event.provider.as_str())
956        .and_then(|rest| rest.strip_prefix('.'))
957        .is_some_and(|kind| kind == event.kind)
958}
959
960pub async fn install_manifest_triggers(
961    specs: Vec<TriggerBindingSpec>,
962) -> Result<(), TriggerRegistryError> {
963    let (event_log, events) = TRIGGER_REGISTRY.with(|slot| {
964        let registry = &mut *slot.borrow_mut();
965        registry.refresh_runtime_context();
966        let mut touched_ids = BTreeSet::new();
967
968        let mut incoming = BTreeMap::new();
969        for spec in specs {
970            let spec_id = spec.id.clone();
971            if spec.source != TriggerBindingSource::Manifest {
972                return Err(TriggerRegistryError::InvalidSpec(format!(
973                    "manifest install received non-manifest trigger '{spec_id}'"
974                )));
975            }
976            if spec_id.trim().is_empty() {
977                return Err(TriggerRegistryError::InvalidSpec(
978                    "manifest trigger id cannot be empty".to_string(),
979                ));
980            }
981            if incoming.insert(spec_id.clone(), spec).is_some() {
982                return Err(TriggerRegistryError::DuplicateId(spec_id));
983            }
984        }
985
986        let mut lifecycle = Vec::new();
987        let existing_ids: Vec<String> = registry
988            .bindings
989            .iter()
990            .filter(|(_, bindings)| {
991                bindings.iter().any(|binding| {
992                    binding.source == TriggerBindingSource::Manifest
993                        && binding.state_snapshot() != TriggerState::Terminated
994                })
995            })
996            .map(|(id, _)| id.clone())
997            .collect();
998
999        for id in existing_ids {
1000            let live_manifest = registry.live_bindings(&id, TriggerBindingSource::Manifest);
1001            let Some(spec) = incoming.remove(&id) else {
1002                for binding in live_manifest {
1003                    registry.transition_binding_to_draining(&binding, &mut lifecycle);
1004                }
1005                touched_ids.insert(id.clone());
1006                continue;
1007            };
1008
1009            let has_matching_active = live_manifest.iter().any(|binding| {
1010                binding.definition_fingerprint == spec.definition_fingerprint
1011                    && matches!(
1012                        binding.state_snapshot(),
1013                        TriggerState::Registering | TriggerState::Active
1014                    )
1015            });
1016            if has_matching_active {
1017                continue;
1018            }
1019
1020            for binding in live_manifest {
1021                registry.transition_binding_to_draining(&binding, &mut lifecycle);
1022            }
1023
1024            let version = registry.next_version_for_spec(&spec);
1025            registry.register_binding(spec, version, &mut lifecycle);
1026            touched_ids.insert(id.clone());
1027        }
1028
1029        for spec in incoming.into_values() {
1030            touched_ids.insert(spec.id.clone());
1031            let version = registry.next_version_for_spec(&spec);
1032            registry.register_binding(spec, version, &mut lifecycle);
1033        }
1034
1035        for id in touched_ids {
1036            registry.gc_terminated_versions(&id);
1037        }
1038
1039        Ok((registry.event_log.clone(), lifecycle))
1040    })?;
1041
1042    append_lifecycle_events(event_log, events).await
1043}
1044
1045pub async fn dynamic_register(
1046    mut spec: TriggerBindingSpec,
1047) -> Result<TriggerId, TriggerRegistryError> {
1048    if spec.id.trim().is_empty() {
1049        spec.id = format!("dynamic_trigger_{}", Uuid::now_v7());
1050    }
1051    spec.source = TriggerBindingSource::Dynamic;
1052    let id = spec.id.clone();
1053    let (event_log, events) = TRIGGER_REGISTRY.with(|slot| {
1054        let registry = &mut *slot.borrow_mut();
1055        registry.refresh_runtime_context();
1056
1057        if registry.bindings.contains_key(id.as_str()) {
1058            return Err(TriggerRegistryError::DuplicateId(id.clone()));
1059        }
1060
1061        let mut lifecycle = Vec::new();
1062        let version = registry.next_version_for_spec(&spec);
1063        registry.register_binding(spec, version, &mut lifecycle);
1064        Ok((registry.event_log.clone(), lifecycle))
1065    })?;
1066
1067    append_lifecycle_events(event_log, events).await?;
1068    Ok(TriggerId::new(id))
1069}
1070
1071pub async fn dynamic_deregister(id: &str) -> Result<(), TriggerRegistryError> {
1072    let (event_log, events) = TRIGGER_REGISTRY.with(|slot| {
1073        let registry = &mut *slot.borrow_mut();
1074        let live_dynamic = registry.live_bindings(id, TriggerBindingSource::Dynamic);
1075        if live_dynamic.is_empty() {
1076            return Err(TriggerRegistryError::UnknownId(id.to_string()));
1077        }
1078
1079        let mut lifecycle = Vec::new();
1080        for binding in live_dynamic {
1081            registry.transition_binding_to_draining(&binding, &mut lifecycle);
1082        }
1083        Ok((registry.event_log.clone(), lifecycle))
1084    })?;
1085
1086    append_lifecycle_events(event_log, events).await
1087}
1088
1089pub async fn drain(id: &str) -> Result<(), TriggerRegistryError> {
1090    let (event_log, events) = TRIGGER_REGISTRY.with(|slot| {
1091        let registry = &mut *slot.borrow_mut();
1092        let live = registry.live_bindings_any_source(id);
1093        if live.is_empty() {
1094            return Err(TriggerRegistryError::UnknownId(id.to_string()));
1095        }
1096
1097        let mut lifecycle = Vec::new();
1098        for binding in live {
1099            registry.transition_binding_to_draining(&binding, &mut lifecycle);
1100        }
1101        Ok((registry.event_log.clone(), lifecycle))
1102    })?;
1103
1104    append_lifecycle_events(event_log, events).await
1105}
1106
1107pub async fn pause(id: &str) -> Result<(), TriggerRegistryError> {
1108    let (event_log, events) = TRIGGER_REGISTRY.with(|slot| {
1109        let registry = &mut *slot.borrow_mut();
1110        let live = registry.live_bindings_any_source(id);
1111        if live.is_empty() {
1112            return Err(TriggerRegistryError::UnknownId(id.to_string()));
1113        }
1114
1115        let mut lifecycle = Vec::new();
1116        for binding in live {
1117            match binding.state_snapshot() {
1118                TriggerState::Registering | TriggerState::Active => {
1119                    registry.transition_binding_state(
1120                        &binding,
1121                        TriggerState::Paused,
1122                        &mut lifecycle,
1123                    );
1124                }
1125                TriggerState::Paused | TriggerState::Draining | TriggerState::Terminated => {}
1126            }
1127        }
1128        Ok((registry.event_log.clone(), lifecycle))
1129    })?;
1130
1131    append_lifecycle_events(event_log, events).await
1132}
1133
1134pub async fn resume(id: &str) -> Result<(), TriggerRegistryError> {
1135    let (event_log, events) = TRIGGER_REGISTRY.with(|slot| {
1136        let registry = &mut *slot.borrow_mut();
1137        let live = registry.live_bindings_any_source(id);
1138        if live.is_empty() {
1139            return Err(TriggerRegistryError::UnknownId(id.to_string()));
1140        }
1141
1142        let mut lifecycle = Vec::new();
1143        for binding in live {
1144            if binding.state_snapshot() == TriggerState::Paused {
1145                registry.transition_binding_state(&binding, TriggerState::Active, &mut lifecycle);
1146            }
1147        }
1148        Ok((registry.event_log.clone(), lifecycle))
1149    })?;
1150
1151    append_lifecycle_events(event_log, events).await
1152}
1153
1154fn pin_trigger_binding_inner(
1155    id: &str,
1156    version: u32,
1157    allow_terminated: bool,
1158) -> Result<(), TriggerRegistryError> {
1159    TRIGGER_REGISTRY.with(|slot| {
1160        let registry = slot.borrow();
1161        let binding = registry.binding(id, version).ok_or_else(|| {
1162            TriggerRegistryError::UnknownBindingVersion {
1163                id: id.to_string(),
1164                version,
1165            }
1166        })?;
1167        match binding.state_snapshot() {
1168            TriggerState::Paused => Err(TriggerRegistryError::InvalidSpec(format!(
1169                "trigger binding '{id}' version {version} is paused"
1170            ))),
1171            TriggerState::Terminated if !allow_terminated => {
1172                Err(TriggerRegistryError::InvalidSpec(format!(
1173                    "trigger binding '{id}' version {version} is terminated"
1174                )))
1175            }
1176            _ => {
1177                binding.in_flight.fetch_add(1, Ordering::Relaxed);
1178                Ok(())
1179            }
1180        }
1181    })
1182}
1183
1184pub fn pin_trigger_binding(id: &str, version: u32) -> Result<(), TriggerRegistryError> {
1185    pin_trigger_binding_inner(id, version, false)
1186}
1187
1188pub async fn unpin_trigger_binding(id: &str, version: u32) -> Result<(), TriggerRegistryError> {
1189    let (event_log, events) = TRIGGER_REGISTRY.with(|slot| {
1190        let registry = &mut *slot.borrow_mut();
1191        let binding = registry.binding(id, version).ok_or_else(|| {
1192            TriggerRegistryError::UnknownBindingVersion {
1193                id: id.to_string(),
1194                version,
1195            }
1196        })?;
1197        let current = binding.in_flight.load(Ordering::Relaxed);
1198        if current == 0 {
1199            return Err(TriggerRegistryError::InvalidSpec(format!(
1200                "trigger binding '{id}' version {version} has no in-flight events"
1201            )));
1202        }
1203        binding.in_flight.fetch_sub(1, Ordering::Relaxed);
1204
1205        let mut lifecycle = Vec::new();
1206        registry.maybe_finalize_draining(&binding, &mut lifecycle);
1207        registry.gc_terminated_versions(binding.id.as_str());
1208        Ok((registry.event_log.clone(), lifecycle))
1209    })?;
1210
1211    append_lifecycle_events(event_log, events).await
1212}
1213
1214pub fn begin_in_flight(id: &str, version: u32) -> Result<(), TriggerRegistryError> {
1215    begin_in_flight_inner(id, version, false)
1216}
1217
1218pub(crate) fn begin_replay_in_flight(id: &str, version: u32) -> Result<(), TriggerRegistryError> {
1219    begin_in_flight_inner(id, version, true)
1220}
1221
1222fn begin_in_flight_inner(
1223    id: &str,
1224    version: u32,
1225    allow_terminated: bool,
1226) -> Result<(), TriggerRegistryError> {
1227    pin_trigger_binding_inner(id, version, allow_terminated)?;
1228    TRIGGER_REGISTRY.with(|slot| {
1229        let registry = slot.borrow();
1230        let binding = registry.binding(id, version).ok_or_else(|| {
1231            TriggerRegistryError::UnknownBindingVersion {
1232                id: id.to_string(),
1233                version,
1234            }
1235        })?;
1236        binding.metrics.received.fetch_add(1, Ordering::Relaxed);
1237        *binding
1238            .metrics
1239            .last_received_ms
1240            .lock()
1241            .expect("trigger metrics poisoned") = Some(now_ms());
1242        Ok(())
1243    })
1244}
1245
1246pub async fn finish_in_flight(
1247    id: &str,
1248    version: u32,
1249    outcome: TriggerDispatchOutcome,
1250) -> Result<(), TriggerRegistryError> {
1251    TRIGGER_REGISTRY.with(|slot| {
1252        let registry = &mut *slot.borrow_mut();
1253        let binding = registry.binding(id, version).ok_or_else(|| {
1254            TriggerRegistryError::UnknownBindingVersion {
1255                id: id.to_string(),
1256                version,
1257            }
1258        })?;
1259        let current = binding.in_flight.load(Ordering::Relaxed);
1260        if current == 0 {
1261            return Err(TriggerRegistryError::InvalidSpec(format!(
1262                "trigger binding '{id}' version {version} has no in-flight events"
1263            )));
1264        }
1265        match outcome {
1266            TriggerDispatchOutcome::Dispatched => {
1267                binding.metrics.dispatched.fetch_add(1, Ordering::Relaxed);
1268            }
1269            TriggerDispatchOutcome::Failed => {
1270                binding.metrics.failed.fetch_add(1, Ordering::Relaxed);
1271            }
1272            TriggerDispatchOutcome::Dlq => {
1273                binding.metrics.dlq.fetch_add(1, Ordering::Relaxed);
1274            }
1275        }
1276        Ok(())
1277    })?;
1278
1279    unpin_trigger_binding(id, version).await
1280}
1281
1282impl TriggerRegistry {
1283    fn refresh_runtime_context(&mut self) {
1284        if self.event_log.is_none() {
1285            self.event_log = active_event_log();
1286        }
1287        if self.secret_provider.is_none() {
1288            self.secret_provider = default_secret_provider();
1289        }
1290    }
1291
1292    fn binding(&self, id: &str, version: u32) -> Option<Arc<TriggerBinding>> {
1293        self.bindings
1294            .get(id)
1295            .and_then(|bindings| bindings.iter().find(|binding| binding.version == version))
1296            .cloned()
1297    }
1298
1299    fn live_bindings(&self, id: &str, source: TriggerBindingSource) -> Vec<Arc<TriggerBinding>> {
1300        self.bindings
1301            .get(id)
1302            .into_iter()
1303            .flat_map(|bindings| bindings.iter())
1304            .filter(|binding| {
1305                binding.source == source && binding.state_snapshot() != TriggerState::Terminated
1306            })
1307            .cloned()
1308            .collect()
1309    }
1310
1311    fn live_bindings_any_source(&self, id: &str) -> Vec<Arc<TriggerBinding>> {
1312        self.bindings
1313            .get(id)
1314            .into_iter()
1315            .flat_map(|bindings| bindings.iter())
1316            .filter(|binding| binding.state_snapshot() != TriggerState::Terminated)
1317            .cloned()
1318            .collect()
1319    }
1320
1321    fn next_version_for_spec(&self, spec: &TriggerBindingSpec) -> u32 {
1322        if let Some(version) = self
1323            .bindings
1324            .get(spec.id.as_str())
1325            .into_iter()
1326            .flat_map(|bindings| bindings.iter())
1327            .find(|binding| binding.definition_fingerprint == spec.definition_fingerprint)
1328            .map(|binding| binding.version)
1329        {
1330            return version;
1331        }
1332
1333        let historical =
1334            self.historical_versions_for(spec.id.as_str(), spec.definition_fingerprint.as_str());
1335        if let Some(version) = historical.matching_version {
1336            return version;
1337        }
1338
1339        self.bindings
1340            .get(spec.id.as_str())
1341            .into_iter()
1342            .flat_map(|bindings| bindings.iter())
1343            .map(|binding| binding.version)
1344            .chain(historical.max_version)
1345            .max()
1346            .unwrap_or(0)
1347            + 1
1348    }
1349
1350    fn gc_terminated_versions(&mut self, id: &str) {
1351        let Some(bindings) = self.bindings.get_mut(id) else {
1352            return;
1353        };
1354
1355        let mut newest_versions: Vec<u32> =
1356            bindings.iter().map(|binding| binding.version).collect();
1357        newest_versions.sort_unstable_by(|left, right| right.cmp(left));
1358        newest_versions.truncate(TERMINATED_VERSION_RETENTION_LIMIT);
1359        let retained_versions: BTreeSet<u32> = newest_versions.into_iter().collect();
1360
1361        bindings.retain(|binding| {
1362            binding.state_snapshot() != TriggerState::Terminated
1363                || retained_versions.contains(&binding.version)
1364        });
1365
1366        if bindings.is_empty() {
1367            self.bindings.remove(id);
1368        }
1369    }
1370
1371    fn historical_versions_for(&self, id: &str, fingerprint: &str) -> HistoricalVersionLookup {
1372        let mut lookup = HistoricalVersionLookup::default();
1373        for record in self.lifecycle_records_for(id) {
1374            lookup.max_version = Some(
1375                lookup
1376                    .max_version
1377                    .unwrap_or(0)
1378                    .max(record.transition.version),
1379            );
1380            if record.transition.definition_fingerprint.as_deref() == Some(fingerprint) {
1381                lookup.matching_version = Some(record.transition.version);
1382            }
1383        }
1384        lookup
1385    }
1386
1387    fn binding_version_as_of(
1388        &self,
1389        id: &str,
1390        as_of: OffsetDateTime,
1391    ) -> Result<u32, TriggerRegistryError> {
1392        let cutoff_ms = harn_clock::offset_datetime_to_ms(as_of);
1393        let mut active_version = None;
1394        for record in self.lifecycle_records_for(id) {
1395            if record.occurred_at_ms > cutoff_ms {
1396                break;
1397            }
1398            match record.transition.to_state {
1399                TriggerState::Active => active_version = Some(record.transition.version),
1400                TriggerState::Paused | TriggerState::Draining | TriggerState::Terminated => {
1401                    if active_version == Some(record.transition.version) {
1402                        active_version = None;
1403                    }
1404                }
1405                TriggerState::Registering => {}
1406            }
1407        }
1408
1409        active_version.ok_or_else(|| {
1410            TriggerRegistryError::InvalidSpec(format!(
1411                "no active trigger binding '{}' at {}",
1412                id,
1413                as_of
1414                    .format(&time::format_description::well_known::Rfc3339)
1415                    .unwrap_or_else(|_| as_of.to_string())
1416            ))
1417        })
1418    }
1419
1420    fn lifecycle_records_for(&self, id: &str) -> Vec<HistoricalLifecycleRecord> {
1421        let Some(event_log) = self.event_log.as_ref() else {
1422            return Vec::new();
1423        };
1424        let topic = Topic::new(TRIGGERS_LIFECYCLE_TOPIC)
1425            .expect("static triggers.lifecycle topic should always be valid");
1426        futures::executor::block_on(event_log.read_range(&topic, None, usize::MAX))
1427            .unwrap_or_default()
1428            .into_iter()
1429            .filter_map(|(_, event)| {
1430                let occurred_at_ms = event.occurred_at_ms;
1431                let transition: LifecycleStateTransitionRecord =
1432                    serde_json::from_value(event.payload).ok()?;
1433                (transition.id == id).then_some(HistoricalLifecycleRecord {
1434                    occurred_at_ms,
1435                    transition,
1436                })
1437            })
1438            .collect()
1439    }
1440
1441    #[allow(clippy::arc_with_non_send_sync)]
1442    fn register_binding(
1443        &mut self,
1444        spec: TriggerBindingSpec,
1445        version: u32,
1446        lifecycle: &mut Vec<LogEvent>,
1447    ) -> Arc<TriggerBinding> {
1448        let binding = Arc::new(TriggerBinding::new(spec, version));
1449        self.by_provider
1450            .entry(binding.provider.as_str().to_string())
1451            .or_default()
1452            .insert(binding.id.as_str().to_string());
1453        self.bindings
1454            .entry(binding.id.as_str().to_string())
1455            .or_default()
1456            .push(binding.clone());
1457        lifecycle.push(lifecycle_event(&binding, None, TriggerState::Registering));
1458        self.transition_binding_state(&binding, TriggerState::Active, lifecycle);
1459        binding
1460    }
1461
1462    fn transition_binding_to_draining(
1463        &self,
1464        binding: &Arc<TriggerBinding>,
1465        lifecycle: &mut Vec<LogEvent>,
1466    ) {
1467        if matches!(binding.state_snapshot(), TriggerState::Terminated) {
1468            return;
1469        }
1470        self.transition_binding_state(binding, TriggerState::Draining, lifecycle);
1471        self.maybe_finalize_draining(binding, lifecycle);
1472    }
1473
1474    fn maybe_finalize_draining(
1475        &self,
1476        binding: &Arc<TriggerBinding>,
1477        lifecycle: &mut Vec<LogEvent>,
1478    ) {
1479        if binding.state_snapshot() == TriggerState::Draining
1480            && binding.in_flight.load(Ordering::Relaxed) == 0
1481        {
1482            self.transition_binding_state(binding, TriggerState::Terminated, lifecycle);
1483        }
1484    }
1485
1486    fn transition_binding_state(
1487        &self,
1488        binding: &Arc<TriggerBinding>,
1489        next: TriggerState,
1490        lifecycle: &mut Vec<LogEvent>,
1491    ) {
1492        let mut state = binding.state.lock().expect("trigger state poisoned");
1493        let previous = *state;
1494        if previous == next {
1495            return;
1496        }
1497        *state = next;
1498        drop(state);
1499        // CH-04 (#1875): drop any pending aggregation buffers when the
1500        // binding terminates so a long-lived buffer doesn't fire after
1501        // the trigger has been removed. Leftover events are discarded —
1502        // matches the in-flight drain contract elsewhere in the
1503        // registry (the trigger is gone; we'd have nowhere to dispatch).
1504        if next == TriggerState::Terminated && binding.aggregation.is_some() {
1505            let _ = super::aggregation::drop_binding_aggregation(&binding.binding_key());
1506        }
1507        lifecycle.push(lifecycle_event(binding, Some(previous), next));
1508    }
1509}
1510
1511fn lifecycle_event(
1512    binding: &TriggerBinding,
1513    from_state: Option<TriggerState>,
1514    to_state: TriggerState,
1515) -> LogEvent {
1516    LogEvent::new(
1517        "state_transition",
1518        serde_json::json!({
1519            "id": binding.id.as_str(),
1520            "binding_key": binding.binding_key(),
1521            "version": binding.version,
1522            "provider": binding.provider.as_str(),
1523            "kind": &binding.kind,
1524            "source": binding.source.as_str(),
1525            "handler_kind": binding.handler.kind(),
1526            "definition_fingerprint": &binding.definition_fingerprint,
1527            "from_state": from_state.map(TriggerState::as_str),
1528            "to_state": to_state.as_str(),
1529        }),
1530    )
1531}
1532
1533async fn append_lifecycle_events(
1534    event_log: Option<Arc<AnyEventLog>>,
1535    events: Vec<LogEvent>,
1536) -> Result<(), TriggerRegistryError> {
1537    let Some(event_log) = event_log else {
1538        return Ok(());
1539    };
1540    if events.is_empty() {
1541        return Ok(());
1542    }
1543
1544    let topic = Topic::new(TRIGGERS_LIFECYCLE_TOPIC)
1545        .expect("static triggers.lifecycle topic should always be valid");
1546    for event in events {
1547        event_log
1548            .append(&topic, event)
1549            .await
1550            .map_err(|error| TriggerRegistryError::EventLog(error.to_string()))?;
1551    }
1552    Ok(())
1553}
1554
1555fn default_secret_provider() -> Option<Arc<dyn SecretProvider>> {
1556    configured_default_chain(default_secret_namespace())
1557        .ok()
1558        .map(|provider| Arc::new(provider) as Arc<dyn SecretProvider>)
1559}
1560
1561fn default_secret_namespace() -> String {
1562    if let Ok(namespace) = std::env::var("HARN_SECRET_NAMESPACE") {
1563        if !namespace.trim().is_empty() {
1564            return namespace;
1565        }
1566    }
1567
1568    let cwd = std::env::current_dir().unwrap_or_default();
1569    let leaf = cwd
1570        .file_name()
1571        .and_then(|name| name.to_str())
1572        .filter(|name| !name.is_empty())
1573        .unwrap_or("workspace");
1574    format!("harn/{leaf}")
1575}
1576
1577fn now_ms() -> i64 {
1578    clock::now_ms()
1579}
1580
1581#[cfg(test)]
1582mod tests {
1583    use super::*;
1584    use crate::event_log::{
1585        install_active_event_log, install_memory_for_current_thread, open_event_log,
1586        pin_test_occurred_at_ms, reset_active_event_log, EventLogBackendKind, EventLogConfig,
1587    };
1588    use crate::events::{add_event_sink, clear_event_sinks, CollectorSink, EventLevel};
1589    use std::path::Path;
1590    use std::rc::Rc;
1591
1592    /// Build the `OffsetDateTime` that corresponds to a pinned event-log
1593    /// `occurred_at_ms`, so `as_of` cutoffs can be expressed in the same
1594    /// reference frame as `pin_test_occurred_at_ms` without touching the wall
1595    /// clock.
1596    fn offset_from_ms(ms: i64) -> OffsetDateTime {
1597        OffsetDateTime::from_unix_timestamp_nanos((ms as i128) * 1_000_000)
1598            .expect("epoch ms within OffsetDateTime range")
1599    }
1600
1601    fn manifest_spec(id: &str, fingerprint: &str) -> TriggerBindingSpec {
1602        TriggerBindingSpec {
1603            id: id.to_string(),
1604            source: TriggerBindingSource::Manifest,
1605            kind: "webhook".to_string(),
1606            provider: ProviderId::from("github"),
1607            autonomy_tier: crate::AutonomyTier::ActAuto,
1608            handler: TriggerHandlerSpec::Worker {
1609                queue: format!("{id}-queue"),
1610            },
1611            dispatch_priority: crate::WorkerQueuePriority::Normal,
1612            when: None,
1613            when_budget: None,
1614            retry: TriggerRetryConfig::default(),
1615            match_events: vec!["issues.opened".to_string()],
1616            dedupe_key: Some("event.dedupe_key".to_string()),
1617            dedupe_retention_days: crate::triggers::DEFAULT_INBOX_RETENTION_DAYS,
1618            filter: Some("event.kind".to_string()),
1619            daily_cost_usd: Some(5.0),
1620            hourly_cost_usd: None,
1621            max_autonomous_decisions_per_hour: None,
1622            max_autonomous_decisions_per_day: None,
1623            on_budget_exhausted: crate::TriggerBudgetExhaustionStrategy::False,
1624            max_concurrent: Some(10),
1625            flow_control: crate::triggers::TriggerFlowControlConfig::default(),
1626            aggregation: None,
1627            manifest_path: None,
1628            package_name: Some("workspace".to_string()),
1629            definition_fingerprint: fingerprint.to_string(),
1630        }
1631    }
1632
1633    fn install_test_memory_event_log() -> Arc<AnyEventLog> {
1634        install_memory_for_current_thread(512)
1635    }
1636
1637    fn install_test_sqlite_event_log(base_dir: &Path) -> Arc<AnyEventLog> {
1638        let config = EventLogConfig {
1639            backend: EventLogBackendKind::Sqlite,
1640            file_dir: base_dir.join("events"),
1641            sqlite_path: base_dir.join("events.sqlite"),
1642            queue_depth: 512,
1643        };
1644        let log = open_event_log(&config).expect("open isolated sqlite event log");
1645        install_active_event_log(log)
1646    }
1647
1648    fn dynamic_spec(id: &str) -> TriggerBindingSpec {
1649        TriggerBindingSpec {
1650            id: id.to_string(),
1651            source: TriggerBindingSource::Dynamic,
1652            kind: "webhook".to_string(),
1653            provider: ProviderId::from("github"),
1654            autonomy_tier: crate::AutonomyTier::ActAuto,
1655            handler: TriggerHandlerSpec::Worker {
1656                queue: format!("{id}-queue"),
1657            },
1658            dispatch_priority: crate::WorkerQueuePriority::Normal,
1659            when: None,
1660            when_budget: None,
1661            retry: TriggerRetryConfig::default(),
1662            match_events: vec!["issues.opened".to_string()],
1663            dedupe_key: None,
1664            dedupe_retention_days: crate::triggers::DEFAULT_INBOX_RETENTION_DAYS,
1665            filter: None,
1666            daily_cost_usd: None,
1667            hourly_cost_usd: None,
1668            max_autonomous_decisions_per_hour: None,
1669            max_autonomous_decisions_per_day: None,
1670            on_budget_exhausted: crate::TriggerBudgetExhaustionStrategy::False,
1671            max_concurrent: None,
1672            flow_control: crate::triggers::TriggerFlowControlConfig::default(),
1673            aggregation: None,
1674            manifest_path: None,
1675            package_name: None,
1676            definition_fingerprint: format!("dynamic:{id}"),
1677        }
1678    }
1679
1680    #[tokio::test(flavor = "current_thread")]
1681    async fn manifest_loaded_trigger_registers_with_zeroed_metrics() {
1682        clear_trigger_registry();
1683
1684        install_manifest_triggers(vec![manifest_spec("github-new-issue", "v1")])
1685            .await
1686            .expect("manifest trigger installs");
1687
1688        let snapshots = snapshot_trigger_bindings();
1689        assert_eq!(snapshots.len(), 1);
1690        let binding = &snapshots[0];
1691        assert_eq!(binding.id, "github-new-issue");
1692        assert_eq!(binding.version, 1);
1693        assert_eq!(binding.state, TriggerState::Active);
1694        assert_eq!(binding.metrics, TriggerMetricsSnapshot::default());
1695
1696        clear_trigger_registry();
1697    }
1698
1699    #[tokio::test(flavor = "current_thread")]
1700    async fn dynamic_register_assigns_unique_ids_and_rejects_duplicates() {
1701        clear_trigger_registry();
1702
1703        let first = dynamic_register(dynamic_spec("dynamic-a"))
1704            .await
1705            .expect("first dynamic trigger");
1706        let second = dynamic_register(dynamic_spec("dynamic-b"))
1707            .await
1708            .expect("second dynamic trigger");
1709        assert_ne!(first, second);
1710
1711        let error = dynamic_register(dynamic_spec("dynamic-a"))
1712            .await
1713            .expect_err("duplicate id should fail");
1714        assert!(matches!(error, TriggerRegistryError::DuplicateId(_)));
1715
1716        clear_trigger_registry();
1717    }
1718
1719    #[test]
1720    fn expected_predicate_cost_average_does_not_overflow() {
1721        let binding = TriggerBinding::new(manifest_spec("costed", "v1"), 1);
1722        record_predicate_cost_sample(&binding, u64::MAX);
1723        record_predicate_cost_sample(&binding, u64::MAX);
1724
1725        assert_eq!(expected_predicate_cost_usd_micros(&binding), u64::MAX);
1726    }
1727
1728    #[tokio::test(flavor = "current_thread")]
1729    async fn drain_waits_for_in_flight_events_before_terminating() {
1730        clear_trigger_registry();
1731
1732        install_manifest_triggers(vec![manifest_spec("github-new-issue", "v1")])
1733            .await
1734            .expect("manifest trigger installs");
1735        begin_in_flight("github-new-issue", 1).expect("start in-flight event");
1736
1737        drain("github-new-issue").await.expect("drain succeeds");
1738        let binding = snapshot_trigger_bindings()
1739            .into_iter()
1740            .find(|binding| binding.id == "github-new-issue" && binding.version == 1)
1741            .expect("binding snapshot");
1742        assert_eq!(binding.state, TriggerState::Draining);
1743        assert_eq!(binding.metrics.in_flight, 1);
1744
1745        finish_in_flight("github-new-issue", 1, TriggerDispatchOutcome::Dispatched)
1746            .await
1747            .expect("finish in-flight event");
1748        let binding = snapshot_trigger_bindings()
1749            .into_iter()
1750            .find(|binding| binding.id == "github-new-issue" && binding.version == 1)
1751            .expect("binding snapshot");
1752        assert_eq!(binding.state, TriggerState::Terminated);
1753        assert_eq!(binding.metrics.in_flight, 0);
1754
1755        clear_trigger_registry();
1756    }
1757
1758    #[tokio::test(flavor = "current_thread")]
1759    async fn hot_reload_registers_new_version_while_old_binding_drains() {
1760        clear_trigger_registry();
1761
1762        install_manifest_triggers(vec![manifest_spec("github-new-issue", "v1")])
1763            .await
1764            .expect("initial manifest trigger installs");
1765        begin_in_flight("github-new-issue", 1).expect("start in-flight event");
1766
1767        install_manifest_triggers(vec![manifest_spec("github-new-issue", "v2")])
1768            .await
1769            .expect("updated manifest trigger installs");
1770
1771        let snapshots = snapshot_trigger_bindings();
1772        assert_eq!(snapshots.len(), 2);
1773        let old = snapshots
1774            .iter()
1775            .find(|binding| binding.id == "github-new-issue" && binding.version == 1)
1776            .expect("old binding");
1777        let new = snapshots
1778            .iter()
1779            .find(|binding| binding.id == "github-new-issue" && binding.version == 2)
1780            .expect("new binding");
1781        assert_eq!(old.state, TriggerState::Draining);
1782        assert_eq!(new.state, TriggerState::Active);
1783
1784        finish_in_flight("github-new-issue", 1, TriggerDispatchOutcome::Dispatched)
1785            .await
1786            .expect("finish old in-flight event");
1787        let old = snapshot_trigger_bindings()
1788            .into_iter()
1789            .find(|binding| binding.id == "github-new-issue" && binding.version == 1)
1790            .expect("old binding");
1791        assert_eq!(old.state, TriggerState::Terminated);
1792
1793        clear_trigger_registry();
1794    }
1795
1796    #[tokio::test(flavor = "current_thread")]
1797    async fn gc_drops_terminated_versions_beyond_retention_limit() {
1798        clear_trigger_registry();
1799
1800        install_manifest_triggers(vec![manifest_spec("github-new-issue", "v1")])
1801            .await
1802            .expect("install v1");
1803        begin_in_flight("github-new-issue", 1).expect("pin v1");
1804
1805        install_manifest_triggers(vec![manifest_spec("github-new-issue", "v2")])
1806            .await
1807            .expect("install v2");
1808        finish_in_flight("github-new-issue", 1, TriggerDispatchOutcome::Dispatched)
1809            .await
1810            .expect("finish v1");
1811
1812        install_manifest_triggers(vec![manifest_spec("github-new-issue", "v3")])
1813            .await
1814            .expect("install v3");
1815
1816        let snapshots = snapshot_trigger_bindings();
1817        let versions: Vec<u32> = snapshots
1818            .into_iter()
1819            .filter(|binding| binding.id == "github-new-issue")
1820            .map(|binding| binding.version)
1821            .collect();
1822        assert_eq!(versions, vec![2, 3]);
1823
1824        clear_trigger_registry();
1825    }
1826
1827    #[tokio::test(flavor = "current_thread")]
1828    async fn lifecycle_transitions_append_to_event_log() {
1829        clear_trigger_registry();
1830        reset_active_event_log();
1831        let log = install_test_memory_event_log();
1832
1833        install_manifest_triggers(vec![manifest_spec("github-new-issue", "v1")])
1834            .await
1835            .expect("manifest trigger installs");
1836        begin_in_flight("github-new-issue", 1).expect("start in-flight event");
1837        drain("github-new-issue").await.expect("drain succeeds");
1838        finish_in_flight("github-new-issue", 1, TriggerDispatchOutcome::Dispatched)
1839            .await
1840            .expect("finish event");
1841
1842        let topic = Topic::new("triggers.lifecycle").expect("valid lifecycle topic");
1843        let events = log
1844            .read_range(&topic, None, 32)
1845            .await
1846            .expect("read lifecycle events");
1847        let states: Vec<String> = events
1848            .into_iter()
1849            .filter_map(|(_, event)| {
1850                event
1851                    .payload
1852                    .get("to_state")
1853                    .and_then(|value| value.as_str())
1854                    .map(|value| value.to_string())
1855            })
1856            .collect();
1857        assert_eq!(
1858            states,
1859            vec![
1860                "registering".to_string(),
1861                "active".to_string(),
1862                "draining".to_string(),
1863                "terminated".to_string(),
1864            ]
1865        );
1866
1867        reset_active_event_log();
1868        clear_trigger_registry();
1869    }
1870
1871    #[tokio::test(flavor = "current_thread")]
1872    async fn version_history_reuses_historical_version_after_restart() {
1873        clear_trigger_registry();
1874        reset_active_event_log();
1875        let tempdir = tempfile::tempdir().expect("tempdir");
1876        install_test_sqlite_event_log(tempdir.path());
1877
1878        install_manifest_triggers(vec![manifest_spec("github-new-issue", "v1")])
1879            .await
1880            .expect("initial manifest trigger installs");
1881        install_manifest_triggers(vec![manifest_spec("github-new-issue", "v2")])
1882            .await
1883            .expect("updated manifest trigger installs");
1884
1885        clear_trigger_registry();
1886        reset_active_event_log();
1887        install_test_sqlite_event_log(tempdir.path());
1888
1889        install_manifest_triggers(vec![manifest_spec("github-new-issue", "v2")])
1890            .await
1891            .expect("manifest reload reuses historical version");
1892
1893        let binding = snapshot_trigger_bindings()
1894            .into_iter()
1895            .find(|binding| binding.id == "github-new-issue")
1896            .expect("binding snapshot");
1897        assert_eq!(binding.version, 2);
1898
1899        reset_active_event_log();
1900        clear_trigger_registry();
1901    }
1902
1903    #[tokio::test(flavor = "current_thread")]
1904    async fn binding_version_as_of_reports_historical_active_version() {
1905        clear_trigger_registry();
1906        reset_active_event_log();
1907        install_test_memory_event_log();
1908
1909        // Pin the event-log timestamp instead of sleeping between captures:
1910        // v1's lifecycle events stamp at T1, v2's at T2 > T1, with zero
1911        // wall-clock dependence. `before_reload` sits at T1 (so v1 is active
1912        // but v2 is not yet), `after_reload` at T2 (so v2 is active).
1913        const T1_MS: i64 = 1_700_000_000_000;
1914        const T2_MS: i64 = T1_MS + 50;
1915        let before_reload = offset_from_ms(T1_MS);
1916        let after_reload = offset_from_ms(T2_MS);
1917
1918        let clock = pin_test_occurred_at_ms(T1_MS);
1919        install_manifest_triggers(vec![manifest_spec("github-new-issue", "v1")])
1920            .await
1921            .expect("initial manifest trigger installs");
1922        drop(clock);
1923
1924        let clock = pin_test_occurred_at_ms(T2_MS);
1925        install_manifest_triggers(vec![manifest_spec("github-new-issue", "v2")])
1926            .await
1927            .expect("updated manifest trigger installs");
1928        drop(clock);
1929
1930        assert_eq!(
1931            binding_version_as_of("github-new-issue", before_reload)
1932                .expect("version before reload"),
1933            1
1934        );
1935        assert_eq!(
1936            binding_version_as_of("github-new-issue", after_reload).expect("version after reload"),
1937            2
1938        );
1939
1940        reset_active_event_log();
1941        clear_trigger_registry();
1942    }
1943
1944    #[tokio::test(flavor = "current_thread")]
1945    async fn resolve_live_or_as_of_logs_structured_gc_fallback() {
1946        clear_trigger_registry();
1947        reset_active_event_log();
1948        let sink = Rc::new(CollectorSink::new());
1949        clear_event_sinks();
1950        add_event_sink(sink.clone());
1951        install_test_memory_event_log();
1952
1953        // Pin the event-log timestamps: v1..v3 land at T1, v4 at T2 > T1, with
1954        // `received_at` at T1 so the gc fallback resolves to v3 (the latest
1955        // binding active at-or-before `received_at`) while v4 is strictly later.
1956        const T1_MS: i64 = 1_700_000_000_000;
1957        const T2_MS: i64 = T1_MS + 50;
1958        let received_at = offset_from_ms(T1_MS);
1959
1960        let clock = pin_test_occurred_at_ms(T1_MS);
1961        install_manifest_triggers(vec![manifest_spec("github-new-issue", "v1")])
1962            .await
1963            .expect("install v1");
1964        install_manifest_triggers(vec![manifest_spec("github-new-issue", "v2")])
1965            .await
1966            .expect("install v2");
1967        install_manifest_triggers(vec![manifest_spec("github-new-issue", "v3")])
1968            .await
1969            .expect("install v3");
1970        drop(clock);
1971
1972        let clock = pin_test_occurred_at_ms(T2_MS);
1973        install_manifest_triggers(vec![manifest_spec("github-new-issue", "v4")])
1974            .await
1975            .expect("install v4");
1976        drop(clock);
1977
1978        let binding = resolve_live_or_as_of(
1979            "github-new-issue",
1980            RecordedTriggerBinding {
1981                version: 1,
1982                received_at,
1983            },
1984        )
1985        .expect("resolve fallback binding");
1986        assert_eq!(binding.version, 3);
1987
1988        let warning = sink
1989            .logs
1990            .borrow()
1991            .iter()
1992            .find(|log| log.category == "replay.binding_version_gc_fallback")
1993            .cloned()
1994            .expect("gc fallback warning");
1995        assert_eq!(warning.level, EventLevel::Warn);
1996        assert_eq!(
1997            warning.metadata.get("trigger_id"),
1998            Some(&serde_json::json!("github-new-issue"))
1999        );
2000        assert_eq!(
2001            warning.metadata.get("recorded_version"),
2002            Some(&serde_json::json!(1))
2003        );
2004        assert_eq!(
2005            warning.metadata.get("received_at"),
2006            Some(&serde_json::json!(received_at
2007                .format(&time::format_description::well_known::Rfc3339)
2008                .unwrap_or_else(|_| received_at.to_string())))
2009        );
2010        assert_eq!(
2011            warning.metadata.get("resolved_version"),
2012            Some(&serde_json::json!(3))
2013        );
2014
2015        clear_event_sinks();
2016        crate::events::reset_event_sinks();
2017        reset_active_event_log();
2018        clear_trigger_registry();
2019    }
2020}