1use serde::{Deserialize, Serialize};
2use std::cell::RefCell;
3use std::collections::{BTreeMap, BTreeSet, VecDeque};
4use std::path::PathBuf;
5use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
6use std::sync::{Arc, Mutex};
7
8use time::OffsetDateTime;
9use uuid::Uuid;
10
11use crate::event_log::{active_event_log, AnyEventLog, EventLog, LogEvent, Topic};
12use crate::llm::trigger_predicate::TriggerPredicateBudget;
13use crate::secrets::{configured_default_chain, SecretProvider};
14use crate::triggers::test_util::clock;
15use crate::trust_graph::AutonomyTier;
16
17use super::aggregation::TriggerAggregationConfig;
18use super::dispatcher::TriggerRetryConfig;
19use super::flow_control::TriggerFlowControlConfig;
20use super::ProviderId;
21
22mod handler;
23mod id;
24
25pub use handler::{AgentScope, TargetExpr, TriggerHandlerSpec};
26pub use id::TriggerId;
27
28#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
29#[serde(rename_all = "snake_case")]
30pub enum TriggerState {
31 Registering,
32 Active,
33 Paused,
34 Draining,
35 Terminated,
36}
37
38impl TriggerState {
39 pub fn as_str(self) -> &'static str {
40 match self {
41 Self::Registering => "registering",
42 Self::Active => "active",
43 Self::Paused => "paused",
44 Self::Draining => "draining",
45 Self::Terminated => "terminated",
46 }
47 }
48}
49
50#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
51#[serde(rename_all = "snake_case")]
52pub enum TriggerBindingSource {
53 Manifest,
54 Dynamic,
55}
56
57impl TriggerBindingSource {
58 pub fn as_str(self) -> &'static str {
59 match self {
60 Self::Manifest => "manifest",
61 Self::Dynamic => "dynamic",
62 }
63 }
64}
65
66#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
67#[serde(rename_all = "snake_case")]
68pub enum TriggerBudgetExhaustionStrategy {
69 #[default]
70 False,
71 RetryLater,
72 Fail,
73 Warn,
74}
75
76impl TriggerBudgetExhaustionStrategy {
77 pub fn as_str(self) -> &'static str {
78 match self {
79 Self::False => "false",
80 Self::RetryLater => "retry_later",
81 Self::Fail => "fail",
82 Self::Warn => "warn",
83 }
84 }
85}
86
87#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
88pub struct OrchestratorBudgetConfig {
89 pub daily_cost_usd: Option<f64>,
90 pub hourly_cost_usd: Option<f64>,
91}
92
93#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
94pub struct OrchestratorBudgetSnapshot {
95 pub daily_cost_usd: Option<f64>,
96 pub hourly_cost_usd: Option<f64>,
97 pub cost_today_usd_micros: u64,
98 pub cost_hour_usd_micros: u64,
99 pub day_utc: i32,
100 pub hour_utc: i64,
101}
102
103#[derive(Debug)]
104struct OrchestratorBudgetState {
105 config: OrchestratorBudgetConfig,
106 day_utc: i32,
107 hour_utc: i64,
108 cost_today_usd_micros: u64,
109 cost_hour_usd_micros: u64,
110}
111
112impl Default for OrchestratorBudgetState {
113 fn default() -> Self {
114 Self {
115 config: OrchestratorBudgetConfig::default(),
116 day_utc: utc_day_key(),
117 hour_utc: utc_hour_key(),
118 cost_today_usd_micros: 0,
119 cost_hour_usd_micros: 0,
120 }
121 }
122}
123
124#[derive(Clone)]
125pub struct TriggerPredicateSpec {
126 pub raw: String,
127 pub callable: crate::value::VmCallable,
128}
129
130impl std::fmt::Debug for TriggerPredicateSpec {
131 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
132 f.debug_struct("TriggerPredicateSpec")
133 .field("raw", &self.raw)
134 .finish()
135 }
136}
137
138#[derive(Clone, Debug)]
139pub struct TriggerBindingSpec {
140 pub id: String,
141 pub source: TriggerBindingSource,
142 pub kind: String,
143 pub provider: ProviderId,
144 pub autonomy_tier: AutonomyTier,
145 pub handler: TriggerHandlerSpec,
146 pub dispatch_priority: super::worker_queue::WorkerQueuePriority,
147 pub when: Option<TriggerPredicateSpec>,
148 pub when_budget: Option<TriggerPredicateBudget>,
149 pub retry: TriggerRetryConfig,
150 pub match_events: Vec<String>,
151 pub dedupe_key: Option<String>,
152 pub dedupe_retention_days: u32,
153 pub filter: Option<String>,
154 pub daily_cost_usd: Option<f64>,
155 pub hourly_cost_usd: Option<f64>,
156 pub max_autonomous_decisions_per_hour: Option<u64>,
157 pub max_autonomous_decisions_per_day: Option<u64>,
158 pub on_budget_exhausted: TriggerBudgetExhaustionStrategy,
159 pub max_concurrent: Option<u32>,
160 pub flow_control: TriggerFlowControlConfig,
161 pub aggregation: Option<TriggerAggregationConfig>,
166 pub manifest_path: Option<PathBuf>,
167 pub package_name: Option<String>,
168 pub definition_fingerprint: String,
169}
170
171#[derive(Debug)]
172pub struct TriggerMetrics {
173 pub received: AtomicU64,
174 pub dispatched: AtomicU64,
175 pub failed: AtomicU64,
176 pub dlq: AtomicU64,
177 pub last_received_ms: Mutex<Option<i64>>,
178 pub cost_total_usd_micros: AtomicU64,
179 pub cost_today_usd_micros: AtomicU64,
180 pub cost_hour_usd_micros: AtomicU64,
181 pub autonomous_decisions_total: AtomicU64,
182 pub autonomous_decisions_today: AtomicU64,
183 pub autonomous_decisions_hour: AtomicU64,
184}
185
186impl Default for TriggerMetrics {
187 fn default() -> Self {
188 Self {
189 received: AtomicU64::new(0),
190 dispatched: AtomicU64::new(0),
191 failed: AtomicU64::new(0),
192 dlq: AtomicU64::new(0),
193 last_received_ms: Mutex::new(None),
194 cost_total_usd_micros: AtomicU64::new(0),
195 cost_today_usd_micros: AtomicU64::new(0),
196 cost_hour_usd_micros: AtomicU64::new(0),
197 autonomous_decisions_total: AtomicU64::new(0),
198 autonomous_decisions_today: AtomicU64::new(0),
199 autonomous_decisions_hour: AtomicU64::new(0),
200 }
201 }
202}
203
204#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
205pub struct TriggerMetricsSnapshot {
206 pub received: u64,
207 pub dispatched: u64,
208 pub failed: u64,
209 pub dlq: u64,
210 pub in_flight: u64,
211 pub last_received_ms: Option<i64>,
212 pub cost_total_usd_micros: u64,
213 pub cost_today_usd_micros: u64,
214 pub cost_hour_usd_micros: u64,
215 pub autonomous_decisions_total: u64,
216 pub autonomous_decisions_today: u64,
217 pub autonomous_decisions_hour: u64,
218}
219
220pub struct TriggerBinding {
221 pub id: TriggerId,
222 pub version: u32,
223 pub source: TriggerBindingSource,
224 pub kind: String,
225 pub provider: ProviderId,
226 pub autonomy_tier: AutonomyTier,
227 pub handler: TriggerHandlerSpec,
228 pub dispatch_priority: super::worker_queue::WorkerQueuePriority,
229 pub when: Option<TriggerPredicateSpec>,
230 pub when_budget: Option<TriggerPredicateBudget>,
231 pub retry: TriggerRetryConfig,
232 pub match_events: Vec<String>,
233 pub dedupe_key: Option<String>,
234 pub dedupe_retention_days: u32,
235 pub filter: Option<String>,
236 pub daily_cost_usd: Option<f64>,
237 pub hourly_cost_usd: Option<f64>,
238 pub max_autonomous_decisions_per_hour: Option<u64>,
239 pub max_autonomous_decisions_per_day: Option<u64>,
240 pub on_budget_exhausted: TriggerBudgetExhaustionStrategy,
241 pub max_concurrent: Option<u32>,
242 pub flow_control: TriggerFlowControlConfig,
243 pub aggregation: Option<TriggerAggregationConfig>,
247 pub manifest_path: Option<PathBuf>,
248 pub package_name: Option<String>,
249 pub definition_fingerprint: String,
250 pub state: Mutex<TriggerState>,
251 pub metrics: TriggerMetrics,
252 pub in_flight: AtomicU64,
253 pub cancel_token: Arc<AtomicBool>,
254 pub predicate_state: Mutex<TriggerPredicateState>,
255}
256
257#[derive(Clone, Debug, Default)]
258pub struct TriggerPredicateState {
259 pub budget_day_utc: Option<i32>,
260 pub budget_hour_utc: Option<i64>,
261 pub consecutive_failures: u32,
262 pub breaker_open_until_ms: Option<i64>,
263 pub recent_cost_usd_micros: VecDeque<u64>,
264}
265
266impl std::fmt::Debug for TriggerBinding {
267 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
268 f.debug_struct("TriggerBinding")
269 .field("id", &self.id)
270 .field("version", &self.version)
271 .field("source", &self.source)
272 .field("kind", &self.kind)
273 .field("provider", &self.provider)
274 .field("handler_kind", &self.handler.kind())
275 .field("state", &self.state_snapshot())
276 .finish()
277 }
278}
279
280impl TriggerBinding {
281 pub fn snapshot(&self) -> TriggerBindingSnapshot {
282 TriggerBindingSnapshot {
283 id: self.id.as_str().to_string(),
284 version: self.version,
285 source: self.source,
286 kind: self.kind.clone(),
287 provider: self.provider.as_str().to_string(),
288 autonomy_tier: self.autonomy_tier,
289 handler_kind: self.handler.kind().to_string(),
290 state: self.state_snapshot(),
291 metrics: self.metrics_snapshot(),
292 daily_cost_usd: self.daily_cost_usd,
293 hourly_cost_usd: self.hourly_cost_usd,
294 max_autonomous_decisions_per_hour: self.max_autonomous_decisions_per_hour,
295 max_autonomous_decisions_per_day: self.max_autonomous_decisions_per_day,
296 on_budget_exhausted: self.on_budget_exhausted,
297 }
298 }
299
300 fn new(spec: TriggerBindingSpec, version: u32) -> Self {
301 Self {
302 id: TriggerId::new(spec.id),
303 version,
304 source: spec.source,
305 kind: spec.kind,
306 provider: spec.provider,
307 autonomy_tier: spec.autonomy_tier,
308 handler: spec.handler,
309 dispatch_priority: spec.dispatch_priority,
310 when: spec.when,
311 when_budget: spec.when_budget,
312 retry: spec.retry,
313 match_events: spec.match_events,
314 dedupe_key: spec.dedupe_key,
315 dedupe_retention_days: spec.dedupe_retention_days,
316 filter: spec.filter,
317 daily_cost_usd: spec.daily_cost_usd,
318 hourly_cost_usd: spec.hourly_cost_usd,
319 max_autonomous_decisions_per_hour: spec.max_autonomous_decisions_per_hour,
320 max_autonomous_decisions_per_day: spec.max_autonomous_decisions_per_day,
321 on_budget_exhausted: spec.on_budget_exhausted,
322 max_concurrent: spec.max_concurrent,
323 flow_control: spec.flow_control,
324 aggregation: spec.aggregation,
325 manifest_path: spec.manifest_path,
326 package_name: spec.package_name,
327 definition_fingerprint: spec.definition_fingerprint,
328 state: Mutex::new(TriggerState::Registering),
329 metrics: TriggerMetrics::default(),
330 in_flight: AtomicU64::new(0),
331 cancel_token: Arc::new(AtomicBool::new(false)),
332 predicate_state: Mutex::new(TriggerPredicateState::default()),
333 }
334 }
335
336 pub fn binding_key(&self) -> String {
337 format!("{}@v{}", self.id.as_str(), self.version)
338 }
339
340 pub fn state_snapshot(&self) -> TriggerState {
341 *self.state.lock().expect("trigger state poisoned")
342 }
343
344 pub fn metrics_snapshot(&self) -> TriggerMetricsSnapshot {
345 TriggerMetricsSnapshot {
346 received: self.metrics.received.load(Ordering::Relaxed),
347 dispatched: self.metrics.dispatched.load(Ordering::Relaxed),
348 failed: self.metrics.failed.load(Ordering::Relaxed),
349 dlq: self.metrics.dlq.load(Ordering::Relaxed),
350 in_flight: self.in_flight.load(Ordering::Relaxed),
351 last_received_ms: *self
352 .metrics
353 .last_received_ms
354 .lock()
355 .expect("trigger metrics poisoned"),
356 cost_total_usd_micros: self.metrics.cost_total_usd_micros.load(Ordering::Relaxed),
357 cost_today_usd_micros: self.metrics.cost_today_usd_micros.load(Ordering::Relaxed),
358 cost_hour_usd_micros: self.metrics.cost_hour_usd_micros.load(Ordering::Relaxed),
359 autonomous_decisions_total: self
360 .metrics
361 .autonomous_decisions_total
362 .load(Ordering::Relaxed),
363 autonomous_decisions_today: self
364 .metrics
365 .autonomous_decisions_today
366 .load(Ordering::Relaxed),
367 autonomous_decisions_hour: self
368 .metrics
369 .autonomous_decisions_hour
370 .load(Ordering::Relaxed),
371 }
372 }
373}
374
375#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
376pub struct TriggerBindingSnapshot {
377 pub id: String,
378 pub version: u32,
379 pub source: TriggerBindingSource,
380 pub kind: String,
381 pub provider: String,
382 pub autonomy_tier: AutonomyTier,
383 pub handler_kind: String,
384 pub state: TriggerState,
385 pub metrics: TriggerMetricsSnapshot,
386 pub daily_cost_usd: Option<f64>,
387 pub hourly_cost_usd: Option<f64>,
388 pub max_autonomous_decisions_per_hour: Option<u64>,
389 pub max_autonomous_decisions_per_day: Option<u64>,
390 pub on_budget_exhausted: TriggerBudgetExhaustionStrategy,
391}
392
393#[derive(Clone, Copy, Debug, PartialEq, Eq)]
394pub enum TriggerDispatchOutcome {
395 Dispatched,
396 Failed,
397 Dlq,
398}
399
400#[derive(Debug)]
401pub enum TriggerRegistryError {
402 DuplicateId(String),
403 InvalidSpec(String),
404 UnknownId(String),
405 UnknownBindingVersion { id: String, version: u32 },
406 EventLog(String),
407}
408
409impl std::fmt::Display for TriggerRegistryError {
410 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
411 match self {
412 Self::DuplicateId(id) => write!(f, "duplicate trigger id '{id}'"),
413 Self::InvalidSpec(message) | Self::EventLog(message) => f.write_str(message),
414 Self::UnknownId(id) => write!(f, "unknown trigger id '{id}'"),
415 Self::UnknownBindingVersion { id, version } => {
416 write!(f, "unknown trigger binding '{id}' version {version}")
417 }
418 }
419 }
420}
421
422impl std::error::Error for TriggerRegistryError {}
423
424#[derive(Default)]
425pub struct TriggerRegistry {
426 bindings: BTreeMap<String, Vec<Arc<TriggerBinding>>>,
427 by_provider: BTreeMap<String, BTreeSet<String>>,
428 event_log: Option<Arc<AnyEventLog>>,
429 secret_provider: Option<Arc<dyn SecretProvider>>,
430}
431
432thread_local! {
433 static TRIGGER_REGISTRY: RefCell<TriggerRegistry> = RefCell::new(TriggerRegistry::default());
434}
435
436thread_local! {
437 static ORCHESTRATOR_BUDGET: RefCell<OrchestratorBudgetState> =
438 RefCell::new(OrchestratorBudgetState::default());
439}
440
441const TERMINATED_VERSION_RETENTION_LIMIT: usize = 2;
442
443const TRIGGERS_LIFECYCLE_TOPIC: &str = "triggers.lifecycle";
444const PREDICATE_COST_WINDOW: usize = 100;
445
446#[derive(Clone, Debug, Deserialize)]
447struct LifecycleStateTransitionRecord {
448 id: String,
449 version: u32,
450 #[serde(default)]
451 definition_fingerprint: Option<String>,
452 to_state: TriggerState,
453}
454
455#[derive(Clone, Debug)]
456struct HistoricalLifecycleRecord {
457 occurred_at_ms: i64,
458 transition: LifecycleStateTransitionRecord,
459}
460
461#[derive(Clone, Copy, Debug, PartialEq, Eq)]
462pub struct RecordedTriggerBinding {
463 pub version: u32,
464 pub received_at: OffsetDateTime,
465}
466
467#[derive(Clone, Copy, Debug, Default)]
468struct HistoricalVersionLookup {
469 matching_version: Option<u32>,
470 max_version: Option<u32>,
471}
472
473pub fn clear_trigger_registry() {
474 TRIGGER_REGISTRY.with(|slot| {
475 *slot.borrow_mut() = TriggerRegistry::default();
476 });
477 clear_orchestrator_budget();
478 super::aggregation::clear_aggregation_state();
479}
480
481pub fn install_orchestrator_budget(config: OrchestratorBudgetConfig) {
482 ORCHESTRATOR_BUDGET.with(|slot| {
483 let mut state = slot.borrow_mut();
484 rollover_orchestrator_budget(&mut state);
485 state.config = config;
486 });
487}
488
489pub fn clear_orchestrator_budget() {
490 ORCHESTRATOR_BUDGET.with(|slot| {
491 *slot.borrow_mut() = OrchestratorBudgetState::default();
492 });
493}
494
495pub fn snapshot_orchestrator_budget() -> OrchestratorBudgetSnapshot {
496 ORCHESTRATOR_BUDGET.with(|slot| {
497 let mut state = slot.borrow_mut();
498 rollover_orchestrator_budget(&mut state);
499 OrchestratorBudgetSnapshot {
500 daily_cost_usd: state.config.daily_cost_usd,
501 hourly_cost_usd: state.config.hourly_cost_usd,
502 cost_today_usd_micros: state.cost_today_usd_micros,
503 cost_hour_usd_micros: state.cost_hour_usd_micros,
504 day_utc: state.day_utc,
505 hour_utc: state.hour_utc,
506 }
507 })
508}
509
510pub fn note_orchestrator_budget_cost(cost_usd_micros: u64) {
511 if cost_usd_micros == 0 {
512 return;
513 }
514 ORCHESTRATOR_BUDGET.with(|slot| {
515 let mut state = slot.borrow_mut();
516 rollover_orchestrator_budget(&mut state);
517 state.cost_today_usd_micros = state.cost_today_usd_micros.saturating_add(cost_usd_micros);
518 state.cost_hour_usd_micros = state.cost_hour_usd_micros.saturating_add(cost_usd_micros);
519 });
520}
521
522pub(crate) fn note_binding_budget_cost(binding: &TriggerBinding, cost_usd_micros: u64) {
523 if cost_usd_micros == 0 {
524 return;
525 }
526 reset_binding_budget_windows(binding);
527 binding
528 .metrics
529 .cost_total_usd_micros
530 .fetch_add(cost_usd_micros, Ordering::Relaxed);
531 binding
532 .metrics
533 .cost_today_usd_micros
534 .fetch_add(cost_usd_micros, Ordering::Relaxed);
535 binding
536 .metrics
537 .cost_hour_usd_micros
538 .fetch_add(cost_usd_micros, Ordering::Relaxed);
539}
540
541pub fn orchestrator_budget_would_exceed(expected_cost_usd_micros: u64) -> Option<&'static str> {
542 ORCHESTRATOR_BUDGET.with(|slot| {
543 let mut state = slot.borrow_mut();
544 rollover_orchestrator_budget(&mut state);
545 if state.config.hourly_cost_usd.is_some_and(|limit| {
546 micros_to_usd(
547 state
548 .cost_hour_usd_micros
549 .saturating_add(expected_cost_usd_micros),
550 ) > limit
551 }) {
552 return Some("orchestrator_hourly_budget_exceeded");
553 }
554 if state.config.daily_cost_usd.is_some_and(|limit| {
555 micros_to_usd(
556 state
557 .cost_today_usd_micros
558 .saturating_add(expected_cost_usd_micros),
559 ) > limit
560 }) {
561 return Some("orchestrator_daily_budget_exceeded");
562 }
563 None
564 })
565}
566
567pub fn reset_binding_budget_windows(binding: &TriggerBinding) {
568 let today = utc_day_key();
569 let hour = utc_hour_key();
570 let mut state = binding
571 .predicate_state
572 .lock()
573 .expect("trigger predicate state poisoned");
574 if state.budget_day_utc != Some(today) {
575 state.budget_day_utc = Some(today);
576 binding
577 .metrics
578 .cost_today_usd_micros
579 .store(0, Ordering::Relaxed);
580 binding
581 .metrics
582 .autonomous_decisions_today
583 .store(0, Ordering::Relaxed);
584 }
585 if state.budget_hour_utc != Some(hour) {
586 state.budget_hour_utc = Some(hour);
587 binding
588 .metrics
589 .cost_hour_usd_micros
590 .store(0, Ordering::Relaxed);
591 binding
592 .metrics
593 .autonomous_decisions_hour
594 .store(0, Ordering::Relaxed);
595 }
596}
597
598pub fn binding_budget_would_exceed(
599 binding: &TriggerBinding,
600 expected_cost_usd_micros: u64,
601) -> Option<&'static str> {
602 reset_binding_budget_windows(binding);
603 if binding.hourly_cost_usd.is_some_and(|limit| {
604 micros_to_usd(
605 binding
606 .metrics
607 .cost_hour_usd_micros
608 .load(Ordering::Relaxed)
609 .saturating_add(expected_cost_usd_micros),
610 ) > limit
611 }) {
612 return Some("hourly_budget_exceeded");
613 }
614 if binding.daily_cost_usd.is_some_and(|limit| {
615 micros_to_usd(
616 binding
617 .metrics
618 .cost_today_usd_micros
619 .load(Ordering::Relaxed)
620 .saturating_add(expected_cost_usd_micros),
621 ) > limit
622 }) {
623 return Some("daily_budget_exceeded");
624 }
625 None
626}
627
628pub fn binding_autonomy_budget_would_exceed(binding: &TriggerBinding) -> Option<&'static str> {
629 reset_binding_budget_windows(binding);
630 if binding
631 .max_autonomous_decisions_per_hour
632 .is_some_and(|limit| {
633 binding
634 .metrics
635 .autonomous_decisions_hour
636 .load(Ordering::Relaxed)
637 .saturating_add(1)
638 > limit
639 })
640 {
641 return Some("hourly_autonomy_budget_exceeded");
642 }
643 if binding
644 .max_autonomous_decisions_per_day
645 .is_some_and(|limit| {
646 binding
647 .metrics
648 .autonomous_decisions_today
649 .load(Ordering::Relaxed)
650 .saturating_add(1)
651 > limit
652 })
653 {
654 return Some("daily_autonomy_budget_exceeded");
655 }
656 None
657}
658
659pub fn note_autonomous_decision(binding: &TriggerBinding) {
660 reset_binding_budget_windows(binding);
661 binding
662 .metrics
663 .autonomous_decisions_total
664 .fetch_add(1, Ordering::Relaxed);
665 binding
666 .metrics
667 .autonomous_decisions_today
668 .fetch_add(1, Ordering::Relaxed);
669 binding
670 .metrics
671 .autonomous_decisions_hour
672 .fetch_add(1, Ordering::Relaxed);
673}
674
675pub fn expected_predicate_cost_usd_micros(binding: &TriggerBinding) -> u64 {
676 let state = binding
677 .predicate_state
678 .lock()
679 .expect("trigger predicate state poisoned");
680 if let Some(average) = average_cost_sample_micros(&state.recent_cost_usd_micros) {
681 return average;
682 }
683 binding
684 .when_budget
685 .as_ref()
686 .and_then(|budget| budget.max_cost_usd)
687 .map(usd_to_micros)
688 .unwrap_or_default()
689}
690
691pub fn record_predicate_cost_sample(binding: &TriggerBinding, cost_usd_micros: u64) {
692 let mut state = binding
693 .predicate_state
694 .lock()
695 .expect("trigger predicate state poisoned");
696 state.recent_cost_usd_micros.push_back(cost_usd_micros);
697 while state.recent_cost_usd_micros.len() > PREDICATE_COST_WINDOW {
698 state.recent_cost_usd_micros.pop_front();
699 }
700}
701
702fn average_cost_sample_micros(samples: &VecDeque<u64>) -> Option<u64> {
703 if samples.is_empty() {
704 return None;
705 }
706 let total: u128 = samples.iter().map(|sample| u128::from(*sample)).sum();
707 Some((total / samples.len() as u128) as u64)
708}
709
710pub fn usd_to_micros(value: f64) -> u64 {
711 if !value.is_finite() || value <= 0.0 {
712 return 0;
713 }
714 (value * 1_000_000.0).ceil() as u64
715}
716
717pub fn micros_to_usd(value: u64) -> f64 {
718 value as f64 / 1_000_000.0
719}
720
721fn rollover_orchestrator_budget(state: &mut OrchestratorBudgetState) {
722 let today = utc_day_key();
723 let hour = utc_hour_key();
724 if state.day_utc != today {
725 state.day_utc = today;
726 state.cost_today_usd_micros = 0;
727 }
728 if state.hour_utc != hour {
729 state.hour_utc = hour;
730 state.cost_hour_usd_micros = 0;
731 }
732}
733
734fn utc_day_key() -> i32 {
735 (clock::now_utc().date()
736 - time::Date::from_calendar_date(1970, time::Month::January, 1).expect("valid epoch date"))
737 .whole_days() as i32
738}
739
740fn utc_hour_key() -> i64 {
741 clock::now_utc().unix_timestamp() / 3_600
742}
743
744pub fn snapshot_trigger_bindings() -> Vec<TriggerBindingSnapshot> {
745 TRIGGER_REGISTRY.with(|slot| {
746 let registry = slot.borrow();
747 let mut snapshots = Vec::new();
748 for bindings in registry.bindings.values() {
749 for binding in bindings {
750 snapshots.push(binding.snapshot());
751 }
752 }
753 snapshots.sort_by(|left, right| {
754 left.id
755 .cmp(&right.id)
756 .then(left.version.cmp(&right.version))
757 .then(left.state.as_str().cmp(right.state.as_str()))
758 });
759 snapshots
760 })
761}
762
763#[allow(clippy::arc_with_non_send_sync)]
764pub fn resolve_trigger_binding_as_of(
765 id: &str,
766 as_of: OffsetDateTime,
767) -> Result<Arc<TriggerBinding>, TriggerRegistryError> {
768 let version = binding_version_as_of(id, as_of)?;
769 resolve_trigger_binding_version(id, version)
770}
771
772#[allow(clippy::arc_with_non_send_sync)]
773pub fn resolve_live_or_as_of(
774 id: &str,
775 recorded: RecordedTriggerBinding,
776) -> Result<Arc<TriggerBinding>, TriggerRegistryError> {
777 match resolve_live_trigger_binding(id, Some(recorded.version)) {
778 Ok(binding) => Ok(binding),
779 Err(TriggerRegistryError::UnknownBindingVersion { .. }) => {
780 let binding = resolve_trigger_binding_as_of(id, recorded.received_at)?;
781 let mut metadata = BTreeMap::new();
782 metadata.insert("trigger_id".to_string(), serde_json::json!(id));
783 metadata.insert(
784 "recorded_version".to_string(),
785 serde_json::json!(recorded.version),
786 );
787 metadata.insert(
788 "received_at".to_string(),
789 serde_json::json!(recorded
790 .received_at
791 .format(&time::format_description::well_known::Rfc3339)
792 .unwrap_or_else(|_| recorded.received_at.to_string())),
793 );
794 metadata.insert(
795 "resolved_version".to_string(),
796 serde_json::json!(binding.version),
797 );
798 crate::events::log_warn_meta(
799 "replay.binding_version_gc_fallback",
800 "trigger replay fell back to lifecycle history after binding version GC",
801 metadata,
802 );
803 Ok(binding)
804 }
805 Err(error) => Err(error),
806 }
807}
808
809pub fn binding_version_as_of(id: &str, as_of: OffsetDateTime) -> Result<u32, TriggerRegistryError> {
810 TRIGGER_REGISTRY.with(|slot| {
811 let registry = slot.borrow();
812 registry.binding_version_as_of(id, as_of)
813 })
814}
815
816#[allow(clippy::arc_with_non_send_sync)]
817fn resolve_trigger_binding_version(
818 id: &str,
819 version: u32,
820) -> Result<Arc<TriggerBinding>, TriggerRegistryError> {
821 TRIGGER_REGISTRY.with(|slot| {
822 let registry = slot.borrow();
823 registry
824 .binding(id, version)
825 .ok_or_else(|| TriggerRegistryError::UnknownBindingVersion {
826 id: id.to_string(),
827 version,
828 })
829 })
830}
831
832#[allow(clippy::arc_with_non_send_sync)]
833pub fn resolve_live_trigger_binding(
834 id: &str,
835 version: Option<u32>,
836) -> Result<Arc<TriggerBinding>, TriggerRegistryError> {
837 TRIGGER_REGISTRY.with(|slot| {
838 let registry = slot.borrow();
839 if let Some(version) = version {
840 let binding = registry.binding(id, version).ok_or_else(|| {
841 TriggerRegistryError::UnknownBindingVersion {
842 id: id.to_string(),
843 version,
844 }
845 })?;
846 if binding.state_snapshot() == TriggerState::Terminated {
847 return Err(TriggerRegistryError::UnknownBindingVersion {
848 id: id.to_string(),
849 version,
850 });
851 }
852 return Ok(binding);
853 }
854
855 registry
856 .live_bindings_any_source(id)
857 .into_iter()
858 .max_by_key(|binding| binding.version)
859 .ok_or_else(|| TriggerRegistryError::UnknownId(id.to_string()))
860 })
861}
862
863pub(crate) fn channel_bindings_matching(
869 scope: &str,
870 scope_id: &str,
871 name: &str,
872) -> Vec<Arc<TriggerBinding>> {
873 TRIGGER_REGISTRY.with(|slot| {
874 let registry = slot.borrow();
875 let Some(binding_ids) = registry.by_provider.get("channel") else {
876 return Vec::new();
877 };
878 let mut bindings = Vec::new();
879 for id in binding_ids {
880 let Some(versions) = registry.bindings.get(id) else {
881 continue;
882 };
883 for binding in versions {
884 if binding.state_snapshot() != TriggerState::Active {
885 continue;
886 }
887 let Some(selector_raw) = binding.match_events.first() else {
888 continue;
889 };
890 let Ok(selector) = crate::channels::ChannelSelector::parse(selector_raw) else {
891 continue;
892 };
893 if !selector.matches(scope, scope_id, name, scope_id) {
897 continue;
898 }
899 bindings.push(binding.clone());
900 }
901 }
902 bindings.sort_by(|left, right| {
903 left.id
904 .as_str()
905 .cmp(right.id.as_str())
906 .then(left.version.cmp(&right.version))
907 });
908 bindings
909 })
910}
911
912pub(crate) fn matching_bindings(event: &super::TriggerEvent) -> Vec<Arc<TriggerBinding>> {
913 TRIGGER_REGISTRY.with(|slot| {
914 let registry = slot.borrow();
915 let Some(binding_ids) = registry.by_provider.get(event.provider.as_str()) else {
916 return Vec::new();
917 };
918
919 let mut bindings = Vec::new();
920 for id in binding_ids {
921 let Some(versions) = registry.bindings.get(id) else {
922 continue;
923 };
924 for binding in versions {
925 if binding.state_snapshot() != TriggerState::Active {
926 continue;
927 }
928 if !binding.match_events.is_empty()
929 && !binding
930 .match_events
931 .iter()
932 .any(|kind| trigger_event_kind_matches(event, kind))
933 {
934 continue;
935 }
936 bindings.push(binding.clone());
937 }
938 }
939
940 bindings.sort_by(|left, right| {
941 left.id
942 .as_str()
943 .cmp(right.id.as_str())
944 .then(left.version.cmp(&right.version))
945 });
946 bindings
947 })
948}
949
950fn trigger_event_kind_matches(event: &super::TriggerEvent, expected: &str) -> bool {
951 if expected == event.kind {
952 return true;
953 }
954 expected
955 .strip_prefix(event.provider.as_str())
956 .and_then(|rest| rest.strip_prefix('.'))
957 .is_some_and(|kind| kind == event.kind)
958}
959
960pub async fn install_manifest_triggers(
961 specs: Vec<TriggerBindingSpec>,
962) -> Result<(), TriggerRegistryError> {
963 let (event_log, events) = TRIGGER_REGISTRY.with(|slot| {
964 let registry = &mut *slot.borrow_mut();
965 registry.refresh_runtime_context();
966 let mut touched_ids = BTreeSet::new();
967
968 let mut incoming = BTreeMap::new();
969 for spec in specs {
970 let spec_id = spec.id.clone();
971 if spec.source != TriggerBindingSource::Manifest {
972 return Err(TriggerRegistryError::InvalidSpec(format!(
973 "manifest install received non-manifest trigger '{spec_id}'"
974 )));
975 }
976 if spec_id.trim().is_empty() {
977 return Err(TriggerRegistryError::InvalidSpec(
978 "manifest trigger id cannot be empty".to_string(),
979 ));
980 }
981 if incoming.insert(spec_id.clone(), spec).is_some() {
982 return Err(TriggerRegistryError::DuplicateId(spec_id));
983 }
984 }
985
986 let mut lifecycle = Vec::new();
987 let existing_ids: Vec<String> = registry
988 .bindings
989 .iter()
990 .filter(|(_, bindings)| {
991 bindings.iter().any(|binding| {
992 binding.source == TriggerBindingSource::Manifest
993 && binding.state_snapshot() != TriggerState::Terminated
994 })
995 })
996 .map(|(id, _)| id.clone())
997 .collect();
998
999 for id in existing_ids {
1000 let live_manifest = registry.live_bindings(&id, TriggerBindingSource::Manifest);
1001 let Some(spec) = incoming.remove(&id) else {
1002 for binding in live_manifest {
1003 registry.transition_binding_to_draining(&binding, &mut lifecycle);
1004 }
1005 touched_ids.insert(id.clone());
1006 continue;
1007 };
1008
1009 let has_matching_active = live_manifest.iter().any(|binding| {
1010 binding.definition_fingerprint == spec.definition_fingerprint
1011 && matches!(
1012 binding.state_snapshot(),
1013 TriggerState::Registering | TriggerState::Active
1014 )
1015 });
1016 if has_matching_active {
1017 continue;
1018 }
1019
1020 for binding in live_manifest {
1021 registry.transition_binding_to_draining(&binding, &mut lifecycle);
1022 }
1023
1024 let version = registry.next_version_for_spec(&spec);
1025 registry.register_binding(spec, version, &mut lifecycle);
1026 touched_ids.insert(id.clone());
1027 }
1028
1029 for spec in incoming.into_values() {
1030 touched_ids.insert(spec.id.clone());
1031 let version = registry.next_version_for_spec(&spec);
1032 registry.register_binding(spec, version, &mut lifecycle);
1033 }
1034
1035 for id in touched_ids {
1036 registry.gc_terminated_versions(&id);
1037 }
1038
1039 Ok((registry.event_log.clone(), lifecycle))
1040 })?;
1041
1042 append_lifecycle_events(event_log, events).await
1043}
1044
1045pub async fn dynamic_register(
1046 mut spec: TriggerBindingSpec,
1047) -> Result<TriggerId, TriggerRegistryError> {
1048 if spec.id.trim().is_empty() {
1049 spec.id = format!("dynamic_trigger_{}", Uuid::now_v7());
1050 }
1051 spec.source = TriggerBindingSource::Dynamic;
1052 let id = spec.id.clone();
1053 let (event_log, events) = TRIGGER_REGISTRY.with(|slot| {
1054 let registry = &mut *slot.borrow_mut();
1055 registry.refresh_runtime_context();
1056
1057 if registry.bindings.contains_key(id.as_str()) {
1058 return Err(TriggerRegistryError::DuplicateId(id.clone()));
1059 }
1060
1061 let mut lifecycle = Vec::new();
1062 let version = registry.next_version_for_spec(&spec);
1063 registry.register_binding(spec, version, &mut lifecycle);
1064 Ok((registry.event_log.clone(), lifecycle))
1065 })?;
1066
1067 append_lifecycle_events(event_log, events).await?;
1068 Ok(TriggerId::new(id))
1069}
1070
1071pub async fn dynamic_deregister(id: &str) -> Result<(), TriggerRegistryError> {
1072 let (event_log, events) = TRIGGER_REGISTRY.with(|slot| {
1073 let registry = &mut *slot.borrow_mut();
1074 let live_dynamic = registry.live_bindings(id, TriggerBindingSource::Dynamic);
1075 if live_dynamic.is_empty() {
1076 return Err(TriggerRegistryError::UnknownId(id.to_string()));
1077 }
1078
1079 let mut lifecycle = Vec::new();
1080 for binding in live_dynamic {
1081 registry.transition_binding_to_draining(&binding, &mut lifecycle);
1082 }
1083 Ok((registry.event_log.clone(), lifecycle))
1084 })?;
1085
1086 append_lifecycle_events(event_log, events).await
1087}
1088
1089pub async fn drain(id: &str) -> Result<(), TriggerRegistryError> {
1090 let (event_log, events) = TRIGGER_REGISTRY.with(|slot| {
1091 let registry = &mut *slot.borrow_mut();
1092 let live = registry.live_bindings_any_source(id);
1093 if live.is_empty() {
1094 return Err(TriggerRegistryError::UnknownId(id.to_string()));
1095 }
1096
1097 let mut lifecycle = Vec::new();
1098 for binding in live {
1099 registry.transition_binding_to_draining(&binding, &mut lifecycle);
1100 }
1101 Ok((registry.event_log.clone(), lifecycle))
1102 })?;
1103
1104 append_lifecycle_events(event_log, events).await
1105}
1106
1107pub async fn pause(id: &str) -> Result<(), TriggerRegistryError> {
1108 let (event_log, events) = TRIGGER_REGISTRY.with(|slot| {
1109 let registry = &mut *slot.borrow_mut();
1110 let live = registry.live_bindings_any_source(id);
1111 if live.is_empty() {
1112 return Err(TriggerRegistryError::UnknownId(id.to_string()));
1113 }
1114
1115 let mut lifecycle = Vec::new();
1116 for binding in live {
1117 match binding.state_snapshot() {
1118 TriggerState::Registering | TriggerState::Active => {
1119 registry.transition_binding_state(
1120 &binding,
1121 TriggerState::Paused,
1122 &mut lifecycle,
1123 );
1124 }
1125 TriggerState::Paused | TriggerState::Draining | TriggerState::Terminated => {}
1126 }
1127 }
1128 Ok((registry.event_log.clone(), lifecycle))
1129 })?;
1130
1131 append_lifecycle_events(event_log, events).await
1132}
1133
1134pub async fn resume(id: &str) -> Result<(), TriggerRegistryError> {
1135 let (event_log, events) = TRIGGER_REGISTRY.with(|slot| {
1136 let registry = &mut *slot.borrow_mut();
1137 let live = registry.live_bindings_any_source(id);
1138 if live.is_empty() {
1139 return Err(TriggerRegistryError::UnknownId(id.to_string()));
1140 }
1141
1142 let mut lifecycle = Vec::new();
1143 for binding in live {
1144 if binding.state_snapshot() == TriggerState::Paused {
1145 registry.transition_binding_state(&binding, TriggerState::Active, &mut lifecycle);
1146 }
1147 }
1148 Ok((registry.event_log.clone(), lifecycle))
1149 })?;
1150
1151 append_lifecycle_events(event_log, events).await
1152}
1153
1154fn pin_trigger_binding_inner(
1155 id: &str,
1156 version: u32,
1157 allow_terminated: bool,
1158) -> Result<(), TriggerRegistryError> {
1159 TRIGGER_REGISTRY.with(|slot| {
1160 let registry = slot.borrow();
1161 let binding = registry.binding(id, version).ok_or_else(|| {
1162 TriggerRegistryError::UnknownBindingVersion {
1163 id: id.to_string(),
1164 version,
1165 }
1166 })?;
1167 match binding.state_snapshot() {
1168 TriggerState::Paused => Err(TriggerRegistryError::InvalidSpec(format!(
1169 "trigger binding '{id}' version {version} is paused"
1170 ))),
1171 TriggerState::Terminated if !allow_terminated => {
1172 Err(TriggerRegistryError::InvalidSpec(format!(
1173 "trigger binding '{id}' version {version} is terminated"
1174 )))
1175 }
1176 _ => {
1177 binding.in_flight.fetch_add(1, Ordering::Relaxed);
1178 Ok(())
1179 }
1180 }
1181 })
1182}
1183
1184pub fn pin_trigger_binding(id: &str, version: u32) -> Result<(), TriggerRegistryError> {
1185 pin_trigger_binding_inner(id, version, false)
1186}
1187
1188pub async fn unpin_trigger_binding(id: &str, version: u32) -> Result<(), TriggerRegistryError> {
1189 let (event_log, events) = TRIGGER_REGISTRY.with(|slot| {
1190 let registry = &mut *slot.borrow_mut();
1191 let binding = registry.binding(id, version).ok_or_else(|| {
1192 TriggerRegistryError::UnknownBindingVersion {
1193 id: id.to_string(),
1194 version,
1195 }
1196 })?;
1197 let current = binding.in_flight.load(Ordering::Relaxed);
1198 if current == 0 {
1199 return Err(TriggerRegistryError::InvalidSpec(format!(
1200 "trigger binding '{id}' version {version} has no in-flight events"
1201 )));
1202 }
1203 binding.in_flight.fetch_sub(1, Ordering::Relaxed);
1204
1205 let mut lifecycle = Vec::new();
1206 registry.maybe_finalize_draining(&binding, &mut lifecycle);
1207 registry.gc_terminated_versions(binding.id.as_str());
1208 Ok((registry.event_log.clone(), lifecycle))
1209 })?;
1210
1211 append_lifecycle_events(event_log, events).await
1212}
1213
1214pub fn begin_in_flight(id: &str, version: u32) -> Result<(), TriggerRegistryError> {
1215 begin_in_flight_inner(id, version, false)
1216}
1217
1218pub(crate) fn begin_replay_in_flight(id: &str, version: u32) -> Result<(), TriggerRegistryError> {
1219 begin_in_flight_inner(id, version, true)
1220}
1221
1222fn begin_in_flight_inner(
1223 id: &str,
1224 version: u32,
1225 allow_terminated: bool,
1226) -> Result<(), TriggerRegistryError> {
1227 pin_trigger_binding_inner(id, version, allow_terminated)?;
1228 TRIGGER_REGISTRY.with(|slot| {
1229 let registry = slot.borrow();
1230 let binding = registry.binding(id, version).ok_or_else(|| {
1231 TriggerRegistryError::UnknownBindingVersion {
1232 id: id.to_string(),
1233 version,
1234 }
1235 })?;
1236 binding.metrics.received.fetch_add(1, Ordering::Relaxed);
1237 *binding
1238 .metrics
1239 .last_received_ms
1240 .lock()
1241 .expect("trigger metrics poisoned") = Some(now_ms());
1242 Ok(())
1243 })
1244}
1245
1246pub async fn finish_in_flight(
1247 id: &str,
1248 version: u32,
1249 outcome: TriggerDispatchOutcome,
1250) -> Result<(), TriggerRegistryError> {
1251 TRIGGER_REGISTRY.with(|slot| {
1252 let registry = &mut *slot.borrow_mut();
1253 let binding = registry.binding(id, version).ok_or_else(|| {
1254 TriggerRegistryError::UnknownBindingVersion {
1255 id: id.to_string(),
1256 version,
1257 }
1258 })?;
1259 let current = binding.in_flight.load(Ordering::Relaxed);
1260 if current == 0 {
1261 return Err(TriggerRegistryError::InvalidSpec(format!(
1262 "trigger binding '{id}' version {version} has no in-flight events"
1263 )));
1264 }
1265 match outcome {
1266 TriggerDispatchOutcome::Dispatched => {
1267 binding.metrics.dispatched.fetch_add(1, Ordering::Relaxed);
1268 }
1269 TriggerDispatchOutcome::Failed => {
1270 binding.metrics.failed.fetch_add(1, Ordering::Relaxed);
1271 }
1272 TriggerDispatchOutcome::Dlq => {
1273 binding.metrics.dlq.fetch_add(1, Ordering::Relaxed);
1274 }
1275 }
1276 Ok(())
1277 })?;
1278
1279 unpin_trigger_binding(id, version).await
1280}
1281
1282impl TriggerRegistry {
1283 fn refresh_runtime_context(&mut self) {
1284 if self.event_log.is_none() {
1285 self.event_log = active_event_log();
1286 }
1287 if self.secret_provider.is_none() {
1288 self.secret_provider = default_secret_provider();
1289 }
1290 }
1291
1292 fn binding(&self, id: &str, version: u32) -> Option<Arc<TriggerBinding>> {
1293 self.bindings
1294 .get(id)
1295 .and_then(|bindings| bindings.iter().find(|binding| binding.version == version))
1296 .cloned()
1297 }
1298
1299 fn live_bindings(&self, id: &str, source: TriggerBindingSource) -> Vec<Arc<TriggerBinding>> {
1300 self.bindings
1301 .get(id)
1302 .into_iter()
1303 .flat_map(|bindings| bindings.iter())
1304 .filter(|binding| {
1305 binding.source == source && binding.state_snapshot() != TriggerState::Terminated
1306 })
1307 .cloned()
1308 .collect()
1309 }
1310
1311 fn live_bindings_any_source(&self, id: &str) -> Vec<Arc<TriggerBinding>> {
1312 self.bindings
1313 .get(id)
1314 .into_iter()
1315 .flat_map(|bindings| bindings.iter())
1316 .filter(|binding| binding.state_snapshot() != TriggerState::Terminated)
1317 .cloned()
1318 .collect()
1319 }
1320
1321 fn next_version_for_spec(&self, spec: &TriggerBindingSpec) -> u32 {
1322 if let Some(version) = self
1323 .bindings
1324 .get(spec.id.as_str())
1325 .into_iter()
1326 .flat_map(|bindings| bindings.iter())
1327 .find(|binding| binding.definition_fingerprint == spec.definition_fingerprint)
1328 .map(|binding| binding.version)
1329 {
1330 return version;
1331 }
1332
1333 let historical =
1334 self.historical_versions_for(spec.id.as_str(), spec.definition_fingerprint.as_str());
1335 if let Some(version) = historical.matching_version {
1336 return version;
1337 }
1338
1339 self.bindings
1340 .get(spec.id.as_str())
1341 .into_iter()
1342 .flat_map(|bindings| bindings.iter())
1343 .map(|binding| binding.version)
1344 .chain(historical.max_version)
1345 .max()
1346 .unwrap_or(0)
1347 + 1
1348 }
1349
1350 fn gc_terminated_versions(&mut self, id: &str) {
1351 let Some(bindings) = self.bindings.get_mut(id) else {
1352 return;
1353 };
1354
1355 let mut newest_versions: Vec<u32> =
1356 bindings.iter().map(|binding| binding.version).collect();
1357 newest_versions.sort_unstable_by(|left, right| right.cmp(left));
1358 newest_versions.truncate(TERMINATED_VERSION_RETENTION_LIMIT);
1359 let retained_versions: BTreeSet<u32> = newest_versions.into_iter().collect();
1360
1361 bindings.retain(|binding| {
1362 binding.state_snapshot() != TriggerState::Terminated
1363 || retained_versions.contains(&binding.version)
1364 });
1365
1366 if bindings.is_empty() {
1367 self.bindings.remove(id);
1368 }
1369 }
1370
1371 fn historical_versions_for(&self, id: &str, fingerprint: &str) -> HistoricalVersionLookup {
1372 let mut lookup = HistoricalVersionLookup::default();
1373 for record in self.lifecycle_records_for(id) {
1374 lookup.max_version = Some(
1375 lookup
1376 .max_version
1377 .unwrap_or(0)
1378 .max(record.transition.version),
1379 );
1380 if record.transition.definition_fingerprint.as_deref() == Some(fingerprint) {
1381 lookup.matching_version = Some(record.transition.version);
1382 }
1383 }
1384 lookup
1385 }
1386
1387 fn binding_version_as_of(
1388 &self,
1389 id: &str,
1390 as_of: OffsetDateTime,
1391 ) -> Result<u32, TriggerRegistryError> {
1392 let cutoff_ms = harn_clock::offset_datetime_to_ms(as_of);
1393 let mut active_version = None;
1394 for record in self.lifecycle_records_for(id) {
1395 if record.occurred_at_ms > cutoff_ms {
1396 break;
1397 }
1398 match record.transition.to_state {
1399 TriggerState::Active => active_version = Some(record.transition.version),
1400 TriggerState::Paused | TriggerState::Draining | TriggerState::Terminated => {
1401 if active_version == Some(record.transition.version) {
1402 active_version = None;
1403 }
1404 }
1405 TriggerState::Registering => {}
1406 }
1407 }
1408
1409 active_version.ok_or_else(|| {
1410 TriggerRegistryError::InvalidSpec(format!(
1411 "no active trigger binding '{}' at {}",
1412 id,
1413 as_of
1414 .format(&time::format_description::well_known::Rfc3339)
1415 .unwrap_or_else(|_| as_of.to_string())
1416 ))
1417 })
1418 }
1419
1420 fn lifecycle_records_for(&self, id: &str) -> Vec<HistoricalLifecycleRecord> {
1421 let Some(event_log) = self.event_log.as_ref() else {
1422 return Vec::new();
1423 };
1424 let topic = Topic::new(TRIGGERS_LIFECYCLE_TOPIC)
1425 .expect("static triggers.lifecycle topic should always be valid");
1426 futures::executor::block_on(event_log.read_range(&topic, None, usize::MAX))
1427 .unwrap_or_default()
1428 .into_iter()
1429 .filter_map(|(_, event)| {
1430 let occurred_at_ms = event.occurred_at_ms;
1431 let transition: LifecycleStateTransitionRecord =
1432 serde_json::from_value(event.payload).ok()?;
1433 (transition.id == id).then_some(HistoricalLifecycleRecord {
1434 occurred_at_ms,
1435 transition,
1436 })
1437 })
1438 .collect()
1439 }
1440
1441 #[allow(clippy::arc_with_non_send_sync)]
1442 fn register_binding(
1443 &mut self,
1444 spec: TriggerBindingSpec,
1445 version: u32,
1446 lifecycle: &mut Vec<LogEvent>,
1447 ) -> Arc<TriggerBinding> {
1448 let binding = Arc::new(TriggerBinding::new(spec, version));
1449 self.by_provider
1450 .entry(binding.provider.as_str().to_string())
1451 .or_default()
1452 .insert(binding.id.as_str().to_string());
1453 self.bindings
1454 .entry(binding.id.as_str().to_string())
1455 .or_default()
1456 .push(binding.clone());
1457 lifecycle.push(lifecycle_event(&binding, None, TriggerState::Registering));
1458 self.transition_binding_state(&binding, TriggerState::Active, lifecycle);
1459 binding
1460 }
1461
1462 fn transition_binding_to_draining(
1463 &self,
1464 binding: &Arc<TriggerBinding>,
1465 lifecycle: &mut Vec<LogEvent>,
1466 ) {
1467 if matches!(binding.state_snapshot(), TriggerState::Terminated) {
1468 return;
1469 }
1470 self.transition_binding_state(binding, TriggerState::Draining, lifecycle);
1471 self.maybe_finalize_draining(binding, lifecycle);
1472 }
1473
1474 fn maybe_finalize_draining(
1475 &self,
1476 binding: &Arc<TriggerBinding>,
1477 lifecycle: &mut Vec<LogEvent>,
1478 ) {
1479 if binding.state_snapshot() == TriggerState::Draining
1480 && binding.in_flight.load(Ordering::Relaxed) == 0
1481 {
1482 self.transition_binding_state(binding, TriggerState::Terminated, lifecycle);
1483 }
1484 }
1485
1486 fn transition_binding_state(
1487 &self,
1488 binding: &Arc<TriggerBinding>,
1489 next: TriggerState,
1490 lifecycle: &mut Vec<LogEvent>,
1491 ) {
1492 let mut state = binding.state.lock().expect("trigger state poisoned");
1493 let previous = *state;
1494 if previous == next {
1495 return;
1496 }
1497 *state = next;
1498 drop(state);
1499 if next == TriggerState::Terminated && binding.aggregation.is_some() {
1505 let _ = super::aggregation::drop_binding_aggregation(&binding.binding_key());
1506 }
1507 lifecycle.push(lifecycle_event(binding, Some(previous), next));
1508 }
1509}
1510
1511fn lifecycle_event(
1512 binding: &TriggerBinding,
1513 from_state: Option<TriggerState>,
1514 to_state: TriggerState,
1515) -> LogEvent {
1516 LogEvent::new(
1517 "state_transition",
1518 serde_json::json!({
1519 "id": binding.id.as_str(),
1520 "binding_key": binding.binding_key(),
1521 "version": binding.version,
1522 "provider": binding.provider.as_str(),
1523 "kind": &binding.kind,
1524 "source": binding.source.as_str(),
1525 "handler_kind": binding.handler.kind(),
1526 "definition_fingerprint": &binding.definition_fingerprint,
1527 "from_state": from_state.map(TriggerState::as_str),
1528 "to_state": to_state.as_str(),
1529 }),
1530 )
1531}
1532
1533async fn append_lifecycle_events(
1534 event_log: Option<Arc<AnyEventLog>>,
1535 events: Vec<LogEvent>,
1536) -> Result<(), TriggerRegistryError> {
1537 let Some(event_log) = event_log else {
1538 return Ok(());
1539 };
1540 if events.is_empty() {
1541 return Ok(());
1542 }
1543
1544 let topic = Topic::new(TRIGGERS_LIFECYCLE_TOPIC)
1545 .expect("static triggers.lifecycle topic should always be valid");
1546 for event in events {
1547 event_log
1548 .append(&topic, event)
1549 .await
1550 .map_err(|error| TriggerRegistryError::EventLog(error.to_string()))?;
1551 }
1552 Ok(())
1553}
1554
1555fn default_secret_provider() -> Option<Arc<dyn SecretProvider>> {
1556 configured_default_chain(default_secret_namespace())
1557 .ok()
1558 .map(|provider| Arc::new(provider) as Arc<dyn SecretProvider>)
1559}
1560
1561fn default_secret_namespace() -> String {
1562 if let Ok(namespace) = std::env::var("HARN_SECRET_NAMESPACE") {
1563 if !namespace.trim().is_empty() {
1564 return namespace;
1565 }
1566 }
1567
1568 let cwd = std::env::current_dir().unwrap_or_default();
1569 let leaf = cwd
1570 .file_name()
1571 .and_then(|name| name.to_str())
1572 .filter(|name| !name.is_empty())
1573 .unwrap_or("workspace");
1574 format!("harn/{leaf}")
1575}
1576
1577fn now_ms() -> i64 {
1578 clock::now_ms()
1579}
1580
1581#[cfg(test)]
1582mod tests {
1583 use super::*;
1584 use crate::event_log::{
1585 install_active_event_log, install_memory_for_current_thread, open_event_log,
1586 pin_test_occurred_at_ms, reset_active_event_log, EventLogBackendKind, EventLogConfig,
1587 };
1588 use crate::events::{add_event_sink, clear_event_sinks, CollectorSink, EventLevel};
1589 use std::path::Path;
1590 use std::rc::Rc;
1591
1592 fn offset_from_ms(ms: i64) -> OffsetDateTime {
1597 OffsetDateTime::from_unix_timestamp_nanos((ms as i128) * 1_000_000)
1598 .expect("epoch ms within OffsetDateTime range")
1599 }
1600
1601 fn manifest_spec(id: &str, fingerprint: &str) -> TriggerBindingSpec {
1602 TriggerBindingSpec {
1603 id: id.to_string(),
1604 source: TriggerBindingSource::Manifest,
1605 kind: "webhook".to_string(),
1606 provider: ProviderId::from("github"),
1607 autonomy_tier: crate::AutonomyTier::ActAuto,
1608 handler: TriggerHandlerSpec::Worker {
1609 queue: format!("{id}-queue"),
1610 },
1611 dispatch_priority: crate::WorkerQueuePriority::Normal,
1612 when: None,
1613 when_budget: None,
1614 retry: TriggerRetryConfig::default(),
1615 match_events: vec!["issues.opened".to_string()],
1616 dedupe_key: Some("event.dedupe_key".to_string()),
1617 dedupe_retention_days: crate::triggers::DEFAULT_INBOX_RETENTION_DAYS,
1618 filter: Some("event.kind".to_string()),
1619 daily_cost_usd: Some(5.0),
1620 hourly_cost_usd: None,
1621 max_autonomous_decisions_per_hour: None,
1622 max_autonomous_decisions_per_day: None,
1623 on_budget_exhausted: crate::TriggerBudgetExhaustionStrategy::False,
1624 max_concurrent: Some(10),
1625 flow_control: crate::triggers::TriggerFlowControlConfig::default(),
1626 aggregation: None,
1627 manifest_path: None,
1628 package_name: Some("workspace".to_string()),
1629 definition_fingerprint: fingerprint.to_string(),
1630 }
1631 }
1632
1633 fn install_test_memory_event_log() -> Arc<AnyEventLog> {
1634 install_memory_for_current_thread(512)
1635 }
1636
1637 fn install_test_sqlite_event_log(base_dir: &Path) -> Arc<AnyEventLog> {
1638 let config = EventLogConfig {
1639 backend: EventLogBackendKind::Sqlite,
1640 file_dir: base_dir.join("events"),
1641 sqlite_path: base_dir.join("events.sqlite"),
1642 queue_depth: 512,
1643 };
1644 let log = open_event_log(&config).expect("open isolated sqlite event log");
1645 install_active_event_log(log)
1646 }
1647
1648 fn dynamic_spec(id: &str) -> TriggerBindingSpec {
1649 TriggerBindingSpec {
1650 id: id.to_string(),
1651 source: TriggerBindingSource::Dynamic,
1652 kind: "webhook".to_string(),
1653 provider: ProviderId::from("github"),
1654 autonomy_tier: crate::AutonomyTier::ActAuto,
1655 handler: TriggerHandlerSpec::Worker {
1656 queue: format!("{id}-queue"),
1657 },
1658 dispatch_priority: crate::WorkerQueuePriority::Normal,
1659 when: None,
1660 when_budget: None,
1661 retry: TriggerRetryConfig::default(),
1662 match_events: vec!["issues.opened".to_string()],
1663 dedupe_key: None,
1664 dedupe_retention_days: crate::triggers::DEFAULT_INBOX_RETENTION_DAYS,
1665 filter: None,
1666 daily_cost_usd: None,
1667 hourly_cost_usd: None,
1668 max_autonomous_decisions_per_hour: None,
1669 max_autonomous_decisions_per_day: None,
1670 on_budget_exhausted: crate::TriggerBudgetExhaustionStrategy::False,
1671 max_concurrent: None,
1672 flow_control: crate::triggers::TriggerFlowControlConfig::default(),
1673 aggregation: None,
1674 manifest_path: None,
1675 package_name: None,
1676 definition_fingerprint: format!("dynamic:{id}"),
1677 }
1678 }
1679
1680 #[tokio::test(flavor = "current_thread")]
1681 async fn manifest_loaded_trigger_registers_with_zeroed_metrics() {
1682 clear_trigger_registry();
1683
1684 install_manifest_triggers(vec![manifest_spec("github-new-issue", "v1")])
1685 .await
1686 .expect("manifest trigger installs");
1687
1688 let snapshots = snapshot_trigger_bindings();
1689 assert_eq!(snapshots.len(), 1);
1690 let binding = &snapshots[0];
1691 assert_eq!(binding.id, "github-new-issue");
1692 assert_eq!(binding.version, 1);
1693 assert_eq!(binding.state, TriggerState::Active);
1694 assert_eq!(binding.metrics, TriggerMetricsSnapshot::default());
1695
1696 clear_trigger_registry();
1697 }
1698
1699 #[tokio::test(flavor = "current_thread")]
1700 async fn dynamic_register_assigns_unique_ids_and_rejects_duplicates() {
1701 clear_trigger_registry();
1702
1703 let first = dynamic_register(dynamic_spec("dynamic-a"))
1704 .await
1705 .expect("first dynamic trigger");
1706 let second = dynamic_register(dynamic_spec("dynamic-b"))
1707 .await
1708 .expect("second dynamic trigger");
1709 assert_ne!(first, second);
1710
1711 let error = dynamic_register(dynamic_spec("dynamic-a"))
1712 .await
1713 .expect_err("duplicate id should fail");
1714 assert!(matches!(error, TriggerRegistryError::DuplicateId(_)));
1715
1716 clear_trigger_registry();
1717 }
1718
1719 #[test]
1720 fn expected_predicate_cost_average_does_not_overflow() {
1721 let binding = TriggerBinding::new(manifest_spec("costed", "v1"), 1);
1722 record_predicate_cost_sample(&binding, u64::MAX);
1723 record_predicate_cost_sample(&binding, u64::MAX);
1724
1725 assert_eq!(expected_predicate_cost_usd_micros(&binding), u64::MAX);
1726 }
1727
1728 #[tokio::test(flavor = "current_thread")]
1729 async fn drain_waits_for_in_flight_events_before_terminating() {
1730 clear_trigger_registry();
1731
1732 install_manifest_triggers(vec![manifest_spec("github-new-issue", "v1")])
1733 .await
1734 .expect("manifest trigger installs");
1735 begin_in_flight("github-new-issue", 1).expect("start in-flight event");
1736
1737 drain("github-new-issue").await.expect("drain succeeds");
1738 let binding = snapshot_trigger_bindings()
1739 .into_iter()
1740 .find(|binding| binding.id == "github-new-issue" && binding.version == 1)
1741 .expect("binding snapshot");
1742 assert_eq!(binding.state, TriggerState::Draining);
1743 assert_eq!(binding.metrics.in_flight, 1);
1744
1745 finish_in_flight("github-new-issue", 1, TriggerDispatchOutcome::Dispatched)
1746 .await
1747 .expect("finish in-flight event");
1748 let binding = snapshot_trigger_bindings()
1749 .into_iter()
1750 .find(|binding| binding.id == "github-new-issue" && binding.version == 1)
1751 .expect("binding snapshot");
1752 assert_eq!(binding.state, TriggerState::Terminated);
1753 assert_eq!(binding.metrics.in_flight, 0);
1754
1755 clear_trigger_registry();
1756 }
1757
1758 #[tokio::test(flavor = "current_thread")]
1759 async fn hot_reload_registers_new_version_while_old_binding_drains() {
1760 clear_trigger_registry();
1761
1762 install_manifest_triggers(vec![manifest_spec("github-new-issue", "v1")])
1763 .await
1764 .expect("initial manifest trigger installs");
1765 begin_in_flight("github-new-issue", 1).expect("start in-flight event");
1766
1767 install_manifest_triggers(vec![manifest_spec("github-new-issue", "v2")])
1768 .await
1769 .expect("updated manifest trigger installs");
1770
1771 let snapshots = snapshot_trigger_bindings();
1772 assert_eq!(snapshots.len(), 2);
1773 let old = snapshots
1774 .iter()
1775 .find(|binding| binding.id == "github-new-issue" && binding.version == 1)
1776 .expect("old binding");
1777 let new = snapshots
1778 .iter()
1779 .find(|binding| binding.id == "github-new-issue" && binding.version == 2)
1780 .expect("new binding");
1781 assert_eq!(old.state, TriggerState::Draining);
1782 assert_eq!(new.state, TriggerState::Active);
1783
1784 finish_in_flight("github-new-issue", 1, TriggerDispatchOutcome::Dispatched)
1785 .await
1786 .expect("finish old in-flight event");
1787 let old = snapshot_trigger_bindings()
1788 .into_iter()
1789 .find(|binding| binding.id == "github-new-issue" && binding.version == 1)
1790 .expect("old binding");
1791 assert_eq!(old.state, TriggerState::Terminated);
1792
1793 clear_trigger_registry();
1794 }
1795
1796 #[tokio::test(flavor = "current_thread")]
1797 async fn gc_drops_terminated_versions_beyond_retention_limit() {
1798 clear_trigger_registry();
1799
1800 install_manifest_triggers(vec![manifest_spec("github-new-issue", "v1")])
1801 .await
1802 .expect("install v1");
1803 begin_in_flight("github-new-issue", 1).expect("pin v1");
1804
1805 install_manifest_triggers(vec![manifest_spec("github-new-issue", "v2")])
1806 .await
1807 .expect("install v2");
1808 finish_in_flight("github-new-issue", 1, TriggerDispatchOutcome::Dispatched)
1809 .await
1810 .expect("finish v1");
1811
1812 install_manifest_triggers(vec![manifest_spec("github-new-issue", "v3")])
1813 .await
1814 .expect("install v3");
1815
1816 let snapshots = snapshot_trigger_bindings();
1817 let versions: Vec<u32> = snapshots
1818 .into_iter()
1819 .filter(|binding| binding.id == "github-new-issue")
1820 .map(|binding| binding.version)
1821 .collect();
1822 assert_eq!(versions, vec![2, 3]);
1823
1824 clear_trigger_registry();
1825 }
1826
1827 #[tokio::test(flavor = "current_thread")]
1828 async fn lifecycle_transitions_append_to_event_log() {
1829 clear_trigger_registry();
1830 reset_active_event_log();
1831 let log = install_test_memory_event_log();
1832
1833 install_manifest_triggers(vec![manifest_spec("github-new-issue", "v1")])
1834 .await
1835 .expect("manifest trigger installs");
1836 begin_in_flight("github-new-issue", 1).expect("start in-flight event");
1837 drain("github-new-issue").await.expect("drain succeeds");
1838 finish_in_flight("github-new-issue", 1, TriggerDispatchOutcome::Dispatched)
1839 .await
1840 .expect("finish event");
1841
1842 let topic = Topic::new("triggers.lifecycle").expect("valid lifecycle topic");
1843 let events = log
1844 .read_range(&topic, None, 32)
1845 .await
1846 .expect("read lifecycle events");
1847 let states: Vec<String> = events
1848 .into_iter()
1849 .filter_map(|(_, event)| {
1850 event
1851 .payload
1852 .get("to_state")
1853 .and_then(|value| value.as_str())
1854 .map(|value| value.to_string())
1855 })
1856 .collect();
1857 assert_eq!(
1858 states,
1859 vec![
1860 "registering".to_string(),
1861 "active".to_string(),
1862 "draining".to_string(),
1863 "terminated".to_string(),
1864 ]
1865 );
1866
1867 reset_active_event_log();
1868 clear_trigger_registry();
1869 }
1870
1871 #[tokio::test(flavor = "current_thread")]
1872 async fn version_history_reuses_historical_version_after_restart() {
1873 clear_trigger_registry();
1874 reset_active_event_log();
1875 let tempdir = tempfile::tempdir().expect("tempdir");
1876 install_test_sqlite_event_log(tempdir.path());
1877
1878 install_manifest_triggers(vec![manifest_spec("github-new-issue", "v1")])
1879 .await
1880 .expect("initial manifest trigger installs");
1881 install_manifest_triggers(vec![manifest_spec("github-new-issue", "v2")])
1882 .await
1883 .expect("updated manifest trigger installs");
1884
1885 clear_trigger_registry();
1886 reset_active_event_log();
1887 install_test_sqlite_event_log(tempdir.path());
1888
1889 install_manifest_triggers(vec![manifest_spec("github-new-issue", "v2")])
1890 .await
1891 .expect("manifest reload reuses historical version");
1892
1893 let binding = snapshot_trigger_bindings()
1894 .into_iter()
1895 .find(|binding| binding.id == "github-new-issue")
1896 .expect("binding snapshot");
1897 assert_eq!(binding.version, 2);
1898
1899 reset_active_event_log();
1900 clear_trigger_registry();
1901 }
1902
1903 #[tokio::test(flavor = "current_thread")]
1904 async fn binding_version_as_of_reports_historical_active_version() {
1905 clear_trigger_registry();
1906 reset_active_event_log();
1907 install_test_memory_event_log();
1908
1909 const T1_MS: i64 = 1_700_000_000_000;
1914 const T2_MS: i64 = T1_MS + 50;
1915 let before_reload = offset_from_ms(T1_MS);
1916 let after_reload = offset_from_ms(T2_MS);
1917
1918 let clock = pin_test_occurred_at_ms(T1_MS);
1919 install_manifest_triggers(vec![manifest_spec("github-new-issue", "v1")])
1920 .await
1921 .expect("initial manifest trigger installs");
1922 drop(clock);
1923
1924 let clock = pin_test_occurred_at_ms(T2_MS);
1925 install_manifest_triggers(vec![manifest_spec("github-new-issue", "v2")])
1926 .await
1927 .expect("updated manifest trigger installs");
1928 drop(clock);
1929
1930 assert_eq!(
1931 binding_version_as_of("github-new-issue", before_reload)
1932 .expect("version before reload"),
1933 1
1934 );
1935 assert_eq!(
1936 binding_version_as_of("github-new-issue", after_reload).expect("version after reload"),
1937 2
1938 );
1939
1940 reset_active_event_log();
1941 clear_trigger_registry();
1942 }
1943
1944 #[tokio::test(flavor = "current_thread")]
1945 async fn resolve_live_or_as_of_logs_structured_gc_fallback() {
1946 clear_trigger_registry();
1947 reset_active_event_log();
1948 let sink = Rc::new(CollectorSink::new());
1949 clear_event_sinks();
1950 add_event_sink(sink.clone());
1951 install_test_memory_event_log();
1952
1953 const T1_MS: i64 = 1_700_000_000_000;
1957 const T2_MS: i64 = T1_MS + 50;
1958 let received_at = offset_from_ms(T1_MS);
1959
1960 let clock = pin_test_occurred_at_ms(T1_MS);
1961 install_manifest_triggers(vec![manifest_spec("github-new-issue", "v1")])
1962 .await
1963 .expect("install v1");
1964 install_manifest_triggers(vec![manifest_spec("github-new-issue", "v2")])
1965 .await
1966 .expect("install v2");
1967 install_manifest_triggers(vec![manifest_spec("github-new-issue", "v3")])
1968 .await
1969 .expect("install v3");
1970 drop(clock);
1971
1972 let clock = pin_test_occurred_at_ms(T2_MS);
1973 install_manifest_triggers(vec![manifest_spec("github-new-issue", "v4")])
1974 .await
1975 .expect("install v4");
1976 drop(clock);
1977
1978 let binding = resolve_live_or_as_of(
1979 "github-new-issue",
1980 RecordedTriggerBinding {
1981 version: 1,
1982 received_at,
1983 },
1984 )
1985 .expect("resolve fallback binding");
1986 assert_eq!(binding.version, 3);
1987
1988 let warning = sink
1989 .logs
1990 .borrow()
1991 .iter()
1992 .find(|log| log.category == "replay.binding_version_gc_fallback")
1993 .cloned()
1994 .expect("gc fallback warning");
1995 assert_eq!(warning.level, EventLevel::Warn);
1996 assert_eq!(
1997 warning.metadata.get("trigger_id"),
1998 Some(&serde_json::json!("github-new-issue"))
1999 );
2000 assert_eq!(
2001 warning.metadata.get("recorded_version"),
2002 Some(&serde_json::json!(1))
2003 );
2004 assert_eq!(
2005 warning.metadata.get("received_at"),
2006 Some(&serde_json::json!(received_at
2007 .format(&time::format_description::well_known::Rfc3339)
2008 .unwrap_or_else(|_| received_at.to_string())))
2009 );
2010 assert_eq!(
2011 warning.metadata.get("resolved_version"),
2012 Some(&serde_json::json!(3))
2013 );
2014
2015 clear_event_sinks();
2016 crate::events::reset_event_sinks();
2017 reset_active_event_log();
2018 clear_trigger_registry();
2019 }
2020}