1use std::collections::HashMap;
2
3use serde_json::json;
4use serde_json::Value;
5use tokio::fs;
6use uuid::Uuid;
7
8use crate::audit::append_protected_audit_event;
9use crate::automation_v2::governance::*;
10use crate::{now_ms, AppState};
11
12const GOVERNANCE_AUDIT_EVENT_PREFIX: &str = "automation.governance";
13
14#[derive(Default)]
15pub struct UnavailableGovernanceEngine;
16
17impl GovernancePolicyEngine for UnavailableGovernanceEngine {
18 fn premium_enabled(&self) -> bool {
19 false
20 }
21
22 fn authorize_create(
23 &self,
24 _snapshot: &GovernanceContextSnapshot,
25 actor: &GovernanceActorRef,
26 _provenance: &AutomationProvenanceRecord,
27 _declared_capabilities: &AutomationDeclaredCapabilities,
28 _now_ms: u64,
29 ) -> Result<(), GovernanceError> {
30 if actor.kind == GovernanceActorKind::Human {
31 return Ok(());
32 }
33 Err(GovernanceError::feature_unavailable(
34 "premium governance is required for agent-authored automation creation",
35 ))
36 }
37
38 fn authorize_capability_escalation(
39 &self,
40 _snapshot: &GovernanceContextSnapshot,
41 actor: &GovernanceActorRef,
42 _previous: &AutomationDeclaredCapabilities,
43 _next: &AutomationDeclaredCapabilities,
44 _now_ms: u64,
45 ) -> Result<(), GovernanceError> {
46 if actor.kind == GovernanceActorKind::Human {
47 return Ok(());
48 }
49 Err(GovernanceError::feature_unavailable(
50 "premium governance is required for agent capability escalation",
51 ))
52 }
53
54 fn authorize_mutation(
55 &self,
56 _record: &AutomationGovernanceRecord,
57 actor: &GovernanceActorRef,
58 _destructive: bool,
59 ) -> Result<(), GovernanceError> {
60 if actor.kind == GovernanceActorKind::Human {
61 return Ok(());
62 }
63 Err(GovernanceError::feature_unavailable(
64 "premium governance is required for agent-owned automation mutation",
65 ))
66 }
67
68 fn create_approval_request(
69 &self,
70 _snapshot: &GovernanceContextSnapshot,
71 _input: GovernanceApprovalDraftInput,
72 _now_ms: u64,
73 ) -> Result<GovernanceApprovalRequest, GovernanceError> {
74 Err(GovernanceError::feature_unavailable(
75 "premium governance approval flows are not available in this build",
76 ))
77 }
78
79 fn decide_approval_request(
80 &self,
81 _existing: &GovernanceApprovalRequest,
82 _reviewer: GovernanceActorRef,
83 _approved: bool,
84 _notes: Option<String>,
85 _now_ms: u64,
86 ) -> Result<GovernanceApprovalRequest, GovernanceError> {
87 Err(GovernanceError::feature_unavailable(
88 "premium governance approval flows are not available in this build",
89 ))
90 }
91
92 fn evaluate_creation_review_progress(
93 &self,
94 _snapshot: &GovernanceContextSnapshot,
95 _agent_id: &str,
96 _automation_id: &str,
97 _now_ms: u64,
98 ) -> Result<GovernanceCreationReviewEvaluation, GovernanceError> {
99 Err(GovernanceError::feature_unavailable(
100 "premium governance review tracking is not available in this build",
101 ))
102 }
103
104 fn evaluate_run_review_progress(
105 &self,
106 _snapshot: &GovernanceContextSnapshot,
107 _automation_id: &str,
108 _reason: AutomationLifecycleReviewKind,
109 _run_id: Option<String>,
110 _detail: Option<String>,
111 _now_ms: u64,
112 ) -> Result<Option<GovernanceAutomationReviewEvaluation>, GovernanceError> {
113 Err(GovernanceError::feature_unavailable(
114 "premium governance review tracking is not available in this build",
115 ))
116 }
117
118 fn evaluate_dependency_revocation(
119 &self,
120 _snapshot: &GovernanceContextSnapshot,
121 _input: GovernanceDependencyRevocationInput,
122 _now_ms: u64,
123 ) -> Result<GovernanceAutomationReviewEvaluation, GovernanceError> {
124 Err(GovernanceError::feature_unavailable(
125 "premium governance dependency revocation is not available in this build",
126 ))
127 }
128
129 fn evaluate_health_check(
130 &self,
131 _snapshot: &GovernanceContextSnapshot,
132 _input: GovernanceHealthCheckInput,
133 _now_ms: u64,
134 ) -> Result<Option<GovernanceHealthCheckEvaluation>, GovernanceError> {
135 Ok(None)
136 }
137
138 fn evaluate_retirement(
139 &self,
140 _input: GovernanceRetirementInput,
141 _now_ms: u64,
142 ) -> Result<AutomationGovernanceRecord, GovernanceError> {
143 Err(GovernanceError::feature_unavailable(
144 "premium governance retirement logic is not available in this build",
145 ))
146 }
147
148 fn evaluate_retirement_extension(
149 &self,
150 _input: GovernanceRetirementExtensionInput,
151 _now_ms: u64,
152 ) -> Result<AutomationGovernanceRecord, GovernanceError> {
153 Err(GovernanceError::feature_unavailable(
154 "premium governance retirement logic is not available in this build",
155 ))
156 }
157
158 fn evaluate_spend_usage(
159 &self,
160 _snapshot: &GovernanceContextSnapshot,
161 _input: &GovernanceSpendInput,
162 _now_ms: u64,
163 ) -> Result<GovernanceSpendEvaluation, GovernanceError> {
164 Err(GovernanceError::feature_unavailable(
165 "premium governance spend tracking is not available in this build",
166 ))
167 }
168}
169
170fn default_human_provenance(
171 creator_id: Option<String>,
172 source: impl Into<String>,
173) -> AutomationProvenanceRecord {
174 AutomationProvenanceRecord::human(creator_id, source)
175}
176
177fn declared_capabilities_for_automation(
178 automation: &crate::AutomationV2Spec,
179) -> AutomationDeclaredCapabilities {
180 AutomationDeclaredCapabilities::from_metadata(automation.metadata.as_ref())
181}
182
183impl AppState {
184 pub fn premium_governance_enabled(&self) -> bool {
185 self.governance_engine.premium_enabled()
186 }
187
188 fn governance_snapshot(&self, state: &GovernanceState) -> GovernanceContextSnapshot {
189 state.snapshot()
190 }
191
192 pub async fn load_automation_governance(&self) -> anyhow::Result<()> {
193 if !self.automation_governance_path.exists() {
194 return Ok(());
195 }
196 let raw = fs::read_to_string(&self.automation_governance_path).await?;
197 let parsed = serde_json::from_str::<GovernanceState>(&raw).unwrap_or_default();
198 *self.automation_governance.write().await = parsed;
199 Ok(())
200 }
201
202 pub async fn persist_automation_governance(&self) -> anyhow::Result<()> {
203 if let Some(parent) = self.automation_governance_path.parent() {
204 fs::create_dir_all(parent).await?;
205 }
206 let payload = {
207 let guard = self.automation_governance.read().await;
208 serde_json::to_string_pretty(&*guard)?
209 };
210 fs::write(&self.automation_governance_path, payload).await?;
211 Ok(())
212 }
213
214 async fn persist_automation_governance_locked(&self) -> anyhow::Result<()> {
215 self.persist_automation_governance().await
216 }
217
218 pub async fn bootstrap_automation_governance(&self) -> anyhow::Result<usize> {
219 let automations = self.list_automations_v2().await;
220 let now = now_ms();
221 let mut inserted = 0usize;
222 {
223 let mut guard = self.automation_governance.write().await;
224 for automation in automations {
225 if guard.records.contains_key(&automation.automation_id) {
226 continue;
227 }
228 guard.records.insert(
229 automation.automation_id.clone(),
230 AutomationGovernanceRecord {
231 automation_id: automation.automation_id.clone(),
232 provenance: default_human_provenance(
233 Some(automation.creator_id.clone()),
234 "migration_or_legacy_default",
235 ),
236 declared_capabilities: declared_capabilities_for_automation(&automation),
237 modify_grants: Vec::new(),
238 capability_grants: Vec::new(),
239 created_at_ms: automation.created_at_ms.max(now),
240 updated_at_ms: now,
241 deleted_at_ms: None,
242 delete_retention_until_ms: None,
243 published_externally: false,
244 creation_paused: false,
245 review_required: false,
246 review_kind: None,
247 review_requested_at_ms: None,
248 review_request_id: None,
249 last_reviewed_at_ms: None,
250 runs_since_review: 0,
251 expires_at_ms: None,
252 expired_at_ms: None,
253 retired_at_ms: None,
254 retire_reason: None,
255 paused_for_lifecycle: false,
256 health_last_checked_at_ms: None,
257 health_findings: Vec::new(),
258 },
259 );
260 inserted += 1;
261 }
262 guard.updated_at_ms = now;
263 }
264 if inserted > 0 {
265 self.persist_automation_governance().await?;
266 }
267 Ok(inserted)
268 }
269
270 pub async fn get_automation_governance(
271 &self,
272 automation_id: &str,
273 ) -> Option<AutomationGovernanceRecord> {
274 self.automation_governance
275 .read()
276 .await
277 .records
278 .get(automation_id)
279 .cloned()
280 }
281
282 pub async fn get_or_bootstrap_automation_governance(
283 &self,
284 automation: &crate::AutomationV2Spec,
285 ) -> AutomationGovernanceRecord {
286 if let Some(record) = self
287 .get_automation_governance(&automation.automation_id)
288 .await
289 {
290 return record;
291 }
292 let record = AutomationGovernanceRecord {
293 automation_id: automation.automation_id.clone(),
294 provenance: default_human_provenance(
295 Some(automation.creator_id.clone()),
296 "legacy_default",
297 ),
298 declared_capabilities: declared_capabilities_for_automation(automation),
299 modify_grants: Vec::new(),
300 capability_grants: Vec::new(),
301 created_at_ms: automation.created_at_ms,
302 updated_at_ms: now_ms(),
303 deleted_at_ms: None,
304 delete_retention_until_ms: None,
305 published_externally: false,
306 creation_paused: false,
307 review_required: false,
308 review_kind: None,
309 review_requested_at_ms: None,
310 review_request_id: None,
311 last_reviewed_at_ms: None,
312 runs_since_review: 0,
313 expires_at_ms: None,
314 expired_at_ms: None,
315 retired_at_ms: None,
316 retire_reason: None,
317 paused_for_lifecycle: false,
318 health_last_checked_at_ms: None,
319 health_findings: Vec::new(),
320 };
321 let _ = self.upsert_automation_governance(record.clone()).await;
322 record
323 }
324
325 pub async fn upsert_automation_governance(
326 &self,
327 mut record: AutomationGovernanceRecord,
328 ) -> anyhow::Result<AutomationGovernanceRecord> {
329 if record.automation_id.trim().is_empty() {
330 anyhow::bail!("automation_id is required");
331 }
332 let now = now_ms();
333 if record.created_at_ms == 0 {
334 record.created_at_ms = now;
335 }
336 record.updated_at_ms = now;
337 {
338 let mut guard = self.automation_governance.write().await;
339 guard
340 .records
341 .insert(record.automation_id.clone(), record.clone());
342 guard.updated_at_ms = now;
343 }
344 self.persist_automation_governance().await?;
345 let _ = append_protected_audit_event(
346 self,
347 format!("{GOVERNANCE_AUDIT_EVENT_PREFIX}.record.updated"),
348 &tandem_types::TenantContext::local_implicit(),
349 record
350 .provenance
351 .creator
352 .actor_id
353 .clone()
354 .or_else(|| record.provenance.creator.source.clone()),
355 json!({
356 "automationID": record.automation_id,
357 "provenance": record.provenance,
358 "declaredCapabilities": record.declared_capabilities,
359 "publishedExternally": record.published_externally,
360 "creationPaused": record.creation_paused,
361 }),
362 )
363 .await;
364 Ok(record)
365 }
366
367 pub async fn set_automation_governance_provenance(
368 &self,
369 automation_id: &str,
370 provenance: AutomationProvenanceRecord,
371 ) -> anyhow::Result<AutomationGovernanceRecord> {
372 let mut record = self
373 .get_automation_governance(automation_id)
374 .await
375 .unwrap_or_else(|| AutomationGovernanceRecord {
376 automation_id: automation_id.to_string(),
377 provenance: provenance.clone(),
378 declared_capabilities: AutomationDeclaredCapabilities::default(),
379 modify_grants: Vec::new(),
380 capability_grants: Vec::new(),
381 created_at_ms: now_ms(),
382 updated_at_ms: now_ms(),
383 deleted_at_ms: None,
384 delete_retention_until_ms: None,
385 published_externally: false,
386 creation_paused: false,
387 review_required: false,
388 review_kind: None,
389 review_requested_at_ms: None,
390 review_request_id: None,
391 last_reviewed_at_ms: None,
392 runs_since_review: 0,
393 expires_at_ms: None,
394 expired_at_ms: None,
395 retired_at_ms: None,
396 retire_reason: None,
397 paused_for_lifecycle: false,
398 health_last_checked_at_ms: None,
399 health_findings: Vec::new(),
400 });
401 record.provenance = provenance;
402 if record.expires_at_ms.is_none()
403 && record.provenance.creator.kind == GovernanceActorKind::Agent
404 {
405 let default_expires_after_ms = self
406 .automation_governance
407 .read()
408 .await
409 .limits
410 .default_expires_after_ms;
411 if default_expires_after_ms > 0 {
412 record.expires_at_ms = Some(now_ms().saturating_add(default_expires_after_ms));
413 }
414 }
415 let stored = self.upsert_automation_governance(record).await?;
416 if let Some(agent_id) = stored
417 .provenance
418 .creator
419 .actor_id
420 .as_deref()
421 .filter(|_| stored.provenance.creator.kind == GovernanceActorKind::Agent)
422 {
423 let _ = self
424 .record_agent_creation_review_progress(agent_id, &stored.automation_id)
425 .await;
426 }
427 Ok(stored)
428 }
429
430 pub async fn sync_automation_governance_from_spec(
431 &self,
432 automation: &crate::AutomationV2Spec,
433 provenance: Option<AutomationProvenanceRecord>,
434 ) -> anyhow::Result<AutomationGovernanceRecord> {
435 let now = now_ms();
436 let mut record = self
437 .get_automation_governance(&automation.automation_id)
438 .await
439 .unwrap_or_else(|| AutomationGovernanceRecord {
440 automation_id: automation.automation_id.clone(),
441 provenance: provenance.clone().unwrap_or_else(|| {
442 default_human_provenance(Some(automation.creator_id.clone()), "sync_default")
443 }),
444 declared_capabilities: declared_capabilities_for_automation(automation),
445 modify_grants: Vec::new(),
446 capability_grants: Vec::new(),
447 created_at_ms: automation.created_at_ms,
448 updated_at_ms: now,
449 deleted_at_ms: None,
450 delete_retention_until_ms: None,
451 published_externally: false,
452 creation_paused: false,
453 review_required: false,
454 review_kind: None,
455 review_requested_at_ms: None,
456 review_request_id: None,
457 last_reviewed_at_ms: None,
458 runs_since_review: 0,
459 expires_at_ms: None,
460 expired_at_ms: None,
461 retired_at_ms: None,
462 retire_reason: None,
463 paused_for_lifecycle: false,
464 health_last_checked_at_ms: None,
465 health_findings: Vec::new(),
466 });
467 if let Some(provenance) = provenance {
468 record.provenance = provenance;
469 }
470 record.declared_capabilities = declared_capabilities_for_automation(automation);
471 if record.created_at_ms == 0 {
472 record.created_at_ms = automation.created_at_ms;
473 }
474 record.updated_at_ms = now;
475 {
476 let mut guard = self.automation_governance.write().await;
477 guard
478 .records
479 .insert(record.automation_id.clone(), record.clone());
480 guard.updated_at_ms = now;
481 }
482 self.persist_automation_governance().await?;
483 if let Some(agent_id) = record
484 .provenance
485 .creator
486 .actor_id
487 .as_deref()
488 .filter(|_| record.provenance.creator.kind == GovernanceActorKind::Agent)
489 {
490 let _ = self
491 .record_agent_creation_review_progress(agent_id, &record.automation_id)
492 .await;
493 }
494 Ok(record)
495 }
496
497 pub async fn pause_automation_creation_for_agent(
498 &self,
499 agent_id: &str,
500 paused: bool,
501 ) -> anyhow::Result<()> {
502 let mut guard = self.automation_governance.write().await;
503 if paused {
504 if !guard.paused_agents.iter().any(|value| value == agent_id) {
505 guard.paused_agents.push(agent_id.to_string());
506 }
507 } else {
508 guard.paused_agents.retain(|value| value != agent_id);
509 }
510 guard.updated_at_ms = now_ms();
511 drop(guard);
512 self.persist_automation_governance().await?;
513 Ok(())
514 }
515
516 pub async fn can_create_automation_for_actor(
517 &self,
518 actor: &GovernanceActorRef,
519 provenance: &AutomationProvenanceRecord,
520 declared_capabilities: &AutomationDeclaredCapabilities,
521 ) -> Result<(), GovernanceError> {
522 let snapshot = {
523 let guard = self.automation_governance.read().await;
524 self.governance_snapshot(&guard)
525 };
526 self.governance_engine.authorize_create(
527 &snapshot,
528 actor,
529 provenance,
530 declared_capabilities,
531 now_ms(),
532 )
533 }
534
535 pub async fn can_escalate_declared_capabilities(
536 &self,
537 actor: &GovernanceActorRef,
538 previous: &AutomationDeclaredCapabilities,
539 next: &AutomationDeclaredCapabilities,
540 ) -> Result<(), GovernanceError> {
541 let snapshot = {
542 let guard = self.automation_governance.read().await;
543 self.governance_snapshot(&guard)
544 };
545 self.governance_engine.authorize_capability_escalation(
546 &snapshot,
547 actor,
548 previous,
549 next,
550 now_ms(),
551 )
552 }
553
554 pub async fn can_mutate_automation(
555 &self,
556 automation_id: &str,
557 actor: &GovernanceActorRef,
558 destructive: bool,
559 ) -> Result<AutomationGovernanceRecord, GovernanceError> {
560 let guard = self.automation_governance.read().await;
561 let Some(record) = guard.records.get(automation_id).cloned() else {
562 return Err(GovernanceError::forbidden(
563 "AUTOMATION_V2_GOVERNANCE_MISSING",
564 "automation governance record not found",
565 ));
566 };
567 self.governance_engine
568 .authorize_mutation(&record, actor, destructive)?;
569 Ok(record)
570 }
571
572 pub async fn record_automation_creation(
573 &self,
574 automation: &crate::AutomationV2Spec,
575 provenance: AutomationProvenanceRecord,
576 ) -> anyhow::Result<AutomationGovernanceRecord> {
577 let mut record = AutomationGovernanceRecord {
578 automation_id: automation.automation_id.clone(),
579 provenance,
580 declared_capabilities: declared_capabilities_for_automation(automation),
581 modify_grants: Vec::new(),
582 capability_grants: Vec::new(),
583 created_at_ms: automation.created_at_ms,
584 updated_at_ms: now_ms(),
585 deleted_at_ms: None,
586 delete_retention_until_ms: None,
587 published_externally: false,
588 creation_paused: false,
589 review_required: false,
590 review_kind: None,
591 review_requested_at_ms: None,
592 review_request_id: None,
593 last_reviewed_at_ms: None,
594 runs_since_review: 0,
595 expires_at_ms: None,
596 expired_at_ms: None,
597 retired_at_ms: None,
598 retire_reason: None,
599 paused_for_lifecycle: false,
600 health_last_checked_at_ms: None,
601 health_findings: Vec::new(),
602 };
603 if record.expires_at_ms.is_none()
604 && record.provenance.creator.kind == GovernanceActorKind::Agent
605 {
606 let default_expires_after_ms = self
607 .automation_governance
608 .read()
609 .await
610 .limits
611 .default_expires_after_ms;
612 if default_expires_after_ms > 0 {
613 record.expires_at_ms = Some(now_ms().saturating_add(default_expires_after_ms));
614 }
615 }
616 let stored = self.upsert_automation_governance(record).await?;
617 if let Some(agent_id) = stored
618 .provenance
619 .creator
620 .actor_id
621 .as_deref()
622 .filter(|_| stored.provenance.creator.kind == GovernanceActorKind::Agent)
623 {
624 let _ = self
625 .record_agent_creation_review_progress(agent_id, &stored.automation_id)
626 .await;
627 }
628 Ok(stored)
629 }
630
631 pub async fn grant_automation_modify_access(
632 &self,
633 automation_id: &str,
634 granted_to: GovernanceActorRef,
635 granted_by: GovernanceActorRef,
636 reason: Option<String>,
637 ) -> anyhow::Result<AutomationGrantRecord> {
638 let grant = {
639 let mut guard = self.automation_governance.write().await;
640 let grant = {
641 let Some(record) = guard.records.get_mut(automation_id) else {
642 anyhow::bail!("automation governance record not found");
643 };
644 let grant = AutomationGrantRecord {
645 grant_id: format!("grant-{}", Uuid::new_v4()),
646 automation_id: automation_id.to_string(),
647 grant_kind: AutomationGrantKind::Modify,
648 granted_to,
649 granted_by,
650 capability_key: None,
651 created_at_ms: now_ms(),
652 revoked_at_ms: None,
653 revoke_reason: reason,
654 };
655 record.modify_grants.push(grant.clone());
656 record.updated_at_ms = now_ms();
657 grant
658 };
659 guard.updated_at_ms = now_ms();
660 grant
661 };
662 self.persist_automation_governance().await?;
663 let _ = append_protected_audit_event(
664 self,
665 format!("{GOVERNANCE_AUDIT_EVENT_PREFIX}.grant.created"),
666 &tandem_types::TenantContext::local_implicit(),
667 grant
668 .granted_by
669 .actor_id
670 .clone()
671 .or_else(|| grant.granted_by.source.clone()),
672 json!({
673 "automationID": automation_id,
674 "grant": grant,
675 }),
676 )
677 .await;
678 Ok(grant)
679 }
680
681 pub async fn revoke_automation_modify_access(
682 &self,
683 automation_id: &str,
684 grant_id: &str,
685 revoked_by: GovernanceActorRef,
686 reason: Option<String>,
687 ) -> anyhow::Result<Option<AutomationGrantRecord>> {
688 let stored = {
689 let mut guard = self.automation_governance.write().await;
690 let stored = {
691 let Some(record) = guard.records.get_mut(automation_id) else {
692 anyhow::bail!("automation governance record not found");
693 };
694 let Some(grant) = record
695 .modify_grants
696 .iter_mut()
697 .find(|grant| grant.grant_id == grant_id && grant.revoked_at_ms.is_none())
698 else {
699 return Ok(None);
700 };
701 grant.revoked_at_ms = Some(now_ms());
702 grant.revoke_reason = reason.clone();
703 record.updated_at_ms = now_ms();
704 grant.clone()
705 };
706 guard.updated_at_ms = now_ms();
707 stored
708 };
709 self.persist_automation_governance().await?;
710 let _ = append_protected_audit_event(
711 self,
712 format!("{GOVERNANCE_AUDIT_EVENT_PREFIX}.grant.revoked"),
713 &tandem_types::TenantContext::local_implicit(),
714 revoked_by
715 .actor_id
716 .clone()
717 .or_else(|| revoked_by.source.clone()),
718 json!({
719 "automationID": automation_id,
720 "grantID": grant_id,
721 "reason": reason,
722 }),
723 )
724 .await;
725 Ok(Some(stored))
726 }
727
728 pub async fn request_approval(
729 &self,
730 request_type: GovernanceApprovalRequestType,
731 requested_by: GovernanceActorRef,
732 target_resource: GovernanceResourceRef,
733 rationale: String,
734 context: Value,
735 expires_at_ms: Option<u64>,
736 ) -> anyhow::Result<GovernanceApprovalRequest> {
737 let now = now_ms();
738 let snapshot = {
739 let guard = self.automation_governance.read().await;
740 self.governance_snapshot(&guard)
741 };
742 let request = self
743 .governance_engine
744 .create_approval_request(
745 &snapshot,
746 GovernanceApprovalDraftInput {
747 request_type,
748 requested_by,
749 target_resource,
750 rationale,
751 context,
752 expires_at_ms,
753 },
754 now,
755 )
756 .map_err(|error| anyhow::anyhow!(error.message))?;
757 {
758 let mut guard = self.automation_governance.write().await;
759 guard
760 .approvals
761 .insert(request.approval_id.clone(), request.clone());
762 guard.updated_at_ms = now;
763 }
764 self.persist_automation_governance().await?;
765 let _ = append_protected_audit_event(
766 self,
767 format!("{GOVERNANCE_AUDIT_EVENT_PREFIX}.approval.requested"),
768 &tandem_types::TenantContext::local_implicit(),
769 request
770 .requested_by
771 .actor_id
772 .clone()
773 .or_else(|| request.requested_by.source.clone()),
774 json!({
775 "approvalID": request.approval_id,
776 "request": request,
777 }),
778 )
779 .await;
780 Ok(request)
781 }
782
783 pub async fn list_approval_requests(
784 &self,
785 request_type: Option<GovernanceApprovalRequestType>,
786 status: Option<GovernanceApprovalStatus>,
787 ) -> Vec<GovernanceApprovalRequest> {
788 let mut rows = self
789 .automation_governance
790 .read()
791 .await
792 .approvals
793 .values()
794 .filter(|request| {
795 request_type
796 .map(|value| request.request_type == value)
797 .unwrap_or(true)
798 && status.map(|value| request.status == value).unwrap_or(true)
799 })
800 .cloned()
801 .collect::<Vec<_>>();
802 rows.sort_by(|a, b| b.updated_at_ms.cmp(&a.updated_at_ms));
803 rows
804 }
805
806 pub async fn decide_approval_request(
807 &self,
808 approval_id: &str,
809 reviewer: GovernanceActorRef,
810 approved: bool,
811 notes: Option<String>,
812 ) -> anyhow::Result<Option<GovernanceApprovalRequest>> {
813 let existing = {
814 let guard = self.automation_governance.read().await;
815 let Some(request) = guard.approvals.get(approval_id).cloned() else {
816 return Ok(None);
817 };
818 request
819 };
820 let stored = self
821 .governance_engine
822 .decide_approval_request(
823 &existing,
824 reviewer.clone(),
825 approved,
826 notes.clone(),
827 now_ms(),
828 )
829 .map_err(|error| anyhow::anyhow!(error.message))?;
830 {
831 let mut guard = self.automation_governance.write().await;
832 guard
833 .approvals
834 .insert(approval_id.to_string(), stored.clone());
835 guard.updated_at_ms = now_ms();
836 }
837 self.persist_automation_governance().await?;
838 let _ = append_protected_audit_event(
839 self,
840 format!(
841 "{GOVERNANCE_AUDIT_EVENT_PREFIX}.approval.{}",
842 if approved { "approved" } else { "denied" }
843 ),
844 &tandem_types::TenantContext::local_implicit(),
845 reviewer
846 .actor_id
847 .clone()
848 .or_else(|| reviewer.source.clone()),
849 json!({
850 "approvalID": approval_id,
851 "approval": stored,
852 }),
853 )
854 .await;
855 Ok(Some(stored))
856 }
857
858 pub async fn delete_automation_v2_with_governance(
859 &self,
860 automation_id: &str,
861 deleted_by: GovernanceActorRef,
862 ) -> anyhow::Result<Option<crate::AutomationV2Spec>> {
863 let _guard = self.automations_v2_persistence.lock().await;
864 let removed = self.automations_v2.write().await.remove(automation_id);
865 if let Some(automation) = removed.clone() {
866 let now = now_ms();
867 {
868 let mut governance = self.automation_governance.write().await;
869 let record = governance
870 .records
871 .entry(automation_id.to_string())
872 .or_insert_with(|| AutomationGovernanceRecord {
873 automation_id: automation_id.to_string(),
874 provenance: default_human_provenance(
875 Some(automation.creator_id.clone()),
876 "delete_default",
877 ),
878 declared_capabilities: declared_capabilities_for_automation(&automation),
879 modify_grants: Vec::new(),
880 capability_grants: Vec::new(),
881 created_at_ms: automation.created_at_ms,
882 updated_at_ms: now,
883 deleted_at_ms: None,
884 delete_retention_until_ms: None,
885 published_externally: false,
886 creation_paused: false,
887 review_required: false,
888 review_kind: None,
889 review_requested_at_ms: None,
890 review_request_id: None,
891 last_reviewed_at_ms: None,
892 runs_since_review: 0,
893 expires_at_ms: None,
894 expired_at_ms: None,
895 retired_at_ms: None,
896 retire_reason: None,
897 paused_for_lifecycle: false,
898 health_last_checked_at_ms: None,
899 health_findings: Vec::new(),
900 });
901 record.deleted_at_ms = Some(now);
902 record.delete_retention_until_ms =
903 Some(now.saturating_add(7 * 24 * 60 * 60 * 1000));
904 record.updated_at_ms = now;
905 governance.deleted_automations.insert(
906 automation_id.to_string(),
907 DeletedAutomationRecord {
908 automation: automation.clone(),
909 deleted_at_ms: now,
910 deleted_by: deleted_by.clone(),
911 restore_until_ms: now.saturating_add(7 * 24 * 60 * 60 * 1000),
912 },
913 );
914 governance.updated_at_ms = now;
915 }
916 self.persist_automation_governance().await?;
917 self.persist_automations_v2_locked().await?;
918 let _ = append_protected_audit_event(
919 self,
920 format!("{GOVERNANCE_AUDIT_EVENT_PREFIX}.deleted"),
921 &tandem_types::TenantContext::local_implicit(),
922 deleted_by
923 .actor_id
924 .clone()
925 .or_else(|| deleted_by.source.clone()),
926 json!({
927 "automationID": automation_id,
928 "deletedBy": deleted_by,
929 "deletedAtMs": now,
930 }),
931 )
932 .await;
933 }
934 Ok(removed)
935 }
936
937 pub async fn restore_deleted_automation_v2(
938 &self,
939 automation_id: &str,
940 ) -> anyhow::Result<Option<crate::AutomationV2Spec>> {
941 let restored = {
942 let mut governance = self.automation_governance.write().await;
943 let Some(deleted) = governance.deleted_automations.remove(automation_id) else {
944 return Ok(None);
945 };
946 let automation = deleted.automation.clone();
947 self.automations_v2
948 .write()
949 .await
950 .insert(automation_id.to_string(), automation.clone());
951 if let Some(record) = governance.records.get_mut(automation_id) {
952 record.deleted_at_ms = None;
953 record.delete_retention_until_ms = None;
954 record.updated_at_ms = now_ms();
955 }
956 governance.updated_at_ms = now_ms();
957 automation
958 };
959 self.persist_automation_governance().await?;
960 self.persist_automations_v2().await?;
961 Ok(Some(restored))
962 }
963
964 pub async fn agent_spend_summary(&self, agent_id: &str) -> Option<AgentSpendSummary> {
965 self.automation_governance
966 .read()
967 .await
968 .agent_spend_summary(agent_id)
969 }
970
971 pub async fn list_agent_spend_summaries(&self) -> Vec<AgentSpendSummary> {
972 self.automation_governance
973 .read()
974 .await
975 .agent_spend_summaries()
976 }
977
978 pub async fn agent_creation_review_summary(
979 &self,
980 agent_id: &str,
981 ) -> Option<AgentCreationReviewSummary> {
982 self.automation_governance
983 .read()
984 .await
985 .agent_creation_review_summary(agent_id)
986 }
987
988 pub async fn list_agent_creation_review_summaries(&self) -> Vec<AgentCreationReviewSummary> {
989 self.automation_governance
990 .read()
991 .await
992 .agent_creation_review_summaries()
993 }
994
995 pub async fn record_agent_creation_review_progress(
996 &self,
997 agent_id: &str,
998 automation_id: &str,
999 ) -> anyhow::Result<()> {
1000 let now = now_ms();
1001 let snapshot = {
1002 let guard = self.automation_governance.read().await;
1003 self.governance_snapshot(&guard)
1004 };
1005 let evaluation = self
1006 .governance_engine
1007 .evaluate_creation_review_progress(&snapshot, agent_id, automation_id, now)
1008 .map_err(|error| anyhow::anyhow!(error.message))?;
1009 let approval = evaluation.approval_request.clone();
1010 {
1011 let mut guard = self.automation_governance.write().await;
1012 guard
1013 .agent_creation_reviews
1014 .insert(agent_id.to_string(), evaluation.summary);
1015 if let Some(approval) = approval.clone() {
1016 guard
1017 .approvals
1018 .insert(approval.approval_id.clone(), approval);
1019 }
1020 guard.updated_at_ms = now;
1021 }
1022 self.persist_automation_governance().await?;
1023 if let Some(approval) = approval {
1024 let _ = append_protected_audit_event(
1025 self,
1026 format!("{GOVERNANCE_AUDIT_EVENT_PREFIX}.approval.requested"),
1027 &tandem_types::TenantContext::local_implicit(),
1028 approval
1029 .requested_by
1030 .actor_id
1031 .clone()
1032 .or_else(|| approval.requested_by.source.clone()),
1033 json!({
1034 "approvalID": approval.approval_id,
1035 "request": approval,
1036 }),
1037 )
1038 .await;
1039 }
1040 Ok(())
1041 }
1042
1043 pub async fn acknowledge_agent_creation_review(
1044 &self,
1045 agent_id: &str,
1046 reviewer: GovernanceActorRef,
1047 notes: Option<String>,
1048 ) -> anyhow::Result<()> {
1049 let now = now_ms();
1050 {
1051 let mut guard = self.automation_governance.write().await;
1052 let summary = guard
1053 .agent_creation_reviews
1054 .entry(agent_id.to_string())
1055 .or_insert_with(|| AgentCreationReviewSummary::new(agent_id.to_string(), now));
1056 summary.created_since_review = 0;
1057 summary.review_required = false;
1058 summary.review_kind = None;
1059 summary.review_requested_at_ms = None;
1060 summary.review_request_id = None;
1061 summary.last_reviewed_at_ms = Some(now);
1062 summary.last_review_notes = notes.clone();
1063 summary.updated_at_ms = now;
1064 guard.updated_at_ms = now;
1065 }
1066 self.persist_automation_governance().await?;
1067 let _ = append_protected_audit_event(
1068 self,
1069 format!("{GOVERNANCE_AUDIT_EVENT_PREFIX}.review.agent_acknowledged"),
1070 &tandem_types::TenantContext::local_implicit(),
1071 reviewer
1072 .actor_id
1073 .clone()
1074 .or_else(|| reviewer.source.clone()),
1075 json!({
1076 "agentID": agent_id,
1077 "reviewer": reviewer,
1078 "notes": notes,
1079 }),
1080 )
1081 .await;
1082 Ok(())
1083 }
1084
1085 pub async fn acknowledge_automation_review(
1086 &self,
1087 automation_id: &str,
1088 reviewer: GovernanceActorRef,
1089 notes: Option<String>,
1090 ) -> anyhow::Result<Option<AutomationGovernanceRecord>> {
1091 let stored = {
1092 let mut guard = self.automation_governance.write().await;
1093 let stored = {
1094 let Some(record) = guard.records.get_mut(automation_id) else {
1095 return Ok(None);
1096 };
1097 let now = now_ms();
1098 record.review_required = false;
1099 record.review_kind = None;
1100 record.review_requested_at_ms = None;
1101 record.review_request_id = None;
1102 record.last_reviewed_at_ms = Some(now);
1103 record.runs_since_review = 0;
1104 record.health_findings.clear();
1105 record.health_last_checked_at_ms = Some(now);
1106 record.updated_at_ms = now;
1107 record.clone()
1108 };
1109 guard.updated_at_ms = now_ms();
1110 stored
1111 };
1112 self.persist_automation_governance().await?;
1113 let _ = append_protected_audit_event(
1114 self,
1115 format!("{GOVERNANCE_AUDIT_EVENT_PREFIX}.review.automation_acknowledged"),
1116 &tandem_types::TenantContext::local_implicit(),
1117 reviewer
1118 .actor_id
1119 .clone()
1120 .or_else(|| reviewer.source.clone()),
1121 json!({
1122 "automationID": automation_id,
1123 "reviewer": reviewer,
1124 "notes": notes,
1125 }),
1126 )
1127 .await;
1128 Ok(Some(stored))
1129 }
1130
1131 pub async fn pause_automation_for_dependency_revocation(
1132 &self,
1133 automation_id: &str,
1134 reason: String,
1135 evidence: Value,
1136 ) -> anyhow::Result<()> {
1137 let Some(automation) = self.get_automation_v2(automation_id).await else {
1138 anyhow::bail!("automation not found");
1139 };
1140 let now = now_ms();
1141 let paused_runs = self
1142 .pause_running_automation_v2_runs(
1143 automation_id,
1144 reason.clone(),
1145 crate::AutomationStopKind::GuardrailStopped,
1146 )
1147 .await;
1148 let dependency_context = json!({
1149 "trigger": "dependency_revoked",
1150 "reason": reason.clone(),
1151 "evidence": evidence,
1152 "pausedRunIDs": paused_runs.clone(),
1153 });
1154 let (evaluation, created_review_id) = {
1155 let guard = self.automation_governance.read().await;
1156 let snapshot = self.governance_snapshot(&guard);
1157 let current_record = guard.records.get(automation_id).cloned();
1158 let evaluation = self
1159 .governance_engine
1160 .evaluate_dependency_revocation(
1161 &snapshot,
1162 GovernanceDependencyRevocationInput {
1163 automation_id: automation_id.to_string(),
1164 current_record,
1165 default_provenance: default_human_provenance(
1166 Some(automation.creator_id.clone()),
1167 "dependency_revocation_default",
1168 ),
1169 declared_capabilities: declared_capabilities_for_automation(&automation),
1170 reason: reason.clone(),
1171 evidence: dependency_context.clone(),
1172 },
1173 now,
1174 )
1175 .map_err(|error| anyhow::anyhow!(error.message))?;
1176 let created_review_id = evaluation
1177 .approval_request
1178 .as_ref()
1179 .map(|approval| approval.approval_id.clone())
1180 .or_else(|| evaluation.record.review_request_id.clone());
1181 (evaluation, created_review_id)
1182 };
1183 {
1184 let mut guard = self.automation_governance.write().await;
1185 guard
1186 .records
1187 .insert(automation_id.to_string(), evaluation.record.clone());
1188 if let Some(approval) = evaluation.approval_request.clone() {
1189 guard
1190 .approvals
1191 .insert(approval.approval_id.clone(), approval);
1192 }
1193 guard.updated_at_ms = now;
1194 }
1195 self.persist_automation_governance().await?;
1196 if let Some(approval) = evaluation.approval_request {
1197 let _ = append_protected_audit_event(
1198 self,
1199 format!("{GOVERNANCE_AUDIT_EVENT_PREFIX}.approval.requested"),
1200 &tandem_types::TenantContext::local_implicit(),
1201 approval
1202 .requested_by
1203 .actor_id
1204 .clone()
1205 .or_else(|| approval.requested_by.source.clone()),
1206 json!({
1207 "approvalID": approval.approval_id,
1208 "request": approval,
1209 }),
1210 )
1211 .await;
1212 }
1213
1214 let _ = append_protected_audit_event(
1215 self,
1216 format!("{GOVERNANCE_AUDIT_EVENT_PREFIX}.dependency_revoked"),
1217 &tandem_types::TenantContext::local_implicit(),
1218 Some("automation_dependency_revocation".to_string()),
1219 json!({
1220 "automationID": automation_id,
1221 "reason": reason,
1222 "pausedRunIDs": paused_runs,
1223 "evidence": dependency_context.clone(),
1224 "reviewRequestID": created_review_id,
1225 }),
1226 )
1227 .await;
1228
1229 Ok(())
1230 }
1231
1232 async fn pause_running_automation_v2_runs(
1233 &self,
1234 automation_id: &str,
1235 reason: String,
1236 stop_kind: crate::AutomationStopKind,
1237 ) -> Vec<String> {
1238 let runs = self.list_automation_v2_runs(Some(automation_id), 100).await;
1239 let mut paused_runs = Vec::new();
1240 for run in runs {
1241 if run.status != crate::AutomationRunStatus::Running {
1242 continue;
1243 }
1244 let session_ids = run.active_session_ids.clone();
1245 let instance_ids = run.active_instance_ids.clone();
1246 let _ = self
1247 .update_automation_v2_run(&run.run_id, |row| {
1248 row.status = crate::AutomationRunStatus::Pausing;
1249 row.pause_reason = Some(reason.clone());
1250 })
1251 .await;
1252 for session_id in &session_ids {
1253 let _ = self.cancellations.cancel(session_id).await;
1254 }
1255 for instance_id in instance_ids {
1256 let _ = self
1257 .agent_teams
1258 .cancel_instance(self, &instance_id, &reason)
1259 .await;
1260 }
1261 self.forget_automation_v2_sessions(&session_ids).await;
1262 let _ = self
1263 .update_automation_v2_run(&run.run_id, |row| {
1264 row.status = crate::AutomationRunStatus::Paused;
1265 row.active_session_ids.clear();
1266 row.active_instance_ids.clear();
1267 row.pause_reason = Some(reason.clone());
1268 row.stop_kind = Some(stop_kind.clone());
1269 row.stop_reason = Some(reason.clone());
1270 crate::app::state::automation::lifecycle::record_automation_lifecycle_event(
1271 row,
1272 "run_paused_governance",
1273 Some(reason.clone()),
1274 Some(stop_kind.clone()),
1275 );
1276 })
1277 .await;
1278 paused_runs.push(run.run_id);
1279 }
1280 paused_runs
1281 }
1282
1283 pub async fn record_automation_review_progress(
1284 &self,
1285 automation_id: &str,
1286 reason: AutomationLifecycleReviewKind,
1287 run_id: Option<String>,
1288 detail: Option<String>,
1289 ) -> anyhow::Result<()> {
1290 let now = now_ms();
1291 let evaluation = {
1292 let guard = self.automation_governance.read().await;
1293 let snapshot = self.governance_snapshot(&guard);
1294 self.governance_engine
1295 .evaluate_run_review_progress(
1296 &snapshot,
1297 automation_id,
1298 reason,
1299 run_id.clone(),
1300 detail.clone(),
1301 now,
1302 )
1303 .map_err(|error| anyhow::anyhow!(error.message))?
1304 };
1305 let Some(evaluation) = evaluation else {
1306 return Ok(());
1307 };
1308 let approval = evaluation.approval_request.clone();
1309 {
1310 let mut guard = self.automation_governance.write().await;
1311 guard
1312 .records
1313 .insert(automation_id.to_string(), evaluation.record);
1314 if let Some(approval) = approval.clone() {
1315 guard
1316 .approvals
1317 .insert(approval.approval_id.clone(), approval);
1318 }
1319 guard.updated_at_ms = now;
1320 }
1321 self.persist_automation_governance().await?;
1322 if let Some(approval) = approval {
1323 let _ = append_protected_audit_event(
1324 self,
1325 format!("{GOVERNANCE_AUDIT_EVENT_PREFIX}.approval.requested"),
1326 &tandem_types::TenantContext::local_implicit(),
1327 approval
1328 .requested_by
1329 .actor_id
1330 .clone()
1331 .or_else(|| approval.requested_by.source.clone()),
1332 json!({
1333 "approvalID": approval.approval_id,
1334 "request": approval,
1335 }),
1336 )
1337 .await;
1338 }
1339 Ok(())
1340 }
1341
1342 pub async fn run_automation_governance_health_check(&self) -> anyhow::Result<usize> {
1343 if !self.premium_governance_enabled() {
1344 return Ok(0);
1345 }
1346 let now = now_ms();
1347 let limits = self.automation_governance.read().await.limits.clone();
1348 let automations = self.list_automations_v2().await;
1349 let mut finding_count = 0usize;
1350
1351 for automation in automations {
1352 let runs = self
1353 .list_automation_v2_runs(
1354 Some(&automation.automation_id),
1355 limits.health_window_run_limit.max(5) as usize,
1356 )
1357 .await;
1358 let terminal_runs = runs
1359 .iter()
1360 .filter(|run| {
1361 matches!(
1362 run.status,
1363 crate::AutomationRunStatus::Completed
1364 | crate::AutomationRunStatus::Blocked
1365 | crate::AutomationRunStatus::Failed
1366 | crate::AutomationRunStatus::Cancelled
1367 )
1368 })
1369 .collect::<Vec<_>>();
1370 let failure_count = terminal_runs
1371 .iter()
1372 .filter(|run| {
1373 matches!(
1374 run.status,
1375 crate::AutomationRunStatus::Failed | crate::AutomationRunStatus::Blocked
1376 )
1377 })
1378 .count();
1379 let empty_output_count = terminal_runs
1380 .iter()
1381 .filter(|run| {
1382 run.status == crate::AutomationRunStatus::Completed
1383 && run.checkpoint.node_outputs.is_empty()
1384 })
1385 .count();
1386 let guardrail_stop_count = terminal_runs
1387 .iter()
1388 .filter(|run| run.stop_kind == Some(crate::AutomationStopKind::GuardrailStopped))
1389 .count();
1390 let evaluation = {
1391 let guard = self.automation_governance.read().await;
1392 let snapshot = self.governance_snapshot(&guard);
1393 self.governance_engine
1394 .evaluate_health_check(
1395 &snapshot,
1396 GovernanceHealthCheckInput {
1397 automation_id: automation.automation_id.clone(),
1398 current_record: guard.records.get(&automation.automation_id).cloned(),
1399 default_provenance: default_human_provenance(
1400 Some(automation.creator_id.clone()),
1401 "health_check_default",
1402 ),
1403 declared_capabilities: declared_capabilities_for_automation(
1404 &automation,
1405 ),
1406 terminal_run_count: terminal_runs.len() as u64,
1407 failure_count: failure_count as u64,
1408 empty_output_count: empty_output_count as u64,
1409 guardrail_stop_count: guardrail_stop_count as u64,
1410 last_terminal_run_id: terminal_runs
1411 .last()
1412 .map(|run| run.run_id.clone()),
1413 },
1414 now,
1415 )
1416 .map_err(|error| anyhow::anyhow!(error.message))?
1417 };
1418 let Some(evaluation) = evaluation else {
1419 continue;
1420 };
1421 {
1422 let mut guard = self.automation_governance.write().await;
1423 guard
1424 .records
1425 .insert(automation.automation_id.clone(), evaluation.record.clone());
1426 for approval in &evaluation.approval_requests {
1427 guard
1428 .approvals
1429 .insert(approval.approval_id.clone(), approval.clone());
1430 }
1431 guard.updated_at_ms = now;
1432 }
1433 self.persist_automation_governance().await?;
1434
1435 if evaluation.pause_automation && automation.status != crate::AutomationV2Status::Paused
1436 {
1437 let mut paused = automation.clone();
1438 paused.status = crate::AutomationV2Status::Paused;
1439 let _ = self.put_automation_v2(paused).await;
1440 let _ = self
1441 .pause_running_automation_v2_runs(
1442 &automation.automation_id,
1443 format!(
1444 "automation expired after reaching {}ms retention",
1445 limits.default_expires_after_ms
1446 ),
1447 crate::AutomationStopKind::GuardrailStopped,
1448 )
1449 .await;
1450 }
1451
1452 for approval in &evaluation.approval_requests {
1453 let _ = append_protected_audit_event(
1454 self,
1455 format!("{GOVERNANCE_AUDIT_EVENT_PREFIX}.approval.requested"),
1456 &tandem_types::TenantContext::local_implicit(),
1457 approval
1458 .requested_by
1459 .actor_id
1460 .clone()
1461 .or_else(|| approval.requested_by.source.clone()),
1462 json!({
1463 "approvalID": approval.approval_id,
1464 "request": approval,
1465 }),
1466 )
1467 .await;
1468 }
1469
1470 finding_count += evaluation.record.health_findings.len();
1471 }
1472
1473 Ok(finding_count)
1474 }
1475
1476 pub async fn retire_automation_v2(
1477 &self,
1478 automation_id: &str,
1479 actor: GovernanceActorRef,
1480 reason: Option<String>,
1481 ) -> anyhow::Result<Option<crate::AutomationV2Spec>> {
1482 let Some(mut automation) = self.get_automation_v2(automation_id).await else {
1483 return Ok(None);
1484 };
1485 let now = now_ms();
1486 let reason = reason.unwrap_or_else(|| "retired by operator".to_string());
1487 automation.status = crate::AutomationV2Status::Paused;
1488 let stored = self.put_automation_v2(automation).await?;
1489 let _ = self
1490 .pause_running_automation_v2_runs(
1491 automation_id,
1492 reason.clone(),
1493 crate::AutomationStopKind::OperatorStopped,
1494 )
1495 .await;
1496 let current_record = self.get_automation_governance(automation_id).await;
1497 let record = self
1498 .governance_engine
1499 .evaluate_retirement(
1500 GovernanceRetirementInput {
1501 automation_id: automation_id.to_string(),
1502 current_record,
1503 default_provenance: default_human_provenance(
1504 Some(stored.creator_id.clone()),
1505 "retire_default",
1506 ),
1507 declared_capabilities: declared_capabilities_for_automation(&stored),
1508 reason: reason.clone(),
1509 },
1510 now,
1511 )
1512 .map_err(|error| anyhow::anyhow!(error.message))?;
1513 {
1514 let mut guard = self.automation_governance.write().await;
1515 guard.records.insert(automation_id.to_string(), record);
1516 guard.updated_at_ms = now;
1517 }
1518 self.persist_automation_governance().await?;
1519 let _ = append_protected_audit_event(
1520 self,
1521 format!("{GOVERNANCE_AUDIT_EVENT_PREFIX}.retired"),
1522 &tandem_types::TenantContext::local_implicit(),
1523 actor.actor_id.clone().or_else(|| actor.source.clone()),
1524 json!({
1525 "automationID": automation_id,
1526 "reason": reason,
1527 "actor": actor,
1528 }),
1529 )
1530 .await;
1531 Ok(Some(stored))
1532 }
1533
1534 pub async fn extend_automation_v2_retirement(
1535 &self,
1536 automation_id: &str,
1537 actor: GovernanceActorRef,
1538 expires_at_ms: Option<u64>,
1539 reason: Option<String>,
1540 ) -> anyhow::Result<Option<crate::AutomationV2Spec>> {
1541 let Some(mut automation) = self.get_automation_v2(automation_id).await else {
1542 return Ok(None);
1543 };
1544 let now = now_ms();
1545 let default_expires_after_ms = self
1546 .automation_governance
1547 .read()
1548 .await
1549 .limits
1550 .default_expires_after_ms;
1551 let next_expires_at_ms =
1552 expires_at_ms.unwrap_or_else(|| now.saturating_add(default_expires_after_ms.max(1)));
1553 automation.status = crate::AutomationV2Status::Active;
1554 let stored = self.put_automation_v2(automation).await?;
1555 let current_record = self.get_automation_governance(automation_id).await;
1556 let record = self
1557 .governance_engine
1558 .evaluate_retirement_extension(
1559 GovernanceRetirementExtensionInput {
1560 automation_id: automation_id.to_string(),
1561 current_record,
1562 default_provenance: default_human_provenance(
1563 Some(stored.creator_id.clone()),
1564 "extend_default",
1565 ),
1566 declared_capabilities: declared_capabilities_for_automation(&stored),
1567 expires_at_ms: next_expires_at_ms,
1568 },
1569 now,
1570 )
1571 .map_err(|error| anyhow::anyhow!(error.message))?;
1572 {
1573 let mut guard = self.automation_governance.write().await;
1574 guard.records.insert(automation_id.to_string(), record);
1575 guard.updated_at_ms = now;
1576 }
1577 self.persist_automation_governance().await?;
1578 let _ = append_protected_audit_event(
1579 self,
1580 format!("{GOVERNANCE_AUDIT_EVENT_PREFIX}.retirement.extended"),
1581 &tandem_types::TenantContext::local_implicit(),
1582 actor.actor_id.clone().or_else(|| actor.source.clone()),
1583 json!({
1584 "automationID": automation_id,
1585 "expiresAtMs": next_expires_at_ms,
1586 "reason": reason,
1587 "actor": actor,
1588 }),
1589 )
1590 .await;
1591 Ok(Some(stored))
1592 }
1593
1594 pub async fn record_automation_v2_spend(
1595 &self,
1596 run_id: &str,
1597 prompt_tokens: u64,
1598 completion_tokens: u64,
1599 total_tokens: u64,
1600 delta_cost_usd: f64,
1601 ) -> anyhow::Result<()> {
1602 let Some(run_snapshot) = self.get_automation_v2_run(run_id).await else {
1603 return Ok(());
1604 };
1605 let automation = if let Some(snapshot) = run_snapshot.automation_snapshot.clone() {
1606 snapshot
1607 } else {
1608 let Some(automation) = self.get_automation_v2(&run_snapshot.automation_id).await else {
1609 return Ok(());
1610 };
1611 automation
1612 };
1613 let governance = self
1614 .get_or_bootstrap_automation_governance(&automation)
1615 .await;
1616 let agent_ids = governance.agent_lineage_ids();
1617 if agent_ids.is_empty() {
1618 return Ok(());
1619 }
1620
1621 let now = now_ms();
1622 let snapshot = {
1623 let guard = self.automation_governance.read().await;
1624 self.governance_snapshot(&guard)
1625 };
1626 let evaluation = self
1627 .governance_engine
1628 .evaluate_spend_usage(
1629 &snapshot,
1630 &GovernanceSpendInput {
1631 automation_id: automation.automation_id.clone(),
1632 run_id: run_id.to_string(),
1633 agent_ids: agent_ids.clone(),
1634 prompt_tokens,
1635 completion_tokens,
1636 total_tokens,
1637 delta_cost_usd,
1638 },
1639 now,
1640 )
1641 .map_err(|error| anyhow::anyhow!(error.message))?;
1642 {
1643 let mut guard = self.automation_governance.write().await;
1644 for summary in &evaluation.updated_summaries {
1645 guard
1646 .agent_spend
1647 .insert(summary.agent_id.clone(), summary.clone());
1648 }
1649 for agent_id in &evaluation.spend_paused_agents {
1650 if !guard
1651 .spend_paused_agents
1652 .iter()
1653 .any(|value| value == agent_id)
1654 {
1655 guard.spend_paused_agents.push(agent_id.clone());
1656 }
1657 }
1658 for approval in &evaluation.approvals {
1659 guard
1660 .approvals
1661 .insert(approval.approval_id.clone(), approval.clone());
1662 }
1663 guard.updated_at_ms = now;
1664 }
1665 self.persist_automation_governance().await?;
1666
1667 for warning in &evaluation.warnings {
1668 let _ = append_protected_audit_event(
1669 self,
1670 format!("{GOVERNANCE_AUDIT_EVENT_PREFIX}.spend.warning"),
1671 &tandem_types::TenantContext::local_implicit(),
1672 governance
1673 .provenance
1674 .creator
1675 .actor_id
1676 .clone()
1677 .or_else(|| Some(automation.creator_id.clone())),
1678 json!({
1679 "automationID": automation.automation_id,
1680 "runID": run_id,
1681 "agentID": warning.agent_id,
1682 "weeklyCostUsd": warning.weekly_cost_usd,
1683 "weeklySpendCapUsd": warning.weekly_spend_cap_usd,
1684 }),
1685 )
1686 .await;
1687 }
1688
1689 let requested_approvals = evaluation
1690 .approvals
1691 .iter()
1692 .map(|approval| approval.approval_id.clone())
1693 .collect::<Vec<_>>();
1694 for approval in &evaluation.approvals {
1695 let _ = append_protected_audit_event(
1696 self,
1697 format!("{GOVERNANCE_AUDIT_EVENT_PREFIX}.approval.requested"),
1698 &tandem_types::TenantContext::local_implicit(),
1699 approval
1700 .requested_by
1701 .actor_id
1702 .clone()
1703 .or_else(|| approval.requested_by.source.clone()),
1704 json!({
1705 "approvalID": approval.approval_id,
1706 "request": approval,
1707 }),
1708 )
1709 .await;
1710 }
1711
1712 if !evaluation.hard_stops.is_empty() {
1713 let session_ids = run_snapshot.active_session_ids.clone();
1714 for session_id in &session_ids {
1715 let _ = self.cancellations.cancel(session_id).await;
1716 }
1717 self.forget_automation_v2_sessions(&session_ids).await;
1718 let instance_ids = run_snapshot.active_instance_ids.clone();
1719 for instance_id in instance_ids {
1720 let _ = self
1721 .agent_teams
1722 .cancel_instance(self, &instance_id, "paused by spend guardrail")
1723 .await;
1724 }
1725 let paused_agent_labels = evaluation
1726 .hard_stops
1727 .iter()
1728 .map(|entry| {
1729 format!(
1730 "{} ({:.4}/{:.4} USD)",
1731 entry.agent_id, entry.weekly_cost_usd, entry.weekly_spend_cap_usd
1732 )
1733 })
1734 .collect::<Vec<_>>()
1735 .join(", ");
1736 let detail = format!("weekly spend cap exceeded for {paused_agent_labels}");
1737 let _ = self
1738 .update_automation_v2_run(run_id, |row| {
1739 row.status = crate::AutomationRunStatus::Paused;
1740 row.detail = Some(detail.clone());
1741 row.pause_reason = Some(detail.clone());
1742 row.stop_kind = Some(crate::AutomationStopKind::GuardrailStopped);
1743 row.stop_reason = Some(detail.clone());
1744 row.active_session_ids.clear();
1745 row.latest_session_id = None;
1746 row.active_instance_ids.clear();
1747 crate::app::state::automation::lifecycle::record_automation_lifecycle_event(
1748 row,
1749 "run_paused_spend_cap_exceeded",
1750 Some(detail.clone()),
1751 Some(crate::AutomationStopKind::GuardrailStopped),
1752 );
1753 })
1754 .await;
1755 let _ = append_protected_audit_event(
1756 self,
1757 format!("{GOVERNANCE_AUDIT_EVENT_PREFIX}.spend.paused"),
1758 &tandem_types::TenantContext::local_implicit(),
1759 governance
1760 .provenance
1761 .creator
1762 .actor_id
1763 .clone()
1764 .or_else(|| Some(automation.creator_id.clone())),
1765 json!({
1766 "automationID": automation.automation_id,
1767 "runID": run_id,
1768 "pausedAgents": evaluation
1769 .hard_stops
1770 .iter()
1771 .map(|entry| entry.agent_id.clone())
1772 .collect::<Vec<_>>(),
1773 "requestedApprovals": requested_approvals,
1774 "detail": detail,
1775 }),
1776 )
1777 .await;
1778 }
1779
1780 Ok(())
1781 }
1782}