stateknot_store_postgres/
agent_deadlines.rs1#[allow(clippy::wildcard_imports)]
7use super::*;
8use stateknot_core::{
9 FailureCategory, FailureCode, FailureId, FailureMessage, FailureOrigin, JournalEventKind,
10 JournalExpectation, RunCancellationRequest, SchemaReference,
11};
12
13#[derive(Clone, Debug)]
15pub struct AgentDeadlineCursor {
16 tenant: TenantId,
17 deadline: Timestamp,
18 run_id: RunId,
19}
20impl AgentDeadlineCursor {
21 #[must_use]
23 pub const fn run_id(&self) -> RunId {
24 self.run_id
25 }
26 #[must_use]
28 pub const fn deadline(&self) -> Timestamp {
29 self.deadline
30 }
31 #[must_use]
33 pub const fn tenant_id(&self) -> &TenantId {
34 &self.tenant
35 }
36}
37
38#[derive(Clone, Debug)]
40pub enum AgentDeadlineCancellationOutcome {
41 FailureClosing,
43 Requested(JournalHead),
45 AlreadyRequested(Box<RunCancellationRequest>),
47 Terminal(RunStatus),
49 NotDue {
51 deadline: Timestamp,
53 observed_at: Timestamp,
55 },
56}
57
58impl PostgresStore {
59 pub async fn due_agent_deadlines_after(
64 &self,
65 tenant: &TenantId,
66 after: Option<&AgentDeadlineCursor>,
67 ) -> Result<Vec<AgentDeadlineCursor>, StoreError> {
68 if after.is_some_and(|cursor| cursor.tenant_id() != tenant) {
69 return Err(StoreError::InvalidAgentDeadline);
70 }
71 let sql = if after.is_some() {
72 "SELECT agent_deadline_at,run_id FROM stateknot.runs WHERE tenant_id=$1 AND agent_deadline_at IS NOT NULL AND lifecycle_status IN ('pending','active','waiting') AND NOT EXISTS (SELECT 1 FROM stateknot.run_failure_closes c WHERE c.tenant_id=stateknot.runs.tenant_id AND c.run_id=stateknot.runs.run_id) AND agent_deadline_at <= statement_timestamp() AND (agent_deadline_at,run_id)>($2,$3) ORDER BY agent_deadline_at,run_id LIMIT 16"
73 } else {
74 "SELECT agent_deadline_at,run_id FROM stateknot.runs WHERE tenant_id=$1 AND agent_deadline_at IS NOT NULL AND lifecycle_status IN ('pending','active','waiting') AND NOT EXISTS (SELECT 1 FROM stateknot.run_failure_closes c WHERE c.tenant_id=stateknot.runs.tenant_id AND c.run_id=stateknot.runs.run_id) AND agent_deadline_at <= statement_timestamp() ORDER BY agent_deadline_at,run_id LIMIT 16"
75 };
76 let mut listing = query_as::<_, (DateTime<Utc>, Uuid)>(sql).bind(tenant.as_str());
77 if let Some(cursor) = after {
78 listing = listing
79 .bind(to_database_time(cursor.deadline)?)
80 .bind(*cursor.run_id.as_uuid());
81 }
82 listing
83 .fetch_all(&self.pool)
84 .await
85 .map_err(|source| StoreError::database("Agent deadline discovery", source))?
86 .into_iter()
87 .map(|(deadline, run)| {
88 Ok(AgentDeadlineCursor {
89 tenant: tenant.clone(),
90 deadline: from_database_time(deadline)?,
91 run_id: RunId::from_uuid(run)
92 .map_err(|_| StoreError::corrupt("Agent deadline run identity"))?,
93 })
94 })
95 .collect()
96 }
97
98 pub async fn request_agent_deadline_cancellation<V: GraphSchemaValidator + ?Sized>(
105 &self,
106 tenant: &TenantId,
107 run_id: RunId,
108 event_id: EventId,
109 failure_id: FailureId,
110 schema: &SchemaReference,
111 schemas: &V,
112 ) -> Result<AgentDeadlineCancellationOutcome, StoreError> {
113 Box::pin(self.request_agent_deadline_cancellation_inner(
114 tenant, run_id, event_id, failure_id, schema, schemas,
115 ))
116 .await
117 }
118
119 #[allow(clippy::too_many_lines)]
120 async fn request_agent_deadline_cancellation_inner<V: GraphSchemaValidator + ?Sized>(
121 &self,
122 tenant: &TenantId,
123 run_id: RunId,
124 event_id: EventId,
125 failure_id: FailureId,
126 schema: &SchemaReference,
127 schemas: &V,
128 ) -> Result<AgentDeadlineCancellationOutcome, StoreError> {
129 let mut tx = self.begin_mutation("Agent deadline cancellation").await?;
132 let run = decode_run(fetch_locked_run_row(&mut tx, tenant, run_id).await?)?;
133 verify_current_wait_set(&mut tx, &run).await?;
134 let row = load_agent_admission_row(&mut tx, tenant, run_id)
135 .await?
136 .ok_or(StoreError::AgentAdmissionNotFound)?;
137 let stored = verify_stored_agent_admission(&mut tx, run, row).await?;
138 let run = stored.run();
139 let deadline = stored.admission().intent().budget().deadline();
140 let indexed = query_scalar::<_, Option<DateTime<Utc>>>(
141 "SELECT agent_deadline_at FROM stateknot.runs WHERE tenant_id=$1 AND run_id=$2",
142 )
143 .bind(tenant.as_str())
144 .bind(*run_id.as_uuid())
145 .fetch_one(&mut *tx)
146 .await
147 .map_err(|source| StoreError::database("Agent deadline projection", source))?;
148 if indexed.map(from_database_time).transpose()? != Some(deadline) {
149 return Err(StoreError::corrupt("Agent deadline projection"));
150 }
151 if let Some(request) = run.lifecycle().cancellation_request() {
152 return Ok(AgentDeadlineCancellationOutcome::AlreadyRequested(
153 Box::new(request.clone()),
154 ));
155 }
156 if run.lifecycle().status().is_terminal() {
157 return Ok(AgentDeadlineCancellationOutcome::Terminal(
158 run.lifecycle().status(),
159 ));
160 }
161 if run.is_quarantined() {
162 return Err(StoreError::RunQuarantined);
163 }
164 if failure_closes::exists(&mut tx, tenant, run_id).await? {
165 return Ok(AgentDeadlineCancellationOutcome::FailureClosing);
166 }
167 let observed_at = database_now(&mut tx, "Agent deadline clock").await?;
168 if observed_at < deadline {
169 return Ok(AgentDeadlineCancellationOutcome::NotDue {
170 deadline,
171 observed_at,
172 });
173 }
174 let head = run.journal_head().ok_or(StoreError::InvalidAgentDeadline)?;
175 let at = observed_at.max(head.recorded_at());
176 let failure = Failure::new(
177 failure_id,
178 FailureCategory::Cancelled,
179 FailureCode::new("agent.deadline.expired").expect("static failure code"),
180 FailureOrigin::new("stateknot.runtime.agent_deadlines").expect("static failure origin"),
181 FailureMessage::new(
182 "The admitted Agent deadline expired; cooperative cancellation was requested.",
183 )
184 .expect("static public-safe failure message"),
185 RetryAdvice::Never,
186 )
187 .expect("static cancellation semantics")
188 .with_caused_by_event(event_id);
189 let request = RunCancellationRequest::new(failure, at)
190 .map_err(|_| StoreError::InvalidAgentDeadline)?;
191 let transition = RunTransition::RequestCancellation { request };
192 let projection = RunProjection::transition(run.lifecycle().revision(), transition.clone());
193 let prepared = prepare_durable_wait_projection(
194 run,
195 tenant,
196 run_id,
197 run.lifecycle().revision(),
198 transition,
199 at,
200 )?;
201 let data = BoundedJson::try_from_value(serde_json::json!({
202 "operation":"agent_deadline_cancellation_requested",
203 "admission_digest": hex_digest(stored.admission().digest()),
204 "deadline":deadline.to_string(), "failure_id":failure_id.to_string(),
205 }))
206 .map_err(|_| StoreError::InvalidAgentDeadline)?;
207 schemas
208 .validate(schema, &data)
209 .map_err(|_| StoreError::InvalidAgentDeadline)?;
210 let payload = JournalPayload::new(
211 schema.clone(),
212 JournalEventKind::new("agent-deadline-cancellation-requested")
213 .expect("static event kind"),
214 data,
215 )
216 .map_err(|_| StoreError::InvalidAgentDeadline)?;
217 let append = JournalAppend::new(
218 JournalExpectation::exact(head.clone()),
219 JournalEventIntent::control_plane(tenant.clone(), run_id, event_id, payload)
220 .map_err(|_| StoreError::InvalidAgentDeadline)?,
221 )
222 .map_err(|_| StoreError::InvalidAgentDeadline)?;
223 let waits = child_cancellation::cancellation_waits(&mut tx, run).await?;
224 let digest = if waits.is_empty() {
225 projection_digest(&projection)?
226 } else {
227 wait_abandonment_projection_digest(&projection, &waits)?
228 };
229 let event =
230 JournalEvent::commit(append, at).map_err(|error| map_event_commit_error(&error))?;
231 insert_event(&mut tx, &event, digest).await?;
232 for wait in waits {
233 let abandonment = materialize_wait_abandonment(
234 wait,
235 WaitAbandonmentReason::RunCancellation,
236 event.head(),
237 )?;
238 insert_wait_abandonment(&mut tx, &abandonment).await?;
239 project_wait_abandonment(&mut tx, &abandonment).await?;
240 }
241 update_run_head(&mut tx, &event, Some(&prepared)).await?;
242 tx.commit()
243 .await
244 .map_err(|source| StoreError::database("Agent deadline commit", source))?;
245 Ok(AgentDeadlineCancellationOutcome::Requested(event.head()))
246 }
247}
248
249fn hex_digest(value: Digest) -> String {
250 use std::fmt::Write;
251 let mut text = String::with_capacity(64);
252 for byte in value.as_bytes() {
253 write!(&mut text, "{byte:02x}").expect("String write");
254 }
255 text
256}
257
258pub(super) async fn verify_schema(pool: &PgPool) -> Result<(), StoreError> {
259 let installed = query_scalar::<_, String>(CATALOG_QUERY)
260 .fetch_one(pool)
261 .await
262 .map_err(|source| StoreError::database("Agent deadline catalog", source))?;
263 let expected: serde_json::Value =
264 serde_json::from_str(include_str!("agent_deadline_catalog.json"))
265 .map_err(|_| StoreError::IncompleteSchema)?;
266 if serde_json::from_str::<serde_json::Value>(&installed)
267 .map_err(|_| StoreError::IncompleteSchema)?
268 != expected
269 {
270 return Err(StoreError::IncompleteSchema);
271 }
272 for function in include_str!("../migrations/0023_agent_deadlines.sql")
273 .split("CREATE FUNCTION stateknot.")
274 .skip(1)
275 {
276 let (name, _) = function
277 .split_once("() RETURNS trigger")
278 .ok_or(StoreError::IncompleteSchema)?;
279 let body = function
280 .split("$$")
281 .nth(1)
282 .ok_or(StoreError::IncompleteSchema)?;
283 let installed = query_scalar::<_,String>("SELECT prosrc FROM pg_proc JOIN pg_language ON pg_language.oid=prolang WHERE pg_proc.oid=to_regprocedure($1) AND NOT prosecdef AND lanname='plpgsql'")
284 .bind(format!("stateknot.{name}()")).fetch_optional(pool).await
285 .map_err(|source| StoreError::database("Agent deadline guard",source))?;
286 if installed.as_deref() != Some(body) {
287 return Err(StoreError::IncompleteSchema);
288 }
289 }
290 Ok(())
291}
292
293const CATALOG_QUERY: &str = r"
294SELECT jsonb_build_object(
295 'column',(SELECT jsonb_build_array(format_type(atttypid,atttypmod),attnotnull,attgenerated,atthasdef) FROM pg_attribute WHERE attrelid='stateknot.runs'::regclass AND attname='agent_deadline_at' AND NOT attisdropped),
296 'constraint',(SELECT jsonb_build_array(pg_get_constraintdef(oid),convalidated,condeferrable) FROM pg_constraint WHERE conrelid='stateknot.runs'::regclass AND conname='runs_agent_deadline_finite'),
297 'index',(SELECT jsonb_build_array(pg_get_indexdef(indexrelid),indisvalid,indisready,indislive) FROM pg_index WHERE indexrelid=to_regclass('stateknot.runs_due_agent_deadlines')),
298 'triggers',(SELECT jsonb_agg(jsonb_build_array(tgname,pg_get_triggerdef(oid),tgenabled) ORDER BY tgname) FROM pg_trigger WHERE NOT tgisinternal AND ((tgrelid='stateknot.runs'::regclass AND tgname='runs_agent_deadline_guard') OR (tgrelid='stateknot.agent_admissions'::regclass AND tgname='agent_admissions_deadline_capture')))
299)::text
300";