Skip to main content

stateknot_store_postgres/
agent_deadlines.rs

1// Copyright 2026 StateKnot contributors
2// SPDX-License-Identifier: Apache-2.0
3
4//! Database-clock deadline cancellation; never terminal acknowledgement.
5
6#[allow(clippy::wildcard_imports)]
7use super::*;
8use stateknot_core::{
9    FailureCategory, FailureCode, FailureId, FailureMessage, FailureOrigin, JournalEventKind,
10    JournalExpectation, RunCancellationRequest, SchemaReference,
11};
12
13/// Tenant-bound range cursor. It remains usable after the candidate closes.
14#[derive(Clone, Debug)]
15pub struct AgentDeadlineCursor {
16    tenant: TenantId,
17    deadline: Timestamp,
18    run_id: RunId,
19}
20impl AgentDeadlineCursor {
21    /// Returns the admitted run identity; not an authorization grant.
22    #[must_use]
23    pub const fn run_id(&self) -> RunId {
24        self.run_id
25    }
26    /// Returns the exact finite deadline used by the index.
27    #[must_use]
28    pub const fn deadline(&self) -> Timestamp {
29        self.deadline
30    }
31    /// Returns the tenant scope.
32    #[must_use]
33    pub const fn tenant_id(&self) -> &TenantId {
34        &self.tenant
35    }
36}
37
38/// Observation after serializing with all lifecycle, spawn and wait writers.
39#[derive(Clone, Debug)]
40pub enum AgentDeadlineCancellationOutcome {
41    /// An earlier sealed failure owns closure; its original reason is retained.
42    FailureClosing,
43    /// Request, wait abandonment and immediate-child cancellation queue committed together.
44    Requested(JournalHead),
45    /// An earlier cancellation won. Its original reason is never replaced.
46    AlreadyRequested(Box<RunCancellationRequest>),
47    /// A terminal writer won. Its outcome and usage remain untouched.
48    Terminal(RunStatus),
49    /// The authoritative database clock has not reached the admitted deadline.
50    NotDue {
51        /// Exact admitted deadline.
52        deadline: Timestamp,
53        /// Database clock after acquiring the run lock.
54        observed_at: Timestamp,
55    },
56}
57
58impl PostgresStore {
59    /// Discovers at most 16 due admitted runs, including Waiting and sealed Join runs.
60    /// Continue past errors using the last candidate, reset to `None` at sweep end,
61    /// and restart at `None` after process loss. This does not acquire a lease.
62    /// The caller must authenticate and authorize tenant-wide maintenance first.
63    pub async fn due_agent_deadlines_after(
64        &self,
65        tenant: &TenantId,
66        after: Option<&AgentDeadlineCursor>,
67    ) -> Result<Vec<AgentDeadlineCursor>, StoreError> {
68        if after.is_some_and(|cursor| cursor.tenant_id() != tenant) {
69            return Err(StoreError::InvalidAgentDeadline);
70        }
71        let sql = if after.is_some() {
72            "SELECT agent_deadline_at,run_id FROM stateknot.runs WHERE tenant_id=$1 AND agent_deadline_at IS NOT NULL AND lifecycle_status IN ('pending','active','waiting') AND NOT EXISTS (SELECT 1 FROM stateknot.run_failure_closes c WHERE c.tenant_id=stateknot.runs.tenant_id AND c.run_id=stateknot.runs.run_id) AND agent_deadline_at <= statement_timestamp() AND (agent_deadline_at,run_id)>($2,$3) ORDER BY agent_deadline_at,run_id LIMIT 16"
73        } else {
74            "SELECT agent_deadline_at,run_id FROM stateknot.runs WHERE tenant_id=$1 AND agent_deadline_at IS NOT NULL AND lifecycle_status IN ('pending','active','waiting') AND NOT EXISTS (SELECT 1 FROM stateknot.run_failure_closes c WHERE c.tenant_id=stateknot.runs.tenant_id AND c.run_id=stateknot.runs.run_id) AND agent_deadline_at <= statement_timestamp() ORDER BY agent_deadline_at,run_id LIMIT 16"
75        };
76        let mut listing = query_as::<_, (DateTime<Utc>, Uuid)>(sql).bind(tenant.as_str());
77        if let Some(cursor) = after {
78            listing = listing
79                .bind(to_database_time(cursor.deadline)?)
80                .bind(*cursor.run_id.as_uuid());
81        }
82        listing
83            .fetch_all(&self.pool)
84            .await
85            .map_err(|source| StoreError::database("Agent deadline discovery", source))?
86            .into_iter()
87            .map(|(deadline, run)| {
88                Ok(AgentDeadlineCursor {
89                    tenant: tenant.clone(),
90                    deadline: from_database_time(deadline)?,
91                    run_id: RunId::from_uuid(run)
92                        .map_err(|_| StoreError::corrupt("Agent deadline run identity"))?,
93                })
94            })
95            .collect()
96    }
97
98    /// Requests cancellation only after verifying the immutable admission and
99    /// observing expiry under the run row lock. No wall-clock input is accepted.
100    /// The deployment-owned schema must accept the documented deadline audit payload.
101    /// New candidate IDs on retry converge on the original cancellation reason.
102    /// A due deadline is not evidence that external work stopped or cost is known.
103    /// This is a trusted control-plane API, not an untrusted tenant endpoint.
104    pub async fn request_agent_deadline_cancellation<V: GraphSchemaValidator + ?Sized>(
105        &self,
106        tenant: &TenantId,
107        run_id: RunId,
108        event_id: EventId,
109        failure_id: FailureId,
110        schema: &SchemaReference,
111        schemas: &V,
112    ) -> Result<AgentDeadlineCancellationOutcome, StoreError> {
113        Box::pin(self.request_agent_deadline_cancellation_inner(
114            tenant, run_id, event_id, failure_id, schema, schemas,
115        ))
116        .await
117    }
118
119    #[allow(clippy::too_many_lines)]
120    async fn request_agent_deadline_cancellation_inner<V: GraphSchemaValidator + ?Sized>(
121        &self,
122        tenant: &TenantId,
123        run_id: RunId,
124        event_id: EventId,
125        failure_id: FailureId,
126        schema: &SchemaReference,
127        schemas: &V,
128    ) -> Result<AgentDeadlineCancellationOutcome, StoreError> {
129        // No tree/ancestor locks: like every ordinary cancellation writer, the
130        // parent row serializes spawn and triggers bounded direct-child capture.
131        let mut tx = self.begin_mutation("Agent deadline cancellation").await?;
132        let run = decode_run(fetch_locked_run_row(&mut tx, tenant, run_id).await?)?;
133        verify_current_wait_set(&mut tx, &run).await?;
134        let row = load_agent_admission_row(&mut tx, tenant, run_id)
135            .await?
136            .ok_or(StoreError::AgentAdmissionNotFound)?;
137        let stored = verify_stored_agent_admission(&mut tx, run, row).await?;
138        let run = stored.run();
139        let deadline = stored.admission().intent().budget().deadline();
140        let indexed = query_scalar::<_, Option<DateTime<Utc>>>(
141            "SELECT agent_deadline_at FROM stateknot.runs WHERE tenant_id=$1 AND run_id=$2",
142        )
143        .bind(tenant.as_str())
144        .bind(*run_id.as_uuid())
145        .fetch_one(&mut *tx)
146        .await
147        .map_err(|source| StoreError::database("Agent deadline projection", source))?;
148        if indexed.map(from_database_time).transpose()? != Some(deadline) {
149            return Err(StoreError::corrupt("Agent deadline projection"));
150        }
151        if let Some(request) = run.lifecycle().cancellation_request() {
152            return Ok(AgentDeadlineCancellationOutcome::AlreadyRequested(
153                Box::new(request.clone()),
154            ));
155        }
156        if run.lifecycle().status().is_terminal() {
157            return Ok(AgentDeadlineCancellationOutcome::Terminal(
158                run.lifecycle().status(),
159            ));
160        }
161        if run.is_quarantined() {
162            return Err(StoreError::RunQuarantined);
163        }
164        if failure_closes::exists(&mut tx, tenant, run_id).await? {
165            return Ok(AgentDeadlineCancellationOutcome::FailureClosing);
166        }
167        let observed_at = database_now(&mut tx, "Agent deadline clock").await?;
168        if observed_at < deadline {
169            return Ok(AgentDeadlineCancellationOutcome::NotDue {
170                deadline,
171                observed_at,
172            });
173        }
174        let head = run.journal_head().ok_or(StoreError::InvalidAgentDeadline)?;
175        let at = observed_at.max(head.recorded_at());
176        let failure = Failure::new(
177            failure_id,
178            FailureCategory::Cancelled,
179            FailureCode::new("agent.deadline.expired").expect("static failure code"),
180            FailureOrigin::new("stateknot.runtime.agent_deadlines").expect("static failure origin"),
181            FailureMessage::new(
182                "The admitted Agent deadline expired; cooperative cancellation was requested.",
183            )
184            .expect("static public-safe failure message"),
185            RetryAdvice::Never,
186        )
187        .expect("static cancellation semantics")
188        .with_caused_by_event(event_id);
189        let request = RunCancellationRequest::new(failure, at)
190            .map_err(|_| StoreError::InvalidAgentDeadline)?;
191        let transition = RunTransition::RequestCancellation { request };
192        let projection = RunProjection::transition(run.lifecycle().revision(), transition.clone());
193        let prepared = prepare_durable_wait_projection(
194            run,
195            tenant,
196            run_id,
197            run.lifecycle().revision(),
198            transition,
199            at,
200        )?;
201        let data = BoundedJson::try_from_value(serde_json::json!({
202            "operation":"agent_deadline_cancellation_requested",
203            "admission_digest": hex_digest(stored.admission().digest()),
204            "deadline":deadline.to_string(), "failure_id":failure_id.to_string(),
205        }))
206        .map_err(|_| StoreError::InvalidAgentDeadline)?;
207        schemas
208            .validate(schema, &data)
209            .map_err(|_| StoreError::InvalidAgentDeadline)?;
210        let payload = JournalPayload::new(
211            schema.clone(),
212            JournalEventKind::new("agent-deadline-cancellation-requested")
213                .expect("static event kind"),
214            data,
215        )
216        .map_err(|_| StoreError::InvalidAgentDeadline)?;
217        let append = JournalAppend::new(
218            JournalExpectation::exact(head.clone()),
219            JournalEventIntent::control_plane(tenant.clone(), run_id, event_id, payload)
220                .map_err(|_| StoreError::InvalidAgentDeadline)?,
221        )
222        .map_err(|_| StoreError::InvalidAgentDeadline)?;
223        let waits = child_cancellation::cancellation_waits(&mut tx, run).await?;
224        let digest = if waits.is_empty() {
225            projection_digest(&projection)?
226        } else {
227            wait_abandonment_projection_digest(&projection, &waits)?
228        };
229        let event =
230            JournalEvent::commit(append, at).map_err(|error| map_event_commit_error(&error))?;
231        insert_event(&mut tx, &event, digest).await?;
232        for wait in waits {
233            let abandonment = materialize_wait_abandonment(
234                wait,
235                WaitAbandonmentReason::RunCancellation,
236                event.head(),
237            )?;
238            insert_wait_abandonment(&mut tx, &abandonment).await?;
239            project_wait_abandonment(&mut tx, &abandonment).await?;
240        }
241        update_run_head(&mut tx, &event, Some(&prepared)).await?;
242        tx.commit()
243            .await
244            .map_err(|source| StoreError::database("Agent deadline commit", source))?;
245        Ok(AgentDeadlineCancellationOutcome::Requested(event.head()))
246    }
247}
248
249fn hex_digest(value: Digest) -> String {
250    use std::fmt::Write;
251    let mut text = String::with_capacity(64);
252    for byte in value.as_bytes() {
253        write!(&mut text, "{byte:02x}").expect("String write");
254    }
255    text
256}
257
258pub(super) async fn verify_schema(pool: &PgPool) -> Result<(), StoreError> {
259    let installed = query_scalar::<_, String>(CATALOG_QUERY)
260        .fetch_one(pool)
261        .await
262        .map_err(|source| StoreError::database("Agent deadline catalog", source))?;
263    let expected: serde_json::Value =
264        serde_json::from_str(include_str!("agent_deadline_catalog.json"))
265            .map_err(|_| StoreError::IncompleteSchema)?;
266    if serde_json::from_str::<serde_json::Value>(&installed)
267        .map_err(|_| StoreError::IncompleteSchema)?
268        != expected
269    {
270        return Err(StoreError::IncompleteSchema);
271    }
272    for function in include_str!("../migrations/0023_agent_deadlines.sql")
273        .split("CREATE FUNCTION stateknot.")
274        .skip(1)
275    {
276        let (name, _) = function
277            .split_once("() RETURNS trigger")
278            .ok_or(StoreError::IncompleteSchema)?;
279        let body = function
280            .split("$$")
281            .nth(1)
282            .ok_or(StoreError::IncompleteSchema)?;
283        let installed = query_scalar::<_,String>("SELECT prosrc FROM pg_proc JOIN pg_language ON pg_language.oid=prolang WHERE pg_proc.oid=to_regprocedure($1) AND NOT prosecdef AND lanname='plpgsql'")
284            .bind(format!("stateknot.{name}()")).fetch_optional(pool).await
285            .map_err(|source| StoreError::database("Agent deadline guard",source))?;
286        if installed.as_deref() != Some(body) {
287            return Err(StoreError::IncompleteSchema);
288        }
289    }
290    Ok(())
291}
292
293const CATALOG_QUERY: &str = r"
294SELECT jsonb_build_object(
295 'column',(SELECT jsonb_build_array(format_type(atttypid,atttypmod),attnotnull,attgenerated,atthasdef) FROM pg_attribute WHERE attrelid='stateknot.runs'::regclass AND attname='agent_deadline_at' AND NOT attisdropped),
296 'constraint',(SELECT jsonb_build_array(pg_get_constraintdef(oid),convalidated,condeferrable) FROM pg_constraint WHERE conrelid='stateknot.runs'::regclass AND conname='runs_agent_deadline_finite'),
297 'index',(SELECT jsonb_build_array(pg_get_indexdef(indexrelid),indisvalid,indisready,indislive) FROM pg_index WHERE indexrelid=to_regclass('stateknot.runs_due_agent_deadlines')),
298 'triggers',(SELECT jsonb_agg(jsonb_build_array(tgname,pg_get_triggerdef(oid),tgenabled) ORDER BY tgname) FROM pg_trigger WHERE NOT tgisinternal AND ((tgrelid='stateknot.runs'::regclass AND tgname='runs_agent_deadline_guard') OR (tgrelid='stateknot.agent_admissions'::regclass AND tgname='agent_admissions_deadline_capture')))
299)::text
300";