1use std::{
5 borrow::Cow,
6 collections::BTreeMap,
7 fmt,
8 future::Future,
9 io::Write,
10 panic::{AssertUnwindSafe, catch_unwind},
11 pin::Pin,
12 sync::LazyLock,
13 task::{Context, Poll},
14 time::Duration,
15};
16
17use chrono::{DateTime, Utc};
18use serde::{Deserialize, Serialize};
19use sqlx_core::{
20 from_row::FromRow,
21 migrate::{Migration, MigrationType, Migrator},
22 query::query,
23 query_as::query_as,
24 query_scalar::query_scalar,
25 row::Row,
26 transaction::Transaction,
27};
28use sqlx_postgres::{PgPool, PgRow, Postgres};
29use stateknot_core::{
30 AgentAdmission, AgentAdmissionAuthority, AgentAdmissionBudgetLayer, AgentAdmissionIntent,
31 AgentDescriptor, AgentRequest, AgentResultProvenance, AgentSubmissionKey, ArtifactId,
32 ArtifactIdentity, ArtifactRef, AttemptId, BarrierResultHeads, BoundedJson, BudgetUsage,
33 CanonicalJson, Checkpoint, CheckpointBarrier, CheckpointHead, CheckpointId,
34 CheckpointLineageVerifier, CheckpointState, CheckpointWrite, CompiledGraph, DeliveryFence,
35 DeliveryId, DestinationId, Digest, DurableTimer, DurableTimerRecord, DurableWait, EventId,
36 Failure, FencingEpoch, GraphBarrierPlanError, GraphNamespace, GraphReducer, GraphReducerError,
37 GraphReference, GraphSchemaValidationError, GraphSchemaValidator, InterruptId, InterruptRecord,
38 InterruptRequest, InterruptResolution, InterruptResolutionIntent, InvocationId, JournalAppend,
39 JournalChainVerifier, JournalEvent, JournalEventError, JournalEventIntent, JournalEventSource,
40 JournalHead, JournalPayload, JournalSequence, JsonLimits, MAX_OUTBOX_ATTEMPTS, ModelInvocation,
41 ModelInvocationHead, ModelInvocationHistoryVerifier, ModelInvocationIntent,
42 ModelInvocationRevision, ModelInvocationState, ModelInvocationStatus,
43 ModelInvocationTransition, ModelInvocationTransitionKind, NodeActivation, NodeAttempt,
44 NodeAttemptCompletion, NodeAttemptHistoryVerifier, NodeAttemptOutcome, NodeAttemptStart,
45 NodeAttemptStartHead, NodeAttemptStatus, NodeControlKind, NodeId, NodeInvocationBinding,
46 NodeInvocationBindingKind, OutboxAttempt, OutboxAttemptCompletion,
47 OutboxAttemptHistoryVerifier, OutboxAttemptOutcome, OutboxAttemptStart, OutboxDelivery,
48 OutboxDeliveryIntent, OutboxDeliveryStatus, OutboxDestinationRef, PendingNodeResult,
49 PendingNodeResultError, PendingNodeResultHead, PendingNodeResultIntent, QuarantineId,
50 ReadyNodeRecoveryPlan, ReadyNodeRecoveryPlanner, ReadyNodes, RecoveryNodeKind, ResolvedBudget,
51 RetryAdvice, RunFence, RunId, RunInterruptKind, RunLease, RunLeaseValidationError,
52 RunLifecycle, RunRevision, RunStatus, RunTimerKind, RunTransition, RunTransitionKind, RunWaits,
53 SchedulerReservationId, SchedulerShardId, Superstep, TenantId, TimerFiring, TimerFiringIntent,
54 TimerId, Timestamp, ToolInvocation, ToolInvocationHead, ToolInvocationHistoryVerifier,
55 ToolInvocationIntent, ToolInvocationRevision, ToolInvocationStatus, ToolInvocationTransition,
56 ToolInvocationTransitionKind, WaitRegistrationIntent,
57};
58use uuid::Uuid;
59
60#[path = "child_runs.rs"]
61mod child_runs;
62pub use child_runs::{ChildRunCommitOutcome, ChildRunRecord, ChildRunSettlementOutcome};
63
64#[path = "child_joins.rs"]
65mod child_joins;
66pub use child_joins::{ChildJoinCommitOutcome, ChildJoinRecord};
67
68#[path = "agent_deadlines.rs"]
69mod agent_deadlines;
70pub use agent_deadlines::{AgentDeadlineCancellationOutcome, AgentDeadlineCursor};
71
72#[path = "failure_closes.rs"]
73mod failure_closes;
74pub use failure_closes::{RunFailureCloseCursor, RunFailureCloseOutcome, RunFailureCloseRecord};
75
76#[path = "child_cancellation.rs"]
77mod child_cancellation;
78pub use child_cancellation::{
79 ChildCancellationDelivery, ChildCancellationOutcome, ChildCancellationReceipt,
80 ChildCancellationRecord,
81};
82
83#[path = "tool_authorization_receipts.rs"]
84mod tool_authorization_receipts;
85pub use tool_authorization_receipts::{
86 StoredToolAuthorizationReceipt, ToolAuthorizationReceiptOutcome, ToolAuthorizationReceiptPage,
87 ToolAuthorizationReceiptPageSize,
88};
89
90#[path = "skill_activation_windows.rs"]
91mod skill_activation_windows;
92
93use crate::{
94 AdmissionOutcome, AgentAdmissionCommitOutcome, AgentSubmissionCommitOutcome, AppendOutcome,
95 ArtifactRegistration, ArtifactRegistrationOutcome, ArtifactStorageLocator,
96 BarrierCommitOutcome, CheckpointCommitOutcome, CheckpointLineagePage,
97 CheckpointLineagePageSize, CheckpointPointer, ConfigurationError, CorruptionQuarantineContext,
98 DelayedRetryScheduleOutcome, DueTimerPage, DueTimerPageCursor, ExpiredInterruptPage,
99 ExpiredInterruptPageCursor, GraphDefinitionRegistrationOutcome, GraphReplayLimits,
100 GraphReplayReport, InterruptResolutionCommitOutcome, JournalPage, JournalPageSize,
101 LeaseClaimOutcome, LeaseReleaseOutcome, LeaseRenewalOutcome, LiveLeaseObservation,
102 ModelInvocationCommitOutcome, ModelInvocationHistoryPage, ModelInvocationHistoryPageSize,
103 NodeAttemptCommitOutcome, NodeAttemptHistoryPage, NodeAttemptHistoryPageSize,
104 OutboxAttemptHistoryPage, OutboxAttemptHistoryPageSize, OutboxClaim, OutboxClaimOutcome,
105 OutboxCompletionOutcome, OutboxDestinationRegistrationOutcome, OutboxEnqueueOutcome,
106 PendingNodeResultCommitOutcome, PendingNodeResultPage, PendingNodeResultPageCursor,
107 PendingNodeResultPageSize, PostgresStoreOptions, RunProjection, RunQuarantine,
108 RunQuarantineCause, RunQuarantineCommitOutcome, RunQuarantineComponent, RunQuarantineRequest,
109 RunnableRunCandidate, RunnableRunPage, RunnableRunPageCursor, RunnableRunPageSize,
110 SchedulerFairnessPolicyRegistration, SchedulerFairnessPolicyRegistrationOutcome,
111 SchedulerFairnessReservation, SchedulerFairnessRetentionPolicy,
112 SchedulerFairnessRetentionReport, StoreError, StoredAgentAdmission, StoredAgentSubmission,
113 StoredArtifact, StoredGraphDefinition, StoredOutboxDestination, StoredRun,
114 StoredSchedulerFairnessPolicy, TimerFiringCommitOutcome, ToolInvocationCommitOutcome,
115 ToolInvocationHistoryPage, ToolInvocationHistoryPageSize, WaitAbandonment,
116 WaitAbandonmentCommitOutcome, WaitAbandonmentReason, WaitCheckpointCommitOutcome,
117 WaitDiscoveryPageSize,
118};
119
120struct CompleteOnDrop<F>
129where
130 F: Future + Send + 'static,
131 F::Output: Send + 'static,
132{
133 future: Option<Pin<Box<F>>>,
134}
135
136impl<F> CompleteOnDrop<F>
137where
138 F: Future + Send + 'static,
139 F::Output: Send + 'static,
140{
141 fn new(future: F) -> Self {
142 Self {
143 future: Some(Box::pin(future)),
144 }
145 }
146}
147
148impl<F> Future for CompleteOnDrop<F>
149where
150 F: Future + Send + 'static,
151 F::Output: Send + 'static,
152{
153 type Output = F::Output;
154
155 fn poll(self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll<Self::Output> {
156 let this = self.get_mut();
157 let result = this
158 .future
159 .as_mut()
160 .expect("complete-on-drop future polled after completion")
161 .as_mut()
162 .poll(context);
163 if result.is_ready() {
164 this.future = None;
165 }
166 result
167 }
168}
169
170impl<F> Drop for CompleteOnDrop<F>
171where
172 F: Future + Send + 'static,
173 F::Output: Send + 'static,
174{
175 fn drop(&mut self) {
176 let Some(future) = self.future.take() else {
177 return;
178 };
179
180 let _cleanup = sqlx_core::rt::spawn(async move {
183 drop(future.await);
184 });
185 }
186}
187
188static MIGRATOR: LazyLock<Migrator> = LazyLock::new(|| Migrator {
189 migrations: Cow::Owned(vec![
190 Migration::new(
191 1,
192 Cow::Borrowed("initial"),
193 MigrationType::Simple,
194 Cow::Borrowed(include_str!("../migrations/0001_initial.sql")),
195 false,
196 ),
197 Migration::new(
198 2,
199 Cow::Borrowed("checkpoints"),
200 MigrationType::Simple,
201 Cow::Borrowed(include_str!("../migrations/0002_checkpoints.sql")),
202 false,
203 ),
204 Migration::new(
205 3,
206 Cow::Borrowed("tool invocations"),
207 MigrationType::Simple,
208 Cow::Borrowed(include_str!("../migrations/0003_tool_invocations.sql")),
209 false,
210 ),
211 Migration::new(
212 4,
213 Cow::Borrowed("model invocations"),
214 MigrationType::Simple,
215 Cow::Borrowed(include_str!("../migrations/0004_model_invocations.sql")),
216 false,
217 ),
218 Migration::new(
219 5,
220 Cow::Borrowed("pending node results"),
221 MigrationType::Simple,
222 Cow::Borrowed(include_str!("../migrations/0005_pending_node_results.sql")),
223 false,
224 ),
225 Migration::new(
226 6,
227 Cow::Borrowed("node attempts"),
228 MigrationType::Simple,
229 Cow::Borrowed(include_str!("../migrations/0006_node_attempts.sql")),
230 false,
231 ),
232 Migration::new(
233 7,
234 Cow::Borrowed("scheduler readiness"),
235 MigrationType::Simple,
236 Cow::Borrowed(include_str!("../migrations/0007_scheduler_readiness.sql")),
237 false,
238 ),
239 Migration::new(
240 8,
241 Cow::Borrowed("transactional outbox"),
242 MigrationType::Simple,
243 Cow::Borrowed(include_str!("../migrations/0008_transactional_outbox.sql")),
244 false,
245 ),
246 Migration::new(
247 9,
248 Cow::Borrowed("durable waits"),
249 MigrationType::Simple,
250 Cow::Borrowed(include_str!("../migrations/0009_durable_waits.sql")),
251 false,
252 ),
253 Migration::new(
254 10,
255 Cow::Borrowed("run quarantines"),
256 MigrationType::Simple,
257 Cow::Borrowed(include_str!("../migrations/0010_run_quarantines.sql")),
258 false,
259 ),
260 Migration::new(
261 11,
262 Cow::Borrowed("fenced recovery quarantines"),
263 MigrationType::Simple,
264 Cow::Borrowed(include_str!(
265 "../migrations/0011_fenced_recovery_quarantines.sql"
266 )),
267 false,
268 ),
269 Migration::new(
270 12,
271 Cow::Borrowed("delayed retry wakeup"),
272 MigrationType::Simple,
273 Cow::Borrowed(include_str!("../migrations/0012_delayed_retry_wakeup.sql")),
274 false,
275 ),
276 Migration::new(
277 13,
278 Cow::Borrowed("compiled graph registry"),
279 MigrationType::Simple,
280 Cow::Borrowed(include_str!("../migrations/0013_graph_registry.sql")),
281 false,
282 ),
283 Migration::new(
284 14,
285 Cow::Borrowed("distributed scheduler fairness"),
286 MigrationType::Simple,
287 Cow::Borrowed(include_str!("../migrations/0014_scheduler_fairness.sql")),
288 false,
289 ),
290 Migration::new(
291 15,
292 Cow::Borrowed("atomic agent admissions"),
293 MigrationType::Simple,
294 Cow::Borrowed(include_str!("../migrations/0015_agent_admissions.sql")),
295 false,
296 ),
297 Migration::new(
298 16,
299 Cow::Borrowed("durable agent submission keys"),
300 MigrationType::Simple,
301 Cow::Borrowed(include_str!("../migrations/0016_agent_submission_keys.sql")),
302 false,
303 ),
304 Migration::new(
305 17,
306 Cow::Borrowed("terminal tool result bindings"),
307 MigrationType::Simple,
308 Cow::Borrowed(include_str!(
309 "../migrations/0017_terminal_tool_result_bindings.sql"
310 )),
311 false,
312 ),
313 Migration::new(
314 18,
315 Cow::Borrowed("immutable artifact registry"),
316 MigrationType::Simple,
317 Cow::Borrowed(include_str!("../migrations/0018_artifact_registry.sql")),
318 false,
319 ),
320 Migration::new(
321 19,
322 Cow::Borrowed("terminal model failure bindings"),
323 MigrationType::Simple,
324 Cow::Borrowed(include_str!(
325 "../migrations/0019_terminal_model_failure_bindings.sql"
326 )),
327 false,
328 ),
329 Migration::new(
330 20,
331 Cow::Borrowed("child run ownership"),
332 MigrationType::Simple,
333 Cow::Borrowed(include_str!("../migrations/0020_child_run_ownership.sql")),
334 false,
335 ),
336 Migration::new(
337 21,
338 Cow::Borrowed("child run cancellation"),
339 MigrationType::Simple,
340 Cow::Borrowed(include_str!(
341 "../migrations/0021_child_run_cancellation.sql"
342 )),
343 false,
344 ),
345 Migration::new(
346 22,
347 Cow::Borrowed("child run joins"),
348 MigrationType::Simple,
349 Cow::Borrowed(include_str!("../migrations/0022_child_run_joins.sql")),
350 false,
351 ),
352 Migration::new(
353 23,
354 Cow::Borrowed("agent deadlines"),
355 MigrationType::Simple,
356 Cow::Borrowed(include_str!("../migrations/0023_agent_deadlines.sql")),
357 false,
358 ),
359 Migration::new(
360 24,
361 Cow::Borrowed("run failure closes"),
362 MigrationType::Simple,
363 Cow::Borrowed(include_str!("../migrations/0024_run_failure_closes.sql")),
364 false,
365 ),
366 Migration::new(
367 25,
368 Cow::Borrowed("Tool authorization receipts"),
369 MigrationType::Simple,
370 Cow::Borrowed(include_str!(
371 "../migrations/0025_tool_authorization_receipts.sql"
372 )),
373 false,
374 ),
375 Migration::new(
376 26,
377 Cow::Borrowed("Skill activation windows"),
378 MigrationType::Simple,
379 Cow::Borrowed(include_str!(
380 "../migrations/0026_skill_activation_windows.sql"
381 )),
382 false,
383 ),
384 ]),
385 ignore_missing: false,
386 locking: true,
387 no_tx: false,
388});
389
390const MIN_POSTGRES_VERSION_NUMBER: i32 = 160_000;
391const MAX_POSTGRES_VERSION_NUMBER: i32 = 179_999;
392const MAX_CHECKPOINT_BYTES: usize = 2_621_440;
393const MAX_COMPILED_GRAPH_BYTES: usize = CompiledGraph::MAX_DEFINITION_BYTES + 128;
394const MAX_AGENT_ADMISSION_BYTES: usize = 16_777_216;
395const MAX_ARTIFACT_REF_BYTES: usize = 262_144;
396const MAX_TOOL_INVOCATION_INTENT_BYTES: usize = 4_194_304;
397const MAX_TOOL_INVOCATION_RECORD_BYTES: usize = 16_777_216;
398const MAX_MODEL_INVOCATION_INTENT_BYTES: usize = 134_217_728;
399const MAX_MODEL_INVOCATION_RECORD_BYTES: usize = 134_217_728;
400const MAX_PENDING_NODE_RESULT_BYTES: usize = 16_777_216;
401const MAX_NODE_ATTEMPT_START_BYTES: usize = 1_048_576;
402const MAX_NODE_ATTEMPT_COMPLETION_BYTES: usize = 16_777_216;
403const MAX_OUTBOX_DESTINATION_BYTES: usize = 2_097_152;
404const MAX_OUTBOX_DELIVERY_BYTES: usize = 4_194_304;
405const MAX_OUTBOX_ATTEMPT_START_BYTES: usize = 1_048_576;
406const MAX_OUTBOX_ATTEMPT_COMPLETION_BYTES: usize = 1_048_576;
407const MAX_WAIT_REGISTRATION_BYTES: usize = 4_194_304;
408const MAX_INTERRUPT_RESOLUTION_BYTES: usize = 4_194_304;
409const MAX_TIMER_FIRING_BYTES: usize = 1_048_576;
410const RUN_QUARANTINE_DIGEST_DOMAIN_V1: &[u8] = b"stateknot.run-quarantine.v1\0";
411const RUN_QUARANTINE_DIGEST_DOMAIN_V2: &[u8] = b"stateknot.run-quarantine.v2\0";
412const MAX_OUTBOX_DELIVERIES_PER_EVENT: usize = 64;
413const OUTBOX_TERMINAL_REAP_BATCH_SIZE: i64 = 64;
414const PENDING_TOOL_BINDING_BATCH_SIZE: usize = ToolInvocationHistoryPageSize::MAX as usize;
415const PENDING_MODEL_BINDING_BATCH_SIZE: usize = ModelInvocationHistoryPageSize::MAX as usize;
416const PENDING_INVOCATION_ANCHOR_BATCH_SIZE: usize = 8;
417const MODEL_INVOCATION_RECORD_SCHEMA: &str =
418 "https://stateknot.github.io/schema/storage/model-invocation-revision/1.0.0";
419const PROJECTION_DIGEST_DOMAIN: &[u8] = b"stateknot-postgres-run-projection-v1\0";
420const AGENT_ADMISSION_PROJECTION_DIGEST_DOMAIN: &[u8] =
421 b"stateknot-postgres-agent-admission-projection-v1\0";
422const AGENT_SUBMISSION_DIGEST_DOMAIN: &[u8] = b"stateknot-postgres-agent-submission-v1\0";
423const BARRIER_PROJECTION_DIGEST_DOMAIN: &[u8] = b"stateknot-postgres-barrier-projection-v1\0";
424const WAIT_SET_DIGEST_DOMAIN: &[u8] = b"stateknot-postgres-wait-set-v1\0";
425const WAIT_REGISTRATION_PROJECTION_DIGEST_DOMAIN: &[u8] =
426 b"stateknot-postgres-wait-registration-projection-v1\0";
427const WAIT_BARRIER_PROJECTION_DIGEST_DOMAIN: &[u8] =
428 b"stateknot-postgres-wait-barrier-projection-v1\0";
429const INTERRUPT_RESOLUTION_PROJECTION_DIGEST_DOMAIN: &[u8] =
430 b"stateknot-postgres-interrupt-resolution-projection-v1\0";
431const TIMER_FIRING_PROJECTION_DIGEST_DOMAIN: &[u8] =
432 b"stateknot-postgres-timer-firing-projection-v1\0";
433const WAIT_ABANDONMENT_DIGEST_DOMAIN: &[u8] = b"stateknot-postgres-wait-abandonment-v1\0";
434const WAIT_ABANDONMENT_PROJECTION_DIGEST_DOMAIN: &[u8] =
435 b"stateknot-postgres-wait-abandonment-projection-v1\0";
436
437const SELECT_ARTIFACT: &str = r"
438SELECT
439 tenant_id,
440 artifact_id,
441 registration_key_digest,
442 artifact_ref_digest,
443 artifact_ref_bytes,
444 content_byte_length,
445 content_digest,
446 provenance_run_id,
447 provenance_event_id,
448 storage_namespace,
449 object_key,
450 object_version,
451 object_etag,
452 registered_at
453FROM stateknot.artifacts
454WHERE tenant_id = $1 AND artifact_id = $2
455";
456
457const SELECT_ARTIFACT_PARENTS: &str = r"
458SELECT parent_artifact_id
459FROM stateknot.artifact_parents
460WHERE tenant_id = $1 AND artifact_id = $2
461ORDER BY parent_artifact_id ASC
462";
463
464const SELECT_GRAPH_DEFINITION: &str = r"
465SELECT
466 tenant_id,
467 owner_issuer,
468 owner_subject,
469 graph_name,
470 graph_version,
471 definition_digest,
472 definition_bytes,
473 registered_at
474FROM stateknot.graph_definitions
475WHERE tenant_id = $1
476 AND owner_issuer = $2
477 AND owner_subject = $3
478 AND graph_name = $4
479 AND graph_version = $5
480";
481
482const SELECT_AGENT_ADMISSION: &str = r"
483SELECT
484 tenant_id,
485 run_id,
486 agent_owner_issuer,
487 agent_owner_subject,
488 agent_name,
489 agent_version,
490 graph_owner_issuer,
491 graph_owner_subject,
492 graph_name,
493 graph_version,
494 graph_definition_digest,
495 policy_owner_issuer,
496 policy_owner_subject,
497 policy_name,
498 policy_version,
499 policy_digest,
500 intent_digest,
501 admission_digest,
502 admitted_at,
503 journal_sequence,
504 journal_event_id,
505 journal_recorded_at,
506 journal_digest,
507 checkpoint_id,
508 checkpoint_superstep,
509 checkpoint_digest,
510 admission_bytes,
511 created_at
512FROM stateknot.agent_admissions
513WHERE tenant_id = $1 AND run_id = $2
514";
515
516const SELECT_AGENT_SUBMISSION: &str = r"
517SELECT
518 tenant_id,
519 key_digest,
520 submission_digest,
521 run_id,
522 admission_digest,
523 created_at
524FROM stateknot.agent_submission_keys
525WHERE tenant_id = $1 AND key_digest = $2
526";
527
528const SELECT_SCHEDULER_FAIRNESS_SHARD: &str = r"
529SELECT
530 shard_id,
531 policy_digest,
532 policy_bytes,
533 cycle_length,
534 next_slot,
535 next_sequence,
536 registered_at,
537 updated_at
538FROM stateknot.scheduler_fairness_shards
539WHERE shard_id = $1
540";
541
542const SELECT_SCHEDULER_FAIRNESS_SHARD_FOR_UPDATE: &str = r"
543SELECT
544 shard_id,
545 policy_digest,
546 policy_bytes,
547 cycle_length,
548 next_slot,
549 next_sequence,
550 registered_at,
551 updated_at
552FROM stateknot.scheduler_fairness_shards
553WHERE shard_id = $1
554FOR UPDATE
555";
556
557const SELECT_SCHEDULER_FAIRNESS_RESERVATION: &str = r"
558SELECT
559 reservation.shard_id,
560 reservation.reservation_id,
561 reservation.policy_digest,
562 reservation.sequence,
563 reservation.slot,
564 reservation.reserved_at,
565 shard.policy_digest AS shard_policy_digest,
566 shard.cycle_length
567FROM stateknot.scheduler_fairness_reservations AS reservation
568JOIN stateknot.scheduler_fairness_shards AS shard
569 ON shard.shard_id = reservation.shard_id
570WHERE reservation.reservation_id = $1
571";
572
573const VERIFY_SCHEMA_OBJECTS: &str = r"
574SELECT to_regclass('stateknot.runs') IS NOT NULL
575 AND to_regclass('stateknot.artifacts') IS NOT NULL
576 AND to_regclass('stateknot.artifact_parents') IS NOT NULL
577 AND to_regclass('stateknot.artifacts_provenance') IS NOT NULL
578 AND to_regclass('stateknot.artifact_parents_reverse_lineage') IS NOT NULL
579 AND to_regclass('stateknot.graph_definitions') IS NOT NULL
580 AND to_regclass('stateknot.graph_definitions_digest_lookup') IS NOT NULL
581 AND to_regclass('stateknot.agent_admissions') IS NOT NULL
582 AND to_regclass('stateknot.agent_admissions_agent_version') IS NOT NULL
583 AND to_regclass('stateknot.agent_admissions_graph_version') IS NOT NULL
584 AND to_regclass('stateknot.agent_admissions_policy_version') IS NOT NULL
585 AND to_regclass('stateknot.agent_admissions_digest_lookup') IS NOT NULL
586 AND to_regclass('stateknot.agent_submission_keys') IS NOT NULL
587 AND to_regclass('stateknot.agent_submission_keys_created') IS NOT NULL
588 AND to_regclass('stateknot.run_events') IS NOT NULL
589 AND to_regclass('stateknot.run_checkpoints') IS NOT NULL
590 AND to_regclass('stateknot.tool_invocations') IS NOT NULL
591 AND to_regclass('stateknot.tool_invocation_revisions') IS NOT NULL
592 AND to_regclass('stateknot.tool_authorization_receipts') IS NOT NULL
593 AND to_regclass('stateknot.tool_authorization_receipts_invocation_history') IS NOT NULL
594 AND to_regclass('stateknot.run_attempt_claims') IS NOT NULL
595 AND to_regclass('stateknot.model_invocations') IS NOT NULL
596 AND to_regclass('stateknot.model_invocation_revisions') IS NOT NULL
597 AND to_regclass('stateknot.pending_node_results') IS NOT NULL
598 AND to_regclass('stateknot.pending_node_result_tool_bindings') IS NOT NULL
599 AND to_regclass('stateknot.pending_node_result_model_bindings') IS NOT NULL
600 AND to_regclass('stateknot.pending_node_result_consumptions') IS NOT NULL
601 AND to_regclass('stateknot.node_attempts') IS NOT NULL
602 AND to_regclass('stateknot.node_attempt_completions') IS NOT NULL
603 AND to_regclass('stateknot.runs_scheduler_ready') IS NOT NULL
604 AND to_regclass('stateknot.outbox_destinations') IS NOT NULL
605 AND to_regclass('stateknot.outbox_deliveries') IS NOT NULL
606 AND to_regclass('stateknot.outbox_attempts') IS NOT NULL
607 AND to_regclass('stateknot.outbox_attempt_completions') IS NOT NULL
608 AND to_regclass('stateknot.outbox_deliveries_ready') IS NOT NULL
609 AND to_regclass('stateknot.outbox_deliveries_expiry') IS NOT NULL
610 AND to_regclass('stateknot.outbox_deliveries_abandoned_limit') IS NOT NULL
611 AND to_regclass('stateknot.run_wait_registrations') IS NOT NULL
612 AND to_regclass('stateknot.interrupt_resolutions') IS NOT NULL
613 AND to_regclass('stateknot.timer_firings') IS NOT NULL
614 AND to_regclass('stateknot.wait_abandonments') IS NOT NULL
615 AND to_regclass('stateknot.run_wait_registrations_due') IS NOT NULL
616 AND to_regclass('stateknot.run_wait_registrations_expiry') IS NOT NULL
617 AND to_regclass('stateknot.run_quarantines') IS NOT NULL
618 AND to_regclass('stateknot.run_quarantines_observed') IS NOT NULL
619 AND to_regclass('stateknot.scheduler_fairness_shards') IS NOT NULL
620 AND to_regclass('stateknot.scheduler_fairness_reservations') IS NOT NULL
621 AND to_regclass('stateknot.scheduler_fairness_reservations_sequence') IS NOT NULL
622 AND to_regclass('stateknot.scheduler_fairness_reservations_retention') IS NOT NULL
623 AND lower(pg_get_indexdef(to_regclass('stateknot.runs_scheduler_ready')))
624 LIKE '%checkpoint_id is not null%'
625 AND EXISTS (
626 SELECT 1 FROM pg_catalog.pg_constraint
627 WHERE conrelid = to_regclass('stateknot.runs')
628 AND conname = 'runs_exact_thread_unique'
629 AND convalidated
630 )
631 AND EXISTS (
632 SELECT 1 FROM pg_catalog.pg_constraint
633 WHERE conrelid = to_regclass('stateknot.tool_invocation_revisions')
634 AND conname = 'tool_invocation_revisions_attempt_event_unique'
635 AND convalidated
636 )
637 AND EXISTS (
638 SELECT 1 FROM pg_catalog.pg_constraint
639 WHERE conrelid = to_regclass('stateknot.tool_authorization_receipts')
640 AND conname = 'tool_authorization_receipts_run_thread_fk'
641 AND convalidated
642 )
643 AND EXISTS (
644 SELECT 1 FROM pg_catalog.pg_constraint
645 WHERE conrelid = to_regclass('stateknot.tool_authorization_receipts')
646 AND conname = 'tool_authorization_receipts_attempt_event_fk'
647 AND convalidated
648 )
649 AND EXISTS (
650 SELECT 1 FROM pg_catalog.pg_trigger
651 WHERE tgrelid = to_regclass('stateknot.tool_authorization_receipts')
652 AND tgname = 'tool_authorization_receipts_immutable'
653 AND tgenabled <> 'D'
654 AND NOT tgisinternal
655 )
656 AND EXISTS (
657 SELECT 1 FROM pg_catalog.pg_constraint
658 WHERE conrelid = to_regclass('stateknot.artifacts')
659 AND conname = 'artifacts_registration_key_unique'
660 AND convalidated
661 )
662 AND EXISTS (
663 SELECT 1 FROM pg_catalog.pg_constraint
664 WHERE conrelid = to_regclass('stateknot.artifacts')
665 AND conname = 'artifacts_object_locator_unique'
666 AND convalidated
667 )
668 AND EXISTS (
669 SELECT 1 FROM pg_catalog.pg_constraint
670 WHERE conrelid = to_regclass('stateknot.artifacts')
671 AND conname = 'artifacts_provenance_event_fk'
672 AND convalidated
673 )
674 AND EXISTS (
675 SELECT 1 FROM pg_catalog.pg_constraint
676 WHERE conrelid = to_regclass('stateknot.artifacts')
677 AND conname = 'artifacts_digest_lengths'
678 AND convalidated
679 )
680 AND EXISTS (
681 SELECT 1 FROM pg_catalog.pg_constraint
682 WHERE conrelid = to_regclass('stateknot.artifacts')
683 AND conname = 'artifacts_ref_bytes_bounded'
684 AND convalidated
685 )
686 AND EXISTS (
687 SELECT 1 FROM pg_catalog.pg_constraint
688 WHERE conrelid = to_regclass('stateknot.artifacts')
689 AND conname = 'artifacts_content_length_valid'
690 AND convalidated
691 )
692 AND EXISTS (
693 SELECT 1 FROM pg_catalog.pg_constraint
694 WHERE conrelid = to_regclass('stateknot.artifact_parents')
695 AND conname = 'artifact_parents_child_fk'
696 AND convalidated
697 )
698 AND EXISTS (
699 SELECT 1 FROM pg_catalog.pg_constraint
700 WHERE conrelid = to_regclass('stateknot.artifact_parents')
701 AND conname = 'artifact_parents_parent_fk'
702 AND convalidated
703 )
704 AND EXISTS (
705 SELECT 1 FROM pg_catalog.pg_constraint
706 WHERE conrelid = to_regclass('stateknot.artifact_parents')
707 AND conname = 'artifact_parents_not_self'
708 AND convalidated
709 )
710 AND EXISTS (
711 SELECT 1 FROM pg_catalog.pg_constraint
712 WHERE conrelid = to_regclass('stateknot.graph_definitions')
713 AND conname = 'graph_definitions_exact_reference_unique'
714 AND convalidated
715 )
716 AND EXISTS (
717 SELECT 1 FROM pg_catalog.pg_constraint
718 WHERE conrelid = to_regclass('stateknot.agent_admissions')
719 AND conname = 'agent_admissions_graph_fk'
720 AND convalidated
721 )
722 AND EXISTS (
723 SELECT 1 FROM pg_catalog.pg_constraint
724 WHERE conrelid = to_regclass('stateknot.agent_admissions')
725 AND conname = 'agent_admissions_event_fk'
726 AND convalidated
727 )
728 AND EXISTS (
729 SELECT 1 FROM pg_catalog.pg_constraint
730 WHERE conrelid = to_regclass('stateknot.agent_admissions')
731 AND conname = 'agent_admissions_checkpoint_fk'
732 AND convalidated
733 )
734 AND EXISTS (
735 SELECT 1 FROM pg_catalog.pg_constraint
736 WHERE conrelid = to_regclass('stateknot.agent_admissions')
737 AND conname = 'agent_admissions_bytes_bounded'
738 AND convalidated
739 )
740 AND EXISTS (
741 SELECT 1 FROM pg_catalog.pg_constraint
742 WHERE conrelid = to_regclass('stateknot.agent_admissions')
743 AND conname = 'agent_admissions_run_digest_unique'
744 AND convalidated
745 )
746 AND EXISTS (
747 SELECT 1 FROM pg_catalog.pg_constraint
748 WHERE conrelid = to_regclass('stateknot.agent_submission_keys')
749 AND conname = 'agent_submission_keys_run_unique'
750 AND convalidated
751 )
752 AND EXISTS (
753 SELECT 1 FROM pg_catalog.pg_constraint
754 WHERE conrelid = to_regclass('stateknot.agent_submission_keys')
755 AND conname = 'agent_submission_keys_admission_fk'
756 AND convalidated
757 )
758 AND EXISTS (
759 SELECT 1 FROM pg_catalog.pg_constraint
760 WHERE conrelid = to_regclass('stateknot.agent_submission_keys')
761 AND conname = 'agent_submission_keys_tenant_id_valid'
762 AND convalidated
763 )
764 AND EXISTS (
765 SELECT 1 FROM pg_catalog.pg_constraint
766 WHERE conrelid = to_regclass('stateknot.agent_submission_keys')
767 AND conname = 'agent_submission_keys_ids_valid'
768 AND convalidated
769 )
770 AND EXISTS (
771 SELECT 1 FROM pg_catalog.pg_constraint
772 WHERE conrelid = to_regclass('stateknot.agent_submission_keys')
773 AND conname = 'agent_submission_keys_digest_lengths'
774 AND convalidated
775 )
776 AND EXISTS (
777 SELECT 1 FROM pg_catalog.pg_constraint
778 WHERE conrelid = to_regclass('stateknot.pending_node_result_tool_bindings')
779 AND conname = 'pending_node_result_tool_bindings_status_valid'
780 AND convalidated
781 )
782 AND EXISTS (
783 SELECT 1 FROM pg_catalog.pg_constraint
784 WHERE conrelid = to_regclass('stateknot.pending_node_result_tool_bindings')
785 AND conname = 'pending_node_result_tool_bindings_revision_fk'
786 AND convalidated
787 )
788 AND EXISTS (
789 SELECT 1 FROM pg_catalog.pg_constraint
790 WHERE conrelid = to_regclass('stateknot.pending_node_result_model_bindings')
791 AND conname = 'pending_node_result_model_bindings_status_valid'
792 AND convalidated
793 AND pg_get_constraintdef(oid) LIKE '%''committed''%'
794 AND pg_get_constraintdef(oid) LIKE '%''failed''%'
795 )
796 AND EXISTS (
797 SELECT 1 FROM pg_catalog.pg_constraint
798 WHERE conrelid = to_regclass('stateknot.pending_node_result_model_bindings')
799 AND conname = 'pending_node_result_model_bindings_revision_fk'
800 AND convalidated
801 )
802 AND 2 = (
803 SELECT count(*)
804 FROM pg_catalog.pg_attribute
805 WHERE attrelid IN (
806 to_regclass('stateknot.pending_node_result_tool_bindings'),
807 to_regclass('stateknot.pending_node_result_model_bindings')
808 )
809 AND attname = 'invocation_status'
810 AND NOT attisdropped
811 AND attgenerated = ''
812 )
813 AND EXISTS (
814 SELECT 1 FROM pg_catalog.pg_constraint
815 WHERE conrelid = to_regclass('stateknot.graph_definitions')
816 AND conname = 'graph_definitions_bytes_bounded'
817 AND convalidated
818 )
819 AND EXISTS (
820 SELECT 1 FROM pg_catalog.pg_constraint
821 WHERE conrelid = to_regclass('stateknot.scheduler_fairness_shards')
822 AND conname = 'scheduler_fairness_shards_policy_bounded'
823 AND convalidated
824 )
825 AND EXISTS (
826 SELECT 1 FROM pg_catalog.pg_constraint
827 WHERE conrelid = to_regclass('stateknot.scheduler_fairness_reservations')
828 AND conname = 'scheduler_fairness_reservations_id_unique'
829 AND convalidated
830 )
831 AND EXISTS (
832 SELECT 1 FROM pg_catalog.pg_constraint
833 WHERE conrelid = to_regclass('stateknot.run_quarantines')
834 AND conname = 'run_quarantines_fence_shape'
835 AND convalidated
836 )
837 AND EXISTS (
838 SELECT 1 FROM pg_catalog.pg_constraint
839 WHERE conrelid = to_regclass('stateknot.runs')
840 AND conname = 'runs_scheduler_ready_shape'
841 AND convalidated
842 )
843 AND EXISTS (
844 SELECT 1 FROM pg_catalog.pg_constraint
845 WHERE conrelid = to_regclass('stateknot.runs')
846 AND conname = 'runs_scheduler_not_before_shape'
847 AND convalidated
848 )
849 AND EXISTS (
850 SELECT 1 FROM pg_catalog.pg_constraint
851 WHERE conrelid = to_regclass('stateknot.runs')
852 AND conname = 'runs_wait_projection_shape'
853 AND convalidated
854 )
855 AND to_regprocedure('stateknot.is_uuid_v7(uuid)') IS NOT NULL
856";
857
858const SELECT_RUN: &str = r"
859SELECT
860 tenant_id,
861 run_id,
862 thread_id,
863 invocation_id,
864 lifecycle_bytes,
865 lifecycle_revision::text AS lifecycle_revision,
866 lifecycle_status,
867 admitted_at,
868 changed_at,
869 journal_sequence,
870 journal_event_id,
871 journal_recorded_at,
872 journal_digest,
873 checkpoint_id,
874 checkpoint_superstep,
875 checkpoint_digest,
876 fencing_epoch,
877 lease_attempt_id,
878 lease_acquired_at,
879 lease_renewed_at,
880 lease_expires_at,
881 scheduler_ready_at,
882 scheduler_not_before,
883 wait_set_digest,
884 unresolved_wait_count,
885 next_timer_due_at,
886 next_interrupt_expiry_at,
887 quarantined_at
888FROM stateknot.runs
889WHERE tenant_id = $1 AND run_id = $2
890";
891
892const SELECT_RUN_FOR_UPDATE: &str = r"
893SELECT
894 tenant_id,
895 run_id,
896 thread_id,
897 invocation_id,
898 lifecycle_bytes,
899 lifecycle_revision::text AS lifecycle_revision,
900 lifecycle_status,
901 admitted_at,
902 changed_at,
903 journal_sequence,
904 journal_event_id,
905 journal_recorded_at,
906 journal_digest,
907 checkpoint_id,
908 checkpoint_superstep,
909 checkpoint_digest,
910 fencing_epoch,
911 lease_attempt_id,
912 lease_acquired_at,
913 lease_renewed_at,
914 lease_expires_at,
915 scheduler_ready_at,
916 scheduler_not_before,
917 wait_set_digest,
918 unresolved_wait_count,
919 next_timer_due_at,
920 next_interrupt_expiry_at,
921 quarantined_at
922FROM stateknot.runs
923WHERE tenant_id = $1 AND run_id = $2
924FOR UPDATE
925";
926
927const SELECT_RUNNABLE_RUN_PAGE: &str = r"
928SELECT
929 tenant_id,
930 run_id,
931 thread_id,
932 invocation_id,
933 lifecycle_bytes,
934 lifecycle_revision::text AS lifecycle_revision,
935 lifecycle_status,
936 admitted_at,
937 changed_at,
938 journal_sequence,
939 journal_event_id,
940 journal_recorded_at,
941 journal_digest,
942 checkpoint_id,
943 checkpoint_superstep,
944 checkpoint_digest,
945 fencing_epoch,
946 lease_attempt_id,
947 lease_acquired_at,
948 lease_renewed_at,
949 lease_expires_at,
950 scheduler_ready_at,
951 scheduler_not_before,
952 wait_set_digest,
953 unresolved_wait_count,
954 next_timer_due_at,
955 next_interrupt_expiry_at,
956 quarantined_at
957FROM stateknot.runs
958WHERE tenant_id = $1
959 AND quarantined_at IS NULL
960 AND scheduler_ready_at IS NOT NULL
961 AND checkpoint_id IS NOT NULL
962 AND lifecycle_status IN ('pending', 'active', 'cancellation_requested')
963 AND NOT EXISTS (SELECT 1 FROM stateknot.run_failure_closes AS closing
964 WHERE closing.tenant_id=stateknot.runs.tenant_id AND closing.run_id=stateknot.runs.run_id)
965 AND (lifecycle_status <> 'active' OR NOT EXISTS (
966 SELECT 1 FROM stateknot.child_run_joins AS joined
967 WHERE joined.tenant_id = stateknot.runs.tenant_id
968 AND joined.parent_run_id = stateknot.runs.run_id AND joined.ready_at IS NULL
969 ))
970 AND (lifecycle_status <> 'cancellation_requested' OR NOT EXISTS (
971 SELECT 1 FROM stateknot.child_run_ownership AS owned
972 WHERE owned.tenant_id = stateknot.runs.tenant_id
973 AND owned.parent_run_id = stateknot.runs.run_id AND NOT owned.settled
974 ))
975 AND GREATEST(
976 scheduler_ready_at,
977 COALESCE(scheduler_not_before, scheduler_ready_at),
978 COALESCE(lease_expires_at, scheduler_ready_at)
979 ) <= $2
980 AND (
981 GREATEST(
982 scheduler_ready_at,
983 COALESCE(scheduler_not_before, scheduler_ready_at),
984 COALESCE(lease_expires_at, scheduler_ready_at)
985 ),
986 run_id
987 ) > (
988 COALESCE($3::timestamptz, '-infinity'::timestamptz),
989 COALESCE($4::uuid, '00000000-0000-0000-0000-000000000000'::uuid)
990 )
991ORDER BY
992 GREATEST(
993 scheduler_ready_at,
994 COALESCE(scheduler_not_before, scheduler_ready_at),
995 COALESCE(lease_expires_at, scheduler_ready_at)
996 ),
997 run_id
998LIMIT $5
999";
1000
1001const SELECT_EVENT_BY_ID: &str = r"
1002SELECT
1003 tenant_id,
1004 run_id,
1005 sequence,
1006 event_id,
1007 recorded_at,
1008 source_kind,
1009 worker_attempt_id,
1010 worker_epoch,
1011 event_kind,
1012 schema_id,
1013 schema_version,
1014 schema_digest,
1015 payload_bytes,
1016 payload_digest,
1017 intent_digest,
1018 projection_digest,
1019 previous_digest,
1020 event_digest
1021FROM stateknot.run_events
1022WHERE tenant_id = $1 AND run_id = $2 AND event_id = $3
1023";
1024
1025const SELECT_EVENT_BY_SEQUENCE: &str = r"
1026SELECT
1027 tenant_id,
1028 run_id,
1029 sequence,
1030 event_id,
1031 recorded_at,
1032 source_kind,
1033 worker_attempt_id,
1034 worker_epoch,
1035 event_kind,
1036 schema_id,
1037 schema_version,
1038 schema_digest,
1039 payload_bytes,
1040 payload_digest,
1041 intent_digest,
1042 projection_digest,
1043 previous_digest,
1044 event_digest
1045FROM stateknot.run_events
1046WHERE tenant_id = $1 AND run_id = $2 AND sequence = $3
1047";
1048
1049const SELECT_EVENT_PAGE: &str = r"
1050SELECT
1051 tenant_id,
1052 run_id,
1053 sequence,
1054 event_id,
1055 recorded_at,
1056 source_kind,
1057 worker_attempt_id,
1058 worker_epoch,
1059 event_kind,
1060 schema_id,
1061 schema_version,
1062 schema_digest,
1063 payload_bytes,
1064 payload_digest,
1065 intent_digest,
1066 projection_digest,
1067 previous_digest,
1068 event_digest
1069FROM stateknot.run_events
1070WHERE tenant_id = $1 AND run_id = $2 AND sequence > $3
1071ORDER BY sequence ASC
1072LIMIT $4
1073";
1074
1075const WAIT_REGISTRATION_COLUMNS: &str = r"
1076 tenant_id,
1077 run_id,
1078 wait_id,
1079 wait_kind,
1080 interrupt_kind,
1081 timer_kind,
1082 registered_at,
1083 due_at,
1084 expires_at,
1085 action_digest,
1086 registration_sequence,
1087 registration_event_id,
1088 registration_event_digest,
1089 intent_digest,
1090 record_digest,
1091 record_bytes,
1092 status,
1093 terminal_sequence,
1094 terminal_event_id,
1095 terminal_recorded_at,
1096 terminal_event_digest,
1097 resolution_digest,
1098 firing_digest,
1099 abandonment_digest,
1100 created_at,
1101 updated_at
1102";
1103
1104static SELECT_WAIT_REGISTRATIONS_BY_ORIGIN: LazyLock<String> = LazyLock::new(|| {
1105 format!(
1106 "SELECT {WAIT_REGISTRATION_COLUMNS} \
1107 FROM stateknot.run_wait_registrations \
1108 WHERE tenant_id = $1 AND run_id = $2 AND registration_sequence = $3 \
1109 ORDER BY wait_id"
1110 )
1111});
1112
1113static SELECT_WAIT_REGISTRATION_BY_ID: LazyLock<String> = LazyLock::new(|| {
1114 format!(
1115 "SELECT {WAIT_REGISTRATION_COLUMNS} \
1116 FROM stateknot.run_wait_registrations \
1117 WHERE tenant_id = $1 AND run_id = $2 AND wait_id = $3"
1118 )
1119});
1120
1121static SELECT_WAIT_REGISTRATION_BY_ID_FOR_UPDATE: LazyLock<String> = LazyLock::new(|| {
1122 format!(
1123 "SELECT {WAIT_REGISTRATION_COLUMNS} \
1124 FROM stateknot.run_wait_registrations \
1125 WHERE tenant_id = $1 AND run_id = $2 AND wait_id = $3 \
1126 FOR UPDATE"
1127 )
1128});
1129
1130static SELECT_OUTSTANDING_WAIT_REGISTRATIONS: LazyLock<String> = LazyLock::new(|| {
1131 format!(
1132 "SELECT {WAIT_REGISTRATION_COLUMNS} \
1133 FROM stateknot.run_wait_registrations \
1134 WHERE tenant_id = $1 AND run_id = $2 AND status = 'outstanding' \
1135 ORDER BY wait_id"
1136 )
1137});
1138
1139static SELECT_OUTSTANDING_WAIT_REGISTRATIONS_FOR_UPDATE: LazyLock<String> = LazyLock::new(|| {
1140 format!(
1141 "SELECT {WAIT_REGISTRATION_COLUMNS} \
1142 FROM stateknot.run_wait_registrations \
1143 WHERE tenant_id = $1 AND run_id = $2 AND status = 'outstanding' \
1144 ORDER BY wait_id FOR UPDATE"
1145 )
1146});
1147
1148const SELECT_INTERRUPT_RESOLUTION: &str = r"
1149SELECT
1150 tenant_id,
1151 run_id,
1152 interrupt_id,
1153 request_digest,
1154 resolution_sequence,
1155 resolution_event_id,
1156 resolved_at,
1157 resolution_event_digest,
1158 intent_digest,
1159 resolution_digest,
1160 resolution_bytes,
1161 created_at
1162FROM stateknot.interrupt_resolutions
1163WHERE tenant_id = $1 AND run_id = $2 AND interrupt_id = $3
1164";
1165
1166const SELECT_TIMER_FIRING: &str = r"
1167SELECT
1168 tenant_id,
1169 run_id,
1170 timer_id,
1171 timer_digest,
1172 firing_sequence,
1173 firing_event_id,
1174 fired_at,
1175 firing_event_digest,
1176 intent_digest,
1177 firing_digest,
1178 firing_bytes,
1179 created_at
1180FROM stateknot.timer_firings
1181WHERE tenant_id = $1 AND run_id = $2 AND timer_id = $3
1182";
1183
1184static SELECT_DUE_TIMER_PAGE: LazyLock<String> = LazyLock::new(|| {
1185 format!(
1186 "SELECT {WAIT_REGISTRATION_COLUMNS} \
1187 FROM stateknot.run_wait_registrations \
1188 WHERE tenant_id = $1 \
1189 AND wait_kind = 'timer' \
1190 AND status = 'outstanding' \
1191 AND due_at <= $2 \
1192 AND (due_at, run_id, wait_id) > ( \
1193 COALESCE($3::timestamptz, '-infinity'::timestamptz), \
1194 COALESCE($4::uuid, '00000000-0000-0000-0000-000000000000'::uuid), \
1195 COALESCE($5::uuid, '00000000-0000-0000-0000-000000000000'::uuid) \
1196 ) \
1197 ORDER BY due_at, run_id, wait_id \
1198 LIMIT $6"
1199 )
1200});
1201
1202static SELECT_EXPIRED_INTERRUPT_PAGE: LazyLock<String> = LazyLock::new(|| {
1203 format!(
1204 "SELECT {WAIT_REGISTRATION_COLUMNS} \
1205 FROM stateknot.run_wait_registrations \
1206 WHERE tenant_id = $1 \
1207 AND wait_kind = 'interrupt' \
1208 AND status = 'outstanding' \
1209 AND expires_at IS NOT NULL \
1210 AND expires_at <= $2 \
1211 AND (expires_at, run_id, wait_id) > ( \
1212 COALESCE($3::timestamptz, '-infinity'::timestamptz), \
1213 COALESCE($4::uuid, '00000000-0000-0000-0000-000000000000'::uuid), \
1214 COALESCE($5::uuid, '00000000-0000-0000-0000-000000000000'::uuid) \
1215 ) \
1216 ORDER BY expires_at, run_id, wait_id \
1217 LIMIT $6"
1218 )
1219});
1220
1221const WAIT_ABANDONMENT_COLUMNS: &str = r"
1222 tenant_id,
1223 run_id,
1224 wait_id,
1225 wait_kind,
1226 registration_digest,
1227 reason_kind,
1228 abandonment_sequence,
1229 abandonment_event_id,
1230 abandoned_at,
1231 abandonment_event_digest,
1232 abandonment_digest,
1233 created_at
1234";
1235
1236static SELECT_WAIT_ABANDONMENTS_BY_EVENT: LazyLock<String> = LazyLock::new(|| {
1237 format!(
1238 "SELECT {WAIT_ABANDONMENT_COLUMNS} \
1239 FROM stateknot.wait_abandonments \
1240 WHERE tenant_id = $1 AND run_id = $2 AND abandonment_sequence = $3 \
1241 ORDER BY wait_id"
1242 )
1243});
1244
1245static SELECT_WAIT_ABANDONMENT_BY_ID: LazyLock<String> = LazyLock::new(|| {
1246 format!(
1247 "SELECT {WAIT_ABANDONMENT_COLUMNS} \
1248 FROM stateknot.wait_abandonments \
1249 WHERE tenant_id = $1 AND run_id = $2 AND wait_id = $3"
1250 )
1251});
1252
1253const SELECT_CHECKPOINT_BY_ID: &str = r"
1254SELECT
1255 tenant_id,
1256 run_id,
1257 checkpoint_id,
1258 superstep,
1259 parent_checkpoint_id,
1260 parent_superstep,
1261 parent_digest,
1262 journal_sequence,
1263 journal_event_id,
1264 journal_recorded_at,
1265 journal_digest,
1266 graph_definition_digest,
1267 state_schema_id,
1268 state_schema_version,
1269 state_schema_digest,
1270 state_digest,
1271 intent_digest,
1272 checkpoint_digest,
1273 checkpoint_bytes
1274FROM stateknot.run_checkpoints
1275WHERE tenant_id = $1 AND run_id = $2 AND checkpoint_id = $3
1276";
1277
1278const SELECT_CHECKPOINT_BY_ANCHOR: &str = r"
1279SELECT
1280 tenant_id,
1281 run_id,
1282 checkpoint_id,
1283 superstep,
1284 parent_checkpoint_id,
1285 parent_superstep,
1286 parent_digest,
1287 journal_sequence,
1288 journal_event_id,
1289 journal_recorded_at,
1290 journal_digest,
1291 graph_definition_digest,
1292 state_schema_id,
1293 state_schema_version,
1294 state_schema_digest,
1295 state_digest,
1296 intent_digest,
1297 checkpoint_digest,
1298 checkpoint_bytes
1299FROM stateknot.run_checkpoints
1300WHERE tenant_id = $1 AND run_id = $2 AND journal_sequence = $3
1301";
1302
1303const SELECT_CHECKPOINT_LINEAGE: &str = r"
1304WITH RECURSIVE checkpoint_lineage AS (
1305 SELECT current_checkpoint.*, 0::bigint AS lineage_depth
1306 FROM stateknot.run_checkpoints AS current_checkpoint
1307 WHERE current_checkpoint.tenant_id = $1
1308 AND current_checkpoint.run_id = $2
1309 AND current_checkpoint.checkpoint_id = $3
1310
1311 UNION ALL
1312
1313 SELECT parent_checkpoint.*, child.lineage_depth + 1
1314 FROM stateknot.run_checkpoints AS parent_checkpoint
1315 JOIN checkpoint_lineage AS child
1316 ON parent_checkpoint.tenant_id = child.tenant_id
1317 AND parent_checkpoint.run_id = child.run_id
1318 AND parent_checkpoint.checkpoint_id = child.parent_checkpoint_id
1319 AND parent_checkpoint.superstep = child.parent_superstep
1320 AND parent_checkpoint.checkpoint_digest = child.parent_digest
1321 WHERE child.lineage_depth + 1 < $4
1322)
1323SELECT
1324 tenant_id,
1325 run_id,
1326 checkpoint_id,
1327 superstep,
1328 parent_checkpoint_id,
1329 parent_superstep,
1330 parent_digest,
1331 journal_sequence,
1332 journal_event_id,
1333 journal_recorded_at,
1334 journal_digest,
1335 graph_definition_digest,
1336 state_schema_id,
1337 state_schema_version,
1338 state_schema_digest,
1339 state_digest,
1340 intent_digest,
1341 checkpoint_digest,
1342 checkpoint_bytes
1343FROM checkpoint_lineage
1344ORDER BY lineage_depth ASC
1345";
1346
1347const SELECT_EVENTS_BY_SEQUENCES: &str = r"
1348SELECT
1349 tenant_id,
1350 run_id,
1351 sequence,
1352 event_id,
1353 recorded_at,
1354 source_kind,
1355 worker_attempt_id,
1356 worker_epoch,
1357 event_kind,
1358 schema_id,
1359 schema_version,
1360 schema_digest,
1361 payload_bytes,
1362 payload_digest,
1363 intent_digest,
1364 projection_digest,
1365 previous_digest,
1366 event_digest
1367FROM stateknot.run_events
1368WHERE tenant_id = $1 AND run_id = $2 AND sequence = ANY($3)
1369ORDER BY sequence DESC
1370";
1371
1372const SELECT_TOOL_INVOCATION: &str = r"
1373SELECT
1374 tenant_id,
1375 run_id,
1376 invocation_id,
1377 base_checkpoint_id,
1378 base_superstep,
1379 base_checkpoint_digest,
1380 graph_namespace,
1381 node_id,
1382 activation_input_digest,
1383 intent_digest,
1384 intent_bytes,
1385 current_revision,
1386 current_status,
1387 current_attempt_id,
1388 current_record_digest,
1389 created_at,
1390 updated_at
1391FROM stateknot.tool_invocations
1392WHERE tenant_id = $1 AND run_id = $2 AND invocation_id = $3
1393";
1394
1395const SELECT_TOOL_INVOCATION_FOR_UPDATE: &str = r"
1396SELECT
1397 tenant_id,
1398 run_id,
1399 invocation_id,
1400 base_checkpoint_id,
1401 base_superstep,
1402 base_checkpoint_digest,
1403 graph_namespace,
1404 node_id,
1405 activation_input_digest,
1406 intent_digest,
1407 intent_bytes,
1408 current_revision,
1409 current_status,
1410 current_attempt_id,
1411 current_record_digest,
1412 created_at,
1413 updated_at
1414FROM stateknot.tool_invocations
1415WHERE tenant_id = $1 AND run_id = $2 AND invocation_id = $3
1416FOR UPDATE
1417";
1418
1419const SELECT_TOOL_INVOCATION_REVISION: &str = r"
1420SELECT
1421 tenant_id,
1422 run_id,
1423 invocation_id,
1424 revision,
1425 previous_revision,
1426 previous_digest,
1427 journal_sequence,
1428 journal_event_id,
1429 journal_recorded_at,
1430 journal_digest,
1431 status,
1432 attempt_id,
1433 transition_kind,
1434 started_attempt_id,
1435 transition_digest,
1436 record_digest,
1437 record_bytes,
1438 created_at
1439FROM stateknot.tool_invocation_revisions
1440WHERE tenant_id = $1 AND run_id = $2 AND invocation_id = $3 AND revision = $4
1441";
1442
1443const SELECT_TOOL_INVOCATION_REVISION_BY_ANCHOR: &str = r"
1444SELECT
1445 tenant_id,
1446 run_id,
1447 invocation_id,
1448 revision,
1449 previous_revision,
1450 previous_digest,
1451 journal_sequence,
1452 journal_event_id,
1453 journal_recorded_at,
1454 journal_digest,
1455 status,
1456 attempt_id,
1457 transition_kind,
1458 started_attempt_id,
1459 transition_digest,
1460 record_digest,
1461 record_bytes,
1462 created_at
1463FROM stateknot.tool_invocation_revisions
1464WHERE tenant_id = $1 AND run_id = $2 AND journal_sequence = $3
1465";
1466
1467const SELECT_NODE_ATTEMPT_BY_ID: &str = r"
1468SELECT
1469 tenant_id,
1470 run_id,
1471 base_checkpoint_id,
1472 base_superstep,
1473 base_checkpoint_digest,
1474 base_journal_sequence,
1475 base_journal_event_id,
1476 base_journal_recorded_at,
1477 base_journal_digest,
1478 graph_namespace,
1479 node_id,
1480 activation_input_digest,
1481 activation_digest,
1482 attempt_id,
1483 fence_attempt_id,
1484 fence_epoch,
1485 journal_sequence,
1486 journal_event_id,
1487 journal_recorded_at,
1488 journal_digest,
1489 start_digest,
1490 start_bytes,
1491 created_at
1492FROM stateknot.node_attempts
1493WHERE tenant_id = $1 AND run_id = $2 AND attempt_id = $3
1494";
1495
1496const SELECT_NODE_ATTEMPT_COMPLETION: &str = r"
1497SELECT
1498 tenant_id,
1499 run_id,
1500 attempt_id,
1501 base_checkpoint_id,
1502 base_superstep,
1503 base_checkpoint_digest,
1504 graph_namespace,
1505 node_id,
1506 activation_input_digest,
1507 activation_digest,
1508 fence_attempt_id,
1509 fence_epoch,
1510 start_journal_sequence,
1511 start_journal_event_id,
1512 start_journal_recorded_at,
1513 start_journal_digest,
1514 start_digest,
1515 status,
1516 journal_sequence,
1517 journal_event_id,
1518 journal_recorded_at,
1519 journal_digest,
1520 result_intent_digest,
1521 result_record_digest,
1522 failure_id,
1523 retry_kind,
1524 retry_not_before,
1525 completion_digest,
1526 completion_bytes,
1527 created_at
1528FROM stateknot.node_attempt_completions
1529WHERE tenant_id = $1 AND run_id = $2 AND attempt_id = $3
1530";
1531
1532const SELECT_NODE_ATTEMPT_HISTORY: &str = r"
1533SELECT
1534 tenant_id,
1535 run_id,
1536 base_checkpoint_id,
1537 base_superstep,
1538 base_checkpoint_digest,
1539 base_journal_sequence,
1540 base_journal_event_id,
1541 base_journal_recorded_at,
1542 base_journal_digest,
1543 graph_namespace,
1544 node_id,
1545 activation_input_digest,
1546 activation_digest,
1547 attempt_id,
1548 fence_attempt_id,
1549 fence_epoch,
1550 journal_sequence,
1551 journal_event_id,
1552 journal_recorded_at,
1553 journal_digest,
1554 start_digest,
1555 start_bytes,
1556 created_at
1557FROM stateknot.node_attempts
1558WHERE tenant_id = $1
1559 AND run_id = $2
1560 AND base_checkpoint_id = $3
1561 AND base_superstep = $4
1562 AND base_checkpoint_digest = $5
1563 AND graph_namespace = $6
1564 AND node_id = $7
1565 AND activation_input_digest = $8
1566 AND journal_sequence > $9
1567ORDER BY journal_sequence ASC
1568LIMIT $10
1569";
1570
1571const SELECT_NODE_ATTEMPT_COUNT: &str = r"
1572SELECT count(*)
1573FROM stateknot.node_attempts
1574WHERE tenant_id = $1
1575 AND run_id = $2
1576 AND base_checkpoint_id = $3
1577 AND base_superstep = $4
1578 AND base_checkpoint_digest = $5
1579 AND graph_namespace = $6
1580 AND node_id = $7
1581 AND activation_input_digest = $8
1582";
1583
1584const SELECT_LATEST_NODE_ATTEMPT: &str = r"
1585SELECT
1586 tenant_id,
1587 run_id,
1588 base_checkpoint_id,
1589 base_superstep,
1590 base_checkpoint_digest,
1591 base_journal_sequence,
1592 base_journal_event_id,
1593 base_journal_recorded_at,
1594 base_journal_digest,
1595 graph_namespace,
1596 node_id,
1597 activation_input_digest,
1598 activation_digest,
1599 attempt_id,
1600 fence_attempt_id,
1601 fence_epoch,
1602 journal_sequence,
1603 journal_event_id,
1604 journal_recorded_at,
1605 journal_digest,
1606 start_digest,
1607 start_bytes,
1608 created_at
1609FROM stateknot.node_attempts
1610WHERE tenant_id = $1
1611 AND run_id = $2
1612 AND base_checkpoint_id = $3
1613 AND base_superstep = $4
1614 AND base_checkpoint_digest = $5
1615 AND graph_namespace = $6
1616 AND node_id = $7
1617 AND activation_input_digest = $8
1618ORDER BY journal_sequence DESC
1619LIMIT 1
1620";
1621
1622const SELECT_TOOL_INVOCATION_HISTORY: &str = r"
1623SELECT
1624 tenant_id,
1625 run_id,
1626 invocation_id,
1627 revision,
1628 previous_revision,
1629 previous_digest,
1630 journal_sequence,
1631 journal_event_id,
1632 journal_recorded_at,
1633 journal_digest,
1634 status,
1635 attempt_id,
1636 transition_kind,
1637 started_attempt_id,
1638 transition_digest,
1639 record_digest,
1640 record_bytes,
1641 created_at
1642FROM stateknot.tool_invocation_revisions
1643WHERE tenant_id = $1
1644 AND run_id = $2
1645 AND invocation_id = $3
1646 AND revision > $4
1647ORDER BY revision ASC
1648LIMIT $5
1649";
1650
1651const SELECT_UNSETTLED_TOOL_INVOCATION_EXISTS: &str = r"
1652SELECT EXISTS (
1653 SELECT 1
1654 FROM stateknot.tool_invocations
1655 WHERE tenant_id = $1
1656 AND run_id = $2
1657 AND base_checkpoint_id = $3
1658 AND base_superstep = $4
1659 AND base_checkpoint_digest = $5
1660 AND current_status NOT IN ('committed', 'failed')
1661)
1662";
1663
1664const SELECT_MODEL_INVOCATION: &str = r"
1665SELECT
1666 tenant_id,
1667 run_id,
1668 invocation_id,
1669 base_checkpoint_id,
1670 base_superstep,
1671 base_checkpoint_digest,
1672 graph_namespace,
1673 node_id,
1674 activation_input_digest,
1675 intent_digest,
1676 intent_bytes,
1677 current_revision,
1678 current_status,
1679 current_attempt_id,
1680 current_record_digest,
1681 created_at,
1682 updated_at
1683FROM stateknot.model_invocations
1684WHERE tenant_id = $1 AND run_id = $2 AND invocation_id = $3
1685";
1686
1687const SELECT_MODEL_INVOCATION_FOR_UPDATE: &str = r"
1688SELECT
1689 tenant_id,
1690 run_id,
1691 invocation_id,
1692 base_checkpoint_id,
1693 base_superstep,
1694 base_checkpoint_digest,
1695 graph_namespace,
1696 node_id,
1697 activation_input_digest,
1698 intent_digest,
1699 intent_bytes,
1700 current_revision,
1701 current_status,
1702 current_attempt_id,
1703 current_record_digest,
1704 created_at,
1705 updated_at
1706FROM stateknot.model_invocations
1707WHERE tenant_id = $1 AND run_id = $2 AND invocation_id = $3
1708FOR UPDATE
1709";
1710
1711const SELECT_MODEL_INVOCATION_REVISION: &str = r"
1712SELECT
1713 tenant_id,
1714 run_id,
1715 invocation_id,
1716 revision,
1717 previous_revision,
1718 previous_digest,
1719 journal_sequence,
1720 journal_event_id,
1721 journal_recorded_at,
1722 journal_digest,
1723 status,
1724 attempt_id,
1725 transition_kind,
1726 started_attempt_id,
1727 transition_digest,
1728 record_digest,
1729 record_bytes,
1730 created_at
1731FROM stateknot.model_invocation_revisions
1732WHERE tenant_id = $1 AND run_id = $2 AND invocation_id = $3 AND revision = $4
1733";
1734
1735const SELECT_MODEL_INVOCATION_REVISION_BY_ANCHOR: &str = r"
1736SELECT
1737 tenant_id,
1738 run_id,
1739 invocation_id,
1740 revision,
1741 previous_revision,
1742 previous_digest,
1743 journal_sequence,
1744 journal_event_id,
1745 journal_recorded_at,
1746 journal_digest,
1747 status,
1748 attempt_id,
1749 transition_kind,
1750 started_attempt_id,
1751 transition_digest,
1752 record_digest,
1753 record_bytes,
1754 created_at
1755FROM stateknot.model_invocation_revisions
1756WHERE tenant_id = $1 AND run_id = $2 AND journal_sequence = $3
1757";
1758
1759const SELECT_MODEL_INVOCATION_HISTORY: &str = r"
1760SELECT
1761 tenant_id,
1762 run_id,
1763 invocation_id,
1764 revision,
1765 previous_revision,
1766 previous_digest,
1767 journal_sequence,
1768 journal_event_id,
1769 journal_recorded_at,
1770 journal_digest,
1771 status,
1772 attempt_id,
1773 transition_kind,
1774 started_attempt_id,
1775 transition_digest,
1776 record_digest,
1777 record_bytes,
1778 created_at
1779FROM stateknot.model_invocation_revisions
1780WHERE tenant_id = $1
1781 AND run_id = $2
1782 AND invocation_id = $3
1783 AND revision > $4
1784ORDER BY revision ASC
1785LIMIT $5
1786";
1787
1788const SELECT_UNSETTLED_MODEL_INVOCATION_EXISTS: &str = r"
1789SELECT EXISTS (
1790 SELECT 1
1791 FROM stateknot.model_invocations
1792 WHERE tenant_id = $1
1793 AND run_id = $2
1794 AND base_checkpoint_id = $3
1795 AND base_superstep = $4
1796 AND base_checkpoint_digest = $5
1797 AND current_status NOT IN ('committed', 'failed')
1798)
1799";
1800
1801const SELECT_PENDING_NODE_RESULT: &str = r"
1802SELECT
1803 tenant_id,
1804 run_id,
1805 base_checkpoint_id,
1806 base_superstep,
1807 base_checkpoint_digest,
1808 base_journal_sequence,
1809 base_journal_event_id,
1810 base_journal_recorded_at,
1811 base_journal_digest,
1812 graph_namespace,
1813 node_id,
1814 activation_input_digest,
1815 node_attempt_id,
1816 intent_digest,
1817 control_kind,
1818 fence_attempt_id,
1819 fence_epoch,
1820 journal_sequence,
1821 journal_event_id,
1822 journal_recorded_at,
1823 journal_digest,
1824 record_digest,
1825 result_bytes,
1826 created_at
1827FROM stateknot.pending_node_results
1828WHERE tenant_id = $1
1829 AND run_id = $2
1830 AND base_checkpoint_id = $3
1831 AND graph_namespace = $4
1832 AND node_id = $5
1833";
1834
1835const SELECT_UNCONSUMED_PENDING_NODE_RESULT_HEADS: &str = r"
1836SELECT
1837 pending.tenant_id,
1838 pending.run_id,
1839 pending.base_checkpoint_id,
1840 pending.base_superstep,
1841 pending.base_checkpoint_digest,
1842 pending.graph_namespace,
1843 pending.node_id,
1844 pending.activation_input_digest,
1845 pending.intent_digest,
1846 pending.fence_attempt_id,
1847 pending.fence_epoch,
1848 pending.journal_sequence,
1849 pending.journal_event_id,
1850 pending.journal_recorded_at,
1851 pending.journal_digest,
1852 pending.record_digest
1853FROM stateknot.pending_node_results AS pending
1854WHERE pending.tenant_id = $1
1855 AND pending.run_id = $2
1856 AND pending.base_checkpoint_id = $3
1857 AND pending.base_superstep = $4
1858 AND pending.base_checkpoint_digest = $5
1859 AND NOT EXISTS (
1860 SELECT 1
1861 FROM stateknot.pending_node_result_consumptions AS consumed
1862 WHERE consumed.tenant_id = pending.tenant_id
1863 AND consumed.run_id = pending.run_id
1864 AND consumed.base_checkpoint_id = pending.base_checkpoint_id
1865 AND consumed.graph_namespace = pending.graph_namespace
1866 AND consumed.node_id = pending.node_id
1867 )
1868ORDER BY pending.graph_namespace ASC, pending.node_id ASC
1869LIMIT $6
1870";
1871
1872const SELECT_UNCONSUMED_PENDING_NODE_RESULT_HEADS_AFTER: &str = r"
1873SELECT
1874 pending.tenant_id,
1875 pending.run_id,
1876 pending.base_checkpoint_id,
1877 pending.base_superstep,
1878 pending.base_checkpoint_digest,
1879 pending.graph_namespace,
1880 pending.node_id,
1881 pending.activation_input_digest,
1882 pending.intent_digest,
1883 pending.fence_attempt_id,
1884 pending.fence_epoch,
1885 pending.journal_sequence,
1886 pending.journal_event_id,
1887 pending.journal_recorded_at,
1888 pending.journal_digest,
1889 pending.record_digest
1890FROM stateknot.pending_node_results AS pending
1891WHERE pending.tenant_id = $1
1892 AND pending.run_id = $2
1893 AND pending.base_checkpoint_id = $3
1894 AND pending.base_superstep = $4
1895 AND pending.base_checkpoint_digest = $5
1896 AND (pending.graph_namespace, pending.node_id) > ($6, $7)
1897 AND NOT EXISTS (
1898 SELECT 1
1899 FROM stateknot.pending_node_result_consumptions AS consumed
1900 WHERE consumed.tenant_id = pending.tenant_id
1901 AND consumed.run_id = pending.run_id
1902 AND consumed.base_checkpoint_id = pending.base_checkpoint_id
1903 AND consumed.graph_namespace = pending.graph_namespace
1904 AND consumed.node_id = pending.node_id
1905 )
1906ORDER BY pending.graph_namespace ASC, pending.node_id ASC
1907LIMIT $8
1908";
1909
1910const SELECT_RUN_QUARANTINE_TARGET_FOR_UPDATE: &str = r"
1911SELECT
1912 tenant_id,
1913 run_id,
1914 journal_sequence,
1915 journal_event_id,
1916 journal_recorded_at,
1917 journal_digest,
1918 lease_attempt_id,
1919 fencing_epoch,
1920 lease_renewed_at,
1921 lease_expires_at,
1922 quarantined_at,
1923 quarantine_reason
1924FROM stateknot.runs
1925WHERE tenant_id = $1 AND run_id = $2
1926FOR UPDATE
1927";
1928
1929const SELECT_RUN_QUARANTINE_BY_RUN: &str = r"
1930SELECT
1931 q.tenant_id,
1932 q.run_id,
1933 q.quarantine_id,
1934 q.quarantined_at,
1935 q.cause_kind,
1936 q.component,
1937 q.evidence_digest,
1938 q.expected_journal_sequence,
1939 q.expected_journal_event_id,
1940 q.expected_journal_recorded_at,
1941 q.expected_journal_digest,
1942 q.expected_fence_attempt_id,
1943 q.expected_fence_epoch,
1944 q.record_digest,
1945 q.created_at,
1946 r.quarantined_at AS run_quarantined_at,
1947 r.quarantine_reason AS run_quarantine_reason,
1948 r.lease_attempt_id AS run_lease_attempt_id,
1949 r.lease_acquired_at AS run_lease_acquired_at,
1950 r.lease_renewed_at AS run_lease_renewed_at,
1951 r.lease_expires_at AS run_lease_expires_at,
1952 r.fencing_epoch AS run_fencing_epoch,
1953 r.scheduler_ready_at AS run_scheduler_ready_at,
1954 r.updated_at AS run_updated_at
1955FROM stateknot.run_quarantines AS q
1956JOIN stateknot.runs AS r
1957 ON r.tenant_id = q.tenant_id AND r.run_id = q.run_id
1958WHERE q.tenant_id = $1 AND q.run_id = $2
1959";
1960
1961const SELECT_RUN_QUARANTINE_BY_ID: &str = r"
1962SELECT
1963 q.tenant_id,
1964 q.run_id,
1965 q.quarantine_id,
1966 q.quarantined_at,
1967 q.cause_kind,
1968 q.component,
1969 q.evidence_digest,
1970 q.expected_journal_sequence,
1971 q.expected_journal_event_id,
1972 q.expected_journal_recorded_at,
1973 q.expected_journal_digest,
1974 q.expected_fence_attempt_id,
1975 q.expected_fence_epoch,
1976 q.record_digest,
1977 q.created_at,
1978 r.quarantined_at AS run_quarantined_at,
1979 r.quarantine_reason AS run_quarantine_reason,
1980 r.lease_attempt_id AS run_lease_attempt_id,
1981 r.lease_acquired_at AS run_lease_acquired_at,
1982 r.lease_renewed_at AS run_lease_renewed_at,
1983 r.lease_expires_at AS run_lease_expires_at,
1984 r.fencing_epoch AS run_fencing_epoch,
1985 r.scheduler_ready_at AS run_scheduler_ready_at,
1986 r.updated_at AS run_updated_at
1987FROM stateknot.run_quarantines AS q
1988JOIN stateknot.runs AS r
1989 ON r.tenant_id = q.tenant_id AND r.run_id = q.run_id
1990WHERE q.tenant_id = $1 AND q.quarantine_id = $2
1991";
1992
1993const SELECT_PENDING_NODE_RESULT_HEAD: &str = r"
1994SELECT
1995 pending.tenant_id,
1996 pending.run_id,
1997 pending.base_checkpoint_id,
1998 pending.base_superstep,
1999 pending.base_checkpoint_digest,
2000 pending.graph_namespace,
2001 pending.node_id,
2002 pending.activation_input_digest,
2003 pending.intent_digest,
2004 pending.fence_attempt_id,
2005 pending.fence_epoch,
2006 pending.journal_sequence,
2007 pending.journal_event_id,
2008 pending.journal_recorded_at,
2009 pending.journal_digest,
2010 pending.record_digest
2011FROM stateknot.pending_node_results AS pending
2012WHERE pending.tenant_id = $1
2013 AND pending.run_id = $2
2014 AND pending.base_checkpoint_id = $3
2015 AND pending.graph_namespace = $4
2016 AND pending.node_id = $5
2017";
2018
2019const SELECT_PENDING_NODE_RESULT_HEADS_FOR_BARRIER: &str = r"
2020SELECT
2021 pending.tenant_id,
2022 pending.run_id,
2023 pending.base_checkpoint_id,
2024 pending.base_superstep,
2025 pending.base_checkpoint_digest,
2026 pending.graph_namespace,
2027 pending.node_id,
2028 pending.activation_input_digest,
2029 pending.intent_digest,
2030 pending.fence_attempt_id,
2031 pending.fence_epoch,
2032 pending.journal_sequence,
2033 pending.journal_event_id,
2034 pending.journal_recorded_at,
2035 pending.journal_digest,
2036 pending.record_digest
2037FROM stateknot.pending_node_results AS pending
2038WHERE pending.tenant_id = $1
2039 AND pending.run_id = $2
2040 AND pending.base_checkpoint_id = $3
2041ORDER BY pending.graph_namespace ASC, pending.node_id ASC
2042LIMIT $4
2043";
2044
2045const SELECT_PENDING_NODE_RESULT_CONSUMPTIONS_BY_BASE: &str = r"
2046SELECT
2047 tenant_id,
2048 run_id,
2049 base_checkpoint_id,
2050 base_superstep,
2051 base_checkpoint_digest,
2052 graph_namespace,
2053 node_id,
2054 result_record_digest,
2055 successor_checkpoint_id,
2056 successor_superstep,
2057 successor_checkpoint_digest,
2058 successor_journal_sequence,
2059 successor_journal_event_id,
2060 successor_journal_recorded_at,
2061 successor_journal_digest,
2062 created_at
2063FROM stateknot.pending_node_result_consumptions
2064WHERE tenant_id = $1
2065 AND run_id = $2
2066 AND base_checkpoint_id = $3
2067ORDER BY graph_namespace ASC, node_id ASC
2068";
2069
2070const SELECT_PENDING_NODE_RESULT_TOOL_BINDINGS: &str = r"
2071SELECT
2072 tenant_id,
2073 run_id,
2074 base_checkpoint_id,
2075 base_superstep,
2076 base_checkpoint_digest,
2077 graph_namespace,
2078 node_id,
2079 activation_input_digest,
2080 result_record_digest,
2081 result_journal_sequence,
2082 result_journal_recorded_at,
2083 result_journal_digest,
2084 invocation_id,
2085 invocation_revision,
2086 invocation_record_digest,
2087 invocation_status,
2088 invocation_journal_sequence,
2089 invocation_journal_recorded_at,
2090 invocation_journal_digest
2091FROM stateknot.pending_node_result_tool_bindings
2092WHERE tenant_id = $1
2093 AND run_id = $2
2094 AND base_checkpoint_id = $3
2095 AND graph_namespace = $4
2096 AND node_id = $5
2097ORDER BY invocation_id ASC
2098";
2099
2100const SELECT_PENDING_NODE_RESULT_MODEL_BINDINGS: &str = r"
2101SELECT
2102 tenant_id,
2103 run_id,
2104 base_checkpoint_id,
2105 base_superstep,
2106 base_checkpoint_digest,
2107 graph_namespace,
2108 node_id,
2109 activation_input_digest,
2110 result_record_digest,
2111 result_journal_sequence,
2112 result_journal_recorded_at,
2113 result_journal_digest,
2114 invocation_id,
2115 invocation_revision,
2116 invocation_record_digest,
2117 invocation_status,
2118 invocation_journal_sequence,
2119 invocation_journal_recorded_at,
2120 invocation_journal_digest
2121FROM stateknot.pending_node_result_model_bindings
2122WHERE tenant_id = $1
2123 AND run_id = $2
2124 AND base_checkpoint_id = $3
2125 AND graph_namespace = $4
2126 AND node_id = $5
2127ORDER BY invocation_id ASC
2128";
2129
2130const SELECT_TOOL_INVOCATIONS_BY_IDS: &str = r"
2131SELECT
2132 tenant_id,
2133 run_id,
2134 invocation_id,
2135 base_checkpoint_id,
2136 base_superstep,
2137 base_checkpoint_digest,
2138 graph_namespace,
2139 node_id,
2140 activation_input_digest,
2141 intent_digest,
2142 intent_bytes,
2143 current_revision,
2144 current_status,
2145 current_attempt_id,
2146 current_record_digest,
2147 created_at,
2148 updated_at
2149FROM stateknot.tool_invocations
2150WHERE tenant_id = $1 AND run_id = $2 AND invocation_id = ANY($3)
2151ORDER BY invocation_id ASC
2152";
2153
2154const SELECT_TOOL_INVOCATION_REVISIONS_BY_HEADS: &str = r"
2155SELECT
2156 tenant_id,
2157 run_id,
2158 invocation_id,
2159 revision,
2160 previous_revision,
2161 previous_digest,
2162 journal_sequence,
2163 journal_event_id,
2164 journal_recorded_at,
2165 journal_digest,
2166 status,
2167 attempt_id,
2168 transition_kind,
2169 started_attempt_id,
2170 transition_digest,
2171 record_digest,
2172 record_bytes,
2173 created_at
2174FROM stateknot.tool_invocation_revisions
2175WHERE tenant_id = $1
2176 AND run_id = $2
2177 AND (invocation_id, revision) IN (
2178 SELECT * FROM UNNEST($3::uuid[], $4::bigint[])
2179 )
2180ORDER BY invocation_id ASC, revision ASC
2181";
2182
2183const SELECT_MODEL_INVOCATIONS_BY_IDS: &str = r"
2184SELECT
2185 tenant_id,
2186 run_id,
2187 invocation_id,
2188 base_checkpoint_id,
2189 base_superstep,
2190 base_checkpoint_digest,
2191 graph_namespace,
2192 node_id,
2193 activation_input_digest,
2194 intent_digest,
2195 intent_bytes,
2196 current_revision,
2197 current_status,
2198 current_attempt_id,
2199 current_record_digest,
2200 created_at,
2201 updated_at
2202FROM stateknot.model_invocations
2203WHERE tenant_id = $1 AND run_id = $2 AND invocation_id = ANY($3)
2204ORDER BY invocation_id ASC
2205";
2206
2207const SELECT_MODEL_INVOCATION_REVISIONS_BY_HEADS: &str = r"
2208SELECT
2209 tenant_id,
2210 run_id,
2211 invocation_id,
2212 revision,
2213 previous_revision,
2214 previous_digest,
2215 journal_sequence,
2216 journal_event_id,
2217 journal_recorded_at,
2218 journal_digest,
2219 status,
2220 attempt_id,
2221 transition_kind,
2222 started_attempt_id,
2223 transition_digest,
2224 record_digest,
2225 record_bytes,
2226 created_at
2227FROM stateknot.model_invocation_revisions
2228WHERE tenant_id = $1
2229 AND run_id = $2
2230 AND (invocation_id, revision) IN (
2231 SELECT * FROM UNNEST($3::uuid[], $4::bigint[])
2232 )
2233ORDER BY invocation_id ASC, revision ASC
2234";
2235
2236const SELECT_OUTBOX_DESTINATION: &str = r"
2237SELECT
2238 tenant_id,
2239 destination_id,
2240 snapshot_digest,
2241 config_kind,
2242 schema_id,
2243 schema_version,
2244 schema_digest,
2245 config_bytes,
2246 created_at
2247FROM stateknot.outbox_destinations
2248WHERE tenant_id = $1
2249 AND destination_id = $2
2250 AND snapshot_digest = $3
2251";
2252
2253const OUTBOX_DELIVERY_COLUMNS: &str = r"
2254 tenant_id,
2255 run_id,
2256 delivery_id,
2257 origin_sequence,
2258 origin_event_id,
2259 origin_recorded_at,
2260 origin_digest,
2261 destination_id,
2262 destination_snapshot_digest,
2263 intent_digest,
2264 expires_at,
2265 delivery_digest,
2266 delivery_bytes,
2267 status,
2268 attempt_count,
2269 current_attempt_id,
2270 current_epoch,
2271 current_attempt_started_at,
2272 current_attempt_expires_at,
2273 next_attempt_at,
2274 last_completion_digest,
2275 terminal_at,
2276 created_at,
2277 updated_at
2278";
2279
2280static SELECT_OUTBOX_DELIVERY: LazyLock<String> = LazyLock::new(|| {
2281 format!(
2282 "SELECT {OUTBOX_DELIVERY_COLUMNS} FROM stateknot.outbox_deliveries \
2283 WHERE tenant_id = $1 AND run_id = $2 AND delivery_id = $3"
2284 )
2285});
2286
2287static SELECT_OUTBOX_DELIVERY_FOR_UPDATE: LazyLock<String> = LazyLock::new(|| {
2288 format!(
2289 "SELECT {OUTBOX_DELIVERY_COLUMNS} FROM stateknot.outbox_deliveries \
2290 WHERE tenant_id = $1 AND run_id = $2 AND delivery_id = $3 FOR UPDATE"
2291 )
2292});
2293
2294static SELECT_OUTBOX_DELIVERIES_BY_ORIGIN: LazyLock<String> = LazyLock::new(|| {
2295 format!(
2296 "SELECT {OUTBOX_DELIVERY_COLUMNS} FROM stateknot.outbox_deliveries \
2297 WHERE tenant_id = $1 AND run_id = $2 AND origin_sequence = $3 \
2298 ORDER BY delivery_id ASC"
2299 )
2300});
2301
2302static SELECT_OUTBOX_CLAIM_CANDIDATE: LazyLock<String> = LazyLock::new(|| {
2303 format!(
2304 "SELECT {OUTBOX_DELIVERY_COLUMNS} FROM stateknot.outbox_deliveries \
2305 WHERE tenant_id = $1 \
2306 AND status IN ('pending', 'delivering', 'retry_scheduled') \
2307 AND attempt_count < 64 \
2308 AND next_attempt_at <= $2 \
2309 AND expires_at > $2 \
2310 ORDER BY next_attempt_at ASC, delivery_id ASC \
2311 FOR UPDATE SKIP LOCKED LIMIT 1"
2312 )
2313});
2314
2315const OUTBOX_ATTEMPT_COLUMNS: &str = r"
2316 tenant_id,
2317 run_id,
2318 delivery_id,
2319 delivery_expires_at,
2320 delivery_digest,
2321 epoch,
2322 attempt_id,
2323 started_at,
2324 expires_at,
2325 start_digest,
2326 start_bytes,
2327 created_at
2328";
2329
2330static SELECT_OUTBOX_ATTEMPT_BY_ID: LazyLock<String> = LazyLock::new(|| {
2331 format!(
2332 "SELECT {OUTBOX_ATTEMPT_COLUMNS} FROM stateknot.outbox_attempts \
2333 WHERE tenant_id = $1 AND attempt_id = $2"
2334 )
2335});
2336
2337static SELECT_OUTBOX_ATTEMPT_BY_FENCE: LazyLock<String> = LazyLock::new(|| {
2338 format!(
2339 "SELECT {OUTBOX_ATTEMPT_COLUMNS} FROM stateknot.outbox_attempts \
2340 WHERE tenant_id = $1 AND run_id = $2 AND delivery_id = $3 \
2341 AND epoch = $4 AND attempt_id = $5"
2342 )
2343});
2344
2345static SELECT_OUTBOX_ATTEMPT_HISTORY: LazyLock<String> = LazyLock::new(|| {
2346 format!(
2347 "SELECT {OUTBOX_ATTEMPT_COLUMNS} FROM stateknot.outbox_attempts \
2348 WHERE tenant_id = $1 AND run_id = $2 AND delivery_id = $3 AND epoch > $4 \
2349 ORDER BY epoch ASC LIMIT $5"
2350 )
2351});
2352
2353const SELECT_OUTBOX_ATTEMPT_COMPLETION: &str = r"
2354SELECT
2355 tenant_id,
2356 run_id,
2357 delivery_id,
2358 epoch,
2359 attempt_id,
2360 started_at,
2361 attempt_expires_at,
2362 start_digest,
2363 outcome_kind,
2364 retry_advice_kind,
2365 retry_delay_millis,
2366 completed_at,
2367 completion_digest,
2368 completion_bytes,
2369 created_at
2370FROM stateknot.outbox_attempt_completions
2371WHERE tenant_id = $1 AND run_id = $2 AND delivery_id = $3 AND epoch = $4
2372";
2373
2374const SELECT_OUTBOX_ATTEMPT_COMPLETION_HISTORY: &str = r"
2375SELECT
2376 tenant_id,
2377 run_id,
2378 delivery_id,
2379 epoch,
2380 attempt_id,
2381 started_at,
2382 attempt_expires_at,
2383 start_digest,
2384 outcome_kind,
2385 retry_advice_kind,
2386 retry_delay_millis,
2387 completed_at,
2388 completion_digest,
2389 completion_bytes,
2390 created_at
2391FROM stateknot.outbox_attempt_completions
2392WHERE tenant_id = $1 AND run_id = $2 AND delivery_id = $3
2393ORDER BY epoch ASC
2394LIMIT $4
2395";
2396
2397#[derive(Clone)]
2402pub struct PostgresStore {
2403 pool: PgPool,
2404 options: PostgresStoreOptions,
2405}
2406
2407pub struct ClaimedRunRecovery<'store> {
2422 store: &'store PostgresStore,
2423 fence: RunFence,
2424 context: CorruptionQuarantineContext,
2425 initial_run: StoredRun,
2426 initial_observed_at: Timestamp,
2427}
2428
2429struct ClaimedRunRecoveryObservation {
2430 run: StoredRun,
2431 observed_at: Timestamp,
2432}
2433
2434impl ClaimedRunRecovery<'_> {
2435 #[must_use]
2437 pub const fn fence(&self) -> &RunFence {
2438 &self.fence
2439 }
2440
2441 #[must_use]
2443 pub const fn quarantine_context(&self) -> &CorruptionQuarantineContext {
2444 &self.context
2445 }
2446
2447 #[must_use]
2449 pub const fn initial_run(&self) -> &StoredRun {
2450 &self.initial_run
2451 }
2452
2453 #[must_use]
2456 pub const fn initial_observed_at(&self) -> Timestamp {
2457 self.initial_observed_at
2458 }
2459
2460 pub async fn revalidate(&self) -> Result<StoredRun, StoreError> {
2472 self.store
2473 .with_corruption_quarantine(
2474 self.context.clone(),
2475 self.store
2476 .load_claimed_run_recovery_snapshot(&self.fence, &self.context),
2477 )
2478 .await
2479 .map(|observation| observation.run)
2480 }
2481
2482 pub async fn load_pinned_graph(&self) -> Result<StoredGraphDefinition, StoreError> {
2496 Box::pin(self.read(self.load_pinned_graph_inner())).await
2497 }
2498
2499 async fn load_pinned_graph_inner(&self) -> Result<StoredGraphDefinition, StoreError> {
2500 let before = self
2501 .store
2502 .load_claimed_run_recovery_snapshot(&self.fence, &self.context)
2503 .await?;
2504 let pointer = before
2505 .run
2506 .checkpoint()
2507 .ok_or(StoreError::ReadyNodeRecoveryCheckpointMissing)?;
2508 if self.initial_run.checkpoint() != Some(pointer) {
2509 return Err(StoreError::corrupt("pinned graph checkpoint projection"));
2510 }
2511 let checkpoint = self.load_recovery_checkpoint(pointer).await?;
2512 let definition = self.load_graph_for_checkpoint(&checkpoint).await?;
2513 let after = self
2514 .store
2515 .load_claimed_run_recovery_snapshot(&self.fence, &self.context)
2516 .await?;
2517 if after.run.checkpoint() != Some(pointer) {
2518 return Err(StoreError::corrupt("pinned graph checkpoint projection"));
2519 }
2520 Ok(definition)
2521 }
2522
2523 pub async fn validate_noninitial_replay<V, R>(
2552 &self,
2553 schemas: &V,
2554 reducer: &R,
2555 limits: GraphReplayLimits,
2556 ) -> Result<GraphReplayReport, StoreError>
2557 where
2558 V: GraphSchemaValidator + ?Sized,
2559 R: GraphReducer + ?Sized,
2560 {
2561 Box::pin(self.read(self.validate_noninitial_replay_inner(schemas, reducer, limits))).await
2562 }
2563
2564 #[allow(clippy::too_many_lines)]
2565 async fn validate_noninitial_replay_inner<V, R>(
2566 &self,
2567 schemas: &V,
2568 reducer: &R,
2569 limits: GraphReplayLimits,
2570 ) -> Result<GraphReplayReport, StoreError>
2571 where
2572 V: GraphSchemaValidator + ?Sized,
2573 R: GraphReducer + ?Sized,
2574 {
2575 let pointer = self
2576 .initial_run
2577 .checkpoint()
2578 .ok_or(StoreError::ReadyNodeRecoveryCheckpointMissing)?;
2579 let definition = self.load_pinned_graph_inner().await?;
2580 let graph = definition.graph();
2581 if reducer.reference() != graph.reducer() {
2582 return Err(StoreError::GraphReplayDependencyUnavailable);
2583 }
2584
2585 let page_size = CheckpointLineagePageSize::new(CheckpointLineagePageSize::MAX)?;
2586 let mut cursor = None;
2587 let mut child: Option<Checkpoint> = None;
2588 let mut checkpoints_validated = 0_u64;
2589 let mut barriers_replayed = 0_u64;
2590 let mut results_replayed = 0_u64;
2591 let mut maximum_barrier_result_bytes = 0_usize;
2592
2593 loop {
2594 let page = self
2595 .store
2596 .load_checkpoint_lineage_page(
2597 self.fence.tenant_id(),
2598 self.fence.run_id(),
2599 cursor.as_ref(),
2600 page_size,
2601 )
2602 .await?;
2603 if page.checkpoints().is_empty() {
2604 return Err(StoreError::corrupt("noninitial graph replay lineage"));
2605 }
2606
2607 for parent in page.checkpoints() {
2608 if checkpoints_validated == 0
2609 && (parent.checkpoint_id() != pointer.checkpoint_id()
2610 || parent.superstep() != pointer.superstep()
2611 || parent.digest() != pointer.digest())
2612 {
2613 return Err(StoreError::corrupt(
2614 "noninitial graph replay checkpoint projection",
2615 ));
2616 }
2617 if parent.graph() != &graph.reference() {
2618 return Err(StoreError::corrupt("noninitial graph replay graph binding"));
2619 }
2620 validate_graph_replay_checkpoint_state(graph, parent, schemas)?;
2621 checkpoints_validated = checkpoints_validated
2622 .checked_add(1)
2623 .ok_or_else(|| StoreError::corrupt("noninitial graph replay count"))?;
2624
2625 if let Some(committed_child) = &child {
2626 let inputs = self
2627 .store
2628 .load_historical_graph_barrier_results(parent, committed_child, limits)
2629 .await?;
2630 let result_count = u64::try_from(inputs.results.len())
2631 .map_err(|_| StoreError::corrupt("noninitial graph replay count"))?;
2632 results_replayed = results_replayed
2633 .checked_add(result_count)
2634 .ok_or_else(|| StoreError::corrupt("noninitial graph replay count"))?;
2635 maximum_barrier_result_bytes =
2636 maximum_barrier_result_bytes.max(inputs.compact_bytes);
2637
2638 let plan = graph
2639 .plan_barrier(
2640 parent,
2641 &inputs.results,
2642 committed_child.checkpoint_id(),
2643 schemas,
2644 reducer,
2645 )
2646 .map_err(|error| map_graph_replay_plan_error(&error))?;
2647 if !committed_child.matches_write(plan.barrier().successor()) {
2648 return Err(StoreError::corrupt(
2649 "noninitial graph replay successor mismatch",
2650 ));
2651 }
2652 barriers_replayed = barriers_replayed
2653 .checked_add(1)
2654 .ok_or_else(|| StoreError::corrupt("noninitial graph replay count"))?;
2655 }
2656 child = Some(parent.clone());
2657 }
2658
2659 let Some(next_cursor) = page.next_cursor() else {
2660 break;
2661 };
2662 cursor = Some(next_cursor);
2663 }
2664
2665 let root = child.ok_or_else(|| StoreError::corrupt("noninitial graph replay lineage"))?;
2666 if root.superstep() != Superstep::INITIAL
2667 || root.parent().is_some()
2668 || root.ready_nodes() != graph.entry_nodes()
2669 {
2670 return Err(StoreError::corrupt("noninitial graph replay root"));
2671 }
2672 if barriers_replayed != checkpoints_validated.saturating_sub(1) {
2673 return Err(StoreError::corrupt("noninitial graph replay coverage"));
2674 }
2675
2676 let after = self
2677 .store
2678 .load_claimed_run_recovery_snapshot(&self.fence, &self.context)
2679 .await?;
2680 if after.run.checkpoint() != Some(pointer) {
2681 return Err(StoreError::corrupt(
2682 "noninitial graph replay checkpoint projection",
2683 ));
2684 }
2685
2686 Ok(GraphReplayReport {
2687 checkpoints_validated,
2688 barriers_replayed,
2689 results_replayed,
2690 maximum_barrier_result_bytes,
2691 })
2692 }
2693
2694 async fn load_graph_for_checkpoint(
2695 &self,
2696 checkpoint: &Checkpoint,
2697 ) -> Result<StoredGraphDefinition, StoreError> {
2698 let definition = match self
2699 .store
2700 .load_graph_definition(self.fence.tenant_id(), checkpoint.graph())
2701 .await
2702 {
2703 Ok(definition) => definition,
2704 Err(StoreError::GraphDefinitionNotFound) => {
2705 return Err(StoreError::corrupt("pinned graph definition"));
2706 }
2707 Err(error) => return Err(error),
2708 };
2709 let graph = definition.graph();
2710 if graph.reference() != *checkpoint.graph()
2711 || checkpoint.state().schema() != graph.state_schema()
2712 || checkpoint.ready_nodes().len() > usize::from(graph.limits().maximum_parallelism())
2713 || checkpoint
2714 .ready_nodes()
2715 .iter()
2716 .any(|node_id| graph.node(node_id).is_none())
2717 || (checkpoint.superstep() == Superstep::INITIAL
2718 && checkpoint.ready_nodes() != graph.entry_nodes())
2719 {
2720 return Err(StoreError::corrupt("pinned graph checkpoint binding"));
2721 }
2722 Ok(definition)
2723 }
2724
2725 pub async fn plan_ready_nodes(&self) -> Result<ReadyNodeRecoveryPlan, StoreError> {
2750 Box::pin(self.read(self.plan_ready_nodes_inner())).await
2751 }
2752
2753 async fn load_ready_node_checkpoint(
2754 &self,
2755 pointer: &CheckpointPointer,
2756 ) -> Result<Checkpoint, StoreError> {
2757 let checkpoint = self.load_recovery_checkpoint(pointer).await?;
2758 if checkpoint.ready_nodes().is_empty() {
2759 return Err(StoreError::corrupt("ready node recovery empty ready set"));
2760 }
2761 Ok(checkpoint)
2762 }
2763
2764 async fn load_recovery_checkpoint(
2765 &self,
2766 pointer: &CheckpointPointer,
2767 ) -> Result<Checkpoint, StoreError> {
2768 let checkpoint = match self
2769 .store
2770 .load_checkpoint(
2771 self.fence.tenant_id(),
2772 self.fence.run_id(),
2773 pointer.checkpoint_id(),
2774 )
2775 .await
2776 {
2777 Ok(checkpoint) => checkpoint,
2778 Err(StoreError::CheckpointNotFound) => {
2779 return Err(StoreError::corrupt(
2780 "ready node recovery checkpoint pointer",
2781 ));
2782 }
2783 Err(error) => return Err(error),
2784 };
2785 if checkpoint.superstep() != pointer.superstep() || checkpoint.digest() != pointer.digest()
2786 {
2787 return Err(StoreError::corrupt(
2788 "ready node recovery checkpoint pointer",
2789 ));
2790 }
2791 Ok(checkpoint)
2792 }
2793
2794 async fn plan_ready_nodes_inner(&self) -> Result<ReadyNodeRecoveryPlan, StoreError> {
2795 let pointer = self
2796 .initial_run
2797 .checkpoint()
2798 .ok_or(StoreError::ReadyNodeRecoveryCheckpointMissing)?;
2799 let checkpoint = self.load_ready_node_checkpoint(pointer).await?;
2800 self.load_graph_for_checkpoint(&checkpoint).await?;
2801
2802 let base = checkpoint.head();
2803 let mut planner = ReadyNodeRecoveryPlanner::new(checkpoint, self.fence.clone())
2804 .map_err(|_| StoreError::corrupt("ready node recovery activation"))?;
2805 let result_page_size = PendingNodeResultPageSize::new(PendingNodeResultPageSize::MAX)?;
2806 let mut result_cursor = None;
2807 loop {
2808 let page = self
2809 .store
2810 .load_unconsumed_pending_node_result_page(
2811 &base,
2812 result_cursor.as_ref(),
2813 result_page_size,
2814 )
2815 .await?;
2816 if self.context.expectation().head() != Some(page.snapshot_journal_head()) {
2817 return Err(StoreError::StaleClaimedRunRecoveryObservation);
2818 }
2819 for result in page.records() {
2820 planner
2821 .observe_result(result)
2822 .map_err(|_| StoreError::corrupt("ready node recovery result set"))?;
2823 }
2824 if !page.has_more() {
2825 break;
2826 }
2827 result_cursor = Some(
2828 page.next_cursor()
2829 .ok_or_else(|| StoreError::corrupt("ready node recovery result cursor"))?,
2830 );
2831 }
2832
2833 let attempt_page_size = NodeAttemptHistoryPageSize::new(NodeAttemptHistoryPageSize::MAX)?;
2834 for activation in planner.activations() {
2835 let mut attempt_cursor = None;
2836 loop {
2837 let page = self
2838 .store
2839 .load_node_attempt_history_page(
2840 &activation,
2841 attempt_cursor.as_ref(),
2842 attempt_page_size,
2843 )
2844 .await?;
2845 for attempt in page.records() {
2846 planner
2847 .observe_attempt(attempt)
2848 .map_err(|_| StoreError::corrupt("ready node recovery attempt history"))?;
2849 }
2850 if !page.has_more() {
2851 break;
2852 }
2853 attempt_cursor =
2854 Some(page.next_cursor().ok_or_else(|| {
2855 StoreError::corrupt("ready node recovery attempt cursor")
2856 })?);
2857 }
2858 }
2859
2860 let observation = self
2861 .store
2862 .load_claimed_run_recovery_snapshot(&self.fence, &self.context)
2863 .await?;
2864 if observation.run.checkpoint() != Some(pointer) {
2865 return Err(StoreError::corrupt(
2866 "ready node recovery checkpoint projection",
2867 ));
2868 }
2869 let journal_head = self
2870 .context
2871 .expectation()
2872 .head()
2873 .cloned()
2874 .ok_or_else(|| StoreError::corrupt("ready node recovery journal observation"))?;
2875 planner
2876 .finish(journal_head, observation.observed_at)
2877 .map_err(|_| StoreError::corrupt("ready node recovery plan"))
2878 }
2879
2880 pub async fn load_checkpoint_lineage_page(
2887 &self,
2888 from: Option<&CheckpointHead>,
2889 page_size: CheckpointLineagePageSize,
2890 ) -> Result<CheckpointLineagePage, StoreError> {
2891 self.read(self.store.load_checkpoint_lineage_page(
2892 self.fence.tenant_id(),
2893 self.fence.run_id(),
2894 from,
2895 page_size,
2896 ))
2897 .await
2898 }
2899
2900 pub async fn load_journal_page(
2907 &self,
2908 after: Option<&JournalHead>,
2909 page_size: JournalPageSize,
2910 ) -> Result<JournalPage, StoreError> {
2911 self.read(self.store.load_journal_page(
2912 self.fence.tenant_id(),
2913 self.fence.run_id(),
2914 after,
2915 page_size,
2916 ))
2917 .await
2918 }
2919
2920 pub async fn load_tool_invocation_history_page(
2927 &self,
2928 invocation_id: InvocationId,
2929 after: Option<&ToolInvocation>,
2930 page_size: ToolInvocationHistoryPageSize,
2931 ) -> Result<ToolInvocationHistoryPage, StoreError> {
2932 Box::pin(self.read(self.store.load_tool_invocation_history_page(
2933 self.fence.tenant_id(),
2934 self.fence.run_id(),
2935 invocation_id,
2936 after,
2937 page_size,
2938 )))
2939 .await
2940 }
2941
2942 pub async fn load_model_invocation_history_page(
2949 &self,
2950 invocation_id: InvocationId,
2951 after: Option<&ModelInvocation>,
2952 page_size: ModelInvocationHistoryPageSize,
2953 ) -> Result<ModelInvocationHistoryPage, StoreError> {
2954 Box::pin(self.read(self.store.load_model_invocation_history_page(
2955 self.fence.tenant_id(),
2956 self.fence.run_id(),
2957 invocation_id,
2958 after,
2959 page_size,
2960 )))
2961 .await
2962 }
2963
2964 pub async fn load_node_attempt_history_page(
2972 &self,
2973 activation: &NodeActivation,
2974 cursor: Option<&NodeAttempt>,
2975 page_size: NodeAttemptHistoryPageSize,
2976 ) -> Result<NodeAttemptHistoryPage, StoreError> {
2977 self.validate_activation_scope(activation)?;
2978 Box::pin(
2979 self.read(
2980 self.store
2981 .load_node_attempt_history_page(activation, cursor, page_size),
2982 ),
2983 )
2984 .await
2985 }
2986
2987 pub async fn load_unconsumed_pending_node_result_page(
2995 &self,
2996 base: &CheckpointHead,
2997 cursor: Option<&PendingNodeResultPageCursor>,
2998 page_size: PendingNodeResultPageSize,
2999 ) -> Result<PendingNodeResultPage, StoreError> {
3000 self.validate_checkpoint_scope(base)?;
3001 Box::pin(
3002 self.read(
3003 self.store
3004 .load_unconsumed_pending_node_result_page(base, cursor, page_size),
3005 ),
3006 )
3007 .await
3008 }
3009
3010 async fn read<T, F>(&self, recovery_read: F) -> Result<T, StoreError>
3011 where
3012 F: Future<Output = Result<T, StoreError>>,
3013 {
3014 self.store
3015 .with_corruption_quarantine(self.context.clone(), recovery_read)
3016 .await
3017 }
3018
3019 fn validate_checkpoint_scope(&self, checkpoint: &CheckpointHead) -> Result<(), StoreError> {
3020 if checkpoint.tenant_id() != self.fence.tenant_id()
3021 || checkpoint.run_id() != self.fence.run_id()
3022 {
3023 return Err(StoreError::InvalidClaimedRunRecoveryContext);
3024 }
3025 Ok(())
3026 }
3027
3028 fn validate_activation_scope(&self, activation: &NodeActivation) -> Result<(), StoreError> {
3029 self.validate_checkpoint_scope(activation.base_checkpoint())
3030 }
3031}
3032
3033struct HistoricalGraphBarrierResults {
3034 results: Vec<PendingNodeResult>,
3035 compact_bytes: usize,
3036}
3037
3038#[derive(Default)]
3039struct CompactByteCounter {
3040 bytes: usize,
3041}
3042
3043impl Write for CompactByteCounter {
3044 fn write(&mut self, buffer: &[u8]) -> std::io::Result<usize> {
3045 self.bytes = self.bytes.saturating_add(buffer.len());
3046 Ok(buffer.len())
3047 }
3048
3049 fn flush(&mut self) -> std::io::Result<()> {
3050 Ok(())
3051 }
3052}
3053
3054fn map_graph_replay_plan_error(error: &GraphBarrierPlanError) -> StoreError {
3055 match error {
3056 GraphBarrierPlanError::ReducerReferenceMismatch
3057 | GraphBarrierPlanError::SchemaValidatorPanicked { .. }
3058 | GraphBarrierPlanError::ReducerPanicked
3059 | GraphBarrierPlanError::SchemaValidation {
3060 source: GraphSchemaValidationError::Unavailable,
3061 ..
3062 }
3063 | GraphBarrierPlanError::ReducerFailed {
3064 source: GraphReducerError::Unavailable | GraphReducerError::ResourceLimit,
3065 } => StoreError::GraphReplayDependencyUnavailable,
3066 _ => StoreError::corrupt("noninitial graph replay plan"),
3067 }
3068}
3069
3070fn validate_graph_replay_checkpoint_state<V>(
3071 graph: &CompiledGraph,
3072 checkpoint: &Checkpoint,
3073 schemas: &V,
3074) -> Result<(), StoreError>
3075where
3076 V: GraphSchemaValidator + ?Sized,
3077{
3078 match catch_unwind(AssertUnwindSafe(|| {
3079 schemas.validate(graph.state_schema(), checkpoint.state().data())
3080 })) {
3081 Err(_) | Ok(Err(GraphSchemaValidationError::Unavailable)) => {
3082 Err(StoreError::GraphReplayDependencyUnavailable)
3083 }
3084 Ok(Err(_)) => Err(StoreError::corrupt(
3085 "noninitial graph replay checkpoint state",
3086 )),
3087 Ok(Ok(())) => Ok(()),
3088 }
3089}
3090
3091fn map_graph_replay_consumption_error(error: StoreError) -> StoreError {
3092 match error {
3093 StoreError::Database { .. } | StoreError::CorruptData { .. } => error,
3094 _ => StoreError::corrupt("noninitial graph replay consumption"),
3095 }
3096}
3097
3098impl PostgresStore {
3099 #[must_use]
3101 pub const fn options(&self) -> &PostgresStoreOptions {
3102 &self.options
3103 }
3104
3105 pub async fn connect(
3116 database_url: &str,
3117 options: PostgresStoreOptions,
3118 ) -> Result<Self, StoreError> {
3119 let store = Self::connect_server(database_url, options).await?;
3120 if let Err(error) = store.verify_schema().await {
3121 store.close().await;
3122 return Err(error);
3123 }
3124 Ok(store)
3125 }
3126
3127 pub async fn connect_config(config: crate::PostgresStoreConfig) -> Result<Self, StoreError> {
3139 let (runtime_url, migration_url, options, auto_migrate) = config.into_parts();
3140 if auto_migrate {
3141 let migration_url = migration_url
3142 .as_deref()
3143 .ok_or(ConfigurationError::MigrationDatabaseUrlRequired)?;
3144 Self::migrate_database(migration_url, options.clone()).await?;
3145 }
3146 Self::connect(&runtime_url, options).await
3147 }
3148
3149 pub async fn connect_from_env() -> Result<Self, StoreError> {
3157 Self::connect_config(crate::PostgresStoreConfig::from_env()?).await
3158 }
3159
3160 pub async fn connect_development(database_url: &str) -> Result<Self, StoreError> {
3171 Self::connect_config(crate::PostgresStoreConfig::development(database_url)?).await
3172 }
3173
3174 pub async fn migrate_database(
3186 database_url: &str,
3187 options: PostgresStoreOptions,
3188 ) -> Result<(), StoreError> {
3189 let store = Self::connect_server(database_url, options).await?;
3190 let result = async {
3191 MIGRATOR
3192 .run(&store.pool)
3193 .await
3194 .map_err(|source| StoreError::Migration { source })?;
3195 store.verify_schema().await
3196 }
3197 .await;
3198 store.close().await;
3199 result
3200 }
3201
3202 pub async fn verify_schema(&self) -> Result<(), StoreError> {
3213 let applied = match query_as::<_, (i64, bool, Vec<u8>)>(
3214 "SELECT version, success, checksum \
3215 FROM public._sqlx_migrations ORDER BY version",
3216 )
3217 .fetch_all(&self.pool)
3218 .await
3219 {
3220 Ok(applied) => applied,
3221 Err(source) if has_database_error_code(&source, "42P01") => {
3222 return Err(StoreError::SchemaNotMigrated);
3223 }
3224 Err(source) => return Err(StoreError::database("schema version check", source)),
3225 };
3226 if applied.is_empty() {
3227 return Err(StoreError::SchemaNotMigrated);
3228 }
3229 if applied.len() != MIGRATOR.iter().len()
3230 || applied.iter().zip(MIGRATOR.iter()).any(
3231 |((version, success, checksum), migration)| {
3232 !success
3233 || *version != migration.version
3234 || checksum.as_slice() != migration.checksum.as_ref()
3235 },
3236 )
3237 {
3238 return Err(StoreError::IncompatibleSchema);
3239 }
3240
3241 let complete = query_scalar::<_, bool>(VERIFY_SCHEMA_OBJECTS)
3242 .fetch_one(&self.pool)
3243 .await
3244 .map_err(|source| StoreError::database("schema object check", source))?;
3245 if !complete {
3246 return Err(StoreError::IncompleteSchema);
3247 }
3248 child_runs::verify_schema(&self.pool).await?;
3249 child_cancellation::verify_schema(&self.pool).await?;
3250 child_joins::verify_schema(&self.pool).await?;
3251 agent_deadlines::verify_schema(&self.pool).await?;
3252 failure_closes::verify_schema(&self.pool).await?;
3253 Ok(())
3254 }
3255
3256 async fn connect_server(
3257 database_url: &str,
3258 options: PostgresStoreOptions,
3259 ) -> Result<Self, StoreError> {
3260 let connect_options = options.connect_options(database_url)?;
3261 let pool = options
3262 .pool_options()
3263 .connect_with(connect_options)
3264 .await
3265 .map_err(|source| StoreError::database("connect", source))?;
3266
3267 let version =
3268 match query_scalar::<_, i32>("SELECT current_setting('server_version_num')::integer")
3269 .fetch_one(&pool)
3270 .await
3271 {
3272 Ok(version) => version,
3273 Err(source) => {
3274 pool.close().await;
3275 return Err(StoreError::database("server version check", source));
3276 }
3277 };
3278 if !(MIN_POSTGRES_VERSION_NUMBER..=MAX_POSTGRES_VERSION_NUMBER).contains(&version) {
3279 pool.close().await;
3280 return Err(StoreError::UnsupportedServerVersion);
3281 }
3282
3283 Ok(Self { pool, options })
3284 }
3285
3286 pub async fn health_check(&self) -> Result<(), StoreError> {
3292 query("SELECT 1")
3293 .execute(&self.pool)
3294 .await
3295 .map_err(|source| StoreError::database("health check", source))?;
3296 Ok(())
3297 }
3298
3299 pub async fn observe_database_clock(&self) -> Result<Timestamp, StoreError> {
3311 let observed_at = query_scalar::<_, DateTime<Utc>>("SELECT clock_timestamp()")
3312 .fetch_one(&self.pool)
3313 .await
3314 .map_err(|source| StoreError::database("database clock observation", source))?;
3315 from_database_time(observed_at)
3316 }
3317
3318 pub async fn close(&self) {
3320 self.pool.close().await;
3321 }
3322
3323 pub async fn register_artifact(
3338 &self,
3339 registration: ArtifactRegistration,
3340 ) -> Result<ArtifactRegistrationOutcome, StoreError> {
3341 let artifact = registration.artifact();
3342 let identity = artifact.identity();
3343 let tenant_id = identity.tenant_id();
3344 let artifact_bytes = encode_artifact_ref(artifact)?;
3345 let artifact_ref_digest = Digest::sha256(&artifact_bytes);
3346 let representation = artifact.representation();
3347 let content_byte_length = i64::try_from(representation.byte_length().get())
3348 .map_err(|_| StoreError::encoding("artifact content byte length"))?;
3349 let provenance = artifact.provenance();
3350 let locator = registration.locator();
3351
3352 let mut transaction = self.begin_mutation("artifact registration").await?;
3353 let registered_at = database_now(&mut transaction, "artifact registration clock").await?;
3354 let inserted = query(
3355 r"
3356INSERT INTO stateknot.artifacts (
3357 tenant_id,
3358 artifact_id,
3359 registration_key_digest,
3360 artifact_ref_digest,
3361 artifact_ref_bytes,
3362 content_byte_length,
3363 content_digest,
3364 provenance_run_id,
3365 provenance_event_id,
3366 storage_namespace,
3367 object_key,
3368 object_version,
3369 object_etag,
3370 registered_at
3371)
3372VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)
3373ON CONFLICT DO NOTHING
3374",
3375 )
3376 .bind(tenant_id.as_str())
3377 .bind(*identity.artifact_id().as_uuid())
3378 .bind(registration.registration_key().as_bytes())
3379 .bind(artifact_ref_digest.as_bytes())
3380 .bind(&artifact_bytes)
3381 .bind(content_byte_length)
3382 .bind(representation.digest().as_bytes())
3383 .bind(*provenance.run_id().as_uuid())
3384 .bind(*provenance.event_id().as_uuid())
3385 .bind(locator.storage_namespace())
3386 .bind(locator.object_key())
3387 .bind(locator.object_version())
3388 .bind(locator.object_etag())
3389 .bind(to_database_time(registered_at)?)
3390 .execute(&mut *transaction)
3391 .await
3392 .map_err(map_artifact_insert_error)?
3393 .rows_affected();
3394
3395 if inserted == 1 {
3396 for parent in artifact.parents().as_slice() {
3397 let result = query(
3398 "INSERT INTO stateknot.artifact_parents (\
3399 tenant_id, artifact_id, parent_artifact_id\
3400 ) VALUES ($1, $2, $3)",
3401 )
3402 .bind(tenant_id.as_str())
3403 .bind(*identity.artifact_id().as_uuid())
3404 .bind(*parent.as_uuid())
3405 .execute(&mut *transaction)
3406 .await;
3407 match result {
3408 Ok(_) => {}
3409 Err(source)
3410 if has_database_constraint(&source, "artifact_parents_parent_fk") =>
3411 {
3412 return Err(StoreError::ArtifactParentNotFound);
3413 }
3414 Err(source) => {
3415 return Err(StoreError::database("artifact parent insert", source));
3416 }
3417 }
3418 }
3419 }
3420
3421 let stored = load_artifact_record(&mut transaction, identity)
3422 .await?
3423 .ok_or(StoreError::ArtifactRegistrationConflict)?;
3424 if stored.registration() != ®istration {
3425 return Err(StoreError::ArtifactRegistrationConflict);
3426 }
3427 transaction
3428 .commit()
3429 .await
3430 .map_err(|source| StoreError::database("artifact registration commit", source))?;
3431 Ok(if inserted == 1 {
3432 ArtifactRegistrationOutcome::Registered(stored)
3433 } else {
3434 ArtifactRegistrationOutcome::Idempotent(stored)
3435 })
3436 }
3437
3438 pub async fn load_artifact(
3449 &self,
3450 identity: &ArtifactIdentity,
3451 ) -> Result<StoredArtifact, StoreError> {
3452 let mut transaction = self.begin_repeatable_read("artifact load").await?;
3453 let stored = load_artifact_record(&mut transaction, identity)
3454 .await?
3455 .ok_or(StoreError::ArtifactNotFound)?;
3456 transaction
3457 .commit()
3458 .await
3459 .map_err(|source| StoreError::database("artifact load commit", source))?;
3460 Ok(stored)
3461 }
3462
3463 pub async fn register_graph_definition(
3476 &self,
3477 tenant_id: TenantId,
3478 graph: CompiledGraph,
3479 ) -> Result<GraphDefinitionRegistrationOutcome, StoreError> {
3480 let definition_bytes = encode_graph_definition(&graph)?;
3481 let identity = graph.identity();
3482 let owner = identity.owner();
3483 let version = identity.version().to_string();
3484 let reference = graph.reference();
3485 let mut transaction = self.begin_mutation("graph definition registration").await?;
3486 let registered_at = database_now(&mut transaction, "graph definition clock").await?;
3487 let inserted = query(
3491 r"
3492INSERT INTO stateknot.graph_definitions (
3493 tenant_id,
3494 owner_issuer,
3495 owner_subject,
3496 graph_name,
3497 graph_version,
3498 definition_digest,
3499 definition_bytes,
3500 registered_at
3501)
3502VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
3503ON CONFLICT DO NOTHING
3504",
3505 )
3506 .bind(tenant_id.as_str())
3507 .bind(owner.issuer().as_str())
3508 .bind(owner.subject().as_str())
3509 .bind(identity.name().as_str())
3510 .bind(&version)
3511 .bind(graph.definition_digest().as_bytes())
3512 .bind(&definition_bytes)
3513 .bind(to_database_time(registered_at)?)
3514 .execute(&mut *transaction)
3515 .await
3516 .map_err(|source| StoreError::database("graph definition insert", source))?
3517 .rows_affected();
3518
3519 let row = load_graph_definition_row(&mut transaction, &tenant_id, &reference)
3520 .await?
3521 .ok_or(StoreError::GraphDefinitionConflict)?;
3522 let stored = decode_graph_definition(row)?;
3523 if stored.tenant_id() != &tenant_id || stored.graph() != &graph {
3524 return Err(StoreError::GraphDefinitionConflict);
3525 }
3526 transaction
3527 .commit()
3528 .await
3529 .map_err(|source| StoreError::database("graph definition commit", source))?;
3530 Ok(if inserted == 1 {
3531 GraphDefinitionRegistrationOutcome::Registered(stored)
3532 } else {
3533 GraphDefinitionRegistrationOutcome::Idempotent(stored)
3534 })
3535 }
3536
3537 pub async fn load_graph_definition(
3547 &self,
3548 tenant_id: &TenantId,
3549 reference: &GraphReference,
3550 ) -> Result<StoredGraphDefinition, StoreError> {
3551 let mut transaction = self.begin_repeatable_read("graph definition load").await?;
3552 let row = load_graph_definition_row(&mut transaction, tenant_id, reference)
3553 .await?
3554 .ok_or(StoreError::GraphDefinitionNotFound)?;
3555 let stored = decode_graph_definition(row)?;
3556 if stored.tenant_id() != tenant_id || stored.graph().reference() != *reference {
3557 return Err(StoreError::GraphDefinitionNotFound);
3558 }
3559 transaction
3560 .commit()
3561 .await
3562 .map_err(|source| StoreError::database("graph definition load commit", source))?;
3563 Ok(stored)
3564 }
3565
3566 pub async fn register_scheduler_fairness_policy(
3578 &self,
3579 registration: SchedulerFairnessPolicyRegistration,
3580 ) -> Result<SchedulerFairnessPolicyRegistrationOutcome, StoreError> {
3581 let mut transaction = self
3582 .begin_mutation("scheduler fairness policy registration")
3583 .await?;
3584 let registered_at =
3585 database_now(&mut transaction, "scheduler fairness policy clock").await?;
3586 let inserted = query(
3587 r"
3588INSERT INTO stateknot.scheduler_fairness_shards (
3589 shard_id,
3590 policy_digest,
3591 policy_bytes,
3592 cycle_length,
3593 registered_at,
3594 updated_at
3595)
3596VALUES ($1, $2, $3, $4, $5, $5)
3597ON CONFLICT (shard_id) DO NOTHING
3598",
3599 )
3600 .bind(registration.shard_id().as_str())
3601 .bind(registration.policy_digest().as_bytes())
3602 .bind(registration.policy_bytes())
3603 .bind(i32::from(registration.cycle_length()))
3604 .bind(to_database_time(registered_at)?)
3605 .execute(&mut *transaction)
3606 .await
3607 .map_err(|source| StoreError::database("scheduler fairness policy insert", source))?
3608 .rows_affected();
3609
3610 let row = query_as::<_, SchedulerFairnessShardRow>(SELECT_SCHEDULER_FAIRNESS_SHARD)
3611 .bind(registration.shard_id().as_str())
3612 .fetch_optional(&mut *transaction)
3613 .await
3614 .map_err(|source| StoreError::database("scheduler fairness policy lookup", source))?
3615 .ok_or(StoreError::SchedulerFairnessPolicyConflict)?;
3616 let stored = decode_scheduler_fairness_policy(row)?;
3617 if stored.registration() != ®istration {
3618 return Err(StoreError::SchedulerFairnessPolicyConflict);
3619 }
3620 transaction.commit().await.map_err(|source| {
3621 StoreError::database("scheduler fairness policy registration commit", source)
3622 })?;
3623 Ok(if inserted == 1 {
3624 SchedulerFairnessPolicyRegistrationOutcome::Registered(stored)
3625 } else {
3626 SchedulerFairnessPolicyRegistrationOutcome::Idempotent(stored)
3627 })
3628 }
3629
3630 pub async fn load_scheduler_fairness_policy(
3637 &self,
3638 shard_id: &SchedulerShardId,
3639 ) -> Result<StoredSchedulerFairnessPolicy, StoreError> {
3640 let mut transaction = self
3641 .begin_repeatable_read("scheduler fairness policy load")
3642 .await?;
3643 let row = query_as::<_, SchedulerFairnessShardRow>(SELECT_SCHEDULER_FAIRNESS_SHARD)
3644 .bind(shard_id.as_str())
3645 .fetch_optional(&mut *transaction)
3646 .await
3647 .map_err(|source| StoreError::database("scheduler fairness policy load", source))?
3648 .ok_or(StoreError::SchedulerFairnessPolicyNotFound)?;
3649 let stored = decode_scheduler_fairness_policy(row)?;
3650 if stored.registration().shard_id() != shard_id {
3651 return Err(StoreError::corrupt("scheduler fairness policy scope"));
3652 }
3653 transaction.commit().await.map_err(|source| {
3654 StoreError::database("scheduler fairness policy load commit", source)
3655 })?;
3656 Ok(stored)
3657 }
3658
3659 pub async fn reserve_scheduler_fairness_slot(
3671 &self,
3672 shard_id: &SchedulerShardId,
3673 policy_digest: Digest,
3674 reservation_id: SchedulerReservationId,
3675 ) -> Result<SchedulerFairnessReservation, StoreError> {
3676 let mut transaction = self
3677 .begin_mutation("scheduler fairness slot reservation")
3678 .await?;
3679
3680 if let Some(reservation) = load_valid_scheduler_fairness_reservation(
3681 &mut transaction,
3682 shard_id,
3683 policy_digest,
3684 reservation_id,
3685 )
3686 .await?
3687 {
3688 transaction.commit().await.map_err(|source| {
3689 StoreError::database("idempotent scheduler fairness reservation commit", source)
3690 })?;
3691 return Ok(reservation);
3692 }
3693
3694 let shard_row =
3695 query_as::<_, SchedulerFairnessShardRow>(SELECT_SCHEDULER_FAIRNESS_SHARD_FOR_UPDATE)
3696 .bind(shard_id.as_str())
3697 .fetch_optional(&mut *transaction)
3698 .await
3699 .map_err(|source| StoreError::database("scheduler fairness cursor lock", source))?
3700 .ok_or(StoreError::SchedulerFairnessPolicyNotFound)?;
3701
3702 if let Some(reservation) = load_valid_scheduler_fairness_reservation(
3705 &mut transaction,
3706 shard_id,
3707 policy_digest,
3708 reservation_id,
3709 )
3710 .await?
3711 {
3712 transaction.commit().await.map_err(|source| {
3713 StoreError::database("raced scheduler fairness reservation commit", source)
3714 })?;
3715 return Ok(reservation);
3716 }
3717
3718 let reservation = insert_scheduler_fairness_reservation(
3719 &mut transaction,
3720 shard_id,
3721 policy_digest,
3722 reservation_id,
3723 shard_row,
3724 )
3725 .await?;
3726 transaction.commit().await.map_err(|source| {
3727 StoreError::database("scheduler fairness reservation commit", source)
3728 })?;
3729 Ok(reservation)
3730 }
3731
3732 pub async fn prune_scheduler_fairness_reservations(
3749 &self,
3750 policy: SchedulerFairnessRetentionPolicy,
3751 ) -> Result<SchedulerFairnessRetentionReport, StoreError> {
3752 let retention_micros = i64::try_from(policy.retain_for().as_micros())
3753 .map_err(|_| StoreError::InvalidSchedulerFairnessRetention)?;
3754 let mut transaction = self
3755 .begin_mutation("scheduler fairness reservation retention")
3756 .await?;
3757 let (observed_at, cutoff) = query_as::<_, (DateTime<Utc>, DateTime<Utc>)>(
3758 r"
3759WITH observed AS MATERIALIZED (
3760 SELECT clock_timestamp() AS observed_at
3761)
3762SELECT
3763 observed_at,
3764 observed_at - ($1::bigint * interval '1 microsecond') AS cutoff
3765FROM observed
3766",
3767 )
3768 .bind(retention_micros)
3769 .fetch_one(&mut *transaction)
3770 .await
3771 .map_err(|source| StoreError::database("scheduler fairness retention clock", source))?;
3772 let deleted = query(
3773 r"
3774WITH candidates AS MATERIALIZED (
3775 SELECT reservation_id
3776 FROM stateknot.scheduler_fairness_reservations
3777 WHERE reserved_at < $1
3778 ORDER BY reserved_at, shard_id, sequence
3779 LIMIT $2
3780 FOR UPDATE SKIP LOCKED
3781)
3782DELETE FROM stateknot.scheduler_fairness_reservations AS reservation
3783USING candidates
3784WHERE reservation.reservation_id = candidates.reservation_id
3785",
3786 )
3787 .bind(cutoff)
3788 .bind(i64::from(policy.batch_size()))
3789 .execute(&mut *transaction)
3790 .await
3791 .map_err(|source| StoreError::database("scheduler fairness retention delete", source))?
3792 .rows_affected();
3793 let deleted = u16::try_from(deleted)
3794 .map_err(|_| StoreError::corrupt("scheduler fairness retention count"))?;
3795 let observed_at = from_database_time(observed_at)?;
3796 let cutoff = from_database_time(cutoff)?;
3797 if cutoff >= observed_at || deleted > policy.batch_size() {
3798 return Err(StoreError::corrupt(
3799 "scheduler fairness retention projection",
3800 ));
3801 }
3802 transaction.commit().await.map_err(|source| {
3803 StoreError::database("scheduler fairness retention commit", source)
3804 })?;
3805 Ok(SchedulerFairnessRetentionReport {
3806 observed_at,
3807 cutoff,
3808 deleted,
3809 })
3810 }
3811
3812 pub async fn admit_run(
3827 &self,
3828 provenance: AgentResultProvenance,
3829 ) -> Result<AdmissionOutcome, StoreError> {
3830 let tenant_id = provenance.tenant_id().clone();
3831 let run_id = provenance.run_id();
3832 let thread_id = provenance.thread_id();
3833 let invocation_id = provenance.invocation_id();
3834
3835 let mut transaction = self.begin_mutation("run admission").await?;
3836 let observed_at = database_now(&mut transaction, "run admission clock").await?;
3837 let lifecycle = RunLifecycle::admitted(provenance, observed_at);
3838 let lifecycle_bytes = encode_lifecycle(&lifecycle)?;
3839 let revision = lifecycle.revision().to_string();
3840 let status = run_status_text(lifecycle.status());
3841 let observed_db = to_database_time(observed_at)?;
3842
3843 let inserted = query(
3844 r"
3845INSERT INTO stateknot.runs (
3846 tenant_id,
3847 run_id,
3848 thread_id,
3849 invocation_id,
3850 lifecycle_bytes,
3851 lifecycle_revision,
3852 lifecycle_status,
3853 admitted_at,
3854 changed_at,
3855 scheduler_ready_at
3856)
3857VALUES ($1, $2, $3, $4, $5, $6::numeric, $7, $8, $8, $8)
3858ON CONFLICT (tenant_id, run_id) DO NOTHING
3859",
3860 )
3861 .bind(tenant_id.as_str())
3862 .bind(*run_id.as_uuid())
3863 .bind(*thread_id.as_uuid())
3864 .bind(*invocation_id.as_uuid())
3865 .bind(&lifecycle_bytes)
3866 .bind(&revision)
3867 .bind(status)
3868 .bind(observed_db)
3869 .execute(&mut *transaction)
3870 .await
3871 .map_err(|source| StoreError::database("run admission insert", source))?
3872 .rows_affected();
3873
3874 if inserted == 1 {
3875 transaction
3876 .commit()
3877 .await
3878 .map_err(|source| StoreError::database("run admission commit", source))?;
3879 return Ok(AdmissionOutcome::Committed(lifecycle));
3880 }
3881
3882 let row = fetch_locked_run_row(&mut transaction, &tenant_id, run_id).await?;
3883 let existing = decode_run(row)?;
3884 if existing.lifecycle().provenance() != lifecycle.provenance() {
3885 return Err(StoreError::RunConflict);
3886 }
3887 transaction
3888 .commit()
3889 .await
3890 .map_err(|source| StoreError::database("idempotent run admission commit", source))?;
3891 Ok(AdmissionOutcome::Idempotent(existing.lifecycle))
3892 }
3893
3894 pub async fn admit_agent_run<V>(
3919 &self,
3920 intent: AgentAdmissionIntent,
3921 append: JournalAppend,
3922 checkpoint_write: CheckpointWrite,
3923 schemas: &V,
3924 ) -> Result<AgentAdmissionCommitOutcome, StoreError>
3925 where
3926 V: GraphSchemaValidator + ?Sized,
3927 {
3928 validate_agent_admission_commit_input(&intent, &append, &checkpoint_write)?;
3929 let tenant_id = intent.provenance().tenant_id().clone();
3930 let run_id = intent.provenance().run_id();
3931
3932 let mut probe = self
3936 .begin_mutation("agent admission idempotency probe")
3937 .await?;
3938 if let Some(stored) = load_locked_agent_admission(&mut probe, &tenant_id, run_id).await? {
3939 validate_agent_admission_retry(&stored, &intent, &append, &checkpoint_write)?;
3940 probe.commit().await.map_err(|source| {
3941 StoreError::database("agent admission idempotency probe commit", source)
3942 })?;
3943 return Ok(AgentAdmissionCommitOutcome::Idempotent(stored));
3944 }
3945 probe.commit().await.map_err(|source| {
3946 StoreError::database("agent admission idempotency probe commit", source)
3947 })?;
3948
3949 let graph = self
3950 .load_graph_definition(&tenant_id, intent.graph())
3951 .await?;
3952 validate_agent_initial_checkpoint(graph.graph(), &checkpoint_write, schemas)?;
3953
3954 let mut transaction = self.begin_mutation("atomic agent admission").await?;
3955 if let Some(stored) =
3956 load_locked_agent_admission(&mut transaction, &tenant_id, run_id).await?
3957 {
3958 validate_agent_admission_retry(&stored, &intent, &append, &checkpoint_write)?;
3959 transaction.commit().await.map_err(|source| {
3960 StoreError::database("idempotent atomic agent admission commit", source)
3961 })?;
3962 return Ok(AgentAdmissionCommitOutcome::Idempotent(stored));
3963 }
3964
3965 let outcome = Box::pin(commit_new_agent_admission(
3966 &mut transaction,
3967 intent,
3968 append,
3969 checkpoint_write,
3970 ))
3971 .await?;
3972 transaction
3973 .commit()
3974 .await
3975 .map_err(|source| StoreError::database("atomic agent admission commit", source))?;
3976 Ok(match outcome {
3977 NewAgentAdmissionOutcome::Committed(stored) => {
3978 AgentAdmissionCommitOutcome::Committed(stored)
3979 }
3980 NewAgentAdmissionOutcome::Idempotent(stored) => {
3981 AgentAdmissionCommitOutcome::Idempotent(stored)
3982 }
3983 })
3984 }
3985
3986 pub async fn submit_agent_run<V>(
4008 &self,
4009 key: &AgentSubmissionKey,
4010 intent: AgentAdmissionIntent,
4011 append: JournalAppend,
4012 checkpoint_write: CheckpointWrite,
4013 schemas: &V,
4014 ) -> Result<AgentSubmissionCommitOutcome, StoreError>
4015 where
4016 V: GraphSchemaValidator + ?Sized,
4017 {
4018 validate_agent_admission_commit_input(&intent, &append, &checkpoint_write)?;
4019 let tenant_id = intent.provenance().tenant_id().clone();
4020 let run_id = intent.provenance().run_id();
4021 let key_digest = key.digest_for(&tenant_id);
4022 let submission_digest = agent_submission_digest(
4023 &intent,
4024 checkpoint_write.state(),
4025 checkpoint_write.ready_nodes(),
4026 )?;
4027
4028 let mut probe = self
4030 .begin_mutation("agent submission idempotency probe")
4031 .await?;
4032 if let Some(stored) =
4033 load_locked_agent_submission(&mut probe, &tenant_id, key_digest).await?
4034 {
4035 if stored.submission_digest() != submission_digest {
4036 return Err(StoreError::AgentSubmissionConflict);
4037 }
4038 probe.commit().await.map_err(|source| {
4039 StoreError::database("agent submission idempotency probe commit", source)
4040 })?;
4041 return Ok(AgentSubmissionCommitOutcome::Idempotent(stored));
4042 }
4043 probe.commit().await.map_err(|source| {
4044 StoreError::database("agent submission idempotency probe commit", source)
4045 })?;
4046
4047 let graph = self
4048 .load_graph_definition(&tenant_id, intent.graph())
4049 .await?;
4050 validate_agent_initial_checkpoint(graph.graph(), &checkpoint_write, schemas)?;
4051
4052 let mut transaction = self.begin_mutation("atomic agent submission").await?;
4053 if let Some(stored) =
4054 load_locked_agent_submission(&mut transaction, &tenant_id, key_digest).await?
4055 {
4056 if stored.submission_digest() != submission_digest {
4057 return Err(StoreError::AgentSubmissionConflict);
4058 }
4059 transaction.commit().await.map_err(|source| {
4060 StoreError::database("idempotent atomic agent submission commit", source)
4061 })?;
4062 return Ok(AgentSubmissionCommitOutcome::Idempotent(stored));
4063 }
4064
4065 let admission = if let Some(stored) =
4066 load_locked_agent_admission(&mut transaction, &tenant_id, run_id).await?
4067 {
4068 validate_agent_admission_retry(&stored, &intent, &append, &checkpoint_write)?;
4069 stored
4070 } else {
4071 match Box::pin(commit_new_agent_admission(
4072 &mut transaction,
4073 intent,
4074 append,
4075 checkpoint_write,
4076 ))
4077 .await?
4078 {
4079 NewAgentAdmissionOutcome::Committed(stored)
4080 | NewAgentAdmissionOutcome::Idempotent(stored) => stored,
4081 }
4082 };
4083 insert_agent_submission(&mut transaction, key_digest, submission_digest, &admission)
4084 .await?;
4085 let row = load_agent_submission_row(&mut transaction, &tenant_id, key_digest)
4086 .await?
4087 .ok_or_else(|| StoreError::corrupt("agent submission committed row"))?;
4088 let stored = verify_agent_submission(&row, key_digest, admission)?;
4089 transaction
4090 .commit()
4091 .await
4092 .map_err(|source| StoreError::database("atomic agent submission commit", source))?;
4093 Ok(AgentSubmissionCommitOutcome::Committed(stored))
4094 }
4095
4096 pub async fn load_agent_submission(
4107 &self,
4108 tenant_id: &TenantId,
4109 key: &AgentSubmissionKey,
4110 ) -> Result<StoredAgentSubmission, StoreError> {
4111 let key_digest = key.digest_for(tenant_id);
4112 let mut transaction = self.begin_repeatable_read("agent submission load").await?;
4113 let row = load_agent_submission_row(&mut transaction, tenant_id, key_digest)
4114 .await?
4115 .ok_or(StoreError::AgentSubmissionNotFound)?;
4116 let run_id = RunId::from_uuid(row.run_id)
4117 .map_err(|_| StoreError::corrupt("agent submission run identity"))?;
4118 let run_row = query_as::<_, RunRow>(SELECT_RUN)
4119 .bind(tenant_id.as_str())
4120 .bind(*run_id.as_uuid())
4121 .fetch_optional(&mut *transaction)
4122 .await
4123 .map_err(|source| StoreError::database("agent submission run load", source))?
4124 .ok_or_else(|| StoreError::corrupt("agent submission run reference"))?;
4125 let run = decode_run(run_row)?;
4126 verify_current_wait_set(&mut transaction, &run).await?;
4127 let admission_row = load_agent_admission_row(&mut transaction, tenant_id, run_id)
4128 .await?
4129 .ok_or_else(|| StoreError::corrupt("agent submission admission reference"))?;
4130 let admission = verify_stored_agent_admission(&mut transaction, run, admission_row).await?;
4131 let stored = verify_agent_submission(&row, key_digest, admission)?;
4132 transaction
4133 .commit()
4134 .await
4135 .map_err(|source| StoreError::database("agent submission load commit", source))?;
4136 Ok(stored)
4137 }
4138
4139 pub async fn load_agent_admission(
4151 &self,
4152 tenant_id: &TenantId,
4153 run_id: RunId,
4154 ) -> Result<StoredAgentAdmission, StoreError> {
4155 let mut transaction = self.begin_repeatable_read("agent admission load").await?;
4156 let run_row = query_as::<_, RunRow>(SELECT_RUN)
4157 .bind(tenant_id.as_str())
4158 .bind(*run_id.as_uuid())
4159 .fetch_optional(&mut *transaction)
4160 .await
4161 .map_err(|source| StoreError::database("agent admission run load", source))?
4162 .ok_or(StoreError::AgentAdmissionNotFound)?;
4163 let run = decode_run(run_row)?;
4164 verify_current_wait_set(&mut transaction, &run).await?;
4165 let admission_row = load_agent_admission_row(&mut transaction, tenant_id, run_id)
4166 .await?
4167 .ok_or(StoreError::AgentAdmissionNotFound)?;
4168 let stored = verify_stored_agent_admission(&mut transaction, run, admission_row).await?;
4169 transaction
4170 .commit()
4171 .await
4172 .map_err(|source| StoreError::database("agent admission load commit", source))?;
4173 Ok(stored)
4174 }
4175
4176 pub async fn load_run(
4182 &self,
4183 tenant_id: &TenantId,
4184 run_id: RunId,
4185 ) -> Result<StoredRun, StoreError> {
4186 let mut transaction = self.begin_repeatable_read("run load").await?;
4187 let row = query_as::<_, RunRow>(SELECT_RUN)
4188 .bind(tenant_id.as_str())
4189 .bind(*run_id.as_uuid())
4190 .fetch_optional(&mut *transaction)
4191 .await
4192 .map_err(|source| StoreError::database("run load", source))?
4193 .ok_or(StoreError::RunNotFound)?;
4194 let stored = decode_run(row)?;
4195 if stored.lifecycle().provenance().tenant_id() != tenant_id
4196 || stored.lifecycle().provenance().run_id() != run_id
4197 {
4198 return Err(StoreError::corrupt("run scope"));
4199 }
4200 verify_current_wait_set(&mut transaction, &stored).await?;
4201 transaction
4202 .commit()
4203 .await
4204 .map_err(|source| StoreError::database("run load commit", source))?;
4205 Ok(stored)
4206 }
4207
4208 pub async fn quarantine_run(
4227 &self,
4228 request: RunQuarantineRequest,
4229 ) -> Result<RunQuarantineCommitOutcome, StoreError> {
4230 let mut transaction = self.begin_mutation("run quarantine").await?;
4231
4232 if let Some(row) = load_run_quarantine_row_by_id(
4233 &mut transaction,
4234 request.tenant_id(),
4235 request.quarantine_id(),
4236 )
4237 .await?
4238 {
4239 let quarantine = decode_run_quarantine(&row)?;
4240 if quarantine.request() != &request {
4241 return Err(StoreError::RunQuarantineIdConflict);
4242 }
4243 transaction.commit().await.map_err(|source| {
4244 StoreError::database("idempotent run quarantine commit", source)
4245 })?;
4246 return Ok(RunQuarantineCommitOutcome::Idempotent(quarantine));
4247 }
4248
4249 let target = query_as::<_, RunQuarantineTargetRow>(SELECT_RUN_QUARANTINE_TARGET_FOR_UPDATE)
4250 .bind(request.tenant_id().as_str())
4251 .bind(*request.run_id().as_uuid())
4252 .fetch_optional(&mut *transaction)
4253 .await
4254 .map_err(|source| StoreError::database("run quarantine target lock", source))?
4255 .ok_or(StoreError::RunNotFound)?;
4256
4257 if target.quarantined_at.is_some() || target.quarantine_reason.is_some() {
4258 if let Some(row) = load_run_quarantine_row_by_id(
4259 &mut transaction,
4260 request.tenant_id(),
4261 request.quarantine_id(),
4262 )
4263 .await?
4264 {
4265 let quarantine = decode_run_quarantine(&row)?;
4266 if quarantine.request() == &request {
4267 transaction.commit().await.map_err(|source| {
4268 StoreError::database("concurrent run quarantine commit", source)
4269 })?;
4270 return Ok(RunQuarantineCommitOutcome::Idempotent(quarantine));
4271 }
4272 return Err(StoreError::RunQuarantineIdConflict);
4273 }
4274 return Err(StoreError::RunQuarantineConflict);
4275 }
4276 let observed_at = database_now(&mut transaction, "run quarantine clock").await?;
4277 authorize_quarantine_fence(&target, request.expected_fence(), observed_at)?;
4278 if !quarantine_expectation_matches(&target, request.expectation())? {
4279 return Err(StoreError::StaleRunQuarantineObservation);
4280 }
4281
4282 let quarantine = materialize_run_quarantine(request, observed_at)?;
4283 insert_run_quarantine(&mut transaction, &quarantine).await?;
4284 commit_run_quarantine_projection(&mut transaction, &quarantine).await?;
4285
4286 let restored_row = load_run_quarantine_row_by_run(
4287 &mut transaction,
4288 quarantine.request().tenant_id(),
4289 quarantine.request().run_id(),
4290 )
4291 .await?
4292 .ok_or(StoreError::RunQuarantineCommitConflict)?;
4293 let restored = decode_run_quarantine(&restored_row)?;
4294 if restored != quarantine {
4295 return Err(StoreError::RunQuarantineCommitConflict);
4296 }
4297
4298 transaction
4299 .commit()
4300 .await
4301 .map_err(|source| StoreError::database("run quarantine commit", source))?;
4302 Ok(RunQuarantineCommitOutcome::Committed(restored))
4303 }
4304
4305 pub async fn with_corruption_quarantine<T, F>(
4325 &self,
4326 context: CorruptionQuarantineContext,
4327 recovery_read: F,
4328 ) -> Result<T, StoreError>
4329 where
4330 F: Future<Output = Result<T, StoreError>>,
4331 {
4332 match recovery_read.await {
4333 Ok(value) => Ok(value),
4334 Err(error) => {
4335 let component = match RunQuarantineComponent::from_corrupt_store_error(&error) {
4336 Ok(component) => component,
4337 Err(StoreError::InvalidRunQuarantineRequest) => return Err(error),
4338 Err(component_error) => return Err(component_error),
4339 };
4340 let request = context.into_request(component)?;
4341 self.quarantine_run(request).await?;
4342 Err(StoreError::RunQuarantined)
4343 }
4344 }
4345 }
4346
4347 pub async fn begin_claimed_run_recovery(
4367 &self,
4368 fence: RunFence,
4369 context: CorruptionQuarantineContext,
4370 ) -> Result<ClaimedRunRecovery<'_>, StoreError> {
4371 if context.tenant_id() != fence.tenant_id() || context.run_id() != fence.run_id() {
4372 return Err(StoreError::InvalidClaimedRunRecoveryContext);
4373 }
4374 let context = context.with_expected_fence(fence.clone())?;
4375 let observation = self
4376 .with_corruption_quarantine(
4377 context.clone(),
4378 self.load_claimed_run_recovery_snapshot(&fence, &context),
4379 )
4380 .await?;
4381 Ok(ClaimedRunRecovery {
4382 store: self,
4383 fence,
4384 context,
4385 initial_run: observation.run,
4386 initial_observed_at: observation.observed_at,
4387 })
4388 }
4389
4390 async fn load_claimed_run_recovery_snapshot(
4391 &self,
4392 fence: &RunFence,
4393 context: &CorruptionQuarantineContext,
4394 ) -> Result<ClaimedRunRecoveryObservation, StoreError> {
4395 let mut transaction = self.begin_repeatable_read("claimed run recovery").await?;
4396 let row = query_as::<_, RunRow>(SELECT_RUN)
4397 .bind(fence.tenant_id().as_str())
4398 .bind(*fence.run_id().as_uuid())
4399 .fetch_optional(&mut *transaction)
4400 .await
4401 .map_err(|source| StoreError::database("claimed run recovery load", source))?
4402 .ok_or(StoreError::RunNotFound)?;
4403 let stored = decode_run(row)?;
4404 if stored.lifecycle().provenance().tenant_id() != fence.tenant_id()
4405 || stored.lifecycle().provenance().run_id() != fence.run_id()
4406 {
4407 return Err(StoreError::corrupt("claimed run recovery scope"));
4408 }
4409 verify_current_wait_set(&mut transaction, &stored).await?;
4410 validate_runnable(&stored)?;
4411 if !journal_expectation_matches_stored(&stored, context.expectation()) {
4412 return Err(StoreError::StaleClaimedRunRecoveryObservation);
4413 }
4414 let observed_at = database_now(&mut transaction, "claimed run recovery clock").await?;
4415 authorize_worker(&stored, fence, observed_at)?;
4416 transaction
4417 .commit()
4418 .await
4419 .map_err(|source| StoreError::database("claimed run recovery commit", source))?;
4420 Ok(ClaimedRunRecoveryObservation {
4421 run: stored,
4422 observed_at,
4423 })
4424 }
4425
4426 pub async fn load_run_quarantine(
4437 &self,
4438 tenant_id: &TenantId,
4439 run_id: RunId,
4440 ) -> Result<RunQuarantine, StoreError> {
4441 let mut transaction = self.begin_repeatable_read("run quarantine load").await?;
4442 let row = load_run_quarantine_row_by_run(&mut transaction, tenant_id, run_id).await?;
4443 let quarantine = if let Some(row) = row {
4444 decode_run_quarantine(&row)?
4445 } else {
4446 let exists = query_scalar::<_, bool>(
4447 "SELECT EXISTS (SELECT 1 FROM stateknot.runs WHERE tenant_id = $1 AND run_id = $2)",
4448 )
4449 .bind(tenant_id.as_str())
4450 .bind(*run_id.as_uuid())
4451 .fetch_one(&mut *transaction)
4452 .await
4453 .map_err(|source| StoreError::database("run quarantine owner lookup", source))?;
4454 return Err(if exists {
4455 StoreError::RunQuarantineNotFound
4456 } else {
4457 StoreError::RunNotFound
4458 });
4459 };
4460 transaction
4461 .commit()
4462 .await
4463 .map_err(|source| StoreError::database("run quarantine load commit", source))?;
4464 Ok(quarantine)
4465 }
4466
4467 pub async fn load_interrupt_request(
4476 &self,
4477 tenant_id: &TenantId,
4478 run_id: RunId,
4479 interrupt_id: InterruptId,
4480 ) -> Result<InterruptRequest, StoreError> {
4481 let mut transaction = self.begin_repeatable_read("interrupt request load").await?;
4482 let wait = load_and_verify_wait_registration(
4483 &mut transaction,
4484 tenant_id,
4485 run_id,
4486 *interrupt_id.as_uuid(),
4487 )
4488 .await?;
4489 let DurableWait::Interrupt { request } = wait else {
4490 return Err(StoreError::WaitRegistrationKindMismatch);
4491 };
4492 transaction
4493 .commit()
4494 .await
4495 .map_err(|source| StoreError::database("interrupt request load commit", source))?;
4496 Ok(*request)
4497 }
4498
4499 pub async fn load_durable_timer(
4505 &self,
4506 tenant_id: &TenantId,
4507 run_id: RunId,
4508 timer_id: TimerId,
4509 ) -> Result<DurableTimer, StoreError> {
4510 let mut transaction = self.begin_repeatable_read("durable timer load").await?;
4511 let wait = load_and_verify_wait_registration(
4512 &mut transaction,
4513 tenant_id,
4514 run_id,
4515 *timer_id.as_uuid(),
4516 )
4517 .await?;
4518 let DurableWait::Timer { timer } = wait else {
4519 return Err(StoreError::WaitRegistrationKindMismatch);
4520 };
4521 transaction
4522 .commit()
4523 .await
4524 .map_err(|source| StoreError::database("durable timer load commit", source))?;
4525 Ok(*timer)
4526 }
4527
4528 pub async fn load_interrupt_record(
4534 &self,
4535 tenant_id: &TenantId,
4536 run_id: RunId,
4537 interrupt_id: InterruptId,
4538 ) -> Result<InterruptRecord, StoreError> {
4539 let mut transaction = self.begin_repeatable_read("interrupt history load").await?;
4540 let row = query_as::<_, WaitRegistrationRow>(SELECT_WAIT_REGISTRATION_BY_ID.as_str())
4541 .bind(tenant_id.as_str())
4542 .bind(*run_id.as_uuid())
4543 .bind(*interrupt_id.as_uuid())
4544 .fetch_optional(&mut *transaction)
4545 .await
4546 .map_err(|source| StoreError::database("interrupt registration load", source))?
4547 .ok_or(StoreError::WaitRegistrationNotFound)?;
4548 let record = load_interrupt_record_from_row(&mut transaction, &row).await?;
4549 transaction
4550 .commit()
4551 .await
4552 .map_err(|source| StoreError::database("interrupt history load commit", source))?;
4553 Ok(record)
4554 }
4555
4556 pub async fn load_durable_timer_record(
4562 &self,
4563 tenant_id: &TenantId,
4564 run_id: RunId,
4565 timer_id: TimerId,
4566 ) -> Result<DurableTimerRecord, StoreError> {
4567 let mut transaction = self.begin_repeatable_read("timer history load").await?;
4568 let row = query_as::<_, WaitRegistrationRow>(SELECT_WAIT_REGISTRATION_BY_ID.as_str())
4569 .bind(tenant_id.as_str())
4570 .bind(*run_id.as_uuid())
4571 .bind(*timer_id.as_uuid())
4572 .fetch_optional(&mut *transaction)
4573 .await
4574 .map_err(|source| StoreError::database("timer registration load", source))?
4575 .ok_or(StoreError::WaitRegistrationNotFound)?;
4576 let record = load_timer_record_from_row(&mut transaction, &row).await?;
4577 transaction
4578 .commit()
4579 .await
4580 .map_err(|source| StoreError::database("timer history load commit", source))?;
4581 Ok(record)
4582 }
4583
4584 pub async fn load_interrupt_abandonment(
4591 &self,
4592 tenant_id: &TenantId,
4593 run_id: RunId,
4594 interrupt_id: InterruptId,
4595 ) -> Result<WaitAbandonment, StoreError> {
4596 let mut transaction = self
4597 .begin_repeatable_read("interrupt abandonment load")
4598 .await?;
4599 let abandonment = load_wait_abandonment_by_id(
4600 &mut transaction,
4601 tenant_id,
4602 run_id,
4603 *interrupt_id.as_uuid(),
4604 )
4605 .await?;
4606 if !matches!(abandonment.wait(), DurableWait::Interrupt { .. }) {
4607 return Err(StoreError::WaitRegistrationKindMismatch);
4608 }
4609 transaction
4610 .commit()
4611 .await
4612 .map_err(|source| StoreError::database("interrupt abandonment load commit", source))?;
4613 Ok(abandonment)
4614 }
4615
4616 pub async fn load_timer_abandonment(
4623 &self,
4624 tenant_id: &TenantId,
4625 run_id: RunId,
4626 timer_id: TimerId,
4627 ) -> Result<WaitAbandonment, StoreError> {
4628 let mut transaction = self.begin_repeatable_read("timer abandonment load").await?;
4629 let abandonment =
4630 load_wait_abandonment_by_id(&mut transaction, tenant_id, run_id, *timer_id.as_uuid())
4631 .await?;
4632 if !matches!(abandonment.wait(), DurableWait::Timer { .. }) {
4633 return Err(StoreError::WaitRegistrationKindMismatch);
4634 }
4635 transaction
4636 .commit()
4637 .await
4638 .map_err(|source| StoreError::database("timer abandonment load commit", source))?;
4639 Ok(abandonment)
4640 }
4641
4642 pub async fn load_due_timer_page(
4653 &self,
4654 tenant_id: &TenantId,
4655 cursor: Option<&DueTimerPageCursor>,
4656 page_size: WaitDiscoveryPageSize,
4657 ) -> Result<DueTimerPage, StoreError> {
4658 if cursor.is_some_and(|cursor| {
4659 &cursor.tenant_id != tenant_id || cursor.due_at > cursor.snapshot_at
4660 }) {
4661 return Err(StoreError::InvalidDueTimerCursor);
4662 }
4663 let mut transaction = self.begin_repeatable_read("due timer page").await?;
4664 let (transaction_started_at, observed_at) =
4665 database_scheduler_times(&mut transaction, "due timer page clock").await?;
4666 let snapshot_at = cursor.map_or(transaction_started_at, |cursor| cursor.snapshot_at);
4667 if observed_at < transaction_started_at || observed_at < snapshot_at {
4668 return Err(StoreError::DatabaseClockRegression);
4669 }
4670 let mut rows = query_as::<_, WaitRegistrationRow>(SELECT_DUE_TIMER_PAGE.as_str())
4671 .bind(tenant_id.as_str())
4672 .bind(to_database_time(snapshot_at)?)
4673 .bind(
4674 cursor
4675 .map(|cursor| to_database_time(cursor.due_at))
4676 .transpose()?,
4677 )
4678 .bind(cursor.map(|cursor| *cursor.run_id.as_uuid()))
4679 .bind(cursor.map(|cursor| *cursor.timer_id.as_uuid()))
4680 .bind(i64::from(page_size.get()) + 1)
4681 .fetch_all(&mut *transaction)
4682 .await
4683 .map_err(|source| StoreError::database("due timer page query", source))?;
4684 let has_more = rows.len() > usize::from(page_size.get());
4685 rows.truncate(usize::from(page_size.get()));
4686 let mut owners = BTreeMap::new();
4687 let mut records = Vec::with_capacity(rows.len());
4688 let mut previous = cursor.map(|cursor| (cursor.due_at, cursor.run_id, cursor.timer_id));
4689 for row in rows {
4690 let sequence = row.registration_sequence;
4691 let wait = decode_wait_registration(&row)?;
4692 let DurableWait::Timer { timer } = wait else {
4693 return Err(StoreError::corrupt("due timer page kind"));
4694 };
4695 verify_wait_registration_event(
4696 &mut transaction,
4697 &DurableWait::Timer {
4698 timer: timer.clone(),
4699 },
4700 sequence,
4701 )
4702 .await?;
4703 let key = (
4704 timer.marker().due_at(),
4705 timer.intent().run_id(),
4706 timer.marker().timer_id(),
4707 );
4708 if key.0 > snapshot_at
4709 || previous
4710 .as_ref()
4711 .is_some_and(|previous| !wait_discovery_key_after(&key, previous))
4712 {
4713 return Err(StoreError::corrupt("due timer page order"));
4714 }
4715 let owner = load_discovery_wait_owner(
4716 &mut transaction,
4717 &mut owners,
4718 tenant_id,
4719 timer.intent().run_id(),
4720 )
4721 .await?;
4722 if owner
4723 .lifecycle()
4724 .waits()
4725 .and_then(|waits| waits.timer(timer.marker().timer_id()))
4726 != Some(timer.marker())
4727 {
4728 return Err(StoreError::corrupt("due timer lifecycle owner"));
4729 }
4730 previous = Some(key);
4731 records.push(*timer);
4732 }
4733 transaction
4734 .commit()
4735 .await
4736 .map_err(|source| StoreError::database("due timer page commit", source))?;
4737 Ok(DueTimerPage {
4738 tenant_id: tenant_id.clone(),
4739 snapshot_at,
4740 records,
4741 has_more,
4742 })
4743 }
4744
4745 pub async fn load_expired_interrupt_page(
4752 &self,
4753 tenant_id: &TenantId,
4754 cursor: Option<&ExpiredInterruptPageCursor>,
4755 page_size: WaitDiscoveryPageSize,
4756 ) -> Result<ExpiredInterruptPage, StoreError> {
4757 if cursor.is_some_and(|cursor| {
4758 &cursor.tenant_id != tenant_id || cursor.expires_at > cursor.snapshot_at
4759 }) {
4760 return Err(StoreError::InvalidExpiredInterruptCursor);
4761 }
4762 let mut transaction = self.begin_repeatable_read("expired interrupt page").await?;
4763 let (transaction_started_at, observed_at) =
4764 database_scheduler_times(&mut transaction, "expired interrupt page clock").await?;
4765 let snapshot_at = cursor.map_or(transaction_started_at, |cursor| cursor.snapshot_at);
4766 if observed_at < transaction_started_at || observed_at < snapshot_at {
4767 return Err(StoreError::DatabaseClockRegression);
4768 }
4769 let mut rows = query_as::<_, WaitRegistrationRow>(SELECT_EXPIRED_INTERRUPT_PAGE.as_str())
4770 .bind(tenant_id.as_str())
4771 .bind(to_database_time(snapshot_at)?)
4772 .bind(
4773 cursor
4774 .map(|cursor| to_database_time(cursor.expires_at))
4775 .transpose()?,
4776 )
4777 .bind(cursor.map(|cursor| *cursor.run_id.as_uuid()))
4778 .bind(cursor.map(|cursor| *cursor.interrupt_id.as_uuid()))
4779 .bind(i64::from(page_size.get()) + 1)
4780 .fetch_all(&mut *transaction)
4781 .await
4782 .map_err(|source| StoreError::database("expired interrupt page query", source))?;
4783 let has_more = rows.len() > usize::from(page_size.get());
4784 rows.truncate(usize::from(page_size.get()));
4785 let mut owners = BTreeMap::new();
4786 let mut records = Vec::with_capacity(rows.len());
4787 let mut previous =
4788 cursor.map(|cursor| (cursor.expires_at, cursor.run_id, cursor.interrupt_id));
4789 for row in rows {
4790 let sequence = row.registration_sequence;
4791 let wait = decode_wait_registration(&row)?;
4792 let DurableWait::Interrupt { request } = wait else {
4793 return Err(StoreError::corrupt("expired interrupt page kind"));
4794 };
4795 verify_wait_registration_event(
4796 &mut transaction,
4797 &DurableWait::Interrupt {
4798 request: request.clone(),
4799 },
4800 sequence,
4801 )
4802 .await?;
4803 let expires_at = request
4804 .marker()
4805 .expires_at()
4806 .ok_or_else(|| StoreError::corrupt("expired interrupt finite expiry"))?;
4807 let key = (
4808 expires_at,
4809 request.intent().run_id(),
4810 request.marker().interrupt_id(),
4811 );
4812 if expires_at > snapshot_at
4813 || previous
4814 .as_ref()
4815 .is_some_and(|previous| !wait_discovery_key_after(&key, previous))
4816 {
4817 return Err(StoreError::corrupt("expired interrupt page order"));
4818 }
4819 let owner = load_discovery_wait_owner(
4820 &mut transaction,
4821 &mut owners,
4822 tenant_id,
4823 request.intent().run_id(),
4824 )
4825 .await?;
4826 if owner
4827 .lifecycle()
4828 .waits()
4829 .and_then(|waits| waits.interrupt(request.marker().interrupt_id()))
4830 != Some(request.marker())
4831 {
4832 return Err(StoreError::corrupt("expired interrupt lifecycle owner"));
4833 }
4834 previous = Some(key);
4835 records.push(*request);
4836 }
4837 transaction
4838 .commit()
4839 .await
4840 .map_err(|source| StoreError::database("expired interrupt page commit", source))?;
4841 Ok(ExpiredInterruptPage {
4842 tenant_id: tenant_id.clone(),
4843 snapshot_at,
4844 records,
4845 has_more,
4846 })
4847 }
4848
4849 pub async fn load_runnable_run_page(
4867 &self,
4868 tenant_id: &TenantId,
4869 cursor: Option<&RunnableRunPageCursor>,
4870 page_size: RunnableRunPageSize,
4871 ) -> Result<RunnableRunPage, StoreError> {
4872 if cursor.is_some_and(|cursor| {
4873 &cursor.tenant_id != tenant_id || cursor.available_at > cursor.snapshot_at
4874 }) {
4875 return Err(StoreError::InvalidRunnableRunCursor);
4876 }
4877
4878 let mut transaction = self.begin_repeatable_read("runnable run page").await?;
4879 let (transaction_started_at, observed_at) =
4880 database_scheduler_times(&mut transaction, "runnable run page clock").await?;
4881 let snapshot_at = cursor.map_or(transaction_started_at, |cursor| cursor.snapshot_at);
4882 if observed_at < transaction_started_at || observed_at < snapshot_at {
4883 return Err(StoreError::DatabaseClockRegression);
4884 }
4885
4886 let after_available_at = cursor
4887 .map(|cursor| to_database_time(cursor.available_at))
4888 .transpose()?;
4889 let after_run_id = cursor.map(|cursor| *cursor.run_id.as_uuid());
4890 let limit = i64::from(page_size.get()) + 1;
4891 let mut rows = query_as::<_, RunRow>(SELECT_RUNNABLE_RUN_PAGE)
4892 .bind(tenant_id.as_str())
4893 .bind(to_database_time(snapshot_at)?)
4894 .bind(after_available_at)
4895 .bind(after_run_id)
4896 .bind(limit)
4897 .fetch_all(&mut *transaction)
4898 .await
4899 .map_err(|source| StoreError::database("runnable run page query", source))?;
4900 transaction
4901 .commit()
4902 .await
4903 .map_err(|source| StoreError::database("runnable run page commit", source))?;
4904
4905 let has_more = rows.len() > usize::from(page_size.get());
4906 rows.truncate(usize::from(page_size.get()));
4907 let mut records = Vec::with_capacity(rows.len());
4908 let mut previous = cursor.map(|cursor| (cursor.available_at, cursor.run_id));
4909 for row in rows {
4910 let stored = decode_run(row)?;
4911 let provenance = stored.lifecycle().provenance();
4912 if provenance.tenant_id() != tenant_id
4913 || stored.is_quarantined()
4914 || !lifecycle_is_scheduler_runnable(stored.lifecycle().status())
4915 {
4916 return Err(StoreError::corrupt("runnable run page scope"));
4917 }
4918 let ready_at = stored
4919 .scheduler_ready_at()
4920 .ok_or_else(|| StoreError::corrupt("runnable run readiness"))?;
4921 let available_at = scheduler_available_at(&stored)?;
4922 let run_id = provenance.run_id();
4923 if available_at > snapshot_at
4924 || previous.is_some_and(|(previous_available_at, previous_run_id)| {
4925 available_at < previous_available_at
4926 || (available_at == previous_available_at
4927 && run_id.as_uuid() <= previous_run_id.as_uuid())
4928 })
4929 {
4930 return Err(StoreError::corrupt("runnable run page order"));
4931 }
4932 previous = Some((available_at, run_id));
4933 records.push(RunnableRunCandidate {
4934 run: stored,
4935 ready_at,
4936 available_at,
4937 });
4938 }
4939
4940 Ok(RunnableRunPage {
4941 tenant_id: tenant_id.clone(),
4942 snapshot_at,
4943 records,
4944 has_more,
4945 })
4946 }
4947
4948 pub async fn register_outbox_destination(
4959 &self,
4960 destination: OutboxDestinationRef,
4961 config: JournalPayload,
4962 ) -> Result<OutboxDestinationRegistrationOutcome, StoreError> {
4963 if destination.snapshot_digest() != config.digest() {
4964 return Err(StoreError::OutboxDestinationSnapshotMismatch);
4965 }
4966 let config_bytes = encode_outbox_destination_config(&config)?;
4967 let schema = config.schema();
4968 let mut transaction = self
4969 .begin_mutation("outbox destination registration")
4970 .await?;
4971 let observed_at = database_now(&mut transaction, "outbox destination clock").await?;
4972 let inserted = query(
4973 r"
4974INSERT INTO stateknot.outbox_destinations (
4975 tenant_id,
4976 destination_id,
4977 snapshot_digest,
4978 config_kind,
4979 schema_id,
4980 schema_version,
4981 schema_digest,
4982 config_bytes,
4983 created_at
4984)
4985VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
4986ON CONFLICT (tenant_id, destination_id, snapshot_digest) DO NOTHING
4987",
4988 )
4989 .bind(destination.tenant_id().as_str())
4990 .bind(*destination.destination_id().as_uuid())
4991 .bind(destination.snapshot_digest().as_bytes())
4992 .bind(config.kind().as_str())
4993 .bind(schema.id().as_str())
4994 .bind(schema.version().to_string())
4995 .bind(schema.digest().as_bytes())
4996 .bind(&config_bytes)
4997 .bind(to_database_time(observed_at)?)
4998 .execute(&mut *transaction)
4999 .await
5000 .map_err(|source| StoreError::database("outbox destination insert", source))?
5001 .rows_affected();
5002
5003 let row = load_outbox_destination_row(&mut transaction, &destination)
5004 .await?
5005 .ok_or(StoreError::OutboxDestinationConflict)?;
5006 let stored = decode_outbox_destination(row)?;
5007 if stored.destination() != &destination || stored.config() != &config {
5008 return Err(StoreError::OutboxDestinationConflict);
5009 }
5010 transaction
5011 .commit()
5012 .await
5013 .map_err(|source| StoreError::database("outbox destination commit", source))?;
5014 Ok(if inserted == 1 {
5015 OutboxDestinationRegistrationOutcome::Registered(stored)
5016 } else {
5017 OutboxDestinationRegistrationOutcome::Idempotent(stored)
5018 })
5019 }
5020
5021 pub async fn load_outbox_destination(
5028 &self,
5029 destination: &OutboxDestinationRef,
5030 ) -> Result<StoredOutboxDestination, StoreError> {
5031 let mut transaction = self
5032 .begin_repeatable_read("outbox destination load")
5033 .await?;
5034 let row = load_outbox_destination_row(&mut transaction, destination)
5035 .await?
5036 .ok_or(StoreError::OutboxDestinationNotFound)?;
5037 let stored = decode_outbox_destination(row)?;
5038 transaction
5039 .commit()
5040 .await
5041 .map_err(|source| StoreError::database("outbox destination load commit", source))?;
5042 Ok(stored)
5043 }
5044
5045 pub async fn load_outbox_delivery(
5052 &self,
5053 tenant_id: &TenantId,
5054 run_id: RunId,
5055 delivery_id: DeliveryId,
5056 ) -> Result<OutboxDelivery, StoreError> {
5057 let mut transaction = self.begin_repeatable_read("outbox delivery load").await?;
5058 let row = load_outbox_delivery_row(&mut transaction, tenant_id, run_id, delivery_id, false)
5059 .await?
5060 .ok_or(StoreError::OutboxDeliveryNotFound)?;
5061 let delivery = decode_outbox_delivery(&row)?;
5062 verify_outbox_projection(&mut transaction, &row, &delivery).await?;
5063 transaction
5064 .commit()
5065 .await
5066 .map_err(|source| StoreError::database("outbox delivery load commit", source))?;
5067 Ok(delivery)
5068 }
5069
5070 pub async fn append_control_plane_with_outbox(
5077 &self,
5078 append: JournalAppend,
5079 projection: RunProjection,
5080 deliveries: Vec<OutboxDeliveryIntent>,
5081 ) -> Result<OutboxEnqueueOutcome, StoreError> {
5082 self.append_with_outbox(
5083 append,
5084 projection,
5085 AppendAuthority::ControlPlane,
5086 deliveries,
5087 )
5088 .await
5089 }
5090
5091 pub async fn append_worker_with_outbox(
5099 &self,
5100 append: JournalAppend,
5101 projection: RunProjection,
5102 deliveries: Vec<OutboxDeliveryIntent>,
5103 ) -> Result<OutboxEnqueueOutcome, StoreError> {
5104 self.append_with_outbox(append, projection, AppendAuthority::Worker, deliveries)
5105 .await
5106 }
5107
5108 pub async fn load_checkpoint(
5118 &self,
5119 tenant_id: &TenantId,
5120 run_id: RunId,
5121 checkpoint_id: CheckpointId,
5122 ) -> Result<Checkpoint, StoreError> {
5123 let mut transaction = self.begin_repeatable_read("checkpoint load").await?;
5124 let row = query_as::<_, CheckpointRow>(SELECT_CHECKPOINT_BY_ID)
5125 .bind(tenant_id.as_str())
5126 .bind(*run_id.as_uuid())
5127 .bind(*checkpoint_id.as_uuid())
5128 .fetch_optional(&mut *transaction)
5129 .await
5130 .map_err(|source| StoreError::database("checkpoint load", source))?
5131 .ok_or(StoreError::CheckpointNotFound)?;
5132 let checkpoint = decode_checkpoint(row)?;
5133 if checkpoint.tenant_id() != tenant_id
5134 || checkpoint.run_id() != run_id
5135 || checkpoint.checkpoint_id() != checkpoint_id
5136 {
5137 return Err(StoreError::corrupt("checkpoint scope"));
5138 }
5139 verify_checkpoint_anchor(&mut transaction, &checkpoint).await?;
5140 transaction
5141 .commit()
5142 .await
5143 .map_err(|source| StoreError::database("checkpoint load commit", source))?;
5144 Ok(checkpoint)
5145 }
5146
5147 pub async fn load_current_checkpoint(
5157 &self,
5158 tenant_id: &TenantId,
5159 run_id: RunId,
5160 ) -> Result<Option<Checkpoint>, StoreError> {
5161 let mut transaction = self.begin_repeatable_read("current checkpoint").await?;
5162 let row = query_as::<_, RunRow>(SELECT_RUN)
5163 .bind(tenant_id.as_str())
5164 .bind(*run_id.as_uuid())
5165 .fetch_optional(&mut *transaction)
5166 .await
5167 .map_err(|source| StoreError::database("checkpoint run snapshot", source))?
5168 .ok_or(StoreError::RunNotFound)?;
5169 let run = decode_run(row)?;
5170 let checkpoint = if let Some(pointer) = run.checkpoint() {
5171 let row = query_as::<_, CheckpointRow>(SELECT_CHECKPOINT_BY_ID)
5172 .bind(tenant_id.as_str())
5173 .bind(*run_id.as_uuid())
5174 .bind(*pointer.checkpoint_id().as_uuid())
5175 .fetch_optional(&mut *transaction)
5176 .await
5177 .map_err(|source| StoreError::database("current checkpoint load", source))?
5178 .ok_or_else(|| StoreError::corrupt("current checkpoint pointer"))?;
5179 let checkpoint = decode_checkpoint(row)?;
5180 if checkpoint.tenant_id() != tenant_id
5181 || checkpoint.run_id() != run_id
5182 || checkpoint.checkpoint_id() != pointer.checkpoint_id()
5183 || checkpoint.superstep() != pointer.superstep()
5184 || checkpoint.digest() != pointer.digest()
5185 {
5186 return Err(StoreError::corrupt("current checkpoint projection"));
5187 }
5188 verify_checkpoint_anchor(&mut transaction, &checkpoint).await?;
5189 Some(checkpoint)
5190 } else {
5191 None
5192 };
5193 transaction
5194 .commit()
5195 .await
5196 .map_err(|source| StoreError::database("current checkpoint commit", source))?;
5197 Ok(checkpoint)
5198 }
5199
5200 pub async fn load_checkpoint_lineage_page(
5218 &self,
5219 tenant_id: &TenantId,
5220 run_id: RunId,
5221 from: Option<&CheckpointHead>,
5222 page_size: CheckpointLineagePageSize,
5223 ) -> Result<CheckpointLineagePage, StoreError> {
5224 if from.is_some_and(|cursor| cursor.tenant_id() != tenant_id || cursor.run_id() != run_id) {
5225 return Err(StoreError::InvalidCheckpointCursor);
5226 }
5227
5228 let mut transaction = self.begin_repeatable_read("checkpoint lineage").await?;
5229 let row = query_as::<_, RunRow>(SELECT_RUN)
5230 .bind(tenant_id.as_str())
5231 .bind(*run_id.as_uuid())
5232 .fetch_optional(&mut *transaction)
5233 .await
5234 .map_err(|source| StoreError::database("checkpoint lineage run snapshot", source))?
5235 .ok_or(StoreError::RunNotFound)?;
5236 let run = decode_run(row)?;
5237 let Some(pointer) = run.checkpoint() else {
5238 if from.is_some() {
5239 return Err(StoreError::InvalidCheckpointCursor);
5240 }
5241 transaction.commit().await.map_err(|source| {
5242 StoreError::database("empty checkpoint lineage commit", source)
5243 })?;
5244 return Ok(CheckpointLineagePage {
5245 checkpoints: Vec::new(),
5246 next_cursor: None,
5247 });
5248 };
5249
5250 if let Some(cursor) = from {
5251 if cursor.superstep() > pointer.superstep()
5252 || (cursor.superstep() == pointer.superstep()
5253 && (cursor.checkpoint_id() != pointer.checkpoint_id()
5254 || cursor.digest() != pointer.digest()))
5255 {
5256 return Err(StoreError::InvalidCheckpointCursor);
5257 }
5258 }
5259 let start_id = from.map_or(pointer.checkpoint_id(), CheckpointHead::checkpoint_id);
5260 let query_limit = i64::from(page_size.get()) + 1;
5261 let rows = query_as::<_, CheckpointRow>(SELECT_CHECKPOINT_LINEAGE)
5262 .bind(tenant_id.as_str())
5263 .bind(*run_id.as_uuid())
5264 .bind(*start_id.as_uuid())
5265 .bind(query_limit)
5266 .fetch_all(&mut *transaction)
5267 .await
5268 .map_err(|source| StoreError::database("checkpoint lineage load", source))?;
5269 if rows.is_empty() {
5270 return Err(if from.is_some() {
5271 StoreError::InvalidCheckpointCursor
5272 } else {
5273 StoreError::corrupt("current checkpoint lineage pointer")
5274 });
5275 }
5276
5277 let (checkpoints, lookahead) =
5278 decode_checkpoint_lineage(rows, tenant_id, run_id, page_size)?;
5279
5280 let first = checkpoints
5281 .first()
5282 .ok_or_else(|| StoreError::corrupt("checkpoint lineage page"))?;
5283 let expected_tip = if let Some(cursor) = from {
5284 if first.head() != *cursor {
5285 return Err(StoreError::InvalidCheckpointCursor);
5286 }
5287 cursor.clone()
5288 } else {
5289 if first.checkpoint_id() != pointer.checkpoint_id()
5290 || first.superstep() != pointer.superstep()
5291 || first.digest() != pointer.digest()
5292 {
5293 return Err(StoreError::corrupt("current checkpoint lineage projection"));
5294 }
5295 first.head()
5296 };
5297
5298 let mut verifier = CheckpointLineageVerifier::from_tip(expected_tip);
5299 for checkpoint in &checkpoints {
5300 verifier
5301 .verify_next(checkpoint)
5302 .map_err(|_| StoreError::corrupt("checkpoint lineage"))?;
5303 }
5304 match (verifier.expected(), lookahead) {
5305 (None, None) => {}
5306 (Some(expected), Some(parent)) if parent.head() == *expected => {}
5307 _ => return Err(StoreError::corrupt("checkpoint lineage parent")),
5308 }
5309 verify_checkpoint_anchors(&mut transaction, &checkpoints).await?;
5310
5311 let next_cursor = verifier.expected().cloned();
5312 transaction
5313 .commit()
5314 .await
5315 .map_err(|source| StoreError::database("checkpoint lineage commit", source))?;
5316 Ok(CheckpointLineagePage {
5317 checkpoints,
5318 next_cursor,
5319 })
5320 }
5321
5322 pub async fn load_tool_invocation(
5334 &self,
5335 tenant_id: &TenantId,
5336 run_id: RunId,
5337 invocation_id: InvocationId,
5338 ) -> Result<ToolInvocation, StoreError> {
5339 let mut transaction = self.begin_repeatable_read("tool invocation load").await?;
5340 let row = query_as::<_, ToolInvocationRow>(SELECT_TOOL_INVOCATION)
5341 .bind(tenant_id.as_str())
5342 .bind(*run_id.as_uuid())
5343 .bind(*invocation_id.as_uuid())
5344 .fetch_optional(&mut *transaction)
5345 .await
5346 .map_err(|source| StoreError::database("tool invocation load", source))?
5347 .ok_or(StoreError::ToolInvocationNotFound)?;
5348 let intent = decode_tool_invocation_intent(&row)?;
5349 if intent.tenant_id() != tenant_id
5350 || intent.run_id() != run_id
5351 || intent.invocation_id() != invocation_id
5352 {
5353 return Err(StoreError::corrupt("tool invocation scope"));
5354 }
5355 let current_revision = nonnegative_tool_invocation_revision(row.current_revision)?;
5356 let revision_row = load_tool_invocation_revision_row(
5357 &mut transaction,
5358 tenant_id,
5359 run_id,
5360 invocation_id,
5361 current_revision,
5362 )
5363 .await?;
5364 let invocation = decode_tool_invocation_revision(revision_row, &intent)?;
5365 validate_tool_invocation_current_projection(&row, &invocation)?;
5366 verify_tool_invocation_base_checkpoint(&mut transaction, &intent).await?;
5367 verify_tool_invocation_anchor(&mut transaction, &invocation).await?;
5368 transaction
5369 .commit()
5370 .await
5371 .map_err(|source| StoreError::database("tool invocation load commit", source))?;
5372 Ok(invocation)
5373 }
5374
5375 pub async fn load_tool_invocation_revision(
5385 &self,
5386 tenant_id: &TenantId,
5387 run_id: RunId,
5388 invocation_id: InvocationId,
5389 revision: ToolInvocationRevision,
5390 ) -> Result<(JournalEvent, ToolInvocation), StoreError> {
5391 let mut transaction = self.begin_repeatable_read("exact tool revision").await?;
5392 let row = query_as::<_, ToolInvocationRow>(SELECT_TOOL_INVOCATION)
5393 .bind(tenant_id.as_str())
5394 .bind(*run_id.as_uuid())
5395 .bind(*invocation_id.as_uuid())
5396 .fetch_optional(&mut *transaction)
5397 .await
5398 .map_err(|source| StoreError::database("exact tool intent", source))?
5399 .ok_or(StoreError::ToolInvocationNotFound)?;
5400 let intent = decode_tool_invocation_intent(&row)?;
5401 if intent.tenant_id() != tenant_id
5402 || intent.run_id() != run_id
5403 || intent.invocation_id() != invocation_id
5404 {
5405 return Err(StoreError::corrupt("exact tool invocation scope"));
5406 }
5407 let current = nonnegative_tool_invocation_revision(row.current_revision)?;
5408 if revision > current {
5409 return Err(StoreError::ToolInvocationNotFound);
5410 }
5411 verify_tool_invocation_base_checkpoint(&mut transaction, &intent).await?;
5412 let stored = load_tool_invocation_revision_row(
5413 &mut transaction,
5414 tenant_id,
5415 run_id,
5416 invocation_id,
5417 revision,
5418 )
5419 .await
5420 .map_err(|error| match error {
5421 StoreError::ToolInvocationNotFound => {
5422 StoreError::corrupt("missing exact tool revision")
5423 }
5424 other => other,
5425 })?;
5426 let invocation = decode_tool_invocation_revision(stored, &intent)?;
5427 if revision == current {
5428 validate_tool_invocation_current_projection(&row, &invocation)?;
5429 }
5430 let mut verifier = if revision.get() == 0 {
5431 ToolInvocationHistoryVerifier::new()
5432 } else {
5433 let previous = ToolInvocationRevision::new(revision.get() - 1)
5434 .map_err(|_| StoreError::corrupt("exact tool predecessor"))?;
5435 let stored = load_tool_invocation_revision_row(
5436 &mut transaction,
5437 tenant_id,
5438 run_id,
5439 invocation_id,
5440 previous,
5441 )
5442 .await
5443 .map_err(|error| match error {
5444 StoreError::ToolInvocationNotFound => {
5445 StoreError::corrupt("missing exact tool predecessor")
5446 }
5447 other => other,
5448 })?;
5449 let previous = decode_tool_invocation_revision(stored, &intent)?;
5450 verify_tool_invocation_anchor(&mut transaction, &previous).await?;
5451 ToolInvocationHistoryVerifier::after(previous)
5452 };
5453 verifier
5454 .verify_next(&invocation)
5455 .map_err(|_| StoreError::corrupt("exact tool transition"))?;
5456 let event = verify_tool_invocation_anchor(&mut transaction, &invocation).await?;
5457 transaction
5458 .commit()
5459 .await
5460 .map_err(|source| StoreError::database("exact tool revision commit", source))?;
5461 Ok((event, invocation))
5462 }
5463
5464 pub async fn load_tool_invocation_history_page(
5478 &self,
5479 tenant_id: &TenantId,
5480 run_id: RunId,
5481 invocation_id: InvocationId,
5482 after: Option<&ToolInvocation>,
5483 page_size: ToolInvocationHistoryPageSize,
5484 ) -> Result<ToolInvocationHistoryPage, StoreError> {
5485 if after.is_some_and(|cursor| {
5486 cursor.intent().tenant_id() != tenant_id
5487 || cursor.intent().run_id() != run_id
5488 || cursor.intent().invocation_id() != invocation_id
5489 }) {
5490 return Err(StoreError::InvalidToolInvocationCursor);
5491 }
5492
5493 let mut transaction = self
5494 .begin_repeatable_read("tool invocation history")
5495 .await?;
5496 let row = query_as::<_, ToolInvocationRow>(SELECT_TOOL_INVOCATION)
5497 .bind(tenant_id.as_str())
5498 .bind(*run_id.as_uuid())
5499 .bind(*invocation_id.as_uuid())
5500 .fetch_optional(&mut *transaction)
5501 .await
5502 .map_err(|source| StoreError::database("tool invocation history intent", source))?
5503 .ok_or(StoreError::ToolInvocationNotFound)?;
5504 let intent = decode_tool_invocation_intent(&row)?;
5505 let current_revision = nonnegative_tool_invocation_revision(row.current_revision)?;
5506 verify_tool_invocation_base_checkpoint(&mut transaction, &intent).await?;
5507
5508 let cursor = if let Some(cursor) = after {
5509 if cursor.revision() > current_revision || cursor.intent() != &intent {
5510 return Err(StoreError::InvalidToolInvocationCursor);
5511 }
5512 let cursor_row = load_tool_invocation_revision_row(
5513 &mut transaction,
5514 tenant_id,
5515 run_id,
5516 invocation_id,
5517 cursor.revision(),
5518 )
5519 .await
5520 .map_err(|error| match error {
5521 StoreError::ToolInvocationNotFound => StoreError::InvalidToolInvocationCursor,
5522 other => other,
5523 })?;
5524 let stored_cursor = decode_tool_invocation_revision(cursor_row, &intent)?;
5525 if encode_tool_invocation_record(&stored_cursor)?
5526 != encode_tool_invocation_record(cursor)?
5527 {
5528 return Err(StoreError::InvalidToolInvocationCursor);
5529 }
5530 verify_tool_invocation_anchor(&mut transaction, &stored_cursor).await?;
5531 Some(stored_cursor)
5532 } else {
5533 None
5534 };
5535
5536 let after_revision = cursor.as_ref().map_or(-1_i64, |cursor| {
5537 i64::try_from(cursor.revision().get()).unwrap_or(i64::MAX)
5538 });
5539 let query_limit = i64::from(page_size.get());
5540 let rows = query_as::<_, ToolInvocationRevisionRow>(SELECT_TOOL_INVOCATION_HISTORY)
5541 .bind(tenant_id.as_str())
5542 .bind(*run_id.as_uuid())
5543 .bind(*invocation_id.as_uuid())
5544 .bind(after_revision)
5545 .bind(query_limit)
5546 .fetch_all(&mut *transaction)
5547 .await
5548 .map_err(|source| StoreError::database("tool invocation history load", source))?;
5549
5550 let mut verifier = cursor
5551 .clone()
5552 .map_or_else(ToolInvocationHistoryVerifier::new, |cursor| {
5553 ToolInvocationHistoryVerifier::after(cursor)
5554 });
5555 let mut records = Vec::with_capacity(rows.len());
5556 for row in rows {
5557 let record = decode_tool_invocation_revision(row, &intent)?;
5558 verifier
5559 .verify_next(&record)
5560 .map_err(|_| StoreError::corrupt("tool invocation history"))?;
5561 verify_tool_invocation_anchor(&mut transaction, &record).await?;
5562 records.push(record);
5563 }
5564 let final_record = records
5565 .last()
5566 .or(cursor.as_ref())
5567 .ok_or_else(|| StoreError::corrupt("tool invocation empty history"))?;
5568 let has_more = final_record.revision() < current_revision;
5569 if has_more && records.is_empty() {
5570 return Err(StoreError::corrupt("tool invocation history gap"));
5571 }
5572 if !has_more {
5573 validate_tool_invocation_current_projection(&row, final_record)?;
5574 if verifier.head() != Some(final_record.head()) {
5575 return Err(StoreError::corrupt("tool invocation history head"));
5576 }
5577 }
5578
5579 transaction
5580 .commit()
5581 .await
5582 .map_err(|source| StoreError::database("tool invocation history commit", source))?;
5583 Ok(ToolInvocationHistoryPage { records, has_more })
5584 }
5585
5586 pub async fn load_model_invocation(
5598 &self,
5599 tenant_id: &TenantId,
5600 run_id: RunId,
5601 invocation_id: InvocationId,
5602 ) -> Result<ModelInvocation, StoreError> {
5603 let mut transaction = self.begin_repeatable_read("model invocation load").await?;
5604 let row = query_as::<_, ModelInvocationRow>(SELECT_MODEL_INVOCATION)
5605 .bind(tenant_id.as_str())
5606 .bind(*run_id.as_uuid())
5607 .bind(*invocation_id.as_uuid())
5608 .fetch_optional(&mut *transaction)
5609 .await
5610 .map_err(|source| StoreError::database("model invocation load", source))?
5611 .ok_or(StoreError::ModelInvocationNotFound)?;
5612 let intent = decode_model_invocation_intent(&row)?;
5613 if intent.tenant_id() != tenant_id
5614 || intent.run_id() != run_id
5615 || intent.invocation_id() != invocation_id
5616 {
5617 return Err(StoreError::corrupt("model invocation scope"));
5618 }
5619 let current_revision = nonnegative_model_invocation_revision(row.current_revision)?;
5620 let revision_row = load_model_invocation_revision_row(
5621 &mut transaction,
5622 tenant_id,
5623 run_id,
5624 invocation_id,
5625 current_revision,
5626 )
5627 .await?;
5628 let invocation = decode_model_invocation_revision(revision_row, &intent)?;
5629 validate_model_invocation_current_projection(&row, &invocation)?;
5630 verify_model_invocation_base_checkpoint(&mut transaction, &intent).await?;
5631 verify_model_invocation_anchor(&mut transaction, &invocation).await?;
5632 transaction
5633 .commit()
5634 .await
5635 .map_err(|source| StoreError::database("model invocation load commit", source))?;
5636 Ok(invocation)
5637 }
5638
5639 pub async fn load_model_invocation_history_page(
5651 &self,
5652 tenant_id: &TenantId,
5653 run_id: RunId,
5654 invocation_id: InvocationId,
5655 after: Option<&ModelInvocation>,
5656 page_size: ModelInvocationHistoryPageSize,
5657 ) -> Result<ModelInvocationHistoryPage, StoreError> {
5658 if after.is_some_and(|cursor| {
5659 cursor.intent().tenant_id() != tenant_id
5660 || cursor.intent().run_id() != run_id
5661 || cursor.intent().invocation_id() != invocation_id
5662 }) {
5663 return Err(StoreError::InvalidModelInvocationCursor);
5664 }
5665
5666 let mut transaction = self
5667 .begin_repeatable_read("model invocation history")
5668 .await?;
5669 let row = query_as::<_, ModelInvocationRow>(SELECT_MODEL_INVOCATION)
5670 .bind(tenant_id.as_str())
5671 .bind(*run_id.as_uuid())
5672 .bind(*invocation_id.as_uuid())
5673 .fetch_optional(&mut *transaction)
5674 .await
5675 .map_err(|source| StoreError::database("model invocation history intent", source))?
5676 .ok_or(StoreError::ModelInvocationNotFound)?;
5677 let intent = decode_model_invocation_intent(&row)?;
5678 let current_revision = nonnegative_model_invocation_revision(row.current_revision)?;
5679 verify_model_invocation_base_checkpoint(&mut transaction, &intent).await?;
5680
5681 let cursor = if let Some(cursor) = after {
5682 if cursor.revision() > current_revision || cursor.intent() != &intent {
5683 return Err(StoreError::InvalidModelInvocationCursor);
5684 }
5685 let cursor_row = load_model_invocation_revision_row(
5686 &mut transaction,
5687 tenant_id,
5688 run_id,
5689 invocation_id,
5690 cursor.revision(),
5691 )
5692 .await
5693 .map_err(|error| match error {
5694 StoreError::ModelInvocationNotFound => StoreError::InvalidModelInvocationCursor,
5695 other => other,
5696 })?;
5697 let stored_cursor = decode_model_invocation_revision(cursor_row, &intent)?;
5698 if encode_model_invocation_record(&stored_cursor)?
5699 != encode_model_invocation_record(cursor)?
5700 {
5701 return Err(StoreError::InvalidModelInvocationCursor);
5702 }
5703 verify_model_invocation_anchor(&mut transaction, &stored_cursor).await?;
5704 Some(stored_cursor)
5705 } else {
5706 None
5707 };
5708
5709 let after_revision = cursor.as_ref().map_or(-1_i64, |cursor| {
5710 i64::try_from(cursor.revision().get()).unwrap_or(i64::MAX)
5711 });
5712 let rows = query_as::<_, ModelInvocationRevisionRow>(SELECT_MODEL_INVOCATION_HISTORY)
5713 .bind(tenant_id.as_str())
5714 .bind(*run_id.as_uuid())
5715 .bind(*invocation_id.as_uuid())
5716 .bind(after_revision)
5717 .bind(i64::from(page_size.get()))
5718 .fetch_all(&mut *transaction)
5719 .await
5720 .map_err(|source| StoreError::database("model invocation history load", source))?;
5721
5722 let mut verifier = cursor
5723 .clone()
5724 .map_or_else(ModelInvocationHistoryVerifier::new, |cursor| {
5725 ModelInvocationHistoryVerifier::after(cursor)
5726 });
5727 let mut records = Vec::with_capacity(rows.len());
5728 for row in rows {
5729 let record = decode_model_invocation_revision(row, &intent)?;
5730 verifier
5731 .verify_next(&record)
5732 .map_err(|_| StoreError::corrupt("model invocation history"))?;
5733 verify_model_invocation_anchor(&mut transaction, &record).await?;
5734 records.push(record);
5735 }
5736 let final_record = records
5737 .last()
5738 .or(cursor.as_ref())
5739 .ok_or_else(|| StoreError::corrupt("model invocation empty history"))?;
5740 let has_more = final_record.revision() < current_revision;
5741 if has_more && records.is_empty() {
5742 return Err(StoreError::corrupt("model invocation history gap"));
5743 }
5744 if !has_more {
5745 validate_model_invocation_current_projection(&row, final_record)?;
5746 if verifier.head() != Some(final_record.head()) {
5747 return Err(StoreError::corrupt("model invocation history head"));
5748 }
5749 }
5750
5751 transaction
5752 .commit()
5753 .await
5754 .map_err(|source| StoreError::database("model invocation history commit", source))?;
5755 Ok(ModelInvocationHistoryPage { records, has_more })
5756 }
5757
5758 pub async fn load_node_attempt(
5770 &self,
5771 tenant_id: &TenantId,
5772 run_id: &RunId,
5773 attempt_id: AttemptId,
5774 ) -> Result<NodeAttempt, StoreError> {
5775 let mut transaction = self.begin_repeatable_read("node attempt load").await?;
5776 let attempt = load_node_attempt_record(&mut transaction, tenant_id, run_id, attempt_id)
5777 .await?
5778 .ok_or(StoreError::NodeAttemptNotFound)?;
5779 verify_node_attempt(&mut transaction, &attempt).await?;
5780 transaction
5781 .commit()
5782 .await
5783 .map_err(|source| StoreError::database("node attempt load commit", source))?;
5784 Ok(attempt)
5785 }
5786
5787 pub async fn load_node_attempt_history_page(
5798 &self,
5799 activation: &NodeActivation,
5800 cursor: Option<&NodeAttempt>,
5801 page_size: NodeAttemptHistoryPageSize,
5802 ) -> Result<NodeAttemptHistoryPage, StoreError> {
5803 if cursor.is_some_and(|cursor| cursor.start().activation() != activation) {
5804 return Err(StoreError::InvalidNodeAttemptCursor);
5805 }
5806 let mut transaction = self
5807 .begin_repeatable_read("node attempt history load")
5808 .await?;
5809 if let Some(cursor) = cursor {
5810 let durable = load_node_attempt_record(
5811 &mut transaction,
5812 activation.tenant_id(),
5813 &activation.run_id(),
5814 cursor.start().attempt_id(),
5815 )
5816 .await?
5817 .ok_or(StoreError::InvalidNodeAttemptCursor)?;
5818 if encode_node_attempt(&durable)? != encode_node_attempt(cursor)? {
5819 return Err(StoreError::InvalidNodeAttemptCursor);
5820 }
5821 verify_node_attempt(&mut transaction, &durable).await?;
5822 }
5823
5824 let base = activation.base_checkpoint();
5825 let base_superstep = i64::try_from(base.superstep().get())
5826 .map_err(|_| StoreError::InvalidNodeAttemptCursor)?;
5827 let after_sequence = cursor.map_or(0_i64, |cursor| {
5828 i64::try_from(cursor.start().journal_head().sequence().get()).unwrap_or(i64::MAX)
5829 });
5830 let query_limit = i64::from(page_size.get()) + 1;
5831 let mut rows = query_as::<_, NodeAttemptStartRow>(SELECT_NODE_ATTEMPT_HISTORY)
5832 .bind(activation.tenant_id().as_str())
5833 .bind(*activation.run_id().as_uuid())
5834 .bind(*base.checkpoint_id().as_uuid())
5835 .bind(base_superstep)
5836 .bind(base.digest().as_bytes())
5837 .bind(activation.graph_namespace().as_str())
5838 .bind(activation.node_id().as_str())
5839 .bind(activation.input_digest().as_bytes())
5840 .bind(after_sequence)
5841 .bind(query_limit)
5842 .fetch_all(&mut *transaction)
5843 .await
5844 .map_err(|source| StoreError::database("node attempt history rows", source))?;
5845 let has_more = rows.len() > usize::from(page_size.get());
5846 rows.truncate(usize::from(page_size.get()));
5847
5848 let mut verifier = cursor.cloned().map_or_else(
5849 NodeAttemptHistoryVerifier::new,
5850 NodeAttemptHistoryVerifier::after,
5851 );
5852 let mut records = Vec::with_capacity(rows.len());
5853 for row in rows {
5854 let start = decode_node_attempt_start(&row)?;
5855 let completion_row =
5856 query_as::<_, NodeAttemptCompletionRow>(SELECT_NODE_ATTEMPT_COMPLETION)
5857 .bind(activation.tenant_id().as_str())
5858 .bind(*activation.run_id().as_uuid())
5859 .bind(*start.attempt_id().as_uuid())
5860 .fetch_optional(&mut *transaction)
5861 .await
5862 .map_err(|source| {
5863 StoreError::database("node attempt history completion", source)
5864 })?;
5865 let completion = completion_row
5866 .map(|row| decode_node_attempt_completion(&row, &start))
5867 .transpose()?;
5868 let attempt = NodeAttempt::restore(start, completion)
5869 .map_err(|_| StoreError::corrupt("node attempt history join"))?;
5870 verifier
5871 .verify_next(&attempt)
5872 .map_err(|_| StoreError::corrupt("node attempt history"))?;
5873 verify_node_attempt(&mut transaction, &attempt).await?;
5874 records.push(attempt);
5875 }
5876 if has_more && records.is_empty() {
5877 return Err(StoreError::corrupt("node attempt history look-ahead"));
5878 }
5879 transaction
5880 .commit()
5881 .await
5882 .map_err(|source| StoreError::database("node attempt history commit", source))?;
5883 Ok(NodeAttemptHistoryPage { records, has_more })
5884 }
5885
5886 pub async fn load_pending_node_result(
5898 &self,
5899 activation: &NodeActivation,
5900 ) -> Result<PendingNodeResult, StoreError> {
5901 let mut transaction = self
5902 .begin_repeatable_read("pending node result load")
5903 .await?;
5904 let row = load_pending_node_result_row(&mut transaction, activation)
5905 .await?
5906 .ok_or(StoreError::PendingNodeResultNotFound)?;
5907 let result = decode_pending_node_result(&row)?;
5908 if result.intent().activation() != activation {
5909 return Err(StoreError::PendingNodeResultNotFound);
5910 }
5911 verify_pending_node_result(&mut transaction, &result).await?;
5912 transaction
5913 .commit()
5914 .await
5915 .map_err(|source| StoreError::database("pending node result load commit", source))?;
5916 Ok(result)
5917 }
5918
5919 async fn load_historical_graph_barrier_results(
5920 &self,
5921 parent: &Checkpoint,
5922 child: &Checkpoint,
5923 limits: GraphReplayLimits,
5924 ) -> Result<HistoricalGraphBarrierResults, StoreError> {
5925 let mut transaction = self
5926 .begin_repeatable_read("historical graph barrier replay")
5927 .await?;
5928 let heads = load_locked_barrier_result_heads(&mut transaction, &parent.head()).await?;
5929 let barrier = CheckpointBarrier::new(parent, child.write_intent(), heads.clone())
5930 .map_err(|_| StoreError::corrupt("noninitial graph replay result set"))?;
5931
5932 let mut results = Vec::with_capacity(heads.len());
5933 let mut compact_bytes = 0_usize;
5934 for head in &heads {
5935 let row = load_pending_node_result_row(&mut transaction, head.activation())
5936 .await?
5937 .ok_or_else(|| StoreError::corrupt("noninitial graph replay result"))?;
5938 let result = decode_pending_node_result(&row)?;
5939 if result.head() != *head {
5940 return Err(StoreError::corrupt("noninitial graph replay result head"));
5941 }
5942 verify_pending_node_result(&mut transaction, &result).await?;
5943
5944 let mut counter = CompactByteCounter::default();
5945 serde_json::to_writer(&mut counter, &result)
5946 .map_err(|_| StoreError::corrupt("noninitial graph replay result encoding"))?;
5947 compact_bytes = compact_bytes.saturating_add(counter.bytes);
5948 if compact_bytes > limits.maximum_barrier_result_bytes() {
5949 return Err(StoreError::GraphReplayResourceLimit);
5950 }
5951 results.push(result);
5952 }
5953 verify_barrier_consumptions(&mut transaction, &barrier, child)
5954 .await
5955 .map_err(map_graph_replay_consumption_error)?;
5956 transaction.commit().await.map_err(|source| {
5957 StoreError::database("historical graph barrier replay commit", source)
5958 })?;
5959 Ok(HistoricalGraphBarrierResults {
5960 results,
5961 compact_bytes,
5962 })
5963 }
5964
5965 pub async fn load_unconsumed_pending_node_result_page(
5981 &self,
5982 base: &CheckpointHead,
5983 cursor: Option<&PendingNodeResultPageCursor>,
5984 page_size: PendingNodeResultPageSize,
5985 ) -> Result<PendingNodeResultPage, StoreError> {
5986 if cursor.is_some_and(|cursor| !pending_result_cursor_matches_base(cursor, base)) {
5987 return Err(StoreError::InvalidPendingNodeResultCursor);
5988 }
5989
5990 let tenant_id = base.tenant_id();
5991 let run_id = base.run_id();
5992 let mut transaction = self
5993 .begin_repeatable_read("unconsumed pending node result page")
5994 .await?;
5995 let run_row = query_as::<_, RunRow>(SELECT_RUN)
5996 .bind(tenant_id.as_str())
5997 .bind(*run_id.as_uuid())
5998 .fetch_optional(&mut *transaction)
5999 .await
6000 .map_err(|source| StoreError::database("pending result page run snapshot", source))?
6001 .ok_or(StoreError::RunNotFound)?;
6002 let stored = decode_run(run_row)?;
6003 let checkpoint =
6004 load_locked_current_checkpoint(&mut transaction, &stored, tenant_id, run_id)
6005 .await?
6006 .ok_or(StoreError::StaleCheckpointHead)?;
6007 if checkpoint.head() != *base {
6008 return Err(StoreError::StaleCheckpointHead);
6009 }
6010 let snapshot_journal_head = stored
6011 .journal_head()
6012 .cloned()
6013 .ok_or_else(|| StoreError::corrupt("pending result page run journal head"))?;
6014 if cursor.is_some_and(|cursor| cursor.snapshot_journal_head != snapshot_journal_head) {
6015 return Err(StoreError::StalePendingNodeResultSnapshot);
6016 }
6017
6018 if let Some(cursor) = cursor {
6019 let row = load_pending_node_result_head_row(&mut transaction, cursor.after()).await?;
6020 let durable = decode_pending_node_result_head(row, base)?;
6021 if durable != cursor.after {
6022 return Err(StoreError::InvalidPendingNodeResultCursor);
6023 }
6024 }
6025
6026 let base_superstep = i64::try_from(base.superstep().get())
6027 .map_err(|_| StoreError::InvalidPendingNodeResultCursor)?;
6028 let query_limit = i64::from(page_size.get()) + 1;
6029 let rows = if let Some(cursor) = cursor {
6030 query_as::<_, PendingNodeResultHeadRow>(
6031 SELECT_UNCONSUMED_PENDING_NODE_RESULT_HEADS_AFTER,
6032 )
6033 .bind(tenant_id.as_str())
6034 .bind(*run_id.as_uuid())
6035 .bind(*base.checkpoint_id().as_uuid())
6036 .bind(base_superstep)
6037 .bind(base.digest().as_bytes())
6038 .bind(cursor.after().activation().graph_namespace().as_str())
6039 .bind(cursor.after().activation().node_id().as_str())
6040 .bind(query_limit)
6041 .fetch_all(&mut *transaction)
6042 .await
6043 .map_err(|source| {
6044 StoreError::database("unconsumed pending result continuation", source)
6045 })?
6046 } else {
6047 query_as::<_, PendingNodeResultHeadRow>(SELECT_UNCONSUMED_PENDING_NODE_RESULT_HEADS)
6048 .bind(tenant_id.as_str())
6049 .bind(*run_id.as_uuid())
6050 .bind(*base.checkpoint_id().as_uuid())
6051 .bind(base_superstep)
6052 .bind(base.digest().as_bytes())
6053 .bind(query_limit)
6054 .fetch_all(&mut *transaction)
6055 .await
6056 .map_err(|source| {
6057 StoreError::database("unconsumed pending result first page", source)
6058 })?
6059 };
6060 let has_more = rows.len() > usize::from(page_size.get());
6061 let retained = rows.into_iter().take(usize::from(page_size.get()));
6062 let mut records = Vec::with_capacity(usize::from(page_size.get()));
6063 for compact_row in retained {
6064 let compact = decode_pending_node_result_head(compact_row, base)?;
6065 let row = load_pending_node_result_row(&mut transaction, compact.activation())
6066 .await?
6067 .ok_or_else(|| StoreError::corrupt("pending result page row"))?;
6068 let result = decode_pending_node_result(&row)?;
6069 if result.head() != compact {
6070 return Err(StoreError::corrupt("pending result page compact head"));
6071 }
6072 verify_pending_node_result(&mut transaction, &result).await?;
6073 records.push(result);
6074 }
6075 if has_more && records.is_empty() {
6076 return Err(StoreError::corrupt("pending result page look-ahead"));
6077 }
6078
6079 transaction.commit().await.map_err(|source| {
6080 StoreError::database("unconsumed pending result page commit", source)
6081 })?;
6082 Ok(PendingNodeResultPage {
6083 base_checkpoint: base.clone(),
6084 snapshot_journal_head,
6085 records,
6086 has_more,
6087 })
6088 }
6089
6090 pub async fn load_outbox_attempt_history_page(
6102 &self,
6103 tenant_id: &TenantId,
6104 run_id: RunId,
6105 delivery_id: DeliveryId,
6106 after: Option<&OutboxAttempt>,
6107 page_size: OutboxAttemptHistoryPageSize,
6108 ) -> Result<OutboxAttemptHistoryPage, StoreError> {
6109 if after.is_some_and(|attempt| {
6110 attempt.start().delivery().tenant_id() != tenant_id
6111 || attempt.start().delivery().run_id() != run_id
6112 || attempt.start().delivery().delivery_id() != delivery_id
6113 }) {
6114 return Err(StoreError::InvalidOutboxAttemptCursor);
6115 }
6116 let mut transaction = self.begin_repeatable_read("outbox attempt history").await?;
6117 let row = load_outbox_delivery_row(&mut transaction, tenant_id, run_id, delivery_id, false)
6118 .await?
6119 .ok_or(StoreError::OutboxDeliveryNotFound)?;
6120 let delivery = decode_outbox_delivery(&row)?;
6121 let all = load_and_verify_outbox_attempts(&mut transaction, &delivery).await?;
6122 verify_outbox_projection_records(&row, &delivery, &all)?;
6123
6124 let start_index = if let Some(cursor) = after {
6125 all.iter()
6126 .position(|record| outbox_attempts_equal(record, cursor))
6127 .map(|index| index + 1)
6128 .ok_or(StoreError::InvalidOutboxAttemptCursor)?
6129 } else {
6130 0
6131 };
6132 let remaining = &all[start_index..];
6133 let has_more = remaining.len() > usize::from(page_size.get());
6134 let records = remaining
6135 .iter()
6136 .take(usize::from(page_size.get()))
6137 .cloned()
6138 .collect();
6139 transaction
6140 .commit()
6141 .await
6142 .map_err(|source| StoreError::database("outbox attempt history commit", source))?;
6143 Ok(OutboxAttemptHistoryPage { records, has_more })
6144 }
6145
6146 pub async fn claim_outbox_delivery(
6158 &self,
6159 tenant_id: &TenantId,
6160 attempt_id: AttemptId,
6161 ) -> Result<OutboxClaimOutcome, StoreError> {
6162 let mut transaction = self.begin_mutation("outbox delivery claim").await?;
6163 let observed_at = database_now(&mut transaction, "outbox claim clock").await?;
6164 reap_outbox_terminals(&mut transaction, tenant_id, observed_at).await?;
6165
6166 if let Some(claim) =
6167 load_idempotent_outbox_claim(&mut transaction, tenant_id, attempt_id).await?
6168 {
6169 transaction
6170 .commit()
6171 .await
6172 .map_err(|source| StoreError::database("idempotent outbox claim commit", source))?;
6173 return Ok(OutboxClaimOutcome::Idempotent(claim));
6174 }
6175
6176 let Some(row) = query_as::<_, OutboxDeliveryRow>(SELECT_OUTBOX_CLAIM_CANDIDATE.as_str())
6177 .bind(tenant_id.as_str())
6178 .bind(to_database_time(observed_at)?)
6179 .fetch_optional(&mut *transaction)
6180 .await
6181 .map_err(|source| StoreError::database("outbox claim candidate", source))?
6182 else {
6183 transaction
6184 .commit()
6185 .await
6186 .map_err(|source| StoreError::database("empty outbox claim commit", source))?;
6187 return Ok(OutboxClaimOutcome::NoWork);
6188 };
6189
6190 let delivery = decode_outbox_delivery(&row)?;
6191 verify_outbox_delivery_anchor(&mut transaction, &delivery).await?;
6192 let destination =
6193 load_and_decode_outbox_destination(&mut transaction, delivery.intent().destination())
6194 .await?;
6195 let history = load_and_verify_outbox_attempts(&mut transaction, &delivery).await?;
6196 verify_outbox_projection_records(&row, &delivery, &history)?;
6197 let mut verifier = OutboxAttemptHistoryVerifier::new(&delivery);
6198 for attempt in &history {
6199 verifier
6200 .verify_next(attempt)
6201 .map_err(|_| StoreError::corrupt("outbox attempt history"))?;
6202 }
6203 if verifier
6204 .status_at(observed_at)
6205 .map_err(|_| StoreError::corrupt("outbox delivery status"))?
6206 != OutboxDeliveryStatus::Pending
6207 {
6208 return Err(StoreError::corrupt("outbox ready projection"));
6209 }
6210
6211 let next_count = row
6212 .attempt_count
6213 .checked_add(1)
6214 .ok_or(StoreError::InvalidOutboxTransition)?;
6215 if usize::try_from(next_count)
6216 .ok()
6217 .is_none_or(|count| count > MAX_OUTBOX_ATTEMPTS)
6218 {
6219 return Err(StoreError::InvalidOutboxTransition);
6220 }
6221 let epoch = FencingEpoch::new(
6222 u64::try_from(next_count).map_err(|_| StoreError::InvalidOutboxTransition)?,
6223 )
6224 .map_err(|_| StoreError::InvalidOutboxTransition)?;
6225 let configured_expiry =
6226 add_duration(observed_at, self.options.outbox_attempt_lease_duration)?;
6227 let expires_at = configured_expiry.min(delivery.intent().expires_at());
6228 let fence = DeliveryFence::new(
6229 tenant_id.clone(),
6230 delivery.intent().run_id(),
6231 delivery.intent().delivery_id(),
6232 attempt_id,
6233 epoch,
6234 );
6235 let start = OutboxAttemptStart::new(&delivery, fence, observed_at, expires_at)
6236 .map_err(|_| StoreError::InvalidOutboxTransition)?;
6237
6238 insert_outbox_attempt_claim(&mut transaction, &delivery, &start).await?;
6239 insert_outbox_attempt_start(&mut transaction, &delivery, &start).await?;
6240 update_outbox_delivery_claim(&mut transaction, &delivery, &start, row.attempt_count)
6241 .await?;
6242 transaction
6243 .commit()
6244 .await
6245 .map_err(|source| StoreError::database("outbox claim commit", source))?;
6246 Ok(OutboxClaimOutcome::Claimed(OutboxClaim {
6247 destination,
6248 delivery,
6249 start,
6250 }))
6251 }
6252
6253 pub async fn acknowledge_outbox_attempt(
6260 &self,
6261 fence: &DeliveryFence,
6262 evidence_digest: Option<Digest>,
6263 ) -> Result<OutboxCompletionOutcome, StoreError> {
6264 self.complete_outbox_attempt(
6265 fence,
6266 OutboxAttemptOutcome::Acknowledged { evidence_digest },
6267 )
6268 .await
6269 }
6270
6271 pub async fn fail_outbox_attempt(
6282 &self,
6283 fence: &DeliveryFence,
6284 failure: Failure,
6285 ) -> Result<OutboxCompletionOutcome, StoreError> {
6286 self.complete_outbox_attempt(fence, OutboxAttemptOutcome::Failed { failure })
6287 .await
6288 }
6289
6290 async fn complete_outbox_attempt(
6291 &self,
6292 fence: &DeliveryFence,
6293 requested_outcome: OutboxAttemptOutcome,
6294 ) -> Result<OutboxCompletionOutcome, StoreError> {
6295 let mut transaction = self.begin_mutation("outbox attempt completion").await?;
6296 let delivery_row = load_outbox_delivery_row(
6297 &mut transaction,
6298 fence.tenant_id(),
6299 fence.run_id(),
6300 fence.delivery_id(),
6301 true,
6302 )
6303 .await?
6304 .ok_or(StoreError::OutboxDeliveryNotFound)?;
6305 let delivery = decode_outbox_delivery(&delivery_row)?;
6306 verify_outbox_projection(&mut transaction, &delivery_row, &delivery).await?;
6307 let epoch =
6308 i64::try_from(fence.epoch().get()).map_err(|_| StoreError::InvalidOutboxTransition)?;
6309 let start_row =
6310 query_as::<_, OutboxAttemptStartRow>(SELECT_OUTBOX_ATTEMPT_BY_FENCE.as_str())
6311 .bind(fence.tenant_id().as_str())
6312 .bind(*fence.run_id().as_uuid())
6313 .bind(*fence.delivery_id().as_uuid())
6314 .bind(epoch)
6315 .bind(*fence.attempt_id().as_uuid())
6316 .fetch_optional(&mut *transaction)
6317 .await
6318 .map_err(|source| StoreError::database("outbox completion start load", source))?
6319 .ok_or(StoreError::OutboxAttemptNotFound)?;
6320 let start = decode_outbox_attempt_start(&start_row)?;
6321 if start.delivery() != &delivery.head() || start.fence() != fence {
6322 return Err(StoreError::StaleOutboxFence);
6323 }
6324
6325 if let Some(row) = load_outbox_attempt_completion_row(
6326 &mut transaction,
6327 fence.tenant_id(),
6328 fence.run_id(),
6329 fence.delivery_id(),
6330 epoch,
6331 )
6332 .await?
6333 {
6334 let completion = decode_outbox_attempt_completion(&row)?;
6335 if completion.start() != &start.head()
6336 || !outbox_outcomes_equal(completion.outcome(), &requested_outcome)
6337 {
6338 return Err(StoreError::OutboxCompletionConflict);
6339 }
6340 let attempt = OutboxAttempt::restore(start, Some(completion))
6341 .map_err(|_| StoreError::corrupt("outbox completed attempt"))?;
6342 transaction.commit().await.map_err(|source| {
6343 StoreError::database("idempotent outbox completion commit", source)
6344 })?;
6345 return Ok(OutboxCompletionOutcome::Idempotent { attempt });
6346 }
6347
6348 if delivery_row.status != "delivering"
6349 || delivery_row.current_attempt_id != Some(*fence.attempt_id().as_uuid())
6350 || delivery_row.current_epoch != Some(epoch)
6351 {
6352 return Err(StoreError::StaleOutboxFence);
6353 }
6354 let observed_at = database_now(&mut transaction, "outbox completion clock").await?;
6355 if observed_at >= start.expires_at() {
6356 return Err(StoreError::OutboxAttemptExpired);
6357 }
6358 let completion = match requested_outcome {
6359 OutboxAttemptOutcome::Acknowledged { evidence_digest } => {
6360 OutboxAttemptCompletion::acknowledge(&start, evidence_digest, observed_at)
6361 }
6362 OutboxAttemptOutcome::Failed { failure } => {
6363 OutboxAttemptCompletion::fail(&start, failure, observed_at)
6364 }
6365 _ => return Err(StoreError::InvalidOutboxTransition),
6366 }
6367 .map_err(|_| StoreError::InvalidOutboxTransition)?;
6368 insert_outbox_attempt_completion(&mut transaction, &completion).await?;
6369 update_outbox_delivery_completion(
6370 &mut transaction,
6371 &delivery,
6372 &start,
6373 &completion,
6374 delivery_row.attempt_count,
6375 )
6376 .await?;
6377 let attempt = OutboxAttempt::restore(start, Some(completion))
6378 .map_err(|_| StoreError::corrupt("outbox completed attempt"))?;
6379 transaction
6380 .commit()
6381 .await
6382 .map_err(|source| StoreError::database("outbox completion commit", source))?;
6383 Ok(OutboxCompletionOutcome::Committed { attempt })
6384 }
6385
6386 pub async fn claim_lease(
6398 &self,
6399 tenant_id: &TenantId,
6400 run_id: RunId,
6401 attempt_id: AttemptId,
6402 ) -> Result<LeaseClaimOutcome, StoreError> {
6403 self.acquire_lease(tenant_id, run_id, attempt_id, false)
6404 .await
6405 }
6406
6407 pub async fn supersede_lease(
6423 &self,
6424 tenant_id: &TenantId,
6425 run_id: RunId,
6426 successor_attempt_id: AttemptId,
6427 ) -> Result<LeaseClaimOutcome, StoreError> {
6428 self.acquire_lease(tenant_id, run_id, successor_attempt_id, true)
6429 .await
6430 }
6431
6432 async fn acquire_lease(
6433 &self,
6434 tenant_id: &TenantId,
6435 run_id: RunId,
6436 attempt_id: AttemptId,
6437 supersede: bool,
6438 ) -> Result<LeaseClaimOutcome, StoreError> {
6439 let operation = if supersede {
6440 "lease supersession"
6441 } else {
6442 "lease claim"
6443 };
6444 let mut transaction = self.begin_mutation(operation).await?;
6445 let row = fetch_locked_run_row(&mut transaction, tenant_id, run_id).await?;
6446 let last_epoch = row.fencing_epoch;
6447 let stored = decode_run(row)?;
6448 let observed_at = database_now(&mut transaction, "lease acquisition clock").await?;
6449
6450 if let Some(lease) = stored.lease() {
6451 if lease.fence().attempt_id() == attempt_id {
6452 if observed_at < lease.renewed_at() {
6453 return Err(StoreError::DatabaseClockRegression);
6454 }
6455 if observed_at >= lease.expires_at() {
6456 return Err(StoreError::LeaseExpired);
6457 }
6458 transaction.commit().await.map_err(|source| {
6459 StoreError::database("idempotent lease acquisition commit", source)
6460 })?;
6461 return Ok(LeaseClaimOutcome::Idempotent(lease.clone()));
6462 }
6463 if observed_at < lease.renewed_at() {
6464 return Err(StoreError::DatabaseClockRegression);
6465 }
6466 }
6467
6468 validate_runnable(&stored)?;
6469
6470 if !supersede
6471 && stored
6472 .lease()
6473 .is_some_and(|lease| observed_at < lease.expires_at())
6474 {
6475 return Err(StoreError::LeaseHeld);
6476 }
6477 if !supersede && observed_at < scheduler_available_at(&stored)? {
6478 return Err(StoreError::RunNotYetAvailable);
6479 }
6480 if last_epoch == i64::MAX {
6481 return Err(StoreError::FencingEpochExhausted);
6482 }
6483
6484 let next_epoch = last_epoch
6485 .checked_add(1)
6486 .and_then(|value| u64::try_from(value).ok())
6487 .and_then(|value| FencingEpoch::new(value).ok())
6488 .ok_or(StoreError::FencingEpochExhausted)?;
6489 let expires_at = add_duration(observed_at, self.options.lease_duration)?;
6490 let observed_db = to_database_time(observed_at)?;
6491 let expires_db = to_database_time(expires_at)?;
6492
6493 let updated = query(
6494 r"
6495UPDATE stateknot.runs
6496SET fencing_epoch = $3,
6497 lease_attempt_id = $4,
6498 lease_acquired_at = $5,
6499 lease_renewed_at = $5,
6500 lease_expires_at = $6,
6501 scheduler_not_before = NULL,
6502 updated_at = $5
6503WHERE tenant_id = $1 AND run_id = $2 AND fencing_epoch = $7
6504",
6505 )
6506 .bind(tenant_id.as_str())
6507 .bind(*run_id.as_uuid())
6508 .bind(i64::try_from(next_epoch.get()).map_err(|_| StoreError::FencingEpochExhausted)?)
6509 .bind(*attempt_id.as_uuid())
6510 .bind(observed_db)
6511 .bind(expires_db)
6512 .bind(last_epoch)
6513 .execute(&mut *transaction)
6514 .await
6515 .map_err(|source| StoreError::database("lease acquisition update", source))?
6516 .rows_affected();
6517 if updated != 1 {
6518 return Err(StoreError::corrupt("lease acquisition row count"));
6519 }
6520
6521 let lease = RunLease::new(
6522 RunFence::new(tenant_id.clone(), run_id, attempt_id, next_epoch),
6523 observed_at,
6524 expires_at,
6525 )
6526 .map_err(|_| StoreError::corrupt("claimed lease"))?;
6527 transaction
6528 .commit()
6529 .await
6530 .map_err(|source| StoreError::database("lease acquisition commit", source))?;
6531 Ok(LeaseClaimOutcome::Claimed(lease))
6532 }
6533
6534 pub async fn renew_lease(
6545 &self,
6546 fence: &RunFence,
6547 desired_expires_at: Timestamp,
6548 ) -> Result<LeaseRenewalOutcome, StoreError> {
6549 let mut transaction = self.begin_mutation("lease renewal").await?;
6550 let row = fetch_locked_run_row(&mut transaction, fence.tenant_id(), fence.run_id()).await?;
6551 let stored = decode_run(row)?;
6552 let observed_at = database_now(&mut transaction, "lease renewal clock").await?;
6553 let lease = stored.lease().ok_or(StoreError::NoActiveLease)?;
6554 if lease.fence() != fence {
6555 return Err(StoreError::StaleFence);
6556 }
6557
6558 if desired_expires_at == lease.expires_at() {
6559 transaction.commit().await.map_err(|source| {
6560 StoreError::database("idempotent lease renewal commit", source)
6561 })?;
6562 return Ok(LeaseRenewalOutcome::Idempotent(lease.clone()));
6563 }
6564 if observed_at < lease.renewed_at() {
6565 return Err(StoreError::DatabaseClockRegression);
6566 }
6567 if observed_at >= lease.expires_at() {
6568 return Err(StoreError::LeaseExpired);
6569 }
6570 if desired_expires_at < lease.expires_at() {
6571 return Err(StoreError::LeaseExpiryNotExtended);
6572 }
6573 let maximum = add_duration(observed_at, self.options.maximum_lease_horizon)?;
6574 if desired_expires_at > maximum {
6575 return Err(StoreError::LeaseHorizonExceeded);
6576 }
6577
6578 let observed_db = to_database_time(observed_at)?;
6579 let desired_db = to_database_time(desired_expires_at)?;
6580 let previous_db = to_database_time(lease.expires_at())?;
6581 let updated = query(
6582 r"
6583UPDATE stateknot.runs
6584SET lease_renewed_at = $5,
6585 lease_expires_at = $6,
6586 updated_at = $5
6587WHERE tenant_id = $1
6588 AND run_id = $2
6589 AND lease_attempt_id = $3
6590 AND fencing_epoch = $4
6591 AND lease_expires_at = $7
6592 AND lease_expires_at > $5
6593",
6594 )
6595 .bind(fence.tenant_id().as_str())
6596 .bind(*fence.run_id().as_uuid())
6597 .bind(*fence.attempt_id().as_uuid())
6598 .bind(i64::try_from(fence.epoch().get()).map_err(|_| StoreError::StaleFence)?)
6599 .bind(observed_db)
6600 .bind(desired_db)
6601 .bind(previous_db)
6602 .execute(&mut *transaction)
6603 .await
6604 .map_err(|source| StoreError::database("lease renewal update", source))?
6605 .rows_affected();
6606 if updated != 1 {
6607 return Err(StoreError::StaleFence);
6608 }
6609
6610 let renewed = lease
6611 .renewed(fence, observed_at, desired_expires_at)
6612 .map_err(|_| StoreError::corrupt("renewed lease"))?;
6613 transaction
6614 .commit()
6615 .await
6616 .map_err(|source| StoreError::database("lease renewal commit", source))?;
6617 Ok(LeaseRenewalOutcome::Renewed(renewed))
6618 }
6619
6620 pub async fn observe_live_lease(
6633 &self,
6634 fence: &RunFence,
6635 ) -> Result<LiveLeaseObservation, StoreError> {
6636 let mut transaction = self.begin_repeatable_read("live lease observation").await?;
6637 let row = query_as::<_, RunRow>(SELECT_RUN)
6638 .bind(fence.tenant_id().as_str())
6639 .bind(*fence.run_id().as_uuid())
6640 .fetch_optional(&mut *transaction)
6641 .await
6642 .map_err(|source| StoreError::database("live lease observation load", source))?
6643 .ok_or(StoreError::RunNotFound)?;
6644 let stored = decode_run(row)?;
6645 if stored.lifecycle().provenance().tenant_id() != fence.tenant_id()
6646 || stored.lifecycle().provenance().run_id() != fence.run_id()
6647 {
6648 return Err(StoreError::corrupt("live lease observation scope"));
6649 }
6650 verify_current_wait_set(&mut transaction, &stored).await?;
6651 validate_runnable(&stored)?;
6652 let observed_at = database_now(&mut transaction, "live lease observation clock").await?;
6653 authorize_worker(&stored, fence, observed_at)?;
6654 let lease = stored.lease().cloned().ok_or(StoreError::NoActiveLease)?;
6655 transaction
6656 .commit()
6657 .await
6658 .map_err(|source| StoreError::database("live lease observation commit", source))?;
6659 Ok(LiveLeaseObservation { lease, observed_at })
6660 }
6661
6662 pub async fn release_lease(&self, fence: &RunFence) -> Result<LeaseReleaseOutcome, StoreError> {
6672 let mut transaction = self.begin_mutation("lease release").await?;
6673 let row = fetch_locked_run_row(&mut transaction, fence.tenant_id(), fence.run_id()).await?;
6674 let last_epoch = row.fencing_epoch;
6675 let stored = decode_run(row)?;
6676
6677 let Some(lease) = stored.lease() else {
6678 if u64::try_from(last_epoch).ok() == Some(fence.epoch().get()) {
6679 transaction.commit().await.map_err(|source| {
6680 StoreError::database("idempotent lease release commit", source)
6681 })?;
6682 return Ok(LeaseReleaseOutcome::Idempotent);
6683 }
6684 return Err(StoreError::StaleFence);
6685 };
6686 if lease.fence() != fence {
6687 return Err(StoreError::StaleFence);
6688 }
6689
6690 let observed_at = database_now(&mut transaction, "lease release clock").await?;
6691 if observed_at < lease.renewed_at() {
6692 return Err(StoreError::DatabaseClockRegression);
6693 }
6694 let updated = query(
6695 r"
6696UPDATE stateknot.runs
6697SET lease_attempt_id = NULL,
6698 lease_acquired_at = NULL,
6699 lease_renewed_at = NULL,
6700 lease_expires_at = NULL,
6701 scheduler_ready_at = CASE
6702 WHEN lifecycle_status IN ('pending', 'active', 'cancellation_requested') THEN $5
6703 ELSE NULL
6704 END,
6705 scheduler_not_before = NULL,
6706 updated_at = $5
6707WHERE tenant_id = $1
6708 AND run_id = $2
6709 AND lease_attempt_id = $3
6710 AND fencing_epoch = $4
6711",
6712 )
6713 .bind(fence.tenant_id().as_str())
6714 .bind(*fence.run_id().as_uuid())
6715 .bind(*fence.attempt_id().as_uuid())
6716 .bind(i64::try_from(fence.epoch().get()).map_err(|_| StoreError::StaleFence)?)
6717 .bind(to_database_time(observed_at)?)
6718 .execute(&mut *transaction)
6719 .await
6720 .map_err(|source| StoreError::database("lease release update", source))?
6721 .rows_affected();
6722 if updated != 1 {
6723 return Err(StoreError::StaleFence);
6724 }
6725 transaction
6726 .commit()
6727 .await
6728 .map_err(|source| StoreError::database("lease release commit", source))?;
6729 Ok(LeaseReleaseOutcome::Released)
6730 }
6731
6732 pub async fn append_control_plane(
6739 &self,
6740 append: JournalAppend,
6741 projection: RunProjection,
6742 ) -> Result<AppendOutcome, StoreError> {
6743 if append.worker_fence().is_some() {
6744 return Err(StoreError::WrongAppendAuthority);
6745 }
6746 self.append(append, projection, AppendAuthority::ControlPlane)
6747 .await
6748 }
6749
6750 pub async fn append_worker(
6760 &self,
6761 append: JournalAppend,
6762 projection: RunProjection,
6763 ) -> Result<AppendOutcome, StoreError> {
6764 if append.worker_fence().is_none() {
6765 return Err(StoreError::WrongAppendAuthority);
6766 }
6767 self.append(append, projection, AppendAuthority::Worker)
6768 .await
6769 }
6770
6771 pub async fn prepare_tool_invocation(
6783 &self,
6784 append: JournalAppend,
6785 intent: ToolInvocationIntent,
6786 ) -> Result<ToolInvocationCommitOutcome, StoreError> {
6787 Box::pin(self.prepare_tool_invocation_inner(append, intent)).await
6788 }
6789
6790 #[allow(clippy::too_many_lines)]
6791 async fn prepare_tool_invocation_inner(
6792 &self,
6793 append: JournalAppend,
6794 intent: ToolInvocationIntent,
6795 ) -> Result<ToolInvocationCommitOutcome, StoreError> {
6796 let fence = append
6797 .worker_fence()
6798 .cloned()
6799 .ok_or(StoreError::WrongAppendAuthority)?;
6800 let tenant_id = append.intent().tenant_id().clone();
6801 let run_id = append.intent().run_id();
6802 let event_id = append.intent().event_id();
6803 if intent.tenant_id() != &tenant_id || intent.run_id() != run_id {
6804 return Err(StoreError::ToolInvocationCommitConflict);
6805 }
6806
6807 let mut transaction = self.begin_mutation("tool invocation prepare").await?;
6808 let run_row = fetch_locked_run_row(&mut transaction, &tenant_id, run_id).await?;
6809 let stored = decode_run(run_row)?;
6810
6811 let existing_event = query_as::<_, EventRow>(SELECT_EVENT_BY_ID)
6812 .bind(tenant_id.as_str())
6813 .bind(*run_id.as_uuid())
6814 .bind(*event_id.as_uuid())
6815 .fetch_optional(&mut *transaction)
6816 .await
6817 .map_err(|source| StoreError::database("tool invocation event lookup", source))?;
6818 if let Some(row) = existing_event {
6819 let projection_digest = row
6820 .projection_digest
6821 .as_deref()
6822 .map(|bytes| decode_digest(bytes, "tool invocation projection digest"))
6823 .transpose()?;
6824 let event = decode_event(row)?;
6825 if !event.matches_intent(append.intent()) {
6826 return Err(StoreError::EventIdConflict);
6827 }
6828 let intent_row = query_as::<_, ToolInvocationRow>(SELECT_TOOL_INVOCATION)
6829 .bind(tenant_id.as_str())
6830 .bind(*run_id.as_uuid())
6831 .bind(*intent.invocation_id().as_uuid())
6832 .fetch_optional(&mut *transaction)
6833 .await
6834 .map_err(|source| {
6835 StoreError::database("tool invocation idempotency intent", source)
6836 })?
6837 .ok_or(StoreError::ToolInvocationCommitConflict)?;
6838 let stored_intent = decode_tool_invocation_intent(&intent_row)?;
6839 if stored_intent != intent {
6840 return Err(StoreError::ToolInvocationIdConflict);
6841 }
6842 let revision_row =
6843 query_as::<_, ToolInvocationRevisionRow>(SELECT_TOOL_INVOCATION_REVISION_BY_ANCHOR)
6844 .bind(tenant_id.as_str())
6845 .bind(*run_id.as_uuid())
6846 .bind(
6847 i64::try_from(event.sequence().get())
6848 .map_err(|_| StoreError::JournalSequenceExhausted)?,
6849 )
6850 .fetch_optional(&mut *transaction)
6851 .await
6852 .map_err(|source| {
6853 StoreError::database("tool invocation idempotency revision", source)
6854 })?
6855 .ok_or(StoreError::ToolInvocationCommitConflict)?;
6856 if revision_row.invocation_id != *intent.invocation_id().as_uuid() {
6857 return Err(StoreError::ToolInvocationCommitConflict);
6858 }
6859 let invocation = decode_tool_invocation_revision(revision_row, &stored_intent)?;
6860 let expected = ToolInvocation::prepare(intent, event.head())
6861 .map_err(|_| StoreError::ToolInvocationCommitConflict)?;
6862 if projection_digest != Some(invocation.digest())
6863 || encode_tool_invocation_record(&invocation)?
6864 != encode_tool_invocation_record(&expected)?
6865 {
6866 return Err(StoreError::ToolInvocationCommitConflict);
6867 }
6868 verify_tool_invocation_base_checkpoint(&mut transaction, &stored_intent).await?;
6869 verify_tool_invocation_anchor(&mut transaction, &invocation).await?;
6870 transaction.commit().await.map_err(|source| {
6871 StoreError::database("idempotent tool invocation prepare commit", source)
6872 })?;
6873 return Ok(ToolInvocationCommitOutcome::Idempotent { event, invocation });
6874 }
6875
6876 let existing_intent = query_as::<_, ToolInvocationRow>(SELECT_TOOL_INVOCATION)
6877 .bind(tenant_id.as_str())
6878 .bind(*run_id.as_uuid())
6879 .bind(*intent.invocation_id().as_uuid())
6880 .fetch_optional(&mut *transaction)
6881 .await
6882 .map_err(|source| StoreError::database("tool invocation identity lookup", source))?;
6883 if existing_intent.is_some() {
6884 return Err(StoreError::ToolInvocationIdConflict);
6885 }
6886 if stored.is_quarantined() {
6887 return Err(StoreError::RunQuarantined);
6888 }
6889 if stored.lifecycle().status() != RunStatus::Active {
6890 return Err(StoreError::RunNotRunnable);
6891 }
6892 if append.expectation().head() != stored.journal_head() {
6893 return Err(StoreError::StaleJournalHead);
6894 }
6895 let current_checkpoint =
6896 load_locked_current_checkpoint(&mut transaction, &stored, &tenant_id, run_id)
6897 .await?
6898 .ok_or(StoreError::StaleCheckpointHead)?;
6899 if current_checkpoint.head() != *intent.activation().base_checkpoint() {
6900 return Err(StoreError::StaleCheckpointHead);
6901 }
6902 if !tool_invocation_activation_is_ready(¤t_checkpoint, &intent) {
6903 return Err(StoreError::InvalidToolInvocationActivation);
6904 }
6905
6906 let observed_at = database_now(&mut transaction, "tool invocation prepare clock").await?;
6907 authorize_worker(&stored, &fence, observed_at)?;
6908 let recorded_at = stored
6909 .journal_head()
6910 .map_or(observed_at, |head| observed_at.max(head.recorded_at()));
6911 let event = JournalEvent::commit(append, recorded_at)
6912 .map_err(|error| map_event_commit_error(&error))?;
6913 let invocation = ToolInvocation::prepare(intent, event.head())
6914 .map_err(|_| StoreError::InvalidToolInvocationTransition)?;
6915
6916 insert_event(&mut transaction, &event, invocation.digest()).await?;
6917 insert_tool_invocation_intent(&mut transaction, &invocation, &fence).await?;
6918 insert_initial_tool_invocation_revision(&mut transaction, &invocation, &fence).await?;
6919 update_run_head(&mut transaction, &event, None).await?;
6920 transaction
6921 .commit()
6922 .await
6923 .map_err(|source| StoreError::database("tool invocation prepare commit", source))?;
6924 Ok(ToolInvocationCommitOutcome::Committed { event, invocation })
6925 }
6926
6927 pub async fn advance_tool_invocation(
6941 &self,
6942 append: JournalAppend,
6943 expected: &ToolInvocationHead,
6944 transition: ToolInvocationTransition,
6945 ) -> Result<ToolInvocationCommitOutcome, StoreError> {
6946 Box::pin(self.advance_tool_invocation_inner(append, expected, transition, None)).await
6947 }
6948
6949 pub async fn advance_tool_invocation_with_child_budget(
6952 &self,
6953 append: JournalAppend,
6954 expected: &ToolInvocationHead,
6955 transition: ToolInvocationTransition,
6956 child_budget: Option<Digest>,
6957 ) -> Result<ToolInvocationCommitOutcome, StoreError> {
6958 Box::pin(self.advance_tool_invocation_inner(append, expected, transition, child_budget))
6959 .await
6960 }
6961
6962 #[allow(clippy::too_many_lines)]
6963 async fn advance_tool_invocation_inner(
6964 &self,
6965 append: JournalAppend,
6966 expected: &ToolInvocationHead,
6967 transition: ToolInvocationTransition,
6968 child_budget: Option<Digest>,
6969 ) -> Result<ToolInvocationCommitOutcome, StoreError> {
6970 let fence = append
6971 .worker_fence()
6972 .cloned()
6973 .ok_or(StoreError::WrongAppendAuthority)?;
6974 let tenant_id = append.intent().tenant_id().clone();
6975 let run_id = append.intent().run_id();
6976 let event_id = append.intent().event_id();
6977 if expected.tenant_id() != &tenant_id || expected.run_id() != run_id {
6978 return Err(StoreError::StaleToolInvocationHead);
6979 }
6980
6981 let mut transaction = self.begin_mutation("tool invocation advance").await?;
6982 let run_row = fetch_locked_run_row(&mut transaction, &tenant_id, run_id).await?;
6983 let stored = decode_run(run_row)?;
6984
6985 let existing_event = query_as::<_, EventRow>(SELECT_EVENT_BY_ID)
6986 .bind(tenant_id.as_str())
6987 .bind(*run_id.as_uuid())
6988 .bind(*event_id.as_uuid())
6989 .fetch_optional(&mut *transaction)
6990 .await
6991 .map_err(|source| StoreError::database("tool invocation event lookup", source))?;
6992 if let Some(row) = existing_event {
6993 let projection_digest = row
6994 .projection_digest
6995 .as_deref()
6996 .map(|bytes| decode_digest(bytes, "tool invocation projection digest"))
6997 .transpose()?;
6998 let event = decode_event(row)?;
6999 if !event.matches_intent(append.intent()) {
7000 return Err(StoreError::EventIdConflict);
7001 }
7002 let intent_row = query_as::<_, ToolInvocationRow>(SELECT_TOOL_INVOCATION)
7003 .bind(tenant_id.as_str())
7004 .bind(*run_id.as_uuid())
7005 .bind(*expected.invocation_id().as_uuid())
7006 .fetch_optional(&mut *transaction)
7007 .await
7008 .map_err(|source| {
7009 StoreError::database("tool invocation idempotency intent", source)
7010 })?
7011 .ok_or(StoreError::ToolInvocationCommitConflict)?;
7012 let intent = decode_tool_invocation_intent(&intent_row)?;
7013 let previous_row = load_tool_invocation_revision_row(
7014 &mut transaction,
7015 &tenant_id,
7016 run_id,
7017 expected.invocation_id(),
7018 expected.revision(),
7019 )
7020 .await
7021 .map_err(|error| match error {
7022 StoreError::ToolInvocationNotFound => StoreError::ToolInvocationCommitConflict,
7023 other => other,
7024 })?;
7025 let previous = decode_tool_invocation_revision(previous_row, &intent)?;
7026 if previous.head() != *expected {
7027 return Err(StoreError::ToolInvocationCommitConflict);
7028 }
7029 let expected_invocation = previous
7030 .advance(transition, event.head())
7031 .map_err(|_| StoreError::ToolInvocationCommitConflict)?;
7032 let revision_row =
7033 query_as::<_, ToolInvocationRevisionRow>(SELECT_TOOL_INVOCATION_REVISION_BY_ANCHOR)
7034 .bind(tenant_id.as_str())
7035 .bind(*run_id.as_uuid())
7036 .bind(
7037 i64::try_from(event.sequence().get())
7038 .map_err(|_| StoreError::JournalSequenceExhausted)?,
7039 )
7040 .fetch_optional(&mut *transaction)
7041 .await
7042 .map_err(|source| {
7043 StoreError::database("tool invocation idempotency revision", source)
7044 })?
7045 .ok_or(StoreError::ToolInvocationCommitConflict)?;
7046 if revision_row.invocation_id != *expected.invocation_id().as_uuid() {
7047 return Err(StoreError::ToolInvocationCommitConflict);
7048 }
7049 let invocation = decode_tool_invocation_revision(revision_row, &intent)?;
7050 if projection_digest != Some(invocation.digest())
7051 || encode_tool_invocation_record(&invocation)?
7052 != encode_tool_invocation_record(&expected_invocation)?
7053 {
7054 return Err(StoreError::ToolInvocationCommitConflict);
7055 }
7056 verify_tool_invocation_base_checkpoint(&mut transaction, &intent).await?;
7057 verify_tool_invocation_anchor(&mut transaction, &invocation).await?;
7058 transaction.commit().await.map_err(|source| {
7059 StoreError::database("idempotent tool invocation advance commit", source)
7060 })?;
7061 return Ok(ToolInvocationCommitOutcome::Idempotent { event, invocation });
7062 }
7063
7064 let intent_row = query_as::<_, ToolInvocationRow>(SELECT_TOOL_INVOCATION_FOR_UPDATE)
7065 .bind(tenant_id.as_str())
7066 .bind(*run_id.as_uuid())
7067 .bind(*expected.invocation_id().as_uuid())
7068 .fetch_optional(&mut *transaction)
7069 .await
7070 .map_err(|source| StoreError::database("tool invocation row lock", source))?
7071 .ok_or(StoreError::ToolInvocationNotFound)?;
7072 let intent = decode_tool_invocation_intent(&intent_row)?;
7073 let current_revision = nonnegative_tool_invocation_revision(intent_row.current_revision)?;
7074 let current_row = load_tool_invocation_revision_row(
7075 &mut transaction,
7076 &tenant_id,
7077 run_id,
7078 expected.invocation_id(),
7079 current_revision,
7080 )
7081 .await?;
7082 let current = decode_tool_invocation_revision(current_row, &intent)?;
7083 validate_tool_invocation_current_projection(&intent_row, ¤t)?;
7084 if current.head() != *expected {
7085 return Err(StoreError::StaleToolInvocationHead);
7086 }
7087 if stored.is_quarantined() {
7088 return Err(StoreError::RunQuarantined);
7089 }
7090 validate_tool_invocation_transition_lifecycle(&stored, transition.kind())?;
7091 if append.expectation().head() != stored.journal_head() {
7092 return Err(StoreError::StaleJournalHead);
7093 }
7094 let current_checkpoint =
7095 load_locked_current_checkpoint(&mut transaction, &stored, &tenant_id, run_id)
7096 .await?
7097 .ok_or(StoreError::StaleCheckpointHead)?;
7098 if current_checkpoint.head() != *intent.activation().base_checkpoint() {
7099 return Err(StoreError::StaleCheckpointHead);
7100 }
7101 if !tool_invocation_activation_is_ready(¤t_checkpoint, &intent) {
7102 return Err(StoreError::corrupt("tool invocation activation"));
7103 }
7104
7105 let observed_at = database_now(&mut transaction, "tool invocation advance clock").await?;
7106 if matches!(&transition, ToolInvocationTransition::StartAttempt { .. }) {
7107 child_runs::authorize_invocation_budget(
7108 &mut transaction,
7109 &tenant_id,
7110 run_id,
7111 child_budget,
7112 )
7113 .await?;
7114 }
7115 authorize_worker(&stored, &fence, observed_at)?;
7116 let recorded_at = stored
7117 .journal_head()
7118 .map_or(observed_at, |head| observed_at.max(head.recorded_at()));
7119 let event = JournalEvent::commit(append, recorded_at)
7120 .map_err(|error| map_event_commit_error(&error))?;
7121 let invocation = current
7122 .advance(transition, event.head())
7123 .map_err(|_| StoreError::InvalidToolInvocationTransition)?;
7124
7125 insert_event(&mut transaction, &event, invocation.digest()).await?;
7126 insert_successor_tool_invocation_revision(&mut transaction, &invocation, expected, &fence)
7127 .await?;
7128 update_tool_invocation_current(&mut transaction, &invocation, expected, &fence).await?;
7129 update_run_head(&mut transaction, &event, None).await?;
7130 transaction
7131 .commit()
7132 .await
7133 .map_err(|source| StoreError::database("tool invocation advance commit", source))?;
7134 Ok(ToolInvocationCommitOutcome::Committed { event, invocation })
7135 }
7136
7137 pub async fn prepare_model_invocation(
7148 &self,
7149 append: JournalAppend,
7150 intent: ModelInvocationIntent,
7151 ) -> Result<ModelInvocationCommitOutcome, StoreError> {
7152 Box::pin(self.prepare_model_invocation_inner(append, intent)).await
7153 }
7154
7155 #[allow(clippy::too_many_lines)]
7156 async fn prepare_model_invocation_inner(
7157 &self,
7158 append: JournalAppend,
7159 intent: ModelInvocationIntent,
7160 ) -> Result<ModelInvocationCommitOutcome, StoreError> {
7161 let fence = append
7162 .worker_fence()
7163 .cloned()
7164 .ok_or(StoreError::WrongAppendAuthority)?;
7165 let tenant_id = append.intent().tenant_id().clone();
7166 let run_id = append.intent().run_id();
7167 let event_id = append.intent().event_id();
7168 if intent.tenant_id() != &tenant_id || intent.run_id() != run_id {
7169 return Err(StoreError::ModelInvocationCommitConflict);
7170 }
7171
7172 let mut transaction = self.begin_mutation("model invocation prepare").await?;
7173 let run_row = fetch_locked_run_row(&mut transaction, &tenant_id, run_id).await?;
7174 let stored = decode_run(run_row)?;
7175
7176 let existing_event = query_as::<_, EventRow>(SELECT_EVENT_BY_ID)
7177 .bind(tenant_id.as_str())
7178 .bind(*run_id.as_uuid())
7179 .bind(*event_id.as_uuid())
7180 .fetch_optional(&mut *transaction)
7181 .await
7182 .map_err(|source| StoreError::database("model invocation event lookup", source))?;
7183 if let Some(row) = existing_event {
7184 let projection_digest = row
7185 .projection_digest
7186 .as_deref()
7187 .map(|bytes| decode_digest(bytes, "model invocation projection digest"))
7188 .transpose()?;
7189 let event = decode_event(row)?;
7190 if !event.matches_intent(append.intent()) {
7191 return Err(StoreError::EventIdConflict);
7192 }
7193 let intent_row = query_as::<_, ModelInvocationRow>(SELECT_MODEL_INVOCATION)
7194 .bind(tenant_id.as_str())
7195 .bind(*run_id.as_uuid())
7196 .bind(*intent.invocation_id().as_uuid())
7197 .fetch_optional(&mut *transaction)
7198 .await
7199 .map_err(|source| {
7200 StoreError::database("model invocation idempotency intent", source)
7201 })?
7202 .ok_or(StoreError::ModelInvocationCommitConflict)?;
7203 let stored_intent = decode_model_invocation_intent(&intent_row)?;
7204 if stored_intent != intent {
7205 return Err(StoreError::ModelInvocationIdConflict);
7206 }
7207 let revision_row = query_as::<_, ModelInvocationRevisionRow>(
7208 SELECT_MODEL_INVOCATION_REVISION_BY_ANCHOR,
7209 )
7210 .bind(tenant_id.as_str())
7211 .bind(*run_id.as_uuid())
7212 .bind(
7213 i64::try_from(event.sequence().get())
7214 .map_err(|_| StoreError::JournalSequenceExhausted)?,
7215 )
7216 .fetch_optional(&mut *transaction)
7217 .await
7218 .map_err(|source| {
7219 StoreError::database("model invocation idempotency revision", source)
7220 })?
7221 .ok_or(StoreError::ModelInvocationCommitConflict)?;
7222 if revision_row.invocation_id != *intent.invocation_id().as_uuid() {
7223 return Err(StoreError::ModelInvocationCommitConflict);
7224 }
7225 let invocation = decode_model_invocation_revision(revision_row, &stored_intent)?;
7226 let expected = ModelInvocation::prepare(intent, event.head())
7227 .map_err(|_| StoreError::ModelInvocationCommitConflict)?;
7228 if projection_digest != Some(invocation.digest())
7229 || encode_model_invocation_record(&invocation)?
7230 != encode_model_invocation_record(&expected)?
7231 {
7232 return Err(StoreError::ModelInvocationCommitConflict);
7233 }
7234 verify_model_invocation_base_checkpoint(&mut transaction, &stored_intent).await?;
7235 verify_model_invocation_anchor(&mut transaction, &invocation).await?;
7236 transaction.commit().await.map_err(|source| {
7237 StoreError::database("idempotent model invocation prepare commit", source)
7238 })?;
7239 return Ok(ModelInvocationCommitOutcome::Idempotent { event, invocation });
7240 }
7241
7242 let existing_intent = query_as::<_, ModelInvocationRow>(SELECT_MODEL_INVOCATION)
7243 .bind(tenant_id.as_str())
7244 .bind(*run_id.as_uuid())
7245 .bind(*intent.invocation_id().as_uuid())
7246 .fetch_optional(&mut *transaction)
7247 .await
7248 .map_err(|source| StoreError::database("model invocation identity lookup", source))?;
7249 if existing_intent.is_some() {
7250 return Err(StoreError::ModelInvocationIdConflict);
7251 }
7252 if stored.is_quarantined() {
7253 return Err(StoreError::RunQuarantined);
7254 }
7255 if stored.lifecycle().status() != RunStatus::Active {
7256 return Err(StoreError::RunNotRunnable);
7257 }
7258 if append.expectation().head() != stored.journal_head() {
7259 return Err(StoreError::StaleJournalHead);
7260 }
7261 let current_checkpoint =
7262 load_locked_current_checkpoint(&mut transaction, &stored, &tenant_id, run_id)
7263 .await?
7264 .ok_or(StoreError::StaleCheckpointHead)?;
7265 if current_checkpoint.head() != *intent.activation().base_checkpoint() {
7266 return Err(StoreError::StaleCheckpointHead);
7267 }
7268 if !model_invocation_activation_is_ready(¤t_checkpoint, &intent) {
7269 return Err(StoreError::InvalidModelInvocationActivation);
7270 }
7271
7272 let observed_at = database_now(&mut transaction, "model invocation prepare clock").await?;
7273 authorize_worker(&stored, &fence, observed_at)?;
7274 let recorded_at = stored
7275 .journal_head()
7276 .map_or(observed_at, |head| observed_at.max(head.recorded_at()));
7277 let event = JournalEvent::commit(append, recorded_at)
7278 .map_err(|error| map_event_commit_error(&error))?;
7279 let invocation = ModelInvocation::prepare(intent, event.head())
7280 .map_err(|_| StoreError::InvalidModelInvocationTransition)?;
7281
7282 insert_event(&mut transaction, &event, invocation.digest()).await?;
7283 insert_model_invocation_intent(&mut transaction, &invocation, &fence).await?;
7284 insert_initial_model_invocation_revision(&mut transaction, &invocation, &fence).await?;
7285 update_run_head(&mut transaction, &event, None).await?;
7286 transaction
7287 .commit()
7288 .await
7289 .map_err(|source| StoreError::database("model invocation prepare commit", source))?;
7290 Ok(ModelInvocationCommitOutcome::Committed { event, invocation })
7291 }
7292
7293 pub async fn advance_model_invocation(
7305 &self,
7306 append: JournalAppend,
7307 expected: &ModelInvocationHead,
7308 transition: ModelInvocationTransition,
7309 ) -> Result<ModelInvocationCommitOutcome, StoreError> {
7310 Box::pin(self.advance_model_invocation_inner(append, expected, transition, None)).await
7311 }
7312
7313 pub async fn advance_model_invocation_with_child_budget(
7316 &self,
7317 append: JournalAppend,
7318 expected: &ModelInvocationHead,
7319 transition: ModelInvocationTransition,
7320 child_budget: Option<Digest>,
7321 ) -> Result<ModelInvocationCommitOutcome, StoreError> {
7322 Box::pin(self.advance_model_invocation_inner(append, expected, transition, child_budget))
7323 .await
7324 }
7325
7326 #[allow(clippy::too_many_lines)]
7327 async fn advance_model_invocation_inner(
7328 &self,
7329 append: JournalAppend,
7330 expected: &ModelInvocationHead,
7331 transition: ModelInvocationTransition,
7332 child_budget: Option<Digest>,
7333 ) -> Result<ModelInvocationCommitOutcome, StoreError> {
7334 let fence = append
7335 .worker_fence()
7336 .cloned()
7337 .ok_or(StoreError::WrongAppendAuthority)?;
7338 let tenant_id = append.intent().tenant_id().clone();
7339 let run_id = append.intent().run_id();
7340 let event_id = append.intent().event_id();
7341 if expected.tenant_id() != &tenant_id || expected.run_id() != run_id {
7342 return Err(StoreError::StaleModelInvocationHead);
7343 }
7344
7345 let mut transaction = self.begin_mutation("model invocation advance").await?;
7346 let run_row = fetch_locked_run_row(&mut transaction, &tenant_id, run_id).await?;
7347 let stored = decode_run(run_row)?;
7348
7349 let existing_event = query_as::<_, EventRow>(SELECT_EVENT_BY_ID)
7350 .bind(tenant_id.as_str())
7351 .bind(*run_id.as_uuid())
7352 .bind(*event_id.as_uuid())
7353 .fetch_optional(&mut *transaction)
7354 .await
7355 .map_err(|source| StoreError::database("model invocation event lookup", source))?;
7356 if let Some(row) = existing_event {
7357 let projection_digest = row
7358 .projection_digest
7359 .as_deref()
7360 .map(|bytes| decode_digest(bytes, "model invocation projection digest"))
7361 .transpose()?;
7362 let event = decode_event(row)?;
7363 if !event.matches_intent(append.intent()) {
7364 return Err(StoreError::EventIdConflict);
7365 }
7366 let intent_row = query_as::<_, ModelInvocationRow>(SELECT_MODEL_INVOCATION)
7367 .bind(tenant_id.as_str())
7368 .bind(*run_id.as_uuid())
7369 .bind(*expected.invocation_id().as_uuid())
7370 .fetch_optional(&mut *transaction)
7371 .await
7372 .map_err(|source| {
7373 StoreError::database("model invocation idempotency intent", source)
7374 })?
7375 .ok_or(StoreError::ModelInvocationCommitConflict)?;
7376 let intent = decode_model_invocation_intent(&intent_row)?;
7377 let previous_row = load_model_invocation_revision_row(
7378 &mut transaction,
7379 &tenant_id,
7380 run_id,
7381 expected.invocation_id(),
7382 expected.revision(),
7383 )
7384 .await
7385 .map_err(|error| match error {
7386 StoreError::ModelInvocationNotFound => StoreError::ModelInvocationCommitConflict,
7387 other => other,
7388 })?;
7389 let previous = decode_model_invocation_revision(previous_row, &intent)?;
7390 if previous.head() != *expected {
7391 return Err(StoreError::ModelInvocationCommitConflict);
7392 }
7393 let expected_invocation = previous
7394 .advance(transition, event.head())
7395 .map_err(|_| StoreError::ModelInvocationCommitConflict)?;
7396 let revision_row = query_as::<_, ModelInvocationRevisionRow>(
7397 SELECT_MODEL_INVOCATION_REVISION_BY_ANCHOR,
7398 )
7399 .bind(tenant_id.as_str())
7400 .bind(*run_id.as_uuid())
7401 .bind(
7402 i64::try_from(event.sequence().get())
7403 .map_err(|_| StoreError::JournalSequenceExhausted)?,
7404 )
7405 .fetch_optional(&mut *transaction)
7406 .await
7407 .map_err(|source| {
7408 StoreError::database("model invocation idempotency revision", source)
7409 })?
7410 .ok_or(StoreError::ModelInvocationCommitConflict)?;
7411 if revision_row.invocation_id != *expected.invocation_id().as_uuid() {
7412 return Err(StoreError::ModelInvocationCommitConflict);
7413 }
7414 let invocation = decode_model_invocation_revision(revision_row, &intent)?;
7415 if projection_digest != Some(invocation.digest())
7416 || encode_model_invocation_record(&invocation)?
7417 != encode_model_invocation_record(&expected_invocation)?
7418 {
7419 return Err(StoreError::ModelInvocationCommitConflict);
7420 }
7421 verify_model_invocation_base_checkpoint(&mut transaction, &intent).await?;
7422 verify_model_invocation_anchor(&mut transaction, &invocation).await?;
7423 transaction.commit().await.map_err(|source| {
7424 StoreError::database("idempotent model invocation advance commit", source)
7425 })?;
7426 return Ok(ModelInvocationCommitOutcome::Idempotent { event, invocation });
7427 }
7428
7429 let intent_row = query_as::<_, ModelInvocationRow>(SELECT_MODEL_INVOCATION_FOR_UPDATE)
7430 .bind(tenant_id.as_str())
7431 .bind(*run_id.as_uuid())
7432 .bind(*expected.invocation_id().as_uuid())
7433 .fetch_optional(&mut *transaction)
7434 .await
7435 .map_err(|source| StoreError::database("model invocation row lock", source))?
7436 .ok_or(StoreError::ModelInvocationNotFound)?;
7437 let intent = decode_model_invocation_intent(&intent_row)?;
7438 let current_revision = nonnegative_model_invocation_revision(intent_row.current_revision)?;
7439 let current_row = load_model_invocation_revision_row(
7440 &mut transaction,
7441 &tenant_id,
7442 run_id,
7443 expected.invocation_id(),
7444 current_revision,
7445 )
7446 .await?;
7447 let current = decode_model_invocation_revision(current_row, &intent)?;
7448 validate_model_invocation_current_projection(&intent_row, ¤t)?;
7449 if current.head() != *expected {
7450 return Err(StoreError::StaleModelInvocationHead);
7451 }
7452 if stored.is_quarantined() {
7453 return Err(StoreError::RunQuarantined);
7454 }
7455 validate_model_invocation_transition_lifecycle(&stored, transition.kind())?;
7456 if append.expectation().head() != stored.journal_head() {
7457 return Err(StoreError::StaleJournalHead);
7458 }
7459 let current_checkpoint =
7460 load_locked_current_checkpoint(&mut transaction, &stored, &tenant_id, run_id)
7461 .await?
7462 .ok_or(StoreError::StaleCheckpointHead)?;
7463 if current_checkpoint.head() != *intent.activation().base_checkpoint() {
7464 return Err(StoreError::StaleCheckpointHead);
7465 }
7466 if !model_invocation_activation_is_ready(¤t_checkpoint, &intent) {
7467 return Err(StoreError::corrupt("model invocation activation"));
7468 }
7469
7470 let observed_at = database_now(&mut transaction, "model invocation advance clock").await?;
7471 if matches!(&transition, ModelInvocationTransition::StartAttempt { .. }) {
7472 child_runs::authorize_invocation_budget(
7473 &mut transaction,
7474 &tenant_id,
7475 run_id,
7476 child_budget,
7477 )
7478 .await?;
7479 }
7480 authorize_worker(&stored, &fence, observed_at)?;
7481 let recorded_at = stored
7482 .journal_head()
7483 .map_or(observed_at, |head| observed_at.max(head.recorded_at()));
7484 let event = JournalEvent::commit(append, recorded_at)
7485 .map_err(|error| map_event_commit_error(&error))?;
7486 let invocation = current
7487 .advance(transition, event.head())
7488 .map_err(|_| StoreError::InvalidModelInvocationTransition)?;
7489
7490 insert_event(&mut transaction, &event, invocation.digest()).await?;
7491 insert_successor_model_invocation_revision(&mut transaction, &invocation, expected, &fence)
7492 .await?;
7493 update_model_invocation_current(&mut transaction, &invocation, expected, &fence).await?;
7494 update_run_head(&mut transaction, &event, None).await?;
7495 transaction
7496 .commit()
7497 .await
7498 .map_err(|source| StoreError::database("model invocation advance commit", source))?;
7499 Ok(ModelInvocationCommitOutcome::Committed { event, invocation })
7500 }
7501
7502 pub async fn start_node_attempt(
7520 &self,
7521 append: JournalAppend,
7522 activation: NodeActivation,
7523 attempt_id: AttemptId,
7524 ) -> Result<NodeAttemptCommitOutcome, StoreError> {
7525 Box::pin(self.start_node_attempt_inner(append, activation, attempt_id)).await
7526 }
7527
7528 pub async fn start_recovered_node_attempt(
7559 &self,
7560 append: JournalAppend,
7561 plan: &ReadyNodeRecoveryPlan,
7562 node_id: &NodeId,
7563 attempt_id: AttemptId,
7564 ) -> Result<NodeAttemptCommitOutcome, StoreError> {
7565 let fence = append
7566 .worker_fence()
7567 .ok_or(StoreError::InvalidReadyNodeDispatchPlan)?;
7568 let expected_head = append
7569 .expectation()
7570 .head()
7571 .ok_or(StoreError::InvalidReadyNodeDispatchPlan)?;
7572 if fence != plan.fence()
7573 || plan.checkpoint().tenant_id() != fence.tenant_id()
7574 || plan.checkpoint().run_id() != fence.run_id()
7575 || plan.journal_head().tenant_id() != fence.tenant_id()
7576 || plan.journal_head().run_id() != fence.run_id()
7577 || expected_head.tenant_id() != fence.tenant_id()
7578 || expected_head.run_id() != fence.run_id()
7579 || expected_head.sequence() < plan.journal_head().sequence()
7580 || expected_head.recorded_at() < plan.journal_head().recorded_at()
7581 {
7582 return Err(StoreError::InvalidReadyNodeDispatchPlan);
7583 }
7584 let decision = plan
7585 .nodes()
7586 .iter()
7587 .find(|decision| decision.activation().node_id() == node_id)
7588 .ok_or(StoreError::ReadyNodeNotDispatchable)?;
7589 if decision.dispatch_reason().is_none()
7590 || decision.activation().base_checkpoint() != &plan.checkpoint().head()
7591 {
7592 return Err(StoreError::ReadyNodeNotDispatchable);
7593 }
7594 self.start_node_attempt(append, decision.activation().clone(), attempt_id)
7595 .await
7596 }
7597
7598 #[allow(clippy::too_many_lines)]
7621 pub async fn schedule_delayed_retry_wakeup(
7622 &self,
7623 plan: &ReadyNodeRecoveryPlan,
7624 ) -> Result<DelayedRetryScheduleOutcome, StoreError> {
7625 let not_before = plan
7626 .earliest_deferred_at()
7627 .ok_or(StoreError::InvalidDelayedRetryPlan)?;
7628 if plan.nodes().is_empty()
7629 || plan.nodes().iter().any(|node| {
7630 !matches!(
7631 node.kind(),
7632 RecoveryNodeKind::Completed | RecoveryNodeKind::Deferred
7633 )
7634 })
7635 || not_before <= plan.observed_at()
7636 || plan.checkpoint().tenant_id() != plan.fence().tenant_id()
7637 || plan.checkpoint().run_id() != plan.fence().run_id()
7638 || plan.journal_head().tenant_id() != plan.fence().tenant_id()
7639 || plan.journal_head().run_id() != plan.fence().run_id()
7640 {
7641 return Err(StoreError::InvalidDelayedRetryPlan);
7642 }
7643
7644 let fence = plan.fence();
7645 let mut transaction = self.begin_mutation("delayed retry wakeup").await?;
7646 let row = fetch_locked_run_row(&mut transaction, fence.tenant_id(), fence.run_id()).await?;
7647 let stored = decode_run(row)?;
7648 validate_runnable(&stored)?;
7649 if stored.lifecycle().status() != RunStatus::Active {
7650 return Err(StoreError::RunNotRunnable);
7651 }
7652 let current_checkpoint = load_locked_current_checkpoint(
7653 &mut transaction,
7654 &stored,
7655 fence.tenant_id(),
7656 fence.run_id(),
7657 )
7658 .await?
7659 .ok_or(StoreError::ReadyNodeRecoveryCheckpointMissing)?;
7660 if current_checkpoint != *plan.checkpoint() {
7661 return Err(StoreError::StaleCheckpointHead);
7662 }
7663 if stored.journal_head() != Some(plan.journal_head()) {
7664 return Err(StoreError::StaleJournalHead);
7665 }
7666
7667 if stored.lease().is_none()
7668 && stored.last_fencing_epoch() == Some(fence.epoch())
7669 && stored.scheduler_not_before() == Some(not_before)
7670 {
7671 transaction.commit().await.map_err(|source| {
7672 StoreError::database("idempotent delayed retry wakeup commit", source)
7673 })?;
7674 return Ok(DelayedRetryScheduleOutcome::Idempotent { not_before });
7675 }
7676
7677 let observed_at = database_now(&mut transaction, "delayed retry wakeup clock").await?;
7678 if observed_at < plan.observed_at() {
7679 return Err(StoreError::DatabaseClockRegression);
7680 }
7681 authorize_worker(&stored, fence, observed_at)?;
7682 if observed_at >= not_before {
7683 transaction.commit().await.map_err(|source| {
7684 StoreError::database("due delayed retry wakeup commit", source)
7685 })?;
7686 return Ok(DelayedRetryScheduleOutcome::Due { not_before });
7687 }
7688
7689 let epoch = i64::try_from(fence.epoch().get()).map_err(|_| StoreError::StaleFence)?;
7690 let journal_sequence = i64::try_from(plan.journal_head().sequence().get())
7691 .map_err(|_| StoreError::JournalSequenceExhausted)?;
7692 let checkpoint_superstep = i64::try_from(plan.checkpoint().superstep().get())
7693 .map_err(|_| StoreError::StaleCheckpointHead)?;
7694 let scheduled_at = query_scalar::<_, DateTime<Utc>>(
7695 r"
7696WITH observation AS MATERIALIZED (
7697 SELECT clock_timestamp() AS observed_at
7698)
7699UPDATE stateknot.runs
7700SET lease_attempt_id = NULL,
7701 lease_acquired_at = NULL,
7702 lease_renewed_at = NULL,
7703 lease_expires_at = NULL,
7704 scheduler_not_before = $5,
7705 updated_at = observation.observed_at
7706FROM observation
7707WHERE tenant_id = $1
7708 AND run_id = $2
7709 AND lease_attempt_id = $3
7710 AND fencing_epoch = $4
7711 AND lease_expires_at > observation.observed_at
7712 AND observation.observed_at < $5
7713 AND journal_sequence = $6
7714 AND journal_event_id = $7
7715 AND journal_recorded_at = $8
7716 AND journal_digest = $9
7717 AND checkpoint_id = $10
7718 AND checkpoint_superstep = $11
7719 AND checkpoint_digest = $12
7720 AND lifecycle_status = 'active'
7721 AND quarantined_at IS NULL
7722RETURNING observation.observed_at
7723",
7724 )
7725 .bind(fence.tenant_id().as_str())
7726 .bind(*fence.run_id().as_uuid())
7727 .bind(*fence.attempt_id().as_uuid())
7728 .bind(epoch)
7729 .bind(to_database_time(not_before)?)
7730 .bind(journal_sequence)
7731 .bind(*plan.journal_head().event_id().as_uuid())
7732 .bind(to_database_time(plan.journal_head().recorded_at())?)
7733 .bind(plan.journal_head().digest().as_bytes())
7734 .bind(*plan.checkpoint().checkpoint_id().as_uuid())
7735 .bind(checkpoint_superstep)
7736 .bind(plan.checkpoint().digest().as_bytes())
7737 .fetch_optional(&mut *transaction)
7738 .await
7739 .map_err(|source| StoreError::database("delayed retry wakeup update", source))?;
7740 if scheduled_at.is_none() {
7741 let final_observed_at =
7742 database_now(&mut transaction, "delayed retry wakeup final clock").await?;
7743 if final_observed_at < observed_at {
7744 return Err(StoreError::DatabaseClockRegression);
7745 }
7746 authorize_worker(&stored, fence, final_observed_at)?;
7747 if final_observed_at >= not_before {
7748 transaction.commit().await.map_err(|source| {
7749 StoreError::database("due delayed retry wakeup commit", source)
7750 })?;
7751 return Ok(DelayedRetryScheduleOutcome::Due { not_before });
7752 }
7753 return Err(StoreError::corrupt("delayed retry wakeup row count"));
7754 }
7755 transaction
7756 .commit()
7757 .await
7758 .map_err(|source| StoreError::database("delayed retry wakeup commit", source))?;
7759 Ok(DelayedRetryScheduleOutcome::Scheduled { not_before })
7760 }
7761
7762 #[allow(clippy::too_many_lines)]
7763 async fn start_node_attempt_inner(
7764 &self,
7765 append: JournalAppend,
7766 activation: NodeActivation,
7767 attempt_id: AttemptId,
7768 ) -> Result<NodeAttemptCommitOutcome, StoreError> {
7769 let fence = append
7770 .worker_fence()
7771 .cloned()
7772 .ok_or(StoreError::WrongAppendAuthority)?;
7773 let tenant_id = append.intent().tenant_id().clone();
7774 let run_id = append.intent().run_id();
7775 let event_id = append.intent().event_id();
7776 if activation.tenant_id() != &tenant_id || activation.run_id() != run_id {
7777 return Err(StoreError::NodeAttemptCommitConflict);
7778 }
7779
7780 let mut transaction = self.begin_mutation("node attempt start").await?;
7781 let run_row = fetch_locked_run_row(&mut transaction, &tenant_id, run_id).await?;
7782 let stored = decode_run(run_row)?;
7783 let existing_event = query_as::<_, EventRow>(SELECT_EVENT_BY_ID)
7784 .bind(tenant_id.as_str())
7785 .bind(*run_id.as_uuid())
7786 .bind(*event_id.as_uuid())
7787 .fetch_optional(&mut *transaction)
7788 .await
7789 .map_err(|source| StoreError::database("node attempt start event lookup", source))?;
7790 if let Some(row) = existing_event {
7791 let projection_digest = row
7792 .projection_digest
7793 .as_deref()
7794 .map(|bytes| decode_digest(bytes, "node attempt start projection digest"))
7795 .transpose()?;
7796 let event = decode_event(row)?;
7797 if !event.matches_intent(append.intent()) {
7798 return Err(StoreError::EventIdConflict);
7799 }
7800 let expected = NodeAttemptStart::new(activation, attempt_id, fence, event.head())
7801 .map_err(|_| StoreError::NodeAttemptCommitConflict)?;
7802 let attempt =
7803 load_node_attempt_record(&mut transaction, &tenant_id, &run_id, attempt_id)
7804 .await?
7805 .ok_or(StoreError::NodeAttemptCommitConflict)?;
7806 if projection_digest != Some(expected.digest())
7807 || encode_node_attempt_start(attempt.start())?
7808 != encode_node_attempt_start(&expected)?
7809 {
7810 return Err(StoreError::NodeAttemptCommitConflict);
7811 }
7812 verify_node_attempt(&mut transaction, &attempt).await?;
7813 transaction
7814 .commit()
7815 .await
7816 .map_err(|source| StoreError::database("node attempt start retry", source))?;
7817 return Ok(NodeAttemptCommitOutcome::Idempotent { event, attempt });
7818 }
7819
7820 if load_node_attempt_record(&mut transaction, &tenant_id, &run_id, attempt_id)
7821 .await?
7822 .is_some()
7823 {
7824 return Err(StoreError::NodeAttemptIdConflict);
7825 }
7826 if load_pending_node_result_row(&mut transaction, &activation)
7827 .await?
7828 .is_some()
7829 {
7830 return Err(StoreError::InvalidNodeAttemptTransition);
7831 }
7832 if stored.is_quarantined() {
7833 return Err(StoreError::RunQuarantined);
7834 }
7835 if stored.lifecycle().status() != RunStatus::Active {
7836 return Err(StoreError::RunNotRunnable);
7837 }
7838 if append.expectation().head() != stored.journal_head() {
7839 return Err(StoreError::StaleJournalHead);
7840 }
7841 let current_checkpoint =
7842 load_locked_current_checkpoint(&mut transaction, &stored, &tenant_id, run_id)
7843 .await?
7844 .ok_or(StoreError::StaleCheckpointHead)?;
7845 if current_checkpoint.head() != *activation.base_checkpoint() {
7846 return Err(StoreError::StaleCheckpointHead);
7847 }
7848 if !node_attempt_activation_is_ready(¤t_checkpoint, &activation) {
7849 return Err(StoreError::InvalidNodeAttemptActivation);
7850 }
7851
7852 let attempt_count = count_node_attempts(&mut transaction, &activation).await?;
7853 if attempt_count > ReadyNodeRecoveryPlanner::MAX_ATTEMPTS_PER_NODE {
7854 return Err(StoreError::corrupt(
7855 "node attempt history exceeds hard limit",
7856 ));
7857 }
7858 if attempt_count == ReadyNodeRecoveryPlanner::MAX_ATTEMPTS_PER_NODE {
7859 return Err(StoreError::NodeAttemptLimitExceeded);
7860 }
7861 let previous = load_latest_locked_node_attempt(&mut transaction, &activation).await?;
7862 if let Some(previous) = previous.as_ref() {
7863 verify_node_attempt(&mut transaction, previous).await?;
7864 }
7865 let observed_at = database_now(&mut transaction, "node attempt start clock").await?;
7866 authorize_worker(&stored, &fence, observed_at)?;
7867 let recorded_at = stored
7868 .journal_head()
7869 .map_or(observed_at, |head| observed_at.max(head.recorded_at()));
7870 let event = JournalEvent::commit(append, recorded_at)
7871 .map_err(|error| map_event_commit_error(&error))?;
7872 let start = NodeAttemptStart::new(activation.clone(), attempt_id, fence, event.head())
7873 .map_err(|_| StoreError::InvalidNodeAttemptTransition)?;
7874 let mut verifier = previous.map_or_else(
7875 NodeAttemptHistoryVerifier::new,
7876 NodeAttemptHistoryVerifier::after,
7877 );
7878 verifier
7879 .verify_next(&NodeAttempt::executing(start.clone()))
7880 .map_err(|_| StoreError::InvalidNodeAttemptTransition)?;
7881 reject_reused_node_worker_attempt(&mut transaction, &activation, start.fence()).await?;
7882
7883 insert_event(&mut transaction, &event, start.digest()).await?;
7884 insert_node_attempt_claim(&mut transaction, &start).await?;
7885 insert_node_attempt_start(&mut transaction, &start).await?;
7886 update_run_head(&mut transaction, &event, None).await?;
7887 transaction
7888 .commit()
7889 .await
7890 .map_err(|source| StoreError::database("node attempt start commit", source))?;
7891 Ok(NodeAttemptCommitOutcome::Committed {
7892 event,
7893 attempt: NodeAttempt::executing(start),
7894 })
7895 }
7896
7897 pub async fn fail_node_attempt(
7908 &self,
7909 append: JournalAppend,
7910 expected: &NodeAttemptStartHead,
7911 failure: Failure,
7912 usage: BudgetUsage,
7913 ) -> Result<NodeAttemptCommitOutcome, StoreError> {
7914 Box::pin(self.fail_node_attempt_inner(append, expected, failure, usage)).await
7915 }
7916
7917 #[allow(clippy::too_many_lines)]
7918 async fn fail_node_attempt_inner(
7919 &self,
7920 append: JournalAppend,
7921 expected: &NodeAttemptStartHead,
7922 failure: Failure,
7923 usage: BudgetUsage,
7924 ) -> Result<NodeAttemptCommitOutcome, StoreError> {
7925 let fence = append
7926 .worker_fence()
7927 .cloned()
7928 .ok_or(StoreError::WrongAppendAuthority)?;
7929 let tenant_id = append.intent().tenant_id().clone();
7930 let run_id = append.intent().run_id();
7931 let event_id = append.intent().event_id();
7932 if expected.activation().tenant_id() != &tenant_id
7933 || expected.activation().run_id() != run_id
7934 {
7935 return Err(StoreError::NodeAttemptCommitConflict);
7936 }
7937 if &fence != expected.fence() {
7938 return Err(StoreError::StaleFence);
7939 }
7940
7941 let mut transaction = self.begin_mutation("node attempt failure").await?;
7942 let run_row = fetch_locked_run_row(&mut transaction, &tenant_id, run_id).await?;
7943 let stored = decode_run(run_row)?;
7944 let existing_event = query_as::<_, EventRow>(SELECT_EVENT_BY_ID)
7945 .bind(tenant_id.as_str())
7946 .bind(*run_id.as_uuid())
7947 .bind(*event_id.as_uuid())
7948 .fetch_optional(&mut *transaction)
7949 .await
7950 .map_err(|source| StoreError::database("node failure event lookup", source))?;
7951 if let Some(row) = existing_event {
7952 let projection_digest = row
7953 .projection_digest
7954 .as_deref()
7955 .map(|bytes| decode_digest(bytes, "node failure projection digest"))
7956 .transpose()?;
7957 let event = decode_event(row)?;
7958 if !event.matches_intent(append.intent()) {
7959 return Err(StoreError::EventIdConflict);
7960 }
7961 let attempt = load_node_attempt_record(
7962 &mut transaction,
7963 &tenant_id,
7964 &run_id,
7965 expected.attempt_id(),
7966 )
7967 .await?
7968 .ok_or(StoreError::NodeAttemptCommitConflict)?;
7969 if attempt.start().head() != *expected {
7970 return Err(StoreError::StaleNodeAttemptStart);
7971 }
7972 let expected_completion = NodeAttemptCompletion::fail(
7973 attempt.start(),
7974 failure.clone(),
7975 usage.clone(),
7976 event.head(),
7977 )
7978 .map_err(|_| StoreError::NodeAttemptCommitConflict)?;
7979 let completion = attempt
7980 .completion()
7981 .ok_or(StoreError::NodeAttemptCommitConflict)?;
7982 if projection_digest != Some(expected_completion.digest())
7983 || encode_node_attempt_completion(completion)?
7984 != encode_node_attempt_completion(&expected_completion)?
7985 {
7986 return Err(StoreError::NodeAttemptCommitConflict);
7987 }
7988 let durable_event = verify_node_attempt(&mut transaction, &attempt).await?;
7989 if durable_event.head() != event.head() {
7990 return Err(StoreError::NodeAttemptCommitConflict);
7991 }
7992 transaction
7993 .commit()
7994 .await
7995 .map_err(|source| StoreError::database("node failure retry", source))?;
7996 return Ok(NodeAttemptCommitOutcome::Idempotent { event, attempt });
7997 }
7998
7999 let attempt = load_locked_node_attempt(&mut transaction, expected).await?;
8000 if let Some(completion) = attempt.completion() {
8001 let expected_completion = NodeAttemptCompletion::fail(
8002 attempt.start(),
8003 failure,
8004 usage,
8005 completion.journal_head().clone(),
8006 )
8007 .map_err(|_| StoreError::InvalidNodeAttemptTransition)?;
8008 if encode_node_attempt_completion(completion)?
8009 != encode_node_attempt_completion(&expected_completion)?
8010 {
8011 return Err(StoreError::InvalidNodeAttemptTransition);
8012 }
8013 let event = verify_node_attempt(&mut transaction, &attempt).await?;
8014 transaction
8015 .commit()
8016 .await
8017 .map_err(|source| StoreError::database("node failure semantic retry", source))?;
8018 return Ok(NodeAttemptCommitOutcome::Idempotent { event, attempt });
8019 }
8020 if load_pending_node_result_row(&mut transaction, expected.activation())
8021 .await?
8022 .is_some()
8023 {
8024 return Err(StoreError::InvalidNodeAttemptTransition);
8025 }
8026 if stored.is_quarantined() {
8027 return Err(StoreError::RunQuarantined);
8028 }
8029 validate_node_attempt_completion_lifecycle(&stored)?;
8030 if append.expectation().head() != stored.journal_head() {
8031 return Err(StoreError::StaleJournalHead);
8032 }
8033 let current_checkpoint =
8034 load_locked_current_checkpoint(&mut transaction, &stored, &tenant_id, run_id)
8035 .await?
8036 .ok_or(StoreError::StaleCheckpointHead)?;
8037 if current_checkpoint.head() != *expected.activation().base_checkpoint() {
8038 return Err(StoreError::StaleCheckpointHead);
8039 }
8040 if !node_attempt_activation_is_ready(¤t_checkpoint, expected.activation()) {
8041 return Err(StoreError::InvalidNodeAttemptActivation);
8042 }
8043
8044 let observed_at = database_now(&mut transaction, "node attempt failure clock").await?;
8045 authorize_worker(&stored, &fence, observed_at)?;
8046 let recorded_at = stored
8047 .journal_head()
8048 .map_or(observed_at, |head| observed_at.max(head.recorded_at()));
8049 let event = JournalEvent::commit(append, recorded_at)
8050 .map_err(|error| map_event_commit_error(&error))?;
8051 let completion = NodeAttemptCompletion::fail(attempt.start(), failure, usage, event.head())
8052 .map_err(|_| StoreError::InvalidNodeAttemptTransition)?;
8053 let completed = NodeAttempt::restore(attempt.start().clone(), Some(completion.clone()))
8054 .map_err(|_| StoreError::InvalidNodeAttemptTransition)?;
8055
8056 insert_event(&mut transaction, &event, completion.digest()).await?;
8057 insert_node_attempt_completion(&mut transaction, attempt.start(), &completion).await?;
8058 update_run_head(&mut transaction, &event, None).await?;
8059 transaction
8060 .commit()
8061 .await
8062 .map_err(|source| StoreError::database("node attempt failure commit", source))?;
8063 Ok(NodeAttemptCommitOutcome::Committed {
8064 event,
8065 attempt: completed,
8066 })
8067 }
8068
8069 pub async fn succeed_node_attempt(
8080 &self,
8081 append: JournalAppend,
8082 expected: &NodeAttemptStartHead,
8083 intent: PendingNodeResultIntent,
8084 usage: BudgetUsage,
8085 ) -> Result<NodeAttemptCommitOutcome, StoreError> {
8086 Box::pin(self.succeed_node_attempt_inner(append, expected, intent, usage)).await
8087 }
8088
8089 #[allow(clippy::too_many_lines)]
8090 async fn succeed_node_attempt_inner(
8091 &self,
8092 append: JournalAppend,
8093 expected: &NodeAttemptStartHead,
8094 intent: PendingNodeResultIntent,
8095 usage: BudgetUsage,
8096 ) -> Result<NodeAttemptCommitOutcome, StoreError> {
8097 let fence = append
8098 .worker_fence()
8099 .cloned()
8100 .ok_or(StoreError::WrongAppendAuthority)?;
8101 let tenant_id = append.intent().tenant_id().clone();
8102 let run_id = append.intent().run_id();
8103 let event_id = append.intent().event_id();
8104 if expected.activation().tenant_id() != &tenant_id
8105 || expected.activation().run_id() != run_id
8106 || intent.activation() != expected.activation()
8107 {
8108 return Err(StoreError::NodeAttemptCommitConflict);
8109 }
8110 if &fence != expected.fence() {
8111 return Err(StoreError::StaleFence);
8112 }
8113
8114 let mut transaction = self.begin_mutation("node attempt success").await?;
8115 let run_row = fetch_locked_run_row(&mut transaction, &tenant_id, run_id).await?;
8116 let stored = decode_run(run_row)?;
8117 let existing_event = query_as::<_, EventRow>(SELECT_EVENT_BY_ID)
8118 .bind(tenant_id.as_str())
8119 .bind(*run_id.as_uuid())
8120 .bind(*event_id.as_uuid())
8121 .fetch_optional(&mut *transaction)
8122 .await
8123 .map_err(|source| StoreError::database("node success event lookup", source))?;
8124 if let Some(row) = existing_event {
8125 let projection_digest = row
8126 .projection_digest
8127 .as_deref()
8128 .map(|bytes| decode_digest(bytes, "node success projection digest"))
8129 .transpose()?;
8130 let event = decode_event(row)?;
8131 if !event.matches_intent(append.intent()) {
8132 return Err(StoreError::EventIdConflict);
8133 }
8134 let attempt = load_node_attempt_record(
8135 &mut transaction,
8136 &tenant_id,
8137 &run_id,
8138 expected.attempt_id(),
8139 )
8140 .await?
8141 .ok_or(StoreError::NodeAttemptCommitConflict)?;
8142 if attempt.start().head() != *expected {
8143 return Err(StoreError::StaleNodeAttemptStart);
8144 }
8145 let expected_result = PendingNodeResult::commit(intent.clone(), fence, event.head())
8146 .map_err(|_| StoreError::NodeAttemptCommitConflict)?;
8147 let expected_completion = NodeAttemptCompletion::succeed(
8148 attempt.start(),
8149 expected_result.head(),
8150 usage.clone(),
8151 )
8152 .map_err(|_| StoreError::NodeAttemptCommitConflict)?;
8153 let completion = attempt
8154 .completion()
8155 .ok_or(StoreError::NodeAttemptCommitConflict)?;
8156 let result_row = load_pending_node_result_row(&mut transaction, intent.activation())
8157 .await?
8158 .ok_or(StoreError::NodeAttemptCommitConflict)?;
8159 let durable_result = decode_pending_node_result(&result_row)?;
8160 if result_row.node_attempt_id != Some(*expected.attempt_id().as_uuid())
8161 || encode_pending_node_result(&durable_result)?
8162 != encode_pending_node_result(&expected_result)?
8163 || projection_digest != Some(expected_completion.digest())
8164 || encode_node_attempt_completion(completion)?
8165 != encode_node_attempt_completion(&expected_completion)?
8166 {
8167 return Err(StoreError::NodeAttemptCommitConflict);
8168 }
8169 let durable_event = verify_node_attempt(&mut transaction, &attempt).await?;
8170 if durable_event.head() != event.head() {
8171 return Err(StoreError::NodeAttemptCommitConflict);
8172 }
8173 transaction
8174 .commit()
8175 .await
8176 .map_err(|source| StoreError::database("node success retry", source))?;
8177 return Ok(NodeAttemptCommitOutcome::Idempotent { event, attempt });
8178 }
8179
8180 let attempt = load_locked_node_attempt(&mut transaction, expected).await?;
8181 if let Some(completion) = attempt.completion() {
8182 let expected_result = PendingNodeResult::commit(
8183 intent,
8184 attempt.start().fence().clone(),
8185 completion.journal_head().clone(),
8186 )
8187 .map_err(|_| StoreError::InvalidNodeAttemptTransition)?;
8188 let expected_completion =
8189 NodeAttemptCompletion::succeed(attempt.start(), expected_result.head(), usage)
8190 .map_err(|_| StoreError::InvalidNodeAttemptTransition)?;
8191 if encode_node_attempt_completion(completion)?
8192 != encode_node_attempt_completion(&expected_completion)?
8193 {
8194 return Err(StoreError::InvalidNodeAttemptTransition);
8195 }
8196 let event = verify_node_attempt(&mut transaction, &attempt).await?;
8197 transaction
8198 .commit()
8199 .await
8200 .map_err(|source| StoreError::database("node success semantic retry", source))?;
8201 return Ok(NodeAttemptCommitOutcome::Idempotent { event, attempt });
8202 }
8203 if let Some(row) =
8204 load_pending_node_result_row(&mut transaction, intent.activation()).await?
8205 {
8206 return if row.node_attempt_id.is_some() {
8207 Err(StoreError::corrupt("node result without completion"))
8208 } else {
8209 Err(StoreError::PendingNodeResultConflict)
8210 };
8211 }
8212 if stored.is_quarantined() {
8213 return Err(StoreError::RunQuarantined);
8214 }
8215 validate_node_attempt_completion_lifecycle(&stored)?;
8216 if append.expectation().head() != stored.journal_head() {
8217 return Err(StoreError::StaleJournalHead);
8218 }
8219 let current_checkpoint =
8220 load_locked_current_checkpoint(&mut transaction, &stored, &tenant_id, run_id)
8221 .await?
8222 .ok_or(StoreError::StaleCheckpointHead)?;
8223 if current_checkpoint.head() != *expected.activation().base_checkpoint() {
8224 return Err(StoreError::StaleCheckpointHead);
8225 }
8226 if !node_attempt_activation_is_ready(¤t_checkpoint, expected.activation()) {
8227 return Err(StoreError::InvalidNodeAttemptActivation);
8228 }
8229
8230 let observed_at = database_now(&mut transaction, "node attempt success clock").await?;
8231 authorize_worker(&stored, &fence, observed_at)?;
8232 let recorded_at = stored
8233 .journal_head()
8234 .map_or(observed_at, |head| observed_at.max(head.recorded_at()));
8235 let event = JournalEvent::commit(append, recorded_at)
8236 .map_err(|error| map_event_commit_error(&error))?;
8237 let result = PendingNodeResult::commit(intent, fence.clone(), event.head())
8238 .map_err(|error| map_pending_node_result_commit_error(&error))?;
8239 let completion = NodeAttemptCompletion::succeed(attempt.start(), result.head(), usage)
8240 .map_err(|_| StoreError::InvalidNodeAttemptTransition)?;
8241 let completed = NodeAttempt::restore(attempt.start().clone(), Some(completion.clone()))
8242 .map_err(|_| StoreError::InvalidNodeAttemptTransition)?;
8243
8244 insert_event(&mut transaction, &event, completion.digest()).await?;
8245 insert_pending_node_result(&mut transaction, &result, expected.attempt_id(), &fence)
8246 .await?;
8247 insert_pending_node_result_bindings(&mut transaction, &result, &fence).await?;
8248 insert_node_attempt_completion(&mut transaction, attempt.start(), &completion).await?;
8249 update_run_head(&mut transaction, &event, None).await?;
8250 transaction
8251 .commit()
8252 .await
8253 .map_err(|source| StoreError::database("node attempt success commit", source))?;
8254 Ok(NodeAttemptCommitOutcome::Committed {
8255 event,
8256 attempt: completed,
8257 })
8258 }
8259
8260 #[allow(clippy::unused_async)] pub async fn commit_pending_node_result(
8272 &self,
8273 _append: JournalAppend,
8274 _intent: PendingNodeResultIntent,
8275 ) -> Result<PendingNodeResultCommitOutcome, StoreError> {
8276 Err(StoreError::NodeAttemptRequired)
8277 }
8278
8279 pub async fn append_control_plane_checkpoint(
8291 &self,
8292 append: JournalAppend,
8293 projection: RunProjection,
8294 checkpoint: CheckpointWrite,
8295 ) -> Result<CheckpointCommitOutcome, StoreError> {
8296 if append.worker_fence().is_some() {
8297 return Err(StoreError::WrongAppendAuthority);
8298 }
8299 if checkpoint.parent().is_some() {
8300 return Err(StoreError::CheckpointBarrierRequired);
8301 }
8302 self.append_checkpoint(
8303 append,
8304 projection,
8305 checkpoint,
8306 AppendAuthority::ControlPlane,
8307 )
8308 .await
8309 }
8310
8311 pub async fn append_worker_checkpoint(
8323 &self,
8324 append: JournalAppend,
8325 projection: RunProjection,
8326 checkpoint: CheckpointWrite,
8327 ) -> Result<CheckpointCommitOutcome, StoreError> {
8328 if append.worker_fence().is_none() {
8329 return Err(StoreError::WrongAppendAuthority);
8330 }
8331 if checkpoint.parent().is_some() {
8332 return Err(StoreError::CheckpointBarrierRequired);
8333 }
8334 self.append_checkpoint(append, projection, checkpoint, AppendAuthority::Worker)
8335 .await
8336 }
8337
8338 pub async fn append_control_plane_initial_wait_checkpoint(
8351 &self,
8352 append: JournalAppend,
8353 expected_revision: RunRevision,
8354 checkpoint: CheckpointWrite,
8355 registrations: Vec<WaitRegistrationIntent>,
8356 ) -> Result<WaitCheckpointCommitOutcome, StoreError> {
8357 if append.worker_fence().is_some() {
8358 return Err(StoreError::WrongAppendAuthority);
8359 }
8360 if checkpoint.parent().is_some() {
8361 return Err(StoreError::CheckpointBarrierRequired);
8362 }
8363 self.append_initial_wait_checkpoint(
8364 append,
8365 expected_revision,
8366 checkpoint,
8367 registrations,
8368 AppendAuthority::ControlPlane,
8369 )
8370 .await
8371 }
8372
8373 pub async fn append_worker_initial_wait_checkpoint(
8381 &self,
8382 append: JournalAppend,
8383 expected_revision: RunRevision,
8384 checkpoint: CheckpointWrite,
8385 registrations: Vec<WaitRegistrationIntent>,
8386 ) -> Result<WaitCheckpointCommitOutcome, StoreError> {
8387 if append.worker_fence().is_none() {
8388 return Err(StoreError::WrongAppendAuthority);
8389 }
8390 if checkpoint.parent().is_some() {
8391 return Err(StoreError::CheckpointBarrierRequired);
8392 }
8393 self.append_initial_wait_checkpoint(
8394 append,
8395 expected_revision,
8396 checkpoint,
8397 registrations,
8398 AppendAuthority::Worker,
8399 )
8400 .await
8401 }
8402
8403 pub async fn resolve_interrupt(
8415 &self,
8416 append: JournalAppend,
8417 expected_revision: RunRevision,
8418 resolution: InterruptResolutionIntent,
8419 ) -> Result<InterruptResolutionCommitOutcome, StoreError> {
8420 if append.worker_fence().is_some() {
8421 return Err(StoreError::WrongAppendAuthority);
8422 }
8423 self.commit_interrupt_resolution(append, expected_revision, resolution)
8424 .await
8425 }
8426
8427 pub async fn fire_timer(
8435 &self,
8436 append: JournalAppend,
8437 expected_revision: RunRevision,
8438 firing: TimerFiringIntent,
8439 ) -> Result<TimerFiringCommitOutcome, StoreError> {
8440 if append.worker_fence().is_some() {
8441 return Err(StoreError::WrongAppendAuthority);
8442 }
8443 self.commit_timer_firing(append, expected_revision, firing)
8444 .await
8445 }
8446
8447 pub async fn append_control_plane_abandon_waits(
8455 &self,
8456 append: JournalAppend,
8457 expected_revision: RunRevision,
8458 transition: RunTransition,
8459 ) -> Result<WaitAbandonmentCommitOutcome, StoreError> {
8460 if append.worker_fence().is_some() {
8461 return Err(StoreError::WrongAppendAuthority);
8462 }
8463 self.commit_wait_abandonment(
8464 append,
8465 expected_revision,
8466 transition,
8467 AppendAuthority::ControlPlane,
8468 )
8469 .await
8470 }
8471
8472 pub async fn append_worker_abandon_waits(
8479 &self,
8480 append: JournalAppend,
8481 expected_revision: RunRevision,
8482 transition: RunTransition,
8483 ) -> Result<WaitAbandonmentCommitOutcome, StoreError> {
8484 if append.worker_fence().is_none() {
8485 return Err(StoreError::WrongAppendAuthority);
8486 }
8487 self.commit_wait_abandonment(
8488 append,
8489 expected_revision,
8490 transition,
8491 AppendAuthority::Worker,
8492 )
8493 .await
8494 }
8495
8496 pub async fn append_control_plane_wait_barrier(
8505 &self,
8506 append: JournalAppend,
8507 expected_revision: RunRevision,
8508 barrier: CheckpointBarrier,
8509 registrations: Vec<WaitRegistrationIntent>,
8510 ) -> Result<WaitCheckpointCommitOutcome, StoreError> {
8511 if append.worker_fence().is_some() {
8512 return Err(StoreError::WrongAppendAuthority);
8513 }
8514 Box::pin(self.append_wait_barrier(
8515 append,
8516 expected_revision,
8517 barrier,
8518 registrations,
8519 AppendAuthority::ControlPlane,
8520 ))
8521 .await
8522 }
8523
8524 pub async fn append_worker_wait_barrier(
8532 &self,
8533 append: JournalAppend,
8534 expected_revision: RunRevision,
8535 barrier: CheckpointBarrier,
8536 registrations: Vec<WaitRegistrationIntent>,
8537 ) -> Result<WaitCheckpointCommitOutcome, StoreError> {
8538 if append.worker_fence().is_none() {
8539 return Err(StoreError::WrongAppendAuthority);
8540 }
8541 Box::pin(self.append_wait_barrier(
8542 append,
8543 expected_revision,
8544 barrier,
8545 registrations,
8546 AppendAuthority::Worker,
8547 ))
8548 .await
8549 }
8550
8551 pub async fn append_control_plane_barrier(
8566 &self,
8567 append: JournalAppend,
8568 projection: RunProjection,
8569 barrier: CheckpointBarrier,
8570 ) -> Result<BarrierCommitOutcome, StoreError> {
8571 if append.worker_fence().is_some() {
8572 return Err(StoreError::WrongAppendAuthority);
8573 }
8574 Box::pin(self.append_barrier(append, projection, barrier, AppendAuthority::ControlPlane))
8575 .await
8576 }
8577
8578 pub async fn append_worker_barrier(
8591 &self,
8592 append: JournalAppend,
8593 projection: RunProjection,
8594 barrier: CheckpointBarrier,
8595 ) -> Result<BarrierCommitOutcome, StoreError> {
8596 if append.worker_fence().is_none() {
8597 return Err(StoreError::WrongAppendAuthority);
8598 }
8599 Box::pin(self.append_barrier(append, projection, barrier, AppendAuthority::Worker)).await
8600 }
8601
8602 pub async fn load_journal_page(
8613 &self,
8614 tenant_id: &TenantId,
8615 run_id: RunId,
8616 after: Option<&JournalHead>,
8617 page_size: JournalPageSize,
8618 ) -> Result<JournalPage, StoreError> {
8619 if let Some(head) = after {
8620 if head.tenant_id() != tenant_id || head.run_id() != run_id {
8621 return Err(StoreError::InvalidJournalCursor);
8622 }
8623 }
8624
8625 let mut transaction = self.begin_repeatable_read("journal page").await?;
8626 let row = query_as::<_, RunRow>(SELECT_RUN)
8627 .bind(tenant_id.as_str())
8628 .bind(*run_id.as_uuid())
8629 .fetch_optional(&mut *transaction)
8630 .await
8631 .map_err(|source| StoreError::database("journal run snapshot", source))?
8632 .ok_or(StoreError::RunNotFound)?;
8633 let run = decode_run(row)?;
8634
8635 let mut verifier = if let Some(head) = after {
8636 let cursor_row = query_as::<_, EventRow>(SELECT_EVENT_BY_SEQUENCE)
8637 .bind(tenant_id.as_str())
8638 .bind(*run_id.as_uuid())
8639 .bind(
8640 i64::try_from(head.sequence().get())
8641 .map_err(|_| StoreError::InvalidJournalCursor)?,
8642 )
8643 .fetch_optional(&mut *transaction)
8644 .await
8645 .map_err(|source| StoreError::database("journal cursor load", source))?
8646 .ok_or(StoreError::InvalidJournalCursor)?;
8647 let cursor = decode_event(cursor_row)?;
8648 if cursor.head() != *head {
8649 return Err(StoreError::InvalidJournalCursor);
8650 }
8651 JournalChainVerifier::after(head.clone())
8652 } else {
8653 JournalChainVerifier::new()
8654 };
8655
8656 let after_sequence = after.map_or(0_i64, |head| {
8657 i64::try_from(head.sequence().get()).unwrap_or(i64::MAX)
8658 });
8659 let query_limit = i64::from(page_size.get()) + 1;
8660 let rows = query_as::<_, EventRow>(SELECT_EVENT_PAGE)
8661 .bind(tenant_id.as_str())
8662 .bind(*run_id.as_uuid())
8663 .bind(after_sequence)
8664 .bind(query_limit)
8665 .fetch_all(&mut *transaction)
8666 .await
8667 .map_err(|source| StoreError::database("journal page load", source))?;
8668
8669 let mut events = Vec::with_capacity(rows.len());
8670 for row in rows {
8671 let event = decode_event(row)?;
8672 verifier
8673 .verify_next(&event)
8674 .map_err(|_| StoreError::corrupt("journal chain"))?;
8675 events.push(event);
8676 }
8677 let has_more = events.len() > usize::from(page_size.get());
8678 if has_more {
8679 events.pop();
8680 } else if verifier.head() != run.journal_head() {
8681 return Err(StoreError::corrupt("run journal head"));
8682 }
8683
8684 transaction
8685 .commit()
8686 .await
8687 .map_err(|source| StoreError::database("journal page commit", source))?;
8688 Ok(JournalPage { events, has_more })
8689 }
8690
8691 async fn append(
8692 &self,
8693 append: JournalAppend,
8694 projection: RunProjection,
8695 authority: AppendAuthority,
8696 ) -> Result<AppendOutcome, StoreError> {
8697 let tenant_id = append.intent().tenant_id().clone();
8698 let run_id = append.intent().run_id();
8699 let event_id = append.intent().event_id();
8700 let projection_digest = projection_digest(&projection)?;
8701 let mut transaction = self.begin_mutation("journal append").await?;
8702 let row = fetch_locked_run_row(&mut transaction, &tenant_id, run_id).await?;
8703 let stored = decode_run(row)?;
8704
8705 let existing = query_as::<_, EventRow>(SELECT_EVENT_BY_ID)
8706 .bind(tenant_id.as_str())
8707 .bind(*run_id.as_uuid())
8708 .bind(*event_id.as_uuid())
8709 .fetch_optional(&mut *transaction)
8710 .await
8711 .map_err(|source| StoreError::database("journal idempotency lookup", source))?;
8712 if let Some(row) = existing {
8713 let committed_projection = row
8714 .projection_digest
8715 .as_deref()
8716 .map(|bytes| decode_digest(bytes, "journal projection digest"))
8717 .transpose()?;
8718 let event = decode_event(row)?;
8719 if !event.matches_intent(append.intent()) {
8720 return Err(StoreError::EventIdConflict);
8721 }
8722 if committed_projection != Some(projection_digest) {
8723 return Err(StoreError::ProjectionIntentConflict);
8724 }
8725 let checkpoint = query_as::<_, CheckpointRow>(SELECT_CHECKPOINT_BY_ANCHOR)
8726 .bind(tenant_id.as_str())
8727 .bind(*run_id.as_uuid())
8728 .bind(
8729 i64::try_from(event.sequence().get())
8730 .map_err(|_| StoreError::JournalSequenceExhausted)?,
8731 )
8732 .fetch_optional(&mut *transaction)
8733 .await
8734 .map_err(|source| StoreError::database("journal checkpoint lookup", source))?;
8735 if checkpoint.is_some() {
8736 return Err(StoreError::CheckpointCommitConflict);
8737 }
8738 transaction.commit().await.map_err(|source| {
8739 StoreError::database("idempotent journal append commit", source)
8740 })?;
8741 return Ok(AppendOutcome::Idempotent(event));
8742 }
8743
8744 if stored.is_quarantined() {
8745 return Err(StoreError::RunQuarantined);
8746 }
8747 if append.expectation().head() != stored.journal_head() {
8748 return Err(StoreError::StaleJournalHead);
8749 }
8750
8751 let observed_at = database_now(&mut transaction, "journal append clock").await?;
8752 match authority {
8753 AppendAuthority::ControlPlane => {
8754 if append.worker_fence().is_some() {
8755 return Err(StoreError::WrongAppendAuthority);
8756 }
8757 }
8758 AppendAuthority::Worker => {
8759 let fence = append
8760 .worker_fence()
8761 .ok_or(StoreError::WrongAppendAuthority)?;
8762 authorize_worker(&stored, fence, observed_at)?;
8763 }
8764 }
8765
8766 let recorded_at = stored
8767 .journal_head()
8768 .map_or(observed_at, |head| observed_at.max(head.recorded_at()));
8769 let prepared_projection = prepare_projection(&stored, &append, projection, recorded_at)?;
8770 let event = JournalEvent::commit(append, recorded_at)
8771 .map_err(|error| map_event_commit_error(&error))?;
8772 insert_event(&mut transaction, &event, projection_digest).await?;
8773 update_run_head(&mut transaction, &event, prepared_projection.as_ref()).await?;
8774
8775 transaction
8776 .commit()
8777 .await
8778 .map_err(|source| StoreError::database("journal append commit", source))?;
8779 Ok(AppendOutcome::Committed(event))
8780 }
8781
8782 #[allow(clippy::too_many_lines)]
8783 async fn append_with_outbox(
8784 &self,
8785 append: JournalAppend,
8786 projection: RunProjection,
8787 authority: AppendAuthority,
8788 intents: Vec<OutboxDeliveryIntent>,
8789 ) -> Result<OutboxEnqueueOutcome, StoreError> {
8790 validate_outbox_batch(&append, &intents)?;
8791 let tenant_id = append.intent().tenant_id().clone();
8792 let run_id = append.intent().run_id();
8793 let event_id = append.intent().event_id();
8794 let projection_digest = projection_digest(&projection)?;
8795 let mut transaction = self.begin_mutation("journal outbox append").await?;
8796 let row = fetch_locked_run_row(&mut transaction, &tenant_id, run_id).await?;
8797 let stored = decode_run(row)?;
8798
8799 let existing = query_as::<_, EventRow>(SELECT_EVENT_BY_ID)
8800 .bind(tenant_id.as_str())
8801 .bind(*run_id.as_uuid())
8802 .bind(*event_id.as_uuid())
8803 .fetch_optional(&mut *transaction)
8804 .await
8805 .map_err(|source| StoreError::database("outbox event idempotency lookup", source))?;
8806 if let Some(row) = existing {
8807 let committed_projection = row
8808 .projection_digest
8809 .as_deref()
8810 .map(|bytes| decode_digest(bytes, "journal projection digest"))
8811 .transpose()?;
8812 let event = decode_event(row)?;
8813 if !event.matches_intent(append.intent()) {
8814 return Err(StoreError::EventIdConflict);
8815 }
8816 if committed_projection != Some(projection_digest) {
8817 return Err(StoreError::ProjectionIntentConflict);
8818 }
8819 let checkpoint = query_as::<_, CheckpointRow>(SELECT_CHECKPOINT_BY_ANCHOR)
8820 .bind(tenant_id.as_str())
8821 .bind(*run_id.as_uuid())
8822 .bind(
8823 i64::try_from(event.sequence().get())
8824 .map_err(|_| StoreError::JournalSequenceExhausted)?,
8825 )
8826 .fetch_optional(&mut *transaction)
8827 .await
8828 .map_err(|source| StoreError::database("outbox checkpoint lookup", source))?;
8829 if checkpoint.is_some() {
8830 return Err(StoreError::CheckpointCommitConflict);
8831 }
8832
8833 let expected = materialize_outbox_deliveries(intents, &event)?;
8834 let rows =
8835 query_as::<_, OutboxDeliveryRow>(SELECT_OUTBOX_DELIVERIES_BY_ORIGIN.as_str())
8836 .bind(tenant_id.as_str())
8837 .bind(*run_id.as_uuid())
8838 .bind(
8839 i64::try_from(event.sequence().get())
8840 .map_err(|_| StoreError::JournalSequenceExhausted)?,
8841 )
8842 .fetch_all(&mut *transaction)
8843 .await
8844 .map_err(|source| {
8845 StoreError::database("outbox idempotency set load", source)
8846 })?;
8847 let mut durable = BTreeMap::new();
8848 for row in rows {
8849 let delivery = decode_outbox_delivery(&row)?;
8850 verify_outbox_projection(&mut transaction, &row, &delivery).await?;
8851 if durable
8852 .insert(delivery.intent().delivery_id(), delivery)
8853 .is_some()
8854 {
8855 return Err(StoreError::corrupt("outbox delivery identity set"));
8856 }
8857 }
8858 if durable.len() != expected.len()
8859 || expected
8860 .iter()
8861 .any(|delivery| durable.get(&delivery.intent().delivery_id()) != Some(delivery))
8862 {
8863 return Err(StoreError::OutboxEnqueueConflict);
8864 }
8865 transaction.commit().await.map_err(|source| {
8866 StoreError::database("idempotent journal outbox append commit", source)
8867 })?;
8868 return Ok(OutboxEnqueueOutcome::Idempotent {
8869 event,
8870 deliveries: expected,
8871 });
8872 }
8873
8874 if stored.is_quarantined() {
8875 return Err(StoreError::RunQuarantined);
8876 }
8877 if append.expectation().head() != stored.journal_head() {
8878 return Err(StoreError::StaleJournalHead);
8879 }
8880 let observed_at = database_now(&mut transaction, "journal outbox append clock").await?;
8881 match authority {
8882 AppendAuthority::ControlPlane => {
8883 if append.worker_fence().is_some() {
8884 return Err(StoreError::WrongAppendAuthority);
8885 }
8886 }
8887 AppendAuthority::Worker => {
8888 let fence = append
8889 .worker_fence()
8890 .ok_or(StoreError::WrongAppendAuthority)?;
8891 authorize_worker(&stored, fence, observed_at)?;
8892 }
8893 }
8894
8895 let recorded_at = stored
8896 .journal_head()
8897 .map_or(observed_at, |head| observed_at.max(head.recorded_at()));
8898 let prepared_projection = prepare_projection(&stored, &append, projection, recorded_at)?;
8899 let event = JournalEvent::commit(append, recorded_at)
8900 .map_err(|error| map_event_commit_error(&error))?;
8901 let deliveries = materialize_outbox_deliveries(intents, &event)?;
8902 for delivery in &deliveries {
8903 if load_outbox_destination_row(&mut transaction, delivery.intent().destination())
8904 .await?
8905 .is_none()
8906 {
8907 return Err(StoreError::OutboxDestinationNotFound);
8908 }
8909 }
8910
8911 insert_event(&mut transaction, &event, projection_digest).await?;
8912 for delivery in &deliveries {
8913 insert_outbox_delivery(&mut transaction, delivery, event.source()).await?;
8914 }
8915 update_run_head(&mut transaction, &event, prepared_projection.as_ref()).await?;
8916 transaction
8917 .commit()
8918 .await
8919 .map_err(|source| StoreError::database("journal outbox append commit", source))?;
8920 Ok(OutboxEnqueueOutcome::Committed { event, deliveries })
8921 }
8922
8923 #[allow(clippy::too_many_lines)]
8924 async fn commit_interrupt_resolution(
8925 &self,
8926 append: JournalAppend,
8927 expected_revision: RunRevision,
8928 resolution_intent: InterruptResolutionIntent,
8929 ) -> Result<InterruptResolutionCommitOutcome, StoreError> {
8930 let tenant_id = append.intent().tenant_id().clone();
8931 let run_id = append.intent().run_id();
8932 let event_id = append.intent().event_id();
8933 let request_head = resolution_intent.request();
8934 if request_head.tenant_id() != &tenant_id
8935 || request_head.run_id() != run_id
8936 || resolution_intent.resolution_event_id() != event_id
8937 {
8938 return Err(StoreError::InvalidInterruptResolution);
8939 }
8940 let interrupt_id = request_head.interrupt_id();
8941 let projection_digest = wait_terminal_projection_digest(
8942 INTERRUPT_RESOLUTION_PROJECTION_DIGEST_DOMAIN,
8943 expected_revision,
8944 resolution_intent.intent_digest(),
8945 )?;
8946 let mut transaction = self.begin_mutation("interrupt resolution").await?;
8947 let row = fetch_locked_run_row(&mut transaction, &tenant_id, run_id).await?;
8948 let stored = decode_run(row)?;
8949 verify_current_wait_set(&mut transaction, &stored).await?;
8950
8951 let existing_event = query_as::<_, EventRow>(SELECT_EVENT_BY_ID)
8952 .bind(tenant_id.as_str())
8953 .bind(*run_id.as_uuid())
8954 .bind(*event_id.as_uuid())
8955 .fetch_optional(&mut *transaction)
8956 .await
8957 .map_err(|source| StoreError::database("interrupt resolution event lookup", source))?;
8958 if let Some(row) = existing_event {
8959 let committed_projection = row
8960 .projection_digest
8961 .as_deref()
8962 .map(|bytes| decode_digest(bytes, "interrupt resolution projection digest"))
8963 .transpose()?;
8964 let event = decode_event(row)?;
8965 if !event.matches_intent(append.intent()) {
8966 return Err(StoreError::EventIdConflict);
8967 }
8968 if committed_projection != Some(projection_digest) {
8969 return Err(StoreError::ProjectionIntentConflict);
8970 }
8971 let row = query_as::<_, WaitRegistrationRow>(SELECT_WAIT_REGISTRATION_BY_ID.as_str())
8972 .bind(tenant_id.as_str())
8973 .bind(*run_id.as_uuid())
8974 .bind(*interrupt_id.as_uuid())
8975 .fetch_optional(&mut *transaction)
8976 .await
8977 .map_err(|source| {
8978 StoreError::database("interrupt resolution registration lookup", source)
8979 })?
8980 .ok_or(StoreError::InterruptResolutionCommitConflict)?;
8981 let record = load_interrupt_record_from_row(&mut transaction, &row).await?;
8982 let expected = InterruptResolution::commit(resolution_intent, event.head())
8983 .map_err(|_| StoreError::InvalidInterruptResolution)?;
8984 if record.request().head() != *expected.intent().request()
8985 || record.resolution() != Some(&expected)
8986 || expected.journal() != &event.head()
8987 {
8988 return Err(StoreError::InterruptResolutionCommitConflict);
8989 }
8990 transaction.commit().await.map_err(|source| {
8991 StoreError::database("idempotent interrupt resolution commit", source)
8992 })?;
8993 return Ok(InterruptResolutionCommitOutcome::Idempotent { event, record });
8994 }
8995
8996 if stored.is_quarantined() {
8997 return Err(StoreError::RunQuarantined);
8998 }
8999 if stored.lifecycle().status() != RunStatus::Waiting {
9000 return Err(StoreError::InvalidInterruptResolution);
9001 }
9002 if append.expectation().head() != stored.journal_head() {
9003 return Err(StoreError::StaleJournalHead);
9004 }
9005 let registration =
9006 query_as::<_, WaitRegistrationRow>(SELECT_WAIT_REGISTRATION_BY_ID_FOR_UPDATE.as_str())
9007 .bind(tenant_id.as_str())
9008 .bind(*run_id.as_uuid())
9009 .bind(*interrupt_id.as_uuid())
9010 .fetch_optional(&mut *transaction)
9011 .await
9012 .map_err(|source| StoreError::database("interrupt registration lock", source))?
9013 .ok_or(StoreError::WaitRegistrationNotFound)?;
9014 let wait = decode_wait_registration(®istration)?;
9015 let DurableWait::Interrupt { request } = wait else {
9016 return Err(StoreError::WaitRegistrationKindMismatch);
9017 };
9018 if registration.status != "outstanding" || request.head() != *request_head {
9019 return Err(StoreError::InvalidInterruptResolution);
9020 }
9021
9022 let observed_at = database_now(&mut transaction, "interrupt resolution clock").await?;
9023 let recorded_at = stored
9024 .journal_head()
9025 .map_or(observed_at, |head| observed_at.max(head.recorded_at()));
9026 let event = JournalEvent::commit(append, recorded_at)
9027 .map_err(|error| map_event_commit_error(&error))?;
9028 let resolution = InterruptResolution::commit(resolution_intent, event.head())
9029 .map_err(|_| StoreError::InvalidInterruptResolution)?;
9030 let prepared_projection = prepare_durable_wait_projection(
9031 &stored,
9032 &tenant_id,
9033 run_id,
9034 expected_revision,
9035 RunTransition::ResolveInterrupt {
9036 interrupt_id,
9037 resolved_at: resolution.resolved_at(),
9038 },
9039 recorded_at,
9040 )?;
9041 let record = InterruptRecord::restore(*request, Some(resolution.clone()))
9042 .map_err(|_| StoreError::InvalidInterruptResolution)?;
9043
9044 insert_event(&mut transaction, &event, projection_digest).await?;
9045 insert_interrupt_resolution(&mut transaction, record.request(), &resolution).await?;
9046 project_interrupt_resolution(&mut transaction, record.request(), &resolution).await?;
9047 update_run_head(&mut transaction, &event, Some(&prepared_projection)).await?;
9048 transaction
9049 .commit()
9050 .await
9051 .map_err(|source| StoreError::database("interrupt resolution commit", source))?;
9052 Ok(InterruptResolutionCommitOutcome::Committed { event, record })
9053 }
9054
9055 #[allow(clippy::too_many_lines)]
9056 async fn commit_timer_firing(
9057 &self,
9058 append: JournalAppend,
9059 expected_revision: RunRevision,
9060 firing_intent: TimerFiringIntent,
9061 ) -> Result<TimerFiringCommitOutcome, StoreError> {
9062 let tenant_id = append.intent().tenant_id().clone();
9063 let run_id = append.intent().run_id();
9064 let event_id = append.intent().event_id();
9065 let timer_head = firing_intent.timer();
9066 if timer_head.tenant_id() != &tenant_id
9067 || timer_head.run_id() != run_id
9068 || firing_intent.firing_event_id() != event_id
9069 {
9070 return Err(StoreError::InvalidTimerFiring);
9071 }
9072 let timer_id = timer_head.timer_id();
9073 let projection_digest = wait_terminal_projection_digest(
9074 TIMER_FIRING_PROJECTION_DIGEST_DOMAIN,
9075 expected_revision,
9076 firing_intent.intent_digest(),
9077 )?;
9078 let mut transaction = self.begin_mutation("timer firing").await?;
9079 let row = fetch_locked_run_row(&mut transaction, &tenant_id, run_id).await?;
9080 let stored = decode_run(row)?;
9081 verify_current_wait_set(&mut transaction, &stored).await?;
9082
9083 let existing_event = query_as::<_, EventRow>(SELECT_EVENT_BY_ID)
9084 .bind(tenant_id.as_str())
9085 .bind(*run_id.as_uuid())
9086 .bind(*event_id.as_uuid())
9087 .fetch_optional(&mut *transaction)
9088 .await
9089 .map_err(|source| StoreError::database("timer firing event lookup", source))?;
9090 if let Some(row) = existing_event {
9091 let committed_projection = row
9092 .projection_digest
9093 .as_deref()
9094 .map(|bytes| decode_digest(bytes, "timer firing projection digest"))
9095 .transpose()?;
9096 let event = decode_event(row)?;
9097 if !event.matches_intent(append.intent()) {
9098 return Err(StoreError::EventIdConflict);
9099 }
9100 if committed_projection != Some(projection_digest) {
9101 return Err(StoreError::ProjectionIntentConflict);
9102 }
9103 let row = query_as::<_, WaitRegistrationRow>(SELECT_WAIT_REGISTRATION_BY_ID.as_str())
9104 .bind(tenant_id.as_str())
9105 .bind(*run_id.as_uuid())
9106 .bind(*timer_id.as_uuid())
9107 .fetch_optional(&mut *transaction)
9108 .await
9109 .map_err(|source| StoreError::database("timer firing registration lookup", source))?
9110 .ok_or(StoreError::TimerFiringCommitConflict)?;
9111 let record = load_timer_record_from_row(&mut transaction, &row).await?;
9112 let expected = TimerFiring::commit(firing_intent, event.head())
9113 .map_err(|_| StoreError::InvalidTimerFiring)?;
9114 if record.timer().head() != *expected.intent().timer()
9115 || record.firing() != Some(&expected)
9116 || expected.journal() != &event.head()
9117 {
9118 return Err(StoreError::TimerFiringCommitConflict);
9119 }
9120 transaction
9121 .commit()
9122 .await
9123 .map_err(|source| StoreError::database("idempotent timer firing commit", source))?;
9124 return Ok(TimerFiringCommitOutcome::Idempotent { event, record });
9125 }
9126
9127 if stored.is_quarantined() {
9128 return Err(StoreError::RunQuarantined);
9129 }
9130 if stored.lifecycle().status() != RunStatus::Waiting {
9131 return Err(StoreError::InvalidTimerFiring);
9132 }
9133 if append.expectation().head() != stored.journal_head() {
9134 return Err(StoreError::StaleJournalHead);
9135 }
9136 let registration =
9137 query_as::<_, WaitRegistrationRow>(SELECT_WAIT_REGISTRATION_BY_ID_FOR_UPDATE.as_str())
9138 .bind(tenant_id.as_str())
9139 .bind(*run_id.as_uuid())
9140 .bind(*timer_id.as_uuid())
9141 .fetch_optional(&mut *transaction)
9142 .await
9143 .map_err(|source| StoreError::database("timer registration lock", source))?
9144 .ok_or(StoreError::WaitRegistrationNotFound)?;
9145 let wait = decode_wait_registration(®istration)?;
9146 let DurableWait::Timer { timer } = wait else {
9147 return Err(StoreError::WaitRegistrationKindMismatch);
9148 };
9149 if registration.status != "outstanding" || timer.head() != *timer_head {
9150 return Err(StoreError::InvalidTimerFiring);
9151 }
9152
9153 let observed_at = database_now(&mut transaction, "timer firing clock").await?;
9154 let recorded_at = stored
9155 .journal_head()
9156 .map_or(observed_at, |head| observed_at.max(head.recorded_at()));
9157 let event = JournalEvent::commit(append, recorded_at)
9158 .map_err(|error| map_event_commit_error(&error))?;
9159 let firing = TimerFiring::commit(firing_intent, event.head())
9160 .map_err(|_| StoreError::InvalidTimerFiring)?;
9161 let prepared_projection = prepare_durable_wait_projection(
9162 &stored,
9163 &tenant_id,
9164 run_id,
9165 expected_revision,
9166 RunTransition::FireTimer {
9167 timer_id,
9168 fired_at: firing.fired_at(),
9169 },
9170 recorded_at,
9171 )?;
9172 let record = DurableTimerRecord::restore(*timer, Some(firing.clone()))
9173 .map_err(|_| StoreError::InvalidTimerFiring)?;
9174
9175 insert_event(&mut transaction, &event, projection_digest).await?;
9176 insert_timer_firing(&mut transaction, record.timer(), &firing).await?;
9177 project_timer_firing(&mut transaction, record.timer(), &firing).await?;
9178 update_run_head(&mut transaction, &event, Some(&prepared_projection)).await?;
9179 transaction
9180 .commit()
9181 .await
9182 .map_err(|source| StoreError::database("timer firing commit", source))?;
9183 Ok(TimerFiringCommitOutcome::Committed { event, record })
9184 }
9185
9186 #[allow(clippy::too_many_lines)]
9187 async fn commit_wait_abandonment(
9188 &self,
9189 append: JournalAppend,
9190 expected_revision: RunRevision,
9191 transition: RunTransition,
9192 authority: AppendAuthority,
9193 ) -> Result<WaitAbandonmentCommitOutcome, StoreError> {
9194 let reason = wait_abandonment_transition_reason(&transition)?;
9195 let tenant_id = append.intent().tenant_id().clone();
9196 let run_id = append.intent().run_id();
9197 let event_id = append.intent().event_id();
9198 let run_projection = RunProjection::transition(expected_revision, transition.clone());
9199
9200 let mut transaction = self.begin_mutation("wait abandonment").await?;
9201 let row = fetch_locked_run_row(&mut transaction, &tenant_id, run_id).await?;
9202 let stored = decode_run(row)?;
9203 verify_current_wait_set(&mut transaction, &stored).await?;
9204
9205 let existing_event = query_as::<_, EventRow>(SELECT_EVENT_BY_ID)
9206 .bind(tenant_id.as_str())
9207 .bind(*run_id.as_uuid())
9208 .bind(*event_id.as_uuid())
9209 .fetch_optional(&mut *transaction)
9210 .await
9211 .map_err(|source| StoreError::database("wait abandonment event lookup", source))?;
9212 if let Some(row) = existing_event {
9213 let committed_projection = row
9214 .projection_digest
9215 .as_deref()
9216 .map(|bytes| decode_digest(bytes, "wait abandonment projection digest"))
9217 .transpose()?;
9218 let event = decode_event(row)?;
9219 if !event.matches_intent(append.intent()) {
9220 return Err(StoreError::EventIdConflict);
9221 }
9222 let abandonments = load_wait_abandonment_set(&mut transaction, &event).await?;
9223 if abandonments
9224 .iter()
9225 .any(|abandonment| abandonment.reason() != reason)
9226 {
9227 return Err(StoreError::WaitAbandonmentCommitConflict);
9228 }
9229 let waits = abandonments
9230 .iter()
9231 .map(|abandonment| abandonment.wait().clone())
9232 .collect::<Vec<_>>();
9233 if committed_projection
9234 != Some(wait_abandonment_projection_digest(&run_projection, &waits)?)
9235 {
9236 return Err(StoreError::ProjectionIntentConflict);
9237 }
9238 transaction.commit().await.map_err(|source| {
9239 StoreError::database("idempotent wait abandonment commit", source)
9240 })?;
9241 return Ok(WaitAbandonmentCommitOutcome::Idempotent {
9242 event,
9243 abandonments,
9244 });
9245 }
9246
9247 if stored.is_quarantined() {
9248 return Err(StoreError::RunQuarantined);
9249 }
9250 if stored.lifecycle().status() != RunStatus::Waiting {
9251 return Err(StoreError::InvalidWaitAbandonment);
9252 }
9253 if append.expectation().head() != stored.journal_head() {
9254 return Err(StoreError::StaleJournalHead);
9255 }
9256 let rows = query_as::<_, WaitRegistrationRow>(
9257 SELECT_OUTSTANDING_WAIT_REGISTRATIONS_FOR_UPDATE.as_str(),
9258 )
9259 .bind(tenant_id.as_str())
9260 .bind(*run_id.as_uuid())
9261 .fetch_all(&mut *transaction)
9262 .await
9263 .map_err(|source| StoreError::database("wait abandonment registration lock", source))?;
9264 if rows.is_empty()
9265 || rows.len() > RunWaits::MAX_LEN
9266 || rows.len() != usize::from(stored.unresolved_wait_count())
9267 {
9268 return Err(StoreError::WaitAbandonmentCommitConflict);
9269 }
9270 let mut waits = Vec::with_capacity(rows.len());
9271 for row in rows {
9272 let sequence = row.registration_sequence;
9273 let wait = decode_wait_registration(&row)?;
9274 if row.status != "outstanding" {
9275 return Err(StoreError::WaitAbandonmentCommitConflict);
9276 }
9277 verify_wait_registration_event(&mut transaction, &wait, sequence).await?;
9278 waits.push(wait);
9279 }
9280
9281 let observed_at = database_now(&mut transaction, "wait abandonment clock").await?;
9282 match authority {
9283 AppendAuthority::ControlPlane => {
9284 if append.worker_fence().is_some() {
9285 return Err(StoreError::WrongAppendAuthority);
9286 }
9287 }
9288 AppendAuthority::Worker => {
9289 let fence = append
9290 .worker_fence()
9291 .ok_or(StoreError::WrongAppendAuthority)?;
9292 authorize_worker(&stored, fence, observed_at)?;
9293 }
9294 }
9295 let recorded_at = stored
9296 .journal_head()
9297 .map_or(observed_at, |head| observed_at.max(head.recorded_at()));
9298 let event = JournalEvent::commit(append, recorded_at)
9299 .map_err(|error| map_event_commit_error(&error))?;
9300 let prepared_projection = prepare_durable_wait_projection(
9301 &stored,
9302 &tenant_id,
9303 run_id,
9304 expected_revision,
9305 transition,
9306 recorded_at,
9307 )?;
9308 let projection_digest = wait_abandonment_projection_digest(&run_projection, &waits)?;
9309 let abandonments = waits
9310 .into_iter()
9311 .map(|wait| materialize_wait_abandonment(wait, reason, event.head()))
9312 .collect::<Result<Vec<_>, _>>()?;
9313
9314 insert_event(&mut transaction, &event, projection_digest).await?;
9315 for abandonment in &abandonments {
9316 insert_wait_abandonment(&mut transaction, abandonment).await?;
9317 project_wait_abandonment(&mut transaction, abandonment).await?;
9318 }
9319 update_run_head(&mut transaction, &event, Some(&prepared_projection)).await?;
9320 transaction
9321 .commit()
9322 .await
9323 .map_err(|source| StoreError::database("wait abandonment commit", source))?;
9324 Ok(WaitAbandonmentCommitOutcome::Committed {
9325 event,
9326 abandonments,
9327 })
9328 }
9329
9330 #[allow(clippy::too_many_lines)]
9331 async fn append_initial_wait_checkpoint(
9332 &self,
9333 append: JournalAppend,
9334 expected_revision: RunRevision,
9335 checkpoint_write: CheckpointWrite,
9336 registrations: Vec<WaitRegistrationIntent>,
9337 authority: AppendAuthority,
9338 ) -> Result<WaitCheckpointCommitOutcome, StoreError> {
9339 validate_wait_registration_batch(&append, ®istrations)?;
9340 let tenant_id = append.intent().tenant_id().clone();
9341 let run_id = append.intent().run_id();
9342 let event_id = append.intent().event_id();
9343 if checkpoint_write.tenant_id() != &tenant_id
9344 || checkpoint_write.run_id() != run_id
9345 || checkpoint_write.parent().is_some()
9346 {
9347 return Err(StoreError::WaitRegistrationCommitConflict);
9348 }
9349 let projection_digest = wait_registration_projection_digest(
9350 expected_revision,
9351 &checkpoint_write,
9352 ®istrations,
9353 )?;
9354
9355 let mut transaction = self.begin_mutation("initial wait checkpoint").await?;
9356 let row = fetch_locked_run_row(&mut transaction, &tenant_id, run_id).await?;
9357 let stored = decode_run(row)?;
9358 verify_current_wait_set(&mut transaction, &stored).await?;
9359
9360 let existing_event = query_as::<_, EventRow>(SELECT_EVENT_BY_ID)
9361 .bind(tenant_id.as_str())
9362 .bind(*run_id.as_uuid())
9363 .bind(*event_id.as_uuid())
9364 .fetch_optional(&mut *transaction)
9365 .await
9366 .map_err(|source| StoreError::database("wait checkpoint event lookup", source))?;
9367 if let Some(row) = existing_event {
9368 let committed_projection = row
9369 .projection_digest
9370 .as_deref()
9371 .map(|bytes| decode_digest(bytes, "wait registration projection digest"))
9372 .transpose()?;
9373 let event = decode_event(row)?;
9374 if !event.matches_intent(append.intent()) {
9375 return Err(StoreError::EventIdConflict);
9376 }
9377 if committed_projection != Some(projection_digest) {
9378 return Err(StoreError::ProjectionIntentConflict);
9379 }
9380 let row = query_as::<_, CheckpointRow>(SELECT_CHECKPOINT_BY_ANCHOR)
9381 .bind(tenant_id.as_str())
9382 .bind(*run_id.as_uuid())
9383 .bind(
9384 i64::try_from(event.sequence().get())
9385 .map_err(|_| StoreError::JournalSequenceExhausted)?,
9386 )
9387 .fetch_optional(&mut *transaction)
9388 .await
9389 .map_err(|source| StoreError::database("wait checkpoint anchor lookup", source))?
9390 .ok_or(StoreError::WaitRegistrationCommitConflict)?;
9391 let checkpoint = decode_checkpoint(row)?;
9392 if checkpoint.checkpoint_id() != checkpoint_write.checkpoint_id()
9393 || !checkpoint.matches_write(&checkpoint_write)
9394 || checkpoint.journal_head() != &event.head()
9395 {
9396 return Err(StoreError::WaitRegistrationCommitConflict);
9397 }
9398 let waits = materialize_wait_registrations(registrations, &event)?;
9399 verify_wait_registration_set(&mut transaction, &event, &waits).await?;
9400 transaction.commit().await.map_err(|source| {
9401 StoreError::database("idempotent initial wait checkpoint commit", source)
9402 })?;
9403 return Ok(WaitCheckpointCommitOutcome::Idempotent {
9404 event,
9405 checkpoint,
9406 waits,
9407 });
9408 }
9409
9410 let existing_checkpoint = query_as::<_, CheckpointRow>(SELECT_CHECKPOINT_BY_ID)
9411 .bind(tenant_id.as_str())
9412 .bind(*run_id.as_uuid())
9413 .bind(*checkpoint_write.checkpoint_id().as_uuid())
9414 .fetch_optional(&mut *transaction)
9415 .await
9416 .map_err(|source| StoreError::database("wait checkpoint identity lookup", source))?;
9417 if existing_checkpoint.is_some() {
9418 return Err(StoreError::CheckpointIdConflict);
9419 }
9420 if stored.is_quarantined() {
9421 return Err(StoreError::RunQuarantined);
9422 }
9423 if stored.lifecycle().status() != RunStatus::Active {
9424 return Err(StoreError::RunNotRunnable);
9425 }
9426 if append.expectation().head() != stored.journal_head() {
9427 return Err(StoreError::StaleJournalHead);
9428 }
9429 if load_locked_current_checkpoint(&mut transaction, &stored, &tenant_id, run_id)
9430 .await?
9431 .is_some()
9432 {
9433 return Err(StoreError::StaleCheckpointHead);
9434 }
9435
9436 let observed_at = database_now(&mut transaction, "initial wait checkpoint clock").await?;
9437 match authority {
9438 AppendAuthority::ControlPlane => {
9439 if append.worker_fence().is_some() {
9440 return Err(StoreError::WrongAppendAuthority);
9441 }
9442 }
9443 AppendAuthority::Worker => {
9444 let fence = append
9445 .worker_fence()
9446 .ok_or(StoreError::WrongAppendAuthority)?;
9447 authorize_worker(&stored, fence, observed_at)?;
9448 }
9449 }
9450 let recorded_at = stored
9451 .journal_head()
9452 .map_or(observed_at, |head| observed_at.max(head.recorded_at()));
9453 let event = JournalEvent::commit(append, recorded_at)
9454 .map_err(|error| map_event_commit_error(&error))?;
9455 let waits = materialize_wait_registrations(registrations, &event)?;
9456 let lifecycle_waits = RunWaits::try_new(waits.iter().map(DurableWait::marker))
9457 .map_err(|_| StoreError::InvalidWaitRegistrationBatch)?;
9458 let prepared_projection = prepare_durable_wait_projection(
9459 &stored,
9460 &tenant_id,
9461 run_id,
9462 expected_revision,
9463 RunTransition::Wait {
9464 waits: lifecycle_waits,
9465 },
9466 recorded_at,
9467 )?;
9468 let checkpoint = Checkpoint::commit(checkpoint_write, event.head())
9469 .map_err(|_| StoreError::encoding("wait checkpoint commit"))?;
9470
9471 insert_event(&mut transaction, &event, projection_digest).await?;
9472 insert_checkpoint(&mut transaction, &checkpoint, event.source()).await?;
9473 for wait in &waits {
9474 insert_wait_registration(&mut transaction, wait).await?;
9475 }
9476 update_checkpoint_pointer(&mut transaction, &checkpoint, event.source()).await?;
9477 update_run_head(&mut transaction, &event, Some(&prepared_projection)).await?;
9478 transaction
9479 .commit()
9480 .await
9481 .map_err(|source| StoreError::database("initial wait checkpoint commit", source))?;
9482 Ok(WaitCheckpointCommitOutcome::Committed {
9483 event,
9484 checkpoint,
9485 waits,
9486 })
9487 }
9488
9489 #[allow(clippy::too_many_lines)]
9490 async fn append_checkpoint(
9491 &self,
9492 append: JournalAppend,
9493 projection: RunProjection,
9494 checkpoint_write: CheckpointWrite,
9495 authority: AppendAuthority,
9496 ) -> Result<CheckpointCommitOutcome, StoreError> {
9497 let tenant_id = append.intent().tenant_id().clone();
9498 let run_id = append.intent().run_id();
9499 let event_id = append.intent().event_id();
9500 if checkpoint_write.tenant_id() != &tenant_id || checkpoint_write.run_id() != run_id {
9501 return Err(StoreError::CheckpointCommitConflict);
9502 }
9503 let projection_digest = projection_digest(&projection)?;
9504
9505 let mut transaction = self.begin_mutation("checkpoint append").await?;
9506 let row = fetch_locked_run_row(&mut transaction, &tenant_id, run_id).await?;
9507 let stored = decode_run(row)?;
9508
9509 let existing_event = query_as::<_, EventRow>(SELECT_EVENT_BY_ID)
9510 .bind(tenant_id.as_str())
9511 .bind(*run_id.as_uuid())
9512 .bind(*event_id.as_uuid())
9513 .fetch_optional(&mut *transaction)
9514 .await
9515 .map_err(|source| StoreError::database("checkpoint event lookup", source))?;
9516 if let Some(row) = existing_event {
9517 let committed_projection = row
9518 .projection_digest
9519 .as_deref()
9520 .map(|bytes| decode_digest(bytes, "journal projection digest"))
9521 .transpose()?;
9522 let event = decode_event(row)?;
9523 if !event.matches_intent(append.intent()) {
9524 return Err(StoreError::EventIdConflict);
9525 }
9526 if committed_projection != Some(projection_digest) {
9527 return Err(StoreError::ProjectionIntentConflict);
9528 }
9529 let row = query_as::<_, CheckpointRow>(SELECT_CHECKPOINT_BY_ANCHOR)
9530 .bind(tenant_id.as_str())
9531 .bind(*run_id.as_uuid())
9532 .bind(
9533 i64::try_from(event.sequence().get())
9534 .map_err(|_| StoreError::JournalSequenceExhausted)?,
9535 )
9536 .fetch_optional(&mut *transaction)
9537 .await
9538 .map_err(|source| StoreError::database("checkpoint anchor lookup", source))?
9539 .ok_or(StoreError::CheckpointCommitConflict)?;
9540 let checkpoint = decode_checkpoint(row)?;
9541 if checkpoint.checkpoint_id() != checkpoint_write.checkpoint_id()
9542 || !checkpoint.matches_write(&checkpoint_write)
9543 || checkpoint.journal_head() != &event.head()
9544 {
9545 return Err(StoreError::CheckpointCommitConflict);
9546 }
9547 transaction.commit().await.map_err(|source| {
9548 StoreError::database("idempotent checkpoint append commit", source)
9549 })?;
9550 return Ok(CheckpointCommitOutcome::Idempotent { event, checkpoint });
9551 }
9552
9553 let existing_checkpoint = query_as::<_, CheckpointRow>(SELECT_CHECKPOINT_BY_ID)
9554 .bind(tenant_id.as_str())
9555 .bind(*run_id.as_uuid())
9556 .bind(*checkpoint_write.checkpoint_id().as_uuid())
9557 .fetch_optional(&mut *transaction)
9558 .await
9559 .map_err(|source| StoreError::database("checkpoint idempotency lookup", source))?;
9560 if existing_checkpoint.is_some() {
9561 return Err(StoreError::CheckpointIdConflict);
9562 }
9563 if stored.is_quarantined() {
9564 return Err(StoreError::RunQuarantined);
9565 }
9566 if append.expectation().head() != stored.journal_head() {
9567 return Err(StoreError::StaleJournalHead);
9568 }
9569
9570 let current_checkpoint =
9571 load_locked_current_checkpoint(&mut transaction, &stored, &tenant_id, run_id).await?;
9572 let expected_parent = current_checkpoint.as_ref().map(Checkpoint::head);
9573 if checkpoint_write.parent() != expected_parent.as_ref() {
9574 return Err(StoreError::StaleCheckpointHead);
9575 }
9576 if let Some(parent) = current_checkpoint.as_ref() {
9577 ensure_no_unsettled_tool_invocations(&mut transaction, parent).await?;
9578 ensure_no_unsettled_model_invocations(&mut transaction, parent).await?;
9579 }
9580
9581 let observed_at = database_now(&mut transaction, "checkpoint append clock").await?;
9582 match authority {
9583 AppendAuthority::ControlPlane => {
9584 if append.worker_fence().is_some() {
9585 return Err(StoreError::WrongAppendAuthority);
9586 }
9587 }
9588 AppendAuthority::Worker => {
9589 let fence = append
9590 .worker_fence()
9591 .ok_or(StoreError::WrongAppendAuthority)?;
9592 authorize_worker(&stored, fence, observed_at)?;
9593 }
9594 }
9595
9596 let recorded_at = stored
9597 .journal_head()
9598 .map_or(observed_at, |head| observed_at.max(head.recorded_at()));
9599 let prepared_projection = prepare_projection(&stored, &append, projection, recorded_at)?;
9600 let event = JournalEvent::commit(append, recorded_at)
9601 .map_err(|error| map_event_commit_error(&error))?;
9602 let checkpoint = Checkpoint::commit(checkpoint_write, event.head())
9603 .map_err(|_| StoreError::encoding("checkpoint commit"))?;
9604
9605 insert_event(&mut transaction, &event, projection_digest).await?;
9606 insert_checkpoint(&mut transaction, &checkpoint, event.source()).await?;
9607 update_checkpoint_pointer(&mut transaction, &checkpoint, event.source()).await?;
9608 update_run_head(&mut transaction, &event, prepared_projection.as_ref()).await?;
9609
9610 transaction
9611 .commit()
9612 .await
9613 .map_err(|source| StoreError::database("checkpoint append commit", source))?;
9614 Ok(CheckpointCommitOutcome::Committed { event, checkpoint })
9615 }
9616
9617 async fn verify_checkpoint_barrier_preflight(
9618 &self,
9619 barrier: &CheckpointBarrier,
9620 ) -> Result<(), StoreError> {
9621 let base = barrier.base_checkpoint();
9622 let mut transaction = self
9623 .begin_repeatable_read("checkpoint barrier preflight")
9624 .await?;
9625 let row = query_as::<_, CheckpointRow>(SELECT_CHECKPOINT_BY_ID)
9626 .bind(base.tenant_id().as_str())
9627 .bind(*base.run_id().as_uuid())
9628 .bind(*base.checkpoint_id().as_uuid())
9629 .fetch_optional(&mut *transaction)
9630 .await
9631 .map_err(|source| StoreError::database("checkpoint barrier base load", source))?
9632 .ok_or(StoreError::InvalidCheckpointBarrier)?;
9633 let checkpoint = decode_checkpoint(row)?;
9634 if checkpoint.head() != *base || checkpoint.ready_nodes() != barrier.base_ready_nodes() {
9635 return Err(StoreError::InvalidCheckpointBarrier);
9636 }
9637 verify_checkpoint_anchor(&mut transaction, &checkpoint).await?;
9638
9639 for expected in barrier.result_heads().iter() {
9640 let row = load_pending_node_result_row(&mut transaction, expected.activation())
9641 .await?
9642 .ok_or(StoreError::CheckpointBarrierIncomplete)?;
9643 let result = decode_pending_node_result(&row)?;
9644 if result.head() != *expected {
9645 return Err(StoreError::CheckpointBarrierResultConflict);
9646 }
9647 verify_pending_node_result(&mut transaction, &result).await?;
9648 }
9649
9650 transaction.commit().await.map_err(|source| {
9651 StoreError::database("checkpoint barrier preflight commit", source)
9652 })?;
9653 Ok(())
9654 }
9655
9656 #[allow(clippy::too_many_lines)]
9657 async fn append_wait_barrier(
9658 &self,
9659 append: JournalAppend,
9660 expected_revision: RunRevision,
9661 barrier: CheckpointBarrier,
9662 registrations: Vec<WaitRegistrationIntent>,
9663 authority: AppendAuthority,
9664 ) -> Result<WaitCheckpointCommitOutcome, StoreError> {
9665 validate_wait_registration_batch(&append, ®istrations)?;
9666 let tenant_id = append.intent().tenant_id().clone();
9667 let run_id = append.intent().run_id();
9668 let event_id = append.intent().event_id();
9669 let base = barrier.base_checkpoint();
9670 let successor = barrier.successor();
9671 if base.tenant_id() != &tenant_id
9672 || base.run_id() != run_id
9673 || successor.tenant_id() != &tenant_id
9674 || successor.run_id() != run_id
9675 {
9676 return Err(StoreError::CheckpointBarrierCommitConflict);
9677 }
9678 let projection_digest = wait_barrier_projection_digest(
9679 expected_revision,
9680 barrier.intent_digest(),
9681 ®istrations,
9682 )?;
9683 self.verify_checkpoint_barrier_preflight(&barrier).await?;
9684
9685 let mut transaction = self
9686 .begin_mutation("wait checkpoint barrier append")
9687 .await?;
9688 let row = fetch_locked_run_row(&mut transaction, &tenant_id, run_id).await?;
9689 let stored = decode_run(row)?;
9690 verify_current_wait_set(&mut transaction, &stored).await?;
9691
9692 let existing_event = query_as::<_, EventRow>(SELECT_EVENT_BY_ID)
9693 .bind(tenant_id.as_str())
9694 .bind(*run_id.as_uuid())
9695 .bind(*event_id.as_uuid())
9696 .fetch_optional(&mut *transaction)
9697 .await
9698 .map_err(|source| StoreError::database("wait barrier event lookup", source))?;
9699 if let Some(row) = existing_event {
9700 let committed_projection = row
9701 .projection_digest
9702 .as_deref()
9703 .map(|bytes| decode_digest(bytes, "wait barrier projection digest"))
9704 .transpose()?;
9705 let event = decode_event(row)?;
9706 if !event.matches_intent(append.intent()) {
9707 return Err(StoreError::EventIdConflict);
9708 }
9709 if committed_projection != Some(projection_digest) {
9710 return Err(StoreError::ProjectionIntentConflict);
9711 }
9712 let row = query_as::<_, CheckpointRow>(SELECT_CHECKPOINT_BY_ANCHOR)
9713 .bind(tenant_id.as_str())
9714 .bind(*run_id.as_uuid())
9715 .bind(
9716 i64::try_from(event.sequence().get())
9717 .map_err(|_| StoreError::JournalSequenceExhausted)?,
9718 )
9719 .fetch_optional(&mut *transaction)
9720 .await
9721 .map_err(|source| StoreError::database("wait barrier anchor lookup", source))?
9722 .ok_or(StoreError::CheckpointBarrierCommitConflict)?;
9723 let checkpoint = decode_checkpoint(row)?;
9724 if checkpoint.checkpoint_id() != successor.checkpoint_id()
9725 || !checkpoint.matches_write(successor)
9726 || checkpoint.journal_head() != &event.head()
9727 {
9728 return Err(StoreError::CheckpointBarrierCommitConflict);
9729 }
9730 let waits = materialize_wait_registrations(registrations, &event)?;
9731 verify_barrier_consumptions(&mut transaction, &barrier, &checkpoint).await?;
9732 verify_wait_registration_set(&mut transaction, &event, &waits).await?;
9733 transaction.commit().await.map_err(|source| {
9734 StoreError::database("idempotent wait checkpoint barrier commit", source)
9735 })?;
9736 return Ok(WaitCheckpointCommitOutcome::Idempotent {
9737 event,
9738 checkpoint,
9739 waits,
9740 });
9741 }
9742
9743 let existing_checkpoint = query_as::<_, CheckpointRow>(SELECT_CHECKPOINT_BY_ID)
9744 .bind(tenant_id.as_str())
9745 .bind(*run_id.as_uuid())
9746 .bind(*successor.checkpoint_id().as_uuid())
9747 .fetch_optional(&mut *transaction)
9748 .await
9749 .map_err(|source| StoreError::database("wait barrier checkpoint lookup", source))?;
9750 if existing_checkpoint.is_some() {
9751 return Err(StoreError::CheckpointIdConflict);
9752 }
9753 if stored.is_quarantined() {
9754 return Err(StoreError::RunQuarantined);
9755 }
9756 if stored.lifecycle().status() != RunStatus::Active {
9757 return Err(StoreError::RunNotRunnable);
9758 }
9759 if append.expectation().head() != stored.journal_head() {
9760 return Err(StoreError::StaleJournalHead);
9761 }
9762
9763 let current_checkpoint =
9764 load_locked_current_checkpoint(&mut transaction, &stored, &tenant_id, run_id)
9765 .await?
9766 .ok_or(StoreError::StaleCheckpointHead)?;
9767 if current_checkpoint.head() != *base {
9768 return Err(StoreError::StaleCheckpointHead);
9769 }
9770 if current_checkpoint.ready_nodes() != barrier.base_ready_nodes() {
9771 return Err(StoreError::InvalidCheckpointBarrier);
9772 }
9773 ensure_no_unsettled_tool_invocations(&mut transaction, ¤t_checkpoint).await?;
9774 ensure_no_unsettled_model_invocations(&mut transaction, ¤t_checkpoint).await?;
9775
9776 let existing_consumptions = load_barrier_consumption_rows(&mut transaction, base).await?;
9777 if !existing_consumptions.is_empty() {
9778 return Err(StoreError::CheckpointBarrierResultConflict);
9779 }
9780 let durable_heads = load_locked_barrier_result_heads(&mut transaction, base).await?;
9781 validate_complete_barrier_result_heads(&durable_heads, barrier.result_heads())?;
9782 let current_journal = stored
9783 .journal_head()
9784 .ok_or_else(|| StoreError::corrupt("wait barrier run journal head"))?;
9785 if barrier.result_heads().iter().any(|head| {
9786 head.journal_head().sequence() > current_journal.sequence()
9787 || head.journal_head().recorded_at() > current_journal.recorded_at()
9788 }) {
9789 return Err(StoreError::CheckpointBarrierResultConflict);
9790 }
9791
9792 let observed_at = database_now(&mut transaction, "wait checkpoint barrier clock").await?;
9793 match authority {
9794 AppendAuthority::ControlPlane => {
9795 if append.worker_fence().is_some() {
9796 return Err(StoreError::WrongAppendAuthority);
9797 }
9798 }
9799 AppendAuthority::Worker => {
9800 let fence = append
9801 .worker_fence()
9802 .ok_or(StoreError::WrongAppendAuthority)?;
9803 authorize_worker(&stored, fence, observed_at)?;
9804 }
9805 }
9806
9807 let recorded_at = observed_at.max(current_journal.recorded_at());
9808 let event = JournalEvent::commit(append, recorded_at)
9809 .map_err(|error| map_event_commit_error(&error))?;
9810 let waits = materialize_wait_registrations(registrations, &event)?;
9811 let lifecycle_waits = RunWaits::try_new(waits.iter().map(DurableWait::marker))
9812 .map_err(|_| StoreError::InvalidWaitRegistrationBatch)?;
9813 let prepared_projection = prepare_durable_wait_projection(
9814 &stored,
9815 &tenant_id,
9816 run_id,
9817 expected_revision,
9818 RunTransition::Wait {
9819 waits: lifecycle_waits,
9820 },
9821 recorded_at,
9822 )?;
9823 let checkpoint = Checkpoint::commit(successor.clone(), event.head())
9824 .map_err(|_| StoreError::encoding("wait checkpoint barrier commit"))?;
9825
9826 insert_event(&mut transaction, &event, projection_digest).await?;
9827 insert_checkpoint(&mut transaction, &checkpoint, event.source()).await?;
9828 for wait in &waits {
9829 insert_wait_registration(&mut transaction, wait).await?;
9830 }
9831 insert_barrier_consumptions(&mut transaction, &barrier, &checkpoint, event.source())
9832 .await?;
9833 update_checkpoint_pointer(&mut transaction, &checkpoint, event.source()).await?;
9834 update_run_head(&mut transaction, &event, Some(&prepared_projection)).await?;
9835 transaction
9836 .commit()
9837 .await
9838 .map_err(|source| StoreError::database("wait checkpoint barrier commit", source))?;
9839 Ok(WaitCheckpointCommitOutcome::Committed {
9840 event,
9841 checkpoint,
9842 waits,
9843 })
9844 }
9845
9846 #[allow(clippy::too_many_lines)]
9847 async fn append_barrier(
9848 &self,
9849 append: JournalAppend,
9850 projection: RunProjection,
9851 barrier: CheckpointBarrier,
9852 authority: AppendAuthority,
9853 ) -> Result<BarrierCommitOutcome, StoreError> {
9854 let tenant_id = append.intent().tenant_id().clone();
9855 let run_id = append.intent().run_id();
9856 let event_id = append.intent().event_id();
9857 let base = barrier.base_checkpoint();
9858 let successor = barrier.successor();
9859 if base.tenant_id() != &tenant_id
9860 || base.run_id() != run_id
9861 || successor.tenant_id() != &tenant_id
9862 || successor.run_id() != run_id
9863 {
9864 return Err(StoreError::CheckpointBarrierCommitConflict);
9865 }
9866 let projection_digest = barrier_projection_digest(&projection, barrier.intent_digest())?;
9867 self.verify_checkpoint_barrier_preflight(&barrier).await?;
9868
9869 let mut transaction = self.begin_mutation("checkpoint barrier append").await?;
9870 let row = fetch_locked_run_row(&mut transaction, &tenant_id, run_id).await?;
9871 let stored = decode_run(row)?;
9872
9873 let existing_event = query_as::<_, EventRow>(SELECT_EVENT_BY_ID)
9874 .bind(tenant_id.as_str())
9875 .bind(*run_id.as_uuid())
9876 .bind(*event_id.as_uuid())
9877 .fetch_optional(&mut *transaction)
9878 .await
9879 .map_err(|source| StoreError::database("checkpoint barrier event lookup", source))?;
9880 if let Some(row) = existing_event {
9881 let committed_projection = row
9882 .projection_digest
9883 .as_deref()
9884 .map(|bytes| decode_digest(bytes, "checkpoint barrier projection digest"))
9885 .transpose()?;
9886 let event = decode_event(row)?;
9887 if !event.matches_intent(append.intent()) {
9888 return Err(StoreError::EventIdConflict);
9889 }
9890 if committed_projection != Some(projection_digest) {
9891 return Err(StoreError::ProjectionIntentConflict);
9892 }
9893 let row = query_as::<_, CheckpointRow>(SELECT_CHECKPOINT_BY_ANCHOR)
9894 .bind(tenant_id.as_str())
9895 .bind(*run_id.as_uuid())
9896 .bind(
9897 i64::try_from(event.sequence().get())
9898 .map_err(|_| StoreError::JournalSequenceExhausted)?,
9899 )
9900 .fetch_optional(&mut *transaction)
9901 .await
9902 .map_err(|source| StoreError::database("checkpoint barrier anchor lookup", source))?
9903 .ok_or(StoreError::CheckpointBarrierCommitConflict)?;
9904 let checkpoint = decode_checkpoint(row)?;
9905 if checkpoint.checkpoint_id() != successor.checkpoint_id()
9906 || !checkpoint.matches_write(successor)
9907 || checkpoint.journal_head() != &event.head()
9908 {
9909 return Err(StoreError::CheckpointBarrierCommitConflict);
9910 }
9911 verify_barrier_consumptions(&mut transaction, &barrier, &checkpoint).await?;
9912 transaction.commit().await.map_err(|source| {
9913 StoreError::database("idempotent checkpoint barrier commit", source)
9914 })?;
9915 return Ok(BarrierCommitOutcome::Idempotent { event, checkpoint });
9916 }
9917
9918 let existing_checkpoint = query_as::<_, CheckpointRow>(SELECT_CHECKPOINT_BY_ID)
9919 .bind(tenant_id.as_str())
9920 .bind(*run_id.as_uuid())
9921 .bind(*successor.checkpoint_id().as_uuid())
9922 .fetch_optional(&mut *transaction)
9923 .await
9924 .map_err(|source| {
9925 StoreError::database("checkpoint barrier idempotency lookup", source)
9926 })?;
9927 if existing_checkpoint.is_some() {
9928 return Err(StoreError::CheckpointIdConflict);
9929 }
9930 if stored.is_quarantined() {
9931 return Err(StoreError::RunQuarantined);
9932 }
9933 if stored.lifecycle().status() != RunStatus::Active {
9934 return Err(StoreError::RunNotRunnable);
9935 }
9936 if append.expectation().head() != stored.journal_head() {
9937 return Err(StoreError::StaleJournalHead);
9938 }
9939
9940 let current_checkpoint =
9941 load_locked_current_checkpoint(&mut transaction, &stored, &tenant_id, run_id)
9942 .await?
9943 .ok_or(StoreError::StaleCheckpointHead)?;
9944 if current_checkpoint.head() != *base {
9945 return Err(StoreError::StaleCheckpointHead);
9946 }
9947 if current_checkpoint.ready_nodes() != barrier.base_ready_nodes() {
9948 return Err(StoreError::InvalidCheckpointBarrier);
9949 }
9950 ensure_no_unsettled_tool_invocations(&mut transaction, ¤t_checkpoint).await?;
9951 ensure_no_unsettled_model_invocations(&mut transaction, ¤t_checkpoint).await?;
9952
9953 let existing_consumptions = load_barrier_consumption_rows(&mut transaction, base).await?;
9954 if !existing_consumptions.is_empty() {
9955 return Err(StoreError::CheckpointBarrierResultConflict);
9956 }
9957 let durable_heads = load_locked_barrier_result_heads(&mut transaction, base).await?;
9958 validate_complete_barrier_result_heads(&durable_heads, barrier.result_heads())?;
9959 let current_journal = stored
9960 .journal_head()
9961 .ok_or_else(|| StoreError::corrupt("checkpoint barrier run journal head"))?;
9962 if barrier.result_heads().iter().any(|head| {
9963 head.journal_head().sequence() > current_journal.sequence()
9964 || head.journal_head().recorded_at() > current_journal.recorded_at()
9965 }) {
9966 return Err(StoreError::CheckpointBarrierResultConflict);
9967 }
9968
9969 let observed_at = database_now(&mut transaction, "checkpoint barrier clock").await?;
9970 match authority {
9971 AppendAuthority::ControlPlane => {
9972 if append.worker_fence().is_some() {
9973 return Err(StoreError::WrongAppendAuthority);
9974 }
9975 }
9976 AppendAuthority::Worker => {
9977 let fence = append
9978 .worker_fence()
9979 .ok_or(StoreError::WrongAppendAuthority)?;
9980 authorize_worker(&stored, fence, observed_at)?;
9981 }
9982 }
9983
9984 let recorded_at = observed_at.max(current_journal.recorded_at());
9985 let prepared_projection = prepare_projection(&stored, &append, projection, recorded_at)?;
9986 let event = JournalEvent::commit(append, recorded_at)
9987 .map_err(|error| map_event_commit_error(&error))?;
9988 let checkpoint = Checkpoint::commit(successor.clone(), event.head())
9989 .map_err(|_| StoreError::encoding("checkpoint barrier commit"))?;
9990
9991 insert_event(&mut transaction, &event, projection_digest).await?;
9992 insert_checkpoint(&mut transaction, &checkpoint, event.source()).await?;
9993 insert_barrier_consumptions(&mut transaction, &barrier, &checkpoint, event.source())
9994 .await?;
9995 update_checkpoint_pointer(&mut transaction, &checkpoint, event.source()).await?;
9996 update_run_head(&mut transaction, &event, prepared_projection.as_ref()).await?;
9997
9998 transaction
9999 .commit()
10000 .await
10001 .map_err(|source| StoreError::database("checkpoint barrier commit", source))?;
10002 Ok(BarrierCommitOutcome::Committed { event, checkpoint })
10003 }
10004
10005 async fn begin_mutation(
10006 &self,
10007 operation: &'static str,
10008 ) -> Result<Transaction<'_, Postgres>, StoreError> {
10009 let pool = self.pool.clone();
10010 let mut transaction = CompleteOnDrop::new(async move {
10011 pool.begin_with("BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED, READ WRITE")
10012 .await
10013 })
10014 .await
10015 .map_err(|source| StoreError::database(operation, source))?;
10016 apply_transaction_timeouts(&mut transaction, &self.options, operation).await?;
10017 query("SET LOCAL stateknot.failure_close_version = '1'")
10018 .execute(&mut *transaction)
10019 .await
10020 .map_err(|source| StoreError::database(operation, source))?;
10021 query("SET LOCAL stateknot.child_join_version = '1'")
10022 .execute(&mut *transaction)
10023 .await
10024 .map_err(|source| StoreError::database("child Join capability", source))?;
10025 query("SET LOCAL stateknot.child_runtime_version = '1'")
10026 .execute(&mut *transaction)
10027 .await
10028 .map_err(|source| StoreError::database(operation, source))?;
10029 Ok(transaction)
10030 }
10031
10032 async fn begin_repeatable_read(
10033 &self,
10034 operation: &'static str,
10035 ) -> Result<Transaction<'_, Postgres>, StoreError> {
10036 let pool = self.pool.clone();
10037 let mut transaction = CompleteOnDrop::new(async move {
10038 pool.begin_with("BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY")
10039 .await
10040 })
10041 .await
10042 .map_err(|source| StoreError::database(operation, source))?;
10043 apply_transaction_timeouts(&mut transaction, &self.options, operation).await?;
10044 Ok(transaction)
10045 }
10046}
10047
10048impl fmt::Debug for PostgresStore {
10049 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
10050 formatter
10051 .debug_struct("PostgresStore")
10052 .field("transport_security", &self.options.transport_security())
10053 .field("max_connections", &self.options.max_connections)
10054 .finish_non_exhaustive()
10055 }
10056}
10057
10058#[derive(Clone, Copy)]
10059enum AppendAuthority {
10060 ControlPlane,
10061 Worker,
10062}
10063
10064#[derive(Clone, Copy)]
10065enum InvocationAttemptKind {
10066 Tool,
10067 Model,
10068}
10069
10070impl InvocationAttemptKind {
10071 const fn as_str(self) -> &'static str {
10072 match self {
10073 Self::Tool => "tool_invocation",
10074 Self::Model => "model_invocation",
10075 }
10076 }
10077
10078 const fn conflict(self) -> StoreError {
10079 match self {
10080 Self::Tool => StoreError::InvalidToolInvocationTransition,
10081 Self::Model => StoreError::InvalidModelInvocationTransition,
10082 }
10083 }
10084}
10085
10086struct GraphDefinitionRow {
10087 tenant_id: String,
10088 owner_issuer: String,
10089 owner_subject: String,
10090 graph_name: String,
10091 graph_version: String,
10092 definition_digest: Vec<u8>,
10093 definition_bytes: Vec<u8>,
10094 registered_at: DateTime<Utc>,
10095}
10096
10097struct ArtifactRow {
10098 tenant_id: String,
10099 artifact_id: Uuid,
10100 registration_key_digest: Vec<u8>,
10101 artifact_ref_digest: Vec<u8>,
10102 artifact_ref_bytes: Vec<u8>,
10103 content_byte_length: i64,
10104 content_digest: Vec<u8>,
10105 provenance_run_id: Uuid,
10106 provenance_event_id: Uuid,
10107 storage_namespace: String,
10108 object_key: String,
10109 object_version: Option<String>,
10110 object_etag: Option<String>,
10111 registered_at: DateTime<Utc>,
10112}
10113
10114struct AgentAdmissionRow {
10115 tenant_id: String,
10116 run_id: Uuid,
10117 agent_owner_issuer: String,
10118 agent_owner_subject: String,
10119 agent_name: String,
10120 agent_version: String,
10121 graph_owner_issuer: String,
10122 graph_owner_subject: String,
10123 graph_name: String,
10124 graph_version: String,
10125 graph_definition_digest: Vec<u8>,
10126 policy_owner_issuer: String,
10127 policy_owner_subject: String,
10128 policy_name: String,
10129 policy_version: String,
10130 policy_digest: Vec<u8>,
10131 intent_digest: Vec<u8>,
10132 admission_digest: Vec<u8>,
10133 admitted_at: DateTime<Utc>,
10134 journal_sequence: i64,
10135 journal_event_id: Uuid,
10136 journal_recorded_at: DateTime<Utc>,
10137 journal_digest: Vec<u8>,
10138 checkpoint_id: Uuid,
10139 checkpoint_superstep: i64,
10140 checkpoint_digest: Vec<u8>,
10141 admission_bytes: Vec<u8>,
10142 created_at: DateTime<Utc>,
10143}
10144
10145struct AgentSubmissionRow {
10146 tenant_id: String,
10147 key_digest: Vec<u8>,
10148 submission_digest: Vec<u8>,
10149 run_id: Uuid,
10150 admission_digest: Vec<u8>,
10151 created_at: DateTime<Utc>,
10152}
10153
10154#[derive(Clone)]
10155struct SchedulerFairnessShardRow {
10156 shard_id: String,
10157 policy_digest: Vec<u8>,
10158 policy_bytes: Vec<u8>,
10159 cycle_length: i32,
10160 next_slot: i32,
10161 next_sequence: i64,
10162 registered_at: DateTime<Utc>,
10163 #[allow(dead_code)]
10164 updated_at: DateTime<Utc>,
10165}
10166
10167struct SchedulerFairnessReservationRow {
10168 shard_id: String,
10169 reservation_id: Uuid,
10170 policy_digest: Vec<u8>,
10171 sequence: i64,
10172 slot: i32,
10173 reserved_at: DateTime<Utc>,
10174 shard_policy_digest: Vec<u8>,
10175 cycle_length: i32,
10176}
10177
10178struct RunQuarantineTargetRow {
10179 tenant_id: String,
10180 run_id: Uuid,
10181 journal_sequence: Option<i64>,
10182 journal_event_id: Option<Uuid>,
10183 journal_recorded_at: Option<DateTime<Utc>>,
10184 journal_digest: Option<Vec<u8>>,
10185 lease_attempt_id: Option<Uuid>,
10186 fencing_epoch: i64,
10187 lease_renewed_at: Option<DateTime<Utc>>,
10188 lease_expires_at: Option<DateTime<Utc>>,
10189 quarantined_at: Option<DateTime<Utc>>,
10190 quarantine_reason: Option<String>,
10191}
10192
10193struct RunQuarantineRow {
10194 tenant_id: String,
10195 run_id: Uuid,
10196 quarantine_id: Uuid,
10197 quarantined_at: DateTime<Utc>,
10198 cause_kind: String,
10199 component: String,
10200 evidence_digest: Vec<u8>,
10201 expected_journal_sequence: Option<i64>,
10202 expected_journal_event_id: Option<Uuid>,
10203 expected_journal_recorded_at: Option<DateTime<Utc>>,
10204 expected_journal_digest: Option<Vec<u8>>,
10205 expected_fence_attempt_id: Option<Uuid>,
10206 expected_fence_epoch: Option<i64>,
10207 record_digest: Vec<u8>,
10208 created_at: DateTime<Utc>,
10209 run_quarantined_at: Option<DateTime<Utc>>,
10210 run_quarantine_reason: Option<String>,
10211 run_lease_attempt_id: Option<Uuid>,
10212 run_lease_acquired_at: Option<DateTime<Utc>>,
10213 run_lease_renewed_at: Option<DateTime<Utc>>,
10214 run_lease_expires_at: Option<DateTime<Utc>>,
10215 run_fencing_epoch: i64,
10216 run_scheduler_ready_at: Option<DateTime<Utc>>,
10217 run_updated_at: DateTime<Utc>,
10218}
10219
10220struct RunRow {
10221 tenant_id: String,
10222 run_id: Uuid,
10223 thread_id: Uuid,
10224 invocation_id: Uuid,
10225 lifecycle_bytes: Vec<u8>,
10226 lifecycle_revision: String,
10227 lifecycle_status: String,
10228 admitted_at: DateTime<Utc>,
10229 changed_at: DateTime<Utc>,
10230 journal_sequence: Option<i64>,
10231 journal_event_id: Option<Uuid>,
10232 journal_recorded_at: Option<DateTime<Utc>>,
10233 journal_digest: Option<Vec<u8>>,
10234 checkpoint_id: Option<Uuid>,
10235 checkpoint_superstep: Option<i64>,
10236 checkpoint_digest: Option<Vec<u8>>,
10237 fencing_epoch: i64,
10238 lease_attempt_id: Option<Uuid>,
10239 lease_acquired_at: Option<DateTime<Utc>>,
10240 lease_renewed_at: Option<DateTime<Utc>>,
10241 lease_expires_at: Option<DateTime<Utc>>,
10242 scheduler_ready_at: Option<DateTime<Utc>>,
10243 scheduler_not_before: Option<DateTime<Utc>>,
10244 wait_set_digest: Option<Vec<u8>>,
10245 unresolved_wait_count: i16,
10246 next_timer_due_at: Option<DateTime<Utc>>,
10247 next_interrupt_expiry_at: Option<DateTime<Utc>>,
10248 quarantined_at: Option<DateTime<Utc>>,
10249}
10250
10251struct EventRow {
10252 tenant_id: String,
10253 run_id: Uuid,
10254 sequence: i64,
10255 event_id: Uuid,
10256 recorded_at: DateTime<Utc>,
10257 source_kind: String,
10258 worker_attempt_id: Option<Uuid>,
10259 worker_epoch: Option<i64>,
10260 event_kind: String,
10261 schema_id: String,
10262 schema_version: String,
10263 schema_digest: Vec<u8>,
10264 payload_bytes: Vec<u8>,
10265 payload_digest: Vec<u8>,
10266 intent_digest: Vec<u8>,
10267 projection_digest: Option<Vec<u8>>,
10268 previous_digest: Option<Vec<u8>>,
10269 event_digest: Vec<u8>,
10270}
10271
10272struct WaitRegistrationRow {
10273 tenant_id: String,
10274 run_id: Uuid,
10275 wait_id: Uuid,
10276 wait_kind: String,
10277 interrupt_kind: Option<String>,
10278 timer_kind: Option<String>,
10279 registered_at: DateTime<Utc>,
10280 due_at: Option<DateTime<Utc>>,
10281 expires_at: Option<DateTime<Utc>>,
10282 action_digest: Option<Vec<u8>>,
10283 registration_sequence: i64,
10284 registration_event_id: Uuid,
10285 registration_event_digest: Vec<u8>,
10286 intent_digest: Vec<u8>,
10287 record_digest: Vec<u8>,
10288 record_bytes: Vec<u8>,
10289 status: String,
10290 terminal_sequence: Option<i64>,
10291 terminal_event_id: Option<Uuid>,
10292 terminal_recorded_at: Option<DateTime<Utc>>,
10293 terminal_event_digest: Option<Vec<u8>>,
10294 resolution_digest: Option<Vec<u8>>,
10295 firing_digest: Option<Vec<u8>>,
10296 abandonment_digest: Option<Vec<u8>>,
10297 created_at: DateTime<Utc>,
10298 updated_at: DateTime<Utc>,
10299}
10300
10301struct InterruptResolutionRow {
10302 tenant_id: String,
10303 run_id: Uuid,
10304 interrupt_id: Uuid,
10305 request_digest: Vec<u8>,
10306 resolution_sequence: i64,
10307 resolution_event_id: Uuid,
10308 resolved_at: DateTime<Utc>,
10309 resolution_event_digest: Vec<u8>,
10310 intent_digest: Vec<u8>,
10311 resolution_digest: Vec<u8>,
10312 resolution_bytes: Vec<u8>,
10313 created_at: DateTime<Utc>,
10314}
10315
10316struct TimerFiringRow {
10317 tenant_id: String,
10318 run_id: Uuid,
10319 timer_id: Uuid,
10320 timer_digest: Vec<u8>,
10321 firing_sequence: i64,
10322 firing_event_id: Uuid,
10323 fired_at: DateTime<Utc>,
10324 firing_event_digest: Vec<u8>,
10325 intent_digest: Vec<u8>,
10326 firing_digest: Vec<u8>,
10327 firing_bytes: Vec<u8>,
10328 created_at: DateTime<Utc>,
10329}
10330
10331struct WaitAbandonmentRow {
10332 tenant_id: String,
10333 run_id: Uuid,
10334 wait_id: Uuid,
10335 wait_kind: String,
10336 registration_digest: Vec<u8>,
10337 reason_kind: String,
10338 abandonment_sequence: i64,
10339 abandonment_event_id: Uuid,
10340 abandoned_at: DateTime<Utc>,
10341 abandonment_event_digest: Vec<u8>,
10342 abandonment_digest: Vec<u8>,
10343 created_at: DateTime<Utc>,
10344}
10345
10346struct CheckpointRow {
10347 tenant_id: String,
10348 run_id: Uuid,
10349 checkpoint_id: Uuid,
10350 superstep: i64,
10351 parent_checkpoint_id: Option<Uuid>,
10352 parent_superstep: Option<i64>,
10353 parent_digest: Option<Vec<u8>>,
10354 journal_sequence: i64,
10355 journal_event_id: Uuid,
10356 journal_recorded_at: DateTime<Utc>,
10357 journal_digest: Vec<u8>,
10358 graph_definition_digest: Vec<u8>,
10359 state_schema_id: String,
10360 state_schema_version: String,
10361 state_schema_digest: Vec<u8>,
10362 state_digest: Vec<u8>,
10363 intent_digest: Vec<u8>,
10364 checkpoint_digest: Vec<u8>,
10365 checkpoint_bytes: Vec<u8>,
10366}
10367
10368struct ToolInvocationRow {
10369 tenant_id: String,
10370 run_id: Uuid,
10371 invocation_id: Uuid,
10372 base_checkpoint_id: Uuid,
10373 base_superstep: i64,
10374 base_checkpoint_digest: Vec<u8>,
10375 graph_namespace: String,
10376 node_id: String,
10377 activation_input_digest: Vec<u8>,
10378 intent_digest: Vec<u8>,
10379 intent_bytes: Vec<u8>,
10380 current_revision: i64,
10381 current_status: String,
10382 current_attempt_id: Option<Uuid>,
10383 current_record_digest: Vec<u8>,
10384 created_at: DateTime<Utc>,
10385 updated_at: DateTime<Utc>,
10386}
10387
10388struct ToolInvocationRevisionRow {
10389 tenant_id: String,
10390 run_id: Uuid,
10391 invocation_id: Uuid,
10392 revision: i64,
10393 previous_revision: Option<i64>,
10394 previous_digest: Option<Vec<u8>>,
10395 journal_sequence: i64,
10396 journal_event_id: Uuid,
10397 journal_recorded_at: DateTime<Utc>,
10398 journal_digest: Vec<u8>,
10399 status: String,
10400 attempt_id: Option<Uuid>,
10401 transition_kind: Option<String>,
10402 started_attempt_id: Option<Uuid>,
10403 transition_digest: Option<Vec<u8>>,
10404 record_digest: Vec<u8>,
10405 record_bytes: Vec<u8>,
10406 created_at: DateTime<Utc>,
10407}
10408
10409struct ModelInvocationRow {
10410 tenant_id: String,
10411 run_id: Uuid,
10412 invocation_id: Uuid,
10413 base_checkpoint_id: Uuid,
10414 base_superstep: i64,
10415 base_checkpoint_digest: Vec<u8>,
10416 graph_namespace: String,
10417 node_id: String,
10418 activation_input_digest: Vec<u8>,
10419 intent_digest: Vec<u8>,
10420 intent_bytes: Vec<u8>,
10421 current_revision: i64,
10422 current_status: String,
10423 current_attempt_id: Option<Uuid>,
10424 current_record_digest: Vec<u8>,
10425 created_at: DateTime<Utc>,
10426 updated_at: DateTime<Utc>,
10427}
10428
10429struct ModelInvocationRevisionRow {
10430 tenant_id: String,
10431 run_id: Uuid,
10432 invocation_id: Uuid,
10433 revision: i64,
10434 previous_revision: Option<i64>,
10435 previous_digest: Option<Vec<u8>>,
10436 journal_sequence: i64,
10437 journal_event_id: Uuid,
10438 journal_recorded_at: DateTime<Utc>,
10439 journal_digest: Vec<u8>,
10440 status: String,
10441 attempt_id: Option<Uuid>,
10442 transition_kind: Option<String>,
10443 started_attempt_id: Option<Uuid>,
10444 transition_digest: Option<Vec<u8>>,
10445 record_digest: Vec<u8>,
10446 record_bytes: Vec<u8>,
10447 created_at: DateTime<Utc>,
10448}
10449
10450struct NodeAttemptStartRow {
10451 tenant_id: String,
10452 run_id: Uuid,
10453 base_checkpoint_id: Uuid,
10454 base_superstep: i64,
10455 base_checkpoint_digest: Vec<u8>,
10456 base_journal_sequence: i64,
10457 base_journal_event_id: Uuid,
10458 base_journal_recorded_at: DateTime<Utc>,
10459 base_journal_digest: Vec<u8>,
10460 graph_namespace: String,
10461 node_id: String,
10462 activation_input_digest: Vec<u8>,
10463 activation_digest: Vec<u8>,
10464 attempt_id: Uuid,
10465 fence_attempt_id: Uuid,
10466 fence_epoch: i64,
10467 journal_sequence: i64,
10468 journal_event_id: Uuid,
10469 journal_recorded_at: DateTime<Utc>,
10470 journal_digest: Vec<u8>,
10471 start_digest: Vec<u8>,
10472 start_bytes: Vec<u8>,
10473 created_at: DateTime<Utc>,
10474}
10475
10476struct NodeAttemptCompletionRow {
10477 tenant_id: String,
10478 run_id: Uuid,
10479 attempt_id: Uuid,
10480 base_checkpoint_id: Uuid,
10481 base_superstep: i64,
10482 base_checkpoint_digest: Vec<u8>,
10483 graph_namespace: String,
10484 node_id: String,
10485 activation_input_digest: Vec<u8>,
10486 activation_digest: Vec<u8>,
10487 fence_attempt_id: Uuid,
10488 fence_epoch: i64,
10489 start_journal_sequence: i64,
10490 start_journal_event_id: Uuid,
10491 start_journal_recorded_at: DateTime<Utc>,
10492 start_journal_digest: Vec<u8>,
10493 start_digest: Vec<u8>,
10494 status: String,
10495 journal_sequence: i64,
10496 journal_event_id: Uuid,
10497 journal_recorded_at: DateTime<Utc>,
10498 journal_digest: Vec<u8>,
10499 result_intent_digest: Option<Vec<u8>>,
10500 result_record_digest: Option<Vec<u8>>,
10501 failure_id: Option<Uuid>,
10502 retry_kind: Option<String>,
10503 retry_not_before: Option<DateTime<Utc>>,
10504 completion_digest: Vec<u8>,
10505 completion_bytes: Vec<u8>,
10506 created_at: DateTime<Utc>,
10507}
10508
10509struct PendingNodeResultRow {
10510 tenant_id: String,
10511 run_id: Uuid,
10512 base_checkpoint_id: Uuid,
10513 base_superstep: i64,
10514 base_checkpoint_digest: Vec<u8>,
10515 base_journal_sequence: i64,
10516 base_journal_event_id: Uuid,
10517 base_journal_recorded_at: DateTime<Utc>,
10518 base_journal_digest: Vec<u8>,
10519 graph_namespace: String,
10520 node_id: String,
10521 activation_input_digest: Vec<u8>,
10522 node_attempt_id: Option<Uuid>,
10523 intent_digest: Vec<u8>,
10524 control_kind: String,
10525 fence_attempt_id: Uuid,
10526 fence_epoch: i64,
10527 journal_sequence: i64,
10528 journal_event_id: Uuid,
10529 journal_recorded_at: DateTime<Utc>,
10530 journal_digest: Vec<u8>,
10531 record_digest: Vec<u8>,
10532 result_bytes: Vec<u8>,
10533 created_at: DateTime<Utc>,
10534}
10535
10536struct PendingNodeResultHeadRow {
10537 tenant_id: String,
10538 run_id: Uuid,
10539 base_checkpoint_id: Uuid,
10540 base_superstep: i64,
10541 base_checkpoint_digest: Vec<u8>,
10542 graph_namespace: String,
10543 node_id: String,
10544 activation_input_digest: Vec<u8>,
10545 intent_digest: Vec<u8>,
10546 fence_attempt_id: Uuid,
10547 fence_epoch: i64,
10548 journal_sequence: i64,
10549 journal_event_id: Uuid,
10550 journal_recorded_at: DateTime<Utc>,
10551 journal_digest: Vec<u8>,
10552 record_digest: Vec<u8>,
10553}
10554
10555struct PendingNodeResultConsumptionRow {
10556 tenant_id: String,
10557 run_id: Uuid,
10558 base_checkpoint_id: Uuid,
10559 base_superstep: i64,
10560 base_checkpoint_digest: Vec<u8>,
10561 graph_namespace: String,
10562 node_id: String,
10563 result_record_digest: Vec<u8>,
10564 successor_checkpoint_id: Uuid,
10565 successor_superstep: i64,
10566 successor_checkpoint_digest: Vec<u8>,
10567 successor_journal_sequence: i64,
10568 successor_journal_event_id: Uuid,
10569 successor_journal_recorded_at: DateTime<Utc>,
10570 successor_journal_digest: Vec<u8>,
10571 created_at: DateTime<Utc>,
10572}
10573
10574struct PendingNodeResultBindingRow {
10575 tenant_id: String,
10576 run_id: Uuid,
10577 base_checkpoint_id: Uuid,
10578 base_superstep: i64,
10579 base_checkpoint_digest: Vec<u8>,
10580 graph_namespace: String,
10581 node_id: String,
10582 activation_input_digest: Vec<u8>,
10583 result_record_digest: Vec<u8>,
10584 result_journal_sequence: i64,
10585 result_journal_recorded_at: DateTime<Utc>,
10586 result_journal_digest: Vec<u8>,
10587 invocation_id: Uuid,
10588 invocation_revision: i64,
10589 invocation_record_digest: Vec<u8>,
10590 invocation_status: String,
10591 invocation_journal_sequence: i64,
10592 invocation_journal_recorded_at: DateTime<Utc>,
10593 invocation_journal_digest: Vec<u8>,
10594}
10595
10596struct OutboxDestinationRow {
10597 tenant_id: String,
10598 destination_id: Uuid,
10599 snapshot_digest: Vec<u8>,
10600 config_kind: String,
10601 schema_id: String,
10602 schema_version: String,
10603 schema_digest: Vec<u8>,
10604 config_bytes: Vec<u8>,
10605 created_at: DateTime<Utc>,
10606}
10607
10608struct OutboxDeliveryRow {
10609 tenant_id: String,
10610 run_id: Uuid,
10611 delivery_id: Uuid,
10612 origin_sequence: i64,
10613 origin_event_id: Uuid,
10614 origin_recorded_at: DateTime<Utc>,
10615 origin_digest: Vec<u8>,
10616 destination_id: Uuid,
10617 destination_snapshot_digest: Vec<u8>,
10618 intent_digest: Vec<u8>,
10619 expires_at: DateTime<Utc>,
10620 delivery_digest: Vec<u8>,
10621 delivery_bytes: Vec<u8>,
10622 status: String,
10623 attempt_count: i64,
10624 current_attempt_id: Option<Uuid>,
10625 current_epoch: Option<i64>,
10626 current_attempt_started_at: Option<DateTime<Utc>>,
10627 current_attempt_expires_at: Option<DateTime<Utc>>,
10628 next_attempt_at: Option<DateTime<Utc>>,
10629 last_completion_digest: Option<Vec<u8>>,
10630 terminal_at: Option<DateTime<Utc>>,
10631 created_at: DateTime<Utc>,
10632 updated_at: DateTime<Utc>,
10633}
10634
10635struct OutboxAttemptStartRow {
10636 tenant_id: String,
10637 run_id: Uuid,
10638 delivery_id: Uuid,
10639 delivery_expires_at: DateTime<Utc>,
10640 delivery_digest: Vec<u8>,
10641 epoch: i64,
10642 attempt_id: Uuid,
10643 started_at: DateTime<Utc>,
10644 expires_at: DateTime<Utc>,
10645 start_digest: Vec<u8>,
10646 start_bytes: Vec<u8>,
10647 created_at: DateTime<Utc>,
10648}
10649
10650struct OutboxAttemptCompletionRow {
10651 tenant_id: String,
10652 run_id: Uuid,
10653 delivery_id: Uuid,
10654 epoch: i64,
10655 attempt_id: Uuid,
10656 started_at: DateTime<Utc>,
10657 attempt_expires_at: DateTime<Utc>,
10658 start_digest: Vec<u8>,
10659 outcome_kind: String,
10660 retry_advice_kind: Option<String>,
10661 retry_delay_millis: Option<i64>,
10662 completed_at: DateTime<Utc>,
10663 completion_digest: Vec<u8>,
10664 completion_bytes: Vec<u8>,
10665 created_at: DateTime<Utc>,
10666}
10667
10668impl<'row> FromRow<'row, PgRow> for GraphDefinitionRow {
10669 fn from_row(row: &'row PgRow) -> Result<Self, sqlx_core::Error> {
10670 Ok(Self {
10671 tenant_id: row.try_get("tenant_id")?,
10672 owner_issuer: row.try_get("owner_issuer")?,
10673 owner_subject: row.try_get("owner_subject")?,
10674 graph_name: row.try_get("graph_name")?,
10675 graph_version: row.try_get("graph_version")?,
10676 definition_digest: row.try_get("definition_digest")?,
10677 definition_bytes: row.try_get("definition_bytes")?,
10678 registered_at: row.try_get("registered_at")?,
10679 })
10680 }
10681}
10682
10683impl<'row> FromRow<'row, PgRow> for ArtifactRow {
10684 fn from_row(row: &'row PgRow) -> Result<Self, sqlx_core::Error> {
10685 Ok(Self {
10686 tenant_id: row.try_get("tenant_id")?,
10687 artifact_id: row.try_get("artifact_id")?,
10688 registration_key_digest: row.try_get("registration_key_digest")?,
10689 artifact_ref_digest: row.try_get("artifact_ref_digest")?,
10690 artifact_ref_bytes: row.try_get("artifact_ref_bytes")?,
10691 content_byte_length: row.try_get("content_byte_length")?,
10692 content_digest: row.try_get("content_digest")?,
10693 provenance_run_id: row.try_get("provenance_run_id")?,
10694 provenance_event_id: row.try_get("provenance_event_id")?,
10695 storage_namespace: row.try_get("storage_namespace")?,
10696 object_key: row.try_get("object_key")?,
10697 object_version: row.try_get("object_version")?,
10698 object_etag: row.try_get("object_etag")?,
10699 registered_at: row.try_get("registered_at")?,
10700 })
10701 }
10702}
10703
10704impl<'row> FromRow<'row, PgRow> for AgentAdmissionRow {
10705 fn from_row(row: &'row PgRow) -> Result<Self, sqlx_core::Error> {
10706 Ok(Self {
10707 tenant_id: row.try_get("tenant_id")?,
10708 run_id: row.try_get("run_id")?,
10709 agent_owner_issuer: row.try_get("agent_owner_issuer")?,
10710 agent_owner_subject: row.try_get("agent_owner_subject")?,
10711 agent_name: row.try_get("agent_name")?,
10712 agent_version: row.try_get("agent_version")?,
10713 graph_owner_issuer: row.try_get("graph_owner_issuer")?,
10714 graph_owner_subject: row.try_get("graph_owner_subject")?,
10715 graph_name: row.try_get("graph_name")?,
10716 graph_version: row.try_get("graph_version")?,
10717 graph_definition_digest: row.try_get("graph_definition_digest")?,
10718 policy_owner_issuer: row.try_get("policy_owner_issuer")?,
10719 policy_owner_subject: row.try_get("policy_owner_subject")?,
10720 policy_name: row.try_get("policy_name")?,
10721 policy_version: row.try_get("policy_version")?,
10722 policy_digest: row.try_get("policy_digest")?,
10723 intent_digest: row.try_get("intent_digest")?,
10724 admission_digest: row.try_get("admission_digest")?,
10725 admitted_at: row.try_get("admitted_at")?,
10726 journal_sequence: row.try_get("journal_sequence")?,
10727 journal_event_id: row.try_get("journal_event_id")?,
10728 journal_recorded_at: row.try_get("journal_recorded_at")?,
10729 journal_digest: row.try_get("journal_digest")?,
10730 checkpoint_id: row.try_get("checkpoint_id")?,
10731 checkpoint_superstep: row.try_get("checkpoint_superstep")?,
10732 checkpoint_digest: row.try_get("checkpoint_digest")?,
10733 admission_bytes: row.try_get("admission_bytes")?,
10734 created_at: row.try_get("created_at")?,
10735 })
10736 }
10737}
10738
10739impl<'row> FromRow<'row, PgRow> for AgentSubmissionRow {
10740 fn from_row(row: &'row PgRow) -> Result<Self, sqlx_core::Error> {
10741 Ok(Self {
10742 tenant_id: row.try_get("tenant_id")?,
10743 key_digest: row.try_get("key_digest")?,
10744 submission_digest: row.try_get("submission_digest")?,
10745 run_id: row.try_get("run_id")?,
10746 admission_digest: row.try_get("admission_digest")?,
10747 created_at: row.try_get("created_at")?,
10748 })
10749 }
10750}
10751
10752impl<'row> FromRow<'row, PgRow> for SchedulerFairnessShardRow {
10753 fn from_row(row: &'row PgRow) -> Result<Self, sqlx_core::Error> {
10754 Ok(Self {
10755 shard_id: row.try_get("shard_id")?,
10756 policy_digest: row.try_get("policy_digest")?,
10757 policy_bytes: row.try_get("policy_bytes")?,
10758 cycle_length: row.try_get("cycle_length")?,
10759 next_slot: row.try_get("next_slot")?,
10760 next_sequence: row.try_get("next_sequence")?,
10761 registered_at: row.try_get("registered_at")?,
10762 updated_at: row.try_get("updated_at")?,
10763 })
10764 }
10765}
10766
10767impl<'row> FromRow<'row, PgRow> for SchedulerFairnessReservationRow {
10768 fn from_row(row: &'row PgRow) -> Result<Self, sqlx_core::Error> {
10769 Ok(Self {
10770 shard_id: row.try_get("shard_id")?,
10771 reservation_id: row.try_get("reservation_id")?,
10772 policy_digest: row.try_get("policy_digest")?,
10773 sequence: row.try_get("sequence")?,
10774 slot: row.try_get("slot")?,
10775 reserved_at: row.try_get("reserved_at")?,
10776 shard_policy_digest: row.try_get("shard_policy_digest")?,
10777 cycle_length: row.try_get("cycle_length")?,
10778 })
10779 }
10780}
10781
10782impl<'row> FromRow<'row, PgRow> for RunQuarantineTargetRow {
10783 fn from_row(row: &'row PgRow) -> Result<Self, sqlx_core::Error> {
10784 Ok(Self {
10785 tenant_id: row.try_get("tenant_id")?,
10786 run_id: row.try_get("run_id")?,
10787 journal_sequence: row.try_get("journal_sequence")?,
10788 journal_event_id: row.try_get("journal_event_id")?,
10789 journal_recorded_at: row.try_get("journal_recorded_at")?,
10790 journal_digest: row.try_get("journal_digest")?,
10791 lease_attempt_id: row.try_get("lease_attempt_id")?,
10792 fencing_epoch: row.try_get("fencing_epoch")?,
10793 lease_renewed_at: row.try_get("lease_renewed_at")?,
10794 lease_expires_at: row.try_get("lease_expires_at")?,
10795 quarantined_at: row.try_get("quarantined_at")?,
10796 quarantine_reason: row.try_get("quarantine_reason")?,
10797 })
10798 }
10799}
10800
10801impl<'row> FromRow<'row, PgRow> for RunQuarantineRow {
10802 fn from_row(row: &'row PgRow) -> Result<Self, sqlx_core::Error> {
10803 Ok(Self {
10804 tenant_id: row.try_get("tenant_id")?,
10805 run_id: row.try_get("run_id")?,
10806 quarantine_id: row.try_get("quarantine_id")?,
10807 quarantined_at: row.try_get("quarantined_at")?,
10808 cause_kind: row.try_get("cause_kind")?,
10809 component: row.try_get("component")?,
10810 evidence_digest: row.try_get("evidence_digest")?,
10811 expected_journal_sequence: row.try_get("expected_journal_sequence")?,
10812 expected_journal_event_id: row.try_get("expected_journal_event_id")?,
10813 expected_journal_recorded_at: row.try_get("expected_journal_recorded_at")?,
10814 expected_journal_digest: row.try_get("expected_journal_digest")?,
10815 expected_fence_attempt_id: row.try_get("expected_fence_attempt_id")?,
10816 expected_fence_epoch: row.try_get("expected_fence_epoch")?,
10817 record_digest: row.try_get("record_digest")?,
10818 created_at: row.try_get("created_at")?,
10819 run_quarantined_at: row.try_get("run_quarantined_at")?,
10820 run_quarantine_reason: row.try_get("run_quarantine_reason")?,
10821 run_lease_attempt_id: row.try_get("run_lease_attempt_id")?,
10822 run_lease_acquired_at: row.try_get("run_lease_acquired_at")?,
10823 run_lease_renewed_at: row.try_get("run_lease_renewed_at")?,
10824 run_lease_expires_at: row.try_get("run_lease_expires_at")?,
10825 run_fencing_epoch: row.try_get("run_fencing_epoch")?,
10826 run_scheduler_ready_at: row.try_get("run_scheduler_ready_at")?,
10827 run_updated_at: row.try_get("run_updated_at")?,
10828 })
10829 }
10830}
10831
10832impl<'row> FromRow<'row, PgRow> for RunRow {
10833 fn from_row(row: &'row PgRow) -> Result<Self, sqlx_core::Error> {
10834 Ok(Self {
10835 tenant_id: row.try_get("tenant_id")?,
10836 run_id: row.try_get("run_id")?,
10837 thread_id: row.try_get("thread_id")?,
10838 invocation_id: row.try_get("invocation_id")?,
10839 lifecycle_bytes: row.try_get("lifecycle_bytes")?,
10840 lifecycle_revision: row.try_get("lifecycle_revision")?,
10841 lifecycle_status: row.try_get("lifecycle_status")?,
10842 admitted_at: row.try_get("admitted_at")?,
10843 changed_at: row.try_get("changed_at")?,
10844 journal_sequence: row.try_get("journal_sequence")?,
10845 journal_event_id: row.try_get("journal_event_id")?,
10846 journal_recorded_at: row.try_get("journal_recorded_at")?,
10847 journal_digest: row.try_get("journal_digest")?,
10848 checkpoint_id: row.try_get("checkpoint_id")?,
10849 checkpoint_superstep: row.try_get("checkpoint_superstep")?,
10850 checkpoint_digest: row.try_get("checkpoint_digest")?,
10851 fencing_epoch: row.try_get("fencing_epoch")?,
10852 lease_attempt_id: row.try_get("lease_attempt_id")?,
10853 lease_acquired_at: row.try_get("lease_acquired_at")?,
10854 lease_renewed_at: row.try_get("lease_renewed_at")?,
10855 lease_expires_at: row.try_get("lease_expires_at")?,
10856 scheduler_ready_at: row.try_get("scheduler_ready_at")?,
10857 scheduler_not_before: row.try_get("scheduler_not_before")?,
10858 wait_set_digest: row.try_get("wait_set_digest")?,
10859 unresolved_wait_count: row.try_get("unresolved_wait_count")?,
10860 next_timer_due_at: row.try_get("next_timer_due_at")?,
10861 next_interrupt_expiry_at: row.try_get("next_interrupt_expiry_at")?,
10862 quarantined_at: row.try_get("quarantined_at")?,
10863 })
10864 }
10865}
10866
10867impl<'row> FromRow<'row, PgRow> for EventRow {
10868 fn from_row(row: &'row PgRow) -> Result<Self, sqlx_core::Error> {
10869 Ok(Self {
10870 tenant_id: row.try_get("tenant_id")?,
10871 run_id: row.try_get("run_id")?,
10872 sequence: row.try_get("sequence")?,
10873 event_id: row.try_get("event_id")?,
10874 recorded_at: row.try_get("recorded_at")?,
10875 source_kind: row.try_get("source_kind")?,
10876 worker_attempt_id: row.try_get("worker_attempt_id")?,
10877 worker_epoch: row.try_get("worker_epoch")?,
10878 event_kind: row.try_get("event_kind")?,
10879 schema_id: row.try_get("schema_id")?,
10880 schema_version: row.try_get("schema_version")?,
10881 schema_digest: row.try_get("schema_digest")?,
10882 payload_bytes: row.try_get("payload_bytes")?,
10883 payload_digest: row.try_get("payload_digest")?,
10884 intent_digest: row.try_get("intent_digest")?,
10885 projection_digest: row.try_get("projection_digest")?,
10886 previous_digest: row.try_get("previous_digest")?,
10887 event_digest: row.try_get("event_digest")?,
10888 })
10889 }
10890}
10891
10892impl<'row> FromRow<'row, PgRow> for WaitRegistrationRow {
10893 fn from_row(row: &'row PgRow) -> Result<Self, sqlx_core::Error> {
10894 Ok(Self {
10895 tenant_id: row.try_get("tenant_id")?,
10896 run_id: row.try_get("run_id")?,
10897 wait_id: row.try_get("wait_id")?,
10898 wait_kind: row.try_get("wait_kind")?,
10899 interrupt_kind: row.try_get("interrupt_kind")?,
10900 timer_kind: row.try_get("timer_kind")?,
10901 registered_at: row.try_get("registered_at")?,
10902 due_at: row.try_get("due_at")?,
10903 expires_at: row.try_get("expires_at")?,
10904 action_digest: row.try_get("action_digest")?,
10905 registration_sequence: row.try_get("registration_sequence")?,
10906 registration_event_id: row.try_get("registration_event_id")?,
10907 registration_event_digest: row.try_get("registration_event_digest")?,
10908 intent_digest: row.try_get("intent_digest")?,
10909 record_digest: row.try_get("record_digest")?,
10910 record_bytes: row.try_get("record_bytes")?,
10911 status: row.try_get("status")?,
10912 terminal_sequence: row.try_get("terminal_sequence")?,
10913 terminal_event_id: row.try_get("terminal_event_id")?,
10914 terminal_recorded_at: row.try_get("terminal_recorded_at")?,
10915 terminal_event_digest: row.try_get("terminal_event_digest")?,
10916 resolution_digest: row.try_get("resolution_digest")?,
10917 firing_digest: row.try_get("firing_digest")?,
10918 abandonment_digest: row.try_get("abandonment_digest")?,
10919 created_at: row.try_get("created_at")?,
10920 updated_at: row.try_get("updated_at")?,
10921 })
10922 }
10923}
10924
10925impl<'row> FromRow<'row, PgRow> for InterruptResolutionRow {
10926 fn from_row(row: &'row PgRow) -> Result<Self, sqlx_core::Error> {
10927 Ok(Self {
10928 tenant_id: row.try_get("tenant_id")?,
10929 run_id: row.try_get("run_id")?,
10930 interrupt_id: row.try_get("interrupt_id")?,
10931 request_digest: row.try_get("request_digest")?,
10932 resolution_sequence: row.try_get("resolution_sequence")?,
10933 resolution_event_id: row.try_get("resolution_event_id")?,
10934 resolved_at: row.try_get("resolved_at")?,
10935 resolution_event_digest: row.try_get("resolution_event_digest")?,
10936 intent_digest: row.try_get("intent_digest")?,
10937 resolution_digest: row.try_get("resolution_digest")?,
10938 resolution_bytes: row.try_get("resolution_bytes")?,
10939 created_at: row.try_get("created_at")?,
10940 })
10941 }
10942}
10943
10944impl<'row> FromRow<'row, PgRow> for TimerFiringRow {
10945 fn from_row(row: &'row PgRow) -> Result<Self, sqlx_core::Error> {
10946 Ok(Self {
10947 tenant_id: row.try_get("tenant_id")?,
10948 run_id: row.try_get("run_id")?,
10949 timer_id: row.try_get("timer_id")?,
10950 timer_digest: row.try_get("timer_digest")?,
10951 firing_sequence: row.try_get("firing_sequence")?,
10952 firing_event_id: row.try_get("firing_event_id")?,
10953 fired_at: row.try_get("fired_at")?,
10954 firing_event_digest: row.try_get("firing_event_digest")?,
10955 intent_digest: row.try_get("intent_digest")?,
10956 firing_digest: row.try_get("firing_digest")?,
10957 firing_bytes: row.try_get("firing_bytes")?,
10958 created_at: row.try_get("created_at")?,
10959 })
10960 }
10961}
10962
10963impl<'row> FromRow<'row, PgRow> for WaitAbandonmentRow {
10964 fn from_row(row: &'row PgRow) -> Result<Self, sqlx_core::Error> {
10965 Ok(Self {
10966 tenant_id: row.try_get("tenant_id")?,
10967 run_id: row.try_get("run_id")?,
10968 wait_id: row.try_get("wait_id")?,
10969 wait_kind: row.try_get("wait_kind")?,
10970 registration_digest: row.try_get("registration_digest")?,
10971 reason_kind: row.try_get("reason_kind")?,
10972 abandonment_sequence: row.try_get("abandonment_sequence")?,
10973 abandonment_event_id: row.try_get("abandonment_event_id")?,
10974 abandoned_at: row.try_get("abandoned_at")?,
10975 abandonment_event_digest: row.try_get("abandonment_event_digest")?,
10976 abandonment_digest: row.try_get("abandonment_digest")?,
10977 created_at: row.try_get("created_at")?,
10978 })
10979 }
10980}
10981
10982impl<'row> FromRow<'row, PgRow> for CheckpointRow {
10983 fn from_row(row: &'row PgRow) -> Result<Self, sqlx_core::Error> {
10984 Ok(Self {
10985 tenant_id: row.try_get("tenant_id")?,
10986 run_id: row.try_get("run_id")?,
10987 checkpoint_id: row.try_get("checkpoint_id")?,
10988 superstep: row.try_get("superstep")?,
10989 parent_checkpoint_id: row.try_get("parent_checkpoint_id")?,
10990 parent_superstep: row.try_get("parent_superstep")?,
10991 parent_digest: row.try_get("parent_digest")?,
10992 journal_sequence: row.try_get("journal_sequence")?,
10993 journal_event_id: row.try_get("journal_event_id")?,
10994 journal_recorded_at: row.try_get("journal_recorded_at")?,
10995 journal_digest: row.try_get("journal_digest")?,
10996 graph_definition_digest: row.try_get("graph_definition_digest")?,
10997 state_schema_id: row.try_get("state_schema_id")?,
10998 state_schema_version: row.try_get("state_schema_version")?,
10999 state_schema_digest: row.try_get("state_schema_digest")?,
11000 state_digest: row.try_get("state_digest")?,
11001 intent_digest: row.try_get("intent_digest")?,
11002 checkpoint_digest: row.try_get("checkpoint_digest")?,
11003 checkpoint_bytes: row.try_get("checkpoint_bytes")?,
11004 })
11005 }
11006}
11007
11008impl<'row> FromRow<'row, PgRow> for ToolInvocationRow {
11009 fn from_row(row: &'row PgRow) -> Result<Self, sqlx_core::Error> {
11010 Ok(Self {
11011 tenant_id: row.try_get("tenant_id")?,
11012 run_id: row.try_get("run_id")?,
11013 invocation_id: row.try_get("invocation_id")?,
11014 base_checkpoint_id: row.try_get("base_checkpoint_id")?,
11015 base_superstep: row.try_get("base_superstep")?,
11016 base_checkpoint_digest: row.try_get("base_checkpoint_digest")?,
11017 graph_namespace: row.try_get("graph_namespace")?,
11018 node_id: row.try_get("node_id")?,
11019 activation_input_digest: row.try_get("activation_input_digest")?,
11020 intent_digest: row.try_get("intent_digest")?,
11021 intent_bytes: row.try_get("intent_bytes")?,
11022 current_revision: row.try_get("current_revision")?,
11023 current_status: row.try_get("current_status")?,
11024 current_attempt_id: row.try_get("current_attempt_id")?,
11025 current_record_digest: row.try_get("current_record_digest")?,
11026 created_at: row.try_get("created_at")?,
11027 updated_at: row.try_get("updated_at")?,
11028 })
11029 }
11030}
11031
11032impl<'row> FromRow<'row, PgRow> for ToolInvocationRevisionRow {
11033 fn from_row(row: &'row PgRow) -> Result<Self, sqlx_core::Error> {
11034 Ok(Self {
11035 tenant_id: row.try_get("tenant_id")?,
11036 run_id: row.try_get("run_id")?,
11037 invocation_id: row.try_get("invocation_id")?,
11038 revision: row.try_get("revision")?,
11039 previous_revision: row.try_get("previous_revision")?,
11040 previous_digest: row.try_get("previous_digest")?,
11041 journal_sequence: row.try_get("journal_sequence")?,
11042 journal_event_id: row.try_get("journal_event_id")?,
11043 journal_recorded_at: row.try_get("journal_recorded_at")?,
11044 journal_digest: row.try_get("journal_digest")?,
11045 status: row.try_get("status")?,
11046 attempt_id: row.try_get("attempt_id")?,
11047 transition_kind: row.try_get("transition_kind")?,
11048 started_attempt_id: row.try_get("started_attempt_id")?,
11049 transition_digest: row.try_get("transition_digest")?,
11050 record_digest: row.try_get("record_digest")?,
11051 record_bytes: row.try_get("record_bytes")?,
11052 created_at: row.try_get("created_at")?,
11053 })
11054 }
11055}
11056
11057impl<'row> FromRow<'row, PgRow> for ModelInvocationRow {
11058 fn from_row(row: &'row PgRow) -> Result<Self, sqlx_core::Error> {
11059 Ok(Self {
11060 tenant_id: row.try_get("tenant_id")?,
11061 run_id: row.try_get("run_id")?,
11062 invocation_id: row.try_get("invocation_id")?,
11063 base_checkpoint_id: row.try_get("base_checkpoint_id")?,
11064 base_superstep: row.try_get("base_superstep")?,
11065 base_checkpoint_digest: row.try_get("base_checkpoint_digest")?,
11066 graph_namespace: row.try_get("graph_namespace")?,
11067 node_id: row.try_get("node_id")?,
11068 activation_input_digest: row.try_get("activation_input_digest")?,
11069 intent_digest: row.try_get("intent_digest")?,
11070 intent_bytes: row.try_get("intent_bytes")?,
11071 current_revision: row.try_get("current_revision")?,
11072 current_status: row.try_get("current_status")?,
11073 current_attempt_id: row.try_get("current_attempt_id")?,
11074 current_record_digest: row.try_get("current_record_digest")?,
11075 created_at: row.try_get("created_at")?,
11076 updated_at: row.try_get("updated_at")?,
11077 })
11078 }
11079}
11080
11081impl<'row> FromRow<'row, PgRow> for ModelInvocationRevisionRow {
11082 fn from_row(row: &'row PgRow) -> Result<Self, sqlx_core::Error> {
11083 Ok(Self {
11084 tenant_id: row.try_get("tenant_id")?,
11085 run_id: row.try_get("run_id")?,
11086 invocation_id: row.try_get("invocation_id")?,
11087 revision: row.try_get("revision")?,
11088 previous_revision: row.try_get("previous_revision")?,
11089 previous_digest: row.try_get("previous_digest")?,
11090 journal_sequence: row.try_get("journal_sequence")?,
11091 journal_event_id: row.try_get("journal_event_id")?,
11092 journal_recorded_at: row.try_get("journal_recorded_at")?,
11093 journal_digest: row.try_get("journal_digest")?,
11094 status: row.try_get("status")?,
11095 attempt_id: row.try_get("attempt_id")?,
11096 transition_kind: row.try_get("transition_kind")?,
11097 started_attempt_id: row.try_get("started_attempt_id")?,
11098 transition_digest: row.try_get("transition_digest")?,
11099 record_digest: row.try_get("record_digest")?,
11100 record_bytes: row.try_get("record_bytes")?,
11101 created_at: row.try_get("created_at")?,
11102 })
11103 }
11104}
11105
11106impl<'row> FromRow<'row, PgRow> for NodeAttemptStartRow {
11107 fn from_row(row: &'row PgRow) -> Result<Self, sqlx_core::Error> {
11108 Ok(Self {
11109 tenant_id: row.try_get("tenant_id")?,
11110 run_id: row.try_get("run_id")?,
11111 base_checkpoint_id: row.try_get("base_checkpoint_id")?,
11112 base_superstep: row.try_get("base_superstep")?,
11113 base_checkpoint_digest: row.try_get("base_checkpoint_digest")?,
11114 base_journal_sequence: row.try_get("base_journal_sequence")?,
11115 base_journal_event_id: row.try_get("base_journal_event_id")?,
11116 base_journal_recorded_at: row.try_get("base_journal_recorded_at")?,
11117 base_journal_digest: row.try_get("base_journal_digest")?,
11118 graph_namespace: row.try_get("graph_namespace")?,
11119 node_id: row.try_get("node_id")?,
11120 activation_input_digest: row.try_get("activation_input_digest")?,
11121 activation_digest: row.try_get("activation_digest")?,
11122 attempt_id: row.try_get("attempt_id")?,
11123 fence_attempt_id: row.try_get("fence_attempt_id")?,
11124 fence_epoch: row.try_get("fence_epoch")?,
11125 journal_sequence: row.try_get("journal_sequence")?,
11126 journal_event_id: row.try_get("journal_event_id")?,
11127 journal_recorded_at: row.try_get("journal_recorded_at")?,
11128 journal_digest: row.try_get("journal_digest")?,
11129 start_digest: row.try_get("start_digest")?,
11130 start_bytes: row.try_get("start_bytes")?,
11131 created_at: row.try_get("created_at")?,
11132 })
11133 }
11134}
11135
11136impl<'row> FromRow<'row, PgRow> for NodeAttemptCompletionRow {
11137 fn from_row(row: &'row PgRow) -> Result<Self, sqlx_core::Error> {
11138 Ok(Self {
11139 tenant_id: row.try_get("tenant_id")?,
11140 run_id: row.try_get("run_id")?,
11141 attempt_id: row.try_get("attempt_id")?,
11142 base_checkpoint_id: row.try_get("base_checkpoint_id")?,
11143 base_superstep: row.try_get("base_superstep")?,
11144 base_checkpoint_digest: row.try_get("base_checkpoint_digest")?,
11145 graph_namespace: row.try_get("graph_namespace")?,
11146 node_id: row.try_get("node_id")?,
11147 activation_input_digest: row.try_get("activation_input_digest")?,
11148 activation_digest: row.try_get("activation_digest")?,
11149 fence_attempt_id: row.try_get("fence_attempt_id")?,
11150 fence_epoch: row.try_get("fence_epoch")?,
11151 start_journal_sequence: row.try_get("start_journal_sequence")?,
11152 start_journal_event_id: row.try_get("start_journal_event_id")?,
11153 start_journal_recorded_at: row.try_get("start_journal_recorded_at")?,
11154 start_journal_digest: row.try_get("start_journal_digest")?,
11155 start_digest: row.try_get("start_digest")?,
11156 status: row.try_get("status")?,
11157 journal_sequence: row.try_get("journal_sequence")?,
11158 journal_event_id: row.try_get("journal_event_id")?,
11159 journal_recorded_at: row.try_get("journal_recorded_at")?,
11160 journal_digest: row.try_get("journal_digest")?,
11161 result_intent_digest: row.try_get("result_intent_digest")?,
11162 result_record_digest: row.try_get("result_record_digest")?,
11163 failure_id: row.try_get("failure_id")?,
11164 retry_kind: row.try_get("retry_kind")?,
11165 retry_not_before: row.try_get("retry_not_before")?,
11166 completion_digest: row.try_get("completion_digest")?,
11167 completion_bytes: row.try_get("completion_bytes")?,
11168 created_at: row.try_get("created_at")?,
11169 })
11170 }
11171}
11172
11173impl<'row> FromRow<'row, PgRow> for PendingNodeResultRow {
11174 fn from_row(row: &'row PgRow) -> Result<Self, sqlx_core::Error> {
11175 Ok(Self {
11176 tenant_id: row.try_get("tenant_id")?,
11177 run_id: row.try_get("run_id")?,
11178 base_checkpoint_id: row.try_get("base_checkpoint_id")?,
11179 base_superstep: row.try_get("base_superstep")?,
11180 base_checkpoint_digest: row.try_get("base_checkpoint_digest")?,
11181 base_journal_sequence: row.try_get("base_journal_sequence")?,
11182 base_journal_event_id: row.try_get("base_journal_event_id")?,
11183 base_journal_recorded_at: row.try_get("base_journal_recorded_at")?,
11184 base_journal_digest: row.try_get("base_journal_digest")?,
11185 graph_namespace: row.try_get("graph_namespace")?,
11186 node_id: row.try_get("node_id")?,
11187 activation_input_digest: row.try_get("activation_input_digest")?,
11188 node_attempt_id: row.try_get("node_attempt_id")?,
11189 intent_digest: row.try_get("intent_digest")?,
11190 control_kind: row.try_get("control_kind")?,
11191 fence_attempt_id: row.try_get("fence_attempt_id")?,
11192 fence_epoch: row.try_get("fence_epoch")?,
11193 journal_sequence: row.try_get("journal_sequence")?,
11194 journal_event_id: row.try_get("journal_event_id")?,
11195 journal_recorded_at: row.try_get("journal_recorded_at")?,
11196 journal_digest: row.try_get("journal_digest")?,
11197 record_digest: row.try_get("record_digest")?,
11198 result_bytes: row.try_get("result_bytes")?,
11199 created_at: row.try_get("created_at")?,
11200 })
11201 }
11202}
11203
11204impl<'row> FromRow<'row, PgRow> for PendingNodeResultHeadRow {
11205 fn from_row(row: &'row PgRow) -> Result<Self, sqlx_core::Error> {
11206 Ok(Self {
11207 tenant_id: row.try_get("tenant_id")?,
11208 run_id: row.try_get("run_id")?,
11209 base_checkpoint_id: row.try_get("base_checkpoint_id")?,
11210 base_superstep: row.try_get("base_superstep")?,
11211 base_checkpoint_digest: row.try_get("base_checkpoint_digest")?,
11212 graph_namespace: row.try_get("graph_namespace")?,
11213 node_id: row.try_get("node_id")?,
11214 activation_input_digest: row.try_get("activation_input_digest")?,
11215 intent_digest: row.try_get("intent_digest")?,
11216 fence_attempt_id: row.try_get("fence_attempt_id")?,
11217 fence_epoch: row.try_get("fence_epoch")?,
11218 journal_sequence: row.try_get("journal_sequence")?,
11219 journal_event_id: row.try_get("journal_event_id")?,
11220 journal_recorded_at: row.try_get("journal_recorded_at")?,
11221 journal_digest: row.try_get("journal_digest")?,
11222 record_digest: row.try_get("record_digest")?,
11223 })
11224 }
11225}
11226
11227impl<'row> FromRow<'row, PgRow> for PendingNodeResultConsumptionRow {
11228 fn from_row(row: &'row PgRow) -> Result<Self, sqlx_core::Error> {
11229 Ok(Self {
11230 tenant_id: row.try_get("tenant_id")?,
11231 run_id: row.try_get("run_id")?,
11232 base_checkpoint_id: row.try_get("base_checkpoint_id")?,
11233 base_superstep: row.try_get("base_superstep")?,
11234 base_checkpoint_digest: row.try_get("base_checkpoint_digest")?,
11235 graph_namespace: row.try_get("graph_namespace")?,
11236 node_id: row.try_get("node_id")?,
11237 result_record_digest: row.try_get("result_record_digest")?,
11238 successor_checkpoint_id: row.try_get("successor_checkpoint_id")?,
11239 successor_superstep: row.try_get("successor_superstep")?,
11240 successor_checkpoint_digest: row.try_get("successor_checkpoint_digest")?,
11241 successor_journal_sequence: row.try_get("successor_journal_sequence")?,
11242 successor_journal_event_id: row.try_get("successor_journal_event_id")?,
11243 successor_journal_recorded_at: row.try_get("successor_journal_recorded_at")?,
11244 successor_journal_digest: row.try_get("successor_journal_digest")?,
11245 created_at: row.try_get("created_at")?,
11246 })
11247 }
11248}
11249
11250impl<'row> FromRow<'row, PgRow> for PendingNodeResultBindingRow {
11251 fn from_row(row: &'row PgRow) -> Result<Self, sqlx_core::Error> {
11252 Ok(Self {
11253 tenant_id: row.try_get("tenant_id")?,
11254 run_id: row.try_get("run_id")?,
11255 base_checkpoint_id: row.try_get("base_checkpoint_id")?,
11256 base_superstep: row.try_get("base_superstep")?,
11257 base_checkpoint_digest: row.try_get("base_checkpoint_digest")?,
11258 graph_namespace: row.try_get("graph_namespace")?,
11259 node_id: row.try_get("node_id")?,
11260 activation_input_digest: row.try_get("activation_input_digest")?,
11261 result_record_digest: row.try_get("result_record_digest")?,
11262 result_journal_sequence: row.try_get("result_journal_sequence")?,
11263 result_journal_recorded_at: row.try_get("result_journal_recorded_at")?,
11264 result_journal_digest: row.try_get("result_journal_digest")?,
11265 invocation_id: row.try_get("invocation_id")?,
11266 invocation_revision: row.try_get("invocation_revision")?,
11267 invocation_record_digest: row.try_get("invocation_record_digest")?,
11268 invocation_status: row.try_get("invocation_status")?,
11269 invocation_journal_sequence: row.try_get("invocation_journal_sequence")?,
11270 invocation_journal_recorded_at: row.try_get("invocation_journal_recorded_at")?,
11271 invocation_journal_digest: row.try_get("invocation_journal_digest")?,
11272 })
11273 }
11274}
11275
11276impl<'row> FromRow<'row, PgRow> for OutboxDestinationRow {
11277 fn from_row(row: &'row PgRow) -> Result<Self, sqlx_core::Error> {
11278 Ok(Self {
11279 tenant_id: row.try_get("tenant_id")?,
11280 destination_id: row.try_get("destination_id")?,
11281 snapshot_digest: row.try_get("snapshot_digest")?,
11282 config_kind: row.try_get("config_kind")?,
11283 schema_id: row.try_get("schema_id")?,
11284 schema_version: row.try_get("schema_version")?,
11285 schema_digest: row.try_get("schema_digest")?,
11286 config_bytes: row.try_get("config_bytes")?,
11287 created_at: row.try_get("created_at")?,
11288 })
11289 }
11290}
11291
11292impl<'row> FromRow<'row, PgRow> for OutboxDeliveryRow {
11293 fn from_row(row: &'row PgRow) -> Result<Self, sqlx_core::Error> {
11294 Ok(Self {
11295 tenant_id: row.try_get("tenant_id")?,
11296 run_id: row.try_get("run_id")?,
11297 delivery_id: row.try_get("delivery_id")?,
11298 origin_sequence: row.try_get("origin_sequence")?,
11299 origin_event_id: row.try_get("origin_event_id")?,
11300 origin_recorded_at: row.try_get("origin_recorded_at")?,
11301 origin_digest: row.try_get("origin_digest")?,
11302 destination_id: row.try_get("destination_id")?,
11303 destination_snapshot_digest: row.try_get("destination_snapshot_digest")?,
11304 intent_digest: row.try_get("intent_digest")?,
11305 expires_at: row.try_get("expires_at")?,
11306 delivery_digest: row.try_get("delivery_digest")?,
11307 delivery_bytes: row.try_get("delivery_bytes")?,
11308 status: row.try_get("status")?,
11309 attempt_count: row.try_get("attempt_count")?,
11310 current_attempt_id: row.try_get("current_attempt_id")?,
11311 current_epoch: row.try_get("current_epoch")?,
11312 current_attempt_started_at: row.try_get("current_attempt_started_at")?,
11313 current_attempt_expires_at: row.try_get("current_attempt_expires_at")?,
11314 next_attempt_at: row.try_get("next_attempt_at")?,
11315 last_completion_digest: row.try_get("last_completion_digest")?,
11316 terminal_at: row.try_get("terminal_at")?,
11317 created_at: row.try_get("created_at")?,
11318 updated_at: row.try_get("updated_at")?,
11319 })
11320 }
11321}
11322
11323impl<'row> FromRow<'row, PgRow> for OutboxAttemptStartRow {
11324 fn from_row(row: &'row PgRow) -> Result<Self, sqlx_core::Error> {
11325 Ok(Self {
11326 tenant_id: row.try_get("tenant_id")?,
11327 run_id: row.try_get("run_id")?,
11328 delivery_id: row.try_get("delivery_id")?,
11329 delivery_expires_at: row.try_get("delivery_expires_at")?,
11330 delivery_digest: row.try_get("delivery_digest")?,
11331 epoch: row.try_get("epoch")?,
11332 attempt_id: row.try_get("attempt_id")?,
11333 started_at: row.try_get("started_at")?,
11334 expires_at: row.try_get("expires_at")?,
11335 start_digest: row.try_get("start_digest")?,
11336 start_bytes: row.try_get("start_bytes")?,
11337 created_at: row.try_get("created_at")?,
11338 })
11339 }
11340}
11341
11342impl<'row> FromRow<'row, PgRow> for OutboxAttemptCompletionRow {
11343 fn from_row(row: &'row PgRow) -> Result<Self, sqlx_core::Error> {
11344 Ok(Self {
11345 tenant_id: row.try_get("tenant_id")?,
11346 run_id: row.try_get("run_id")?,
11347 delivery_id: row.try_get("delivery_id")?,
11348 epoch: row.try_get("epoch")?,
11349 attempt_id: row.try_get("attempt_id")?,
11350 started_at: row.try_get("started_at")?,
11351 attempt_expires_at: row.try_get("attempt_expires_at")?,
11352 start_digest: row.try_get("start_digest")?,
11353 outcome_kind: row.try_get("outcome_kind")?,
11354 retry_advice_kind: row.try_get("retry_advice_kind")?,
11355 retry_delay_millis: row.try_get("retry_delay_millis")?,
11356 completed_at: row.try_get("completed_at")?,
11357 completion_digest: row.try_get("completion_digest")?,
11358 completion_bytes: row.try_get("completion_bytes")?,
11359 created_at: row.try_get("created_at")?,
11360 })
11361 }
11362}
11363
11364struct PreparedProjection {
11365 lifecycle_bytes: Vec<u8>,
11366 revision: String,
11367 status: &'static str,
11368 changed_at: DateTime<Utc>,
11369 wait_set_digest: Option<Digest>,
11370 unresolved_wait_count: i16,
11371 next_timer_due_at: Option<DateTime<Utc>>,
11372 next_interrupt_expiry_at: Option<DateTime<Utc>>,
11373}
11374
11375struct WaitSetProjection {
11376 digest: Option<Digest>,
11377 count: i16,
11378 next_timer_due_at: Option<Timestamp>,
11379 next_interrupt_expiry_at: Option<Timestamp>,
11380}
11381
11382#[derive(Serialize)]
11383#[serde(tag = "kind", rename_all = "snake_case")]
11384enum ProjectionDigestWire<'a> {
11385 Unchanged,
11386 Transition {
11387 expected_revision: &'a RunRevision,
11388 transition: &'a RunTransition,
11389 },
11390}
11391
11392#[derive(Serialize)]
11393struct AgentAdmissionProjectionDigestWire {
11394 admission: Digest,
11395 event_intent: Digest,
11396 checkpoint_intent: Digest,
11397 lifecycle: Digest,
11398}
11399
11400#[derive(Serialize)]
11401struct BarrierProjectionDigestWire {
11402 run_projection_digest: Digest,
11403 barrier_intent_digest: Digest,
11404}
11405
11406#[derive(Serialize)]
11407struct WaitRegistrationProjectionDigestWire<'a> {
11408 expected_revision: &'a RunRevision,
11409 checkpoint_intent_digest: Digest,
11410 registrations: &'a [WaitRegistrationProjectionItem],
11411}
11412
11413#[derive(Serialize)]
11414struct WaitBarrierProjectionDigestWire<'a> {
11415 expected_revision: &'a RunRevision,
11416 barrier_intent_digest: Digest,
11417 registrations: &'a [WaitRegistrationProjectionItem],
11418}
11419
11420#[derive(Serialize)]
11421struct WaitRegistrationProjectionItem {
11422 wait_kind: &'static str,
11423 wait_id: String,
11424 intent_digest: Digest,
11425}
11426
11427#[derive(Serialize)]
11428struct WaitTerminalProjectionDigestWire<'a> {
11429 expected_revision: &'a RunRevision,
11430 intent_digest: Digest,
11431}
11432
11433#[derive(Serialize)]
11434struct WaitAbandonmentProjectionDigestWire<'a> {
11435 run_projection_digest: Digest,
11436 registrations: &'a [WaitAbandonmentProjectionItem],
11437}
11438
11439#[derive(Serialize)]
11440struct WaitAbandonmentProjectionItem {
11441 wait_kind: &'static str,
11442 wait_id: String,
11443 registration_digest: Digest,
11444}
11445
11446#[derive(Serialize)]
11447struct WaitAbandonmentDigestWire<'a> {
11448 registration_digest: Digest,
11449 reason: WaitAbandonmentReason,
11450 journal: &'a JournalHead,
11451}
11452
11453#[derive(Serialize)]
11454struct RunQuarantineDigestWireV1<'a> {
11455 schema_version: u8,
11456 tenant_id: &'a TenantId,
11457 run_id: RunId,
11458 quarantine_id: QuarantineId,
11459 expectation: &'a stateknot_core::JournalExpectation,
11460 cause: RunQuarantineCause,
11461 component: &'a str,
11462 evidence_digest: Digest,
11463 quarantined_at: Timestamp,
11464}
11465
11466#[derive(Serialize)]
11467struct RunQuarantineDigestWireV2<'a> {
11468 schema_version: u8,
11469 tenant_id: &'a TenantId,
11470 run_id: RunId,
11471 quarantine_id: QuarantineId,
11472 expectation: &'a stateknot_core::JournalExpectation,
11473 expected_fence: &'a RunFence,
11474 cause: RunQuarantineCause,
11475 component: &'a str,
11476 evidence_digest: Digest,
11477 quarantined_at: Timestamp,
11478}
11479
11480#[derive(Deserialize, Serialize)]
11481#[serde(deny_unknown_fields)]
11482struct ModelInvocationRecordWire {
11483 schema: String,
11484 intent_digest: Digest,
11485 revision: ModelInvocationRevision,
11486 state: ModelInvocationState,
11487 #[serde(skip_serializing_if = "Option::is_none")]
11488 previous: Option<ModelInvocationHead>,
11489 journal_head: JournalHead,
11490 #[serde(skip_serializing_if = "Option::is_none")]
11491 transition: Option<ModelInvocationTransition>,
11492 #[serde(skip_serializing_if = "Option::is_none")]
11493 transition_digest: Option<Digest>,
11494 digest: Digest,
11495}
11496
11497async fn apply_transaction_timeouts(
11498 transaction: &mut Transaction<'_, Postgres>,
11499 options: &PostgresStoreOptions,
11500 operation: &'static str,
11501) -> Result<(), StoreError> {
11502 query(
11503 "SELECT set_config('lock_timeout', $1, true), \
11504 set_config('statement_timeout', $2, true), \
11505 set_config('idle_in_transaction_session_timeout', $2, true), \
11506 set_config('synchronous_commit', 'on', true)",
11507 )
11508 .bind(options.lock_timeout_setting())
11509 .bind(options.statement_timeout_setting())
11510 .execute(&mut **transaction)
11511 .await
11512 .map_err(|source| StoreError::database(operation, source))?;
11513 Ok(())
11514}
11515
11516async fn database_now(
11517 transaction: &mut Transaction<'_, Postgres>,
11518 operation: &'static str,
11519) -> Result<Timestamp, StoreError> {
11520 let value = query_scalar::<_, DateTime<Utc>>("SELECT clock_timestamp()")
11521 .fetch_one(&mut **transaction)
11522 .await
11523 .map_err(|source| StoreError::database(operation, source))?;
11524 from_database_time(value)
11525}
11526
11527async fn database_scheduler_times(
11528 transaction: &mut Transaction<'_, Postgres>,
11529 operation: &'static str,
11530) -> Result<(Timestamp, Timestamp), StoreError> {
11531 let (transaction_started_at, observed_at) = query_as::<_, (DateTime<Utc>, DateTime<Utc>)>(
11532 "SELECT transaction_timestamp(), clock_timestamp()",
11533 )
11534 .fetch_one(&mut **transaction)
11535 .await
11536 .map_err(|source| StoreError::database(operation, source))?;
11537 Ok((
11538 from_database_time(transaction_started_at)?,
11539 from_database_time(observed_at)?,
11540 ))
11541}
11542
11543async fn fetch_locked_run_row(
11544 transaction: &mut Transaction<'_, Postgres>,
11545 tenant_id: &TenantId,
11546 run_id: RunId,
11547) -> Result<RunRow, StoreError> {
11548 query_as::<_, RunRow>(SELECT_RUN_FOR_UPDATE)
11549 .bind(tenant_id.as_str())
11550 .bind(*run_id.as_uuid())
11551 .fetch_optional(&mut **transaction)
11552 .await
11553 .map_err(|source| StoreError::database("run row lock", source))?
11554 .ok_or(StoreError::RunNotFound)
11555}
11556
11557async fn load_run_quarantine_row_by_run(
11558 transaction: &mut Transaction<'_, Postgres>,
11559 tenant_id: &TenantId,
11560 run_id: RunId,
11561) -> Result<Option<RunQuarantineRow>, StoreError> {
11562 query_as::<_, RunQuarantineRow>(SELECT_RUN_QUARANTINE_BY_RUN)
11563 .bind(tenant_id.as_str())
11564 .bind(*run_id.as_uuid())
11565 .fetch_optional(&mut **transaction)
11566 .await
11567 .map_err(|source| StoreError::database("run quarantine evidence load", source))
11568}
11569
11570async fn load_run_quarantine_row_by_id(
11571 transaction: &mut Transaction<'_, Postgres>,
11572 tenant_id: &TenantId,
11573 quarantine_id: QuarantineId,
11574) -> Result<Option<RunQuarantineRow>, StoreError> {
11575 query_as::<_, RunQuarantineRow>(SELECT_RUN_QUARANTINE_BY_ID)
11576 .bind(tenant_id.as_str())
11577 .bind(*quarantine_id.as_uuid())
11578 .fetch_optional(&mut **transaction)
11579 .await
11580 .map_err(|source| StoreError::database("run quarantine identity load", source))
11581}
11582
11583fn quarantine_expectation_matches(
11584 target: &RunQuarantineTargetRow,
11585 expectation: &stateknot_core::JournalExpectation,
11586) -> Result<bool, StoreError> {
11587 match expectation.head() {
11588 None => Ok(target.journal_sequence.is_none()
11589 && target.journal_event_id.is_none()
11590 && target.journal_recorded_at.is_none()
11591 && target.journal_digest.is_none()),
11592 Some(head) => {
11593 let sequence = i64::try_from(head.sequence().get())
11594 .map_err(|_| StoreError::JournalSequenceExhausted)?;
11595 Ok(target.tenant_id == head.tenant_id().as_str()
11596 && target.run_id == *head.run_id().as_uuid()
11597 && target.journal_sequence == Some(sequence)
11598 && target.journal_event_id == Some(*head.event_id().as_uuid())
11599 && target.journal_recorded_at == Some(to_database_time(head.recorded_at())?)
11600 && target.journal_digest.as_deref() == Some(head.digest().as_bytes()))
11601 }
11602 }
11603}
11604
11605fn authorize_quarantine_fence(
11606 target: &RunQuarantineTargetRow,
11607 expected: Option<&RunFence>,
11608 observed_at: Timestamp,
11609) -> Result<(), StoreError> {
11610 let Some(expected) = expected else {
11611 return Ok(());
11612 };
11613 if target.tenant_id != expected.tenant_id().as_str()
11614 || target.run_id != *expected.run_id().as_uuid()
11615 {
11616 return Err(StoreError::InvalidRunQuarantineRequest);
11617 }
11618 let expected_epoch =
11619 i64::try_from(expected.epoch().get()).map_err(|_| StoreError::StaleFence)?;
11620 match (
11621 target.lease_attempt_id,
11622 target.lease_renewed_at,
11623 target.lease_expires_at,
11624 ) {
11625 (Some(attempt_id), Some(renewed_at), Some(expires_at)) => {
11626 if attempt_id != *expected.attempt_id().as_uuid()
11627 || target.fencing_epoch != expected_epoch
11628 {
11629 return Err(StoreError::StaleFence);
11630 }
11631 let renewed_at = from_database_time(renewed_at)?;
11632 let expires_at = from_database_time(expires_at)?;
11633 if observed_at < renewed_at {
11634 return Err(StoreError::DatabaseClockRegression);
11635 }
11636 if observed_at >= expires_at {
11637 return Err(StoreError::LeaseExpired);
11638 }
11639 Ok(())
11640 }
11641 (None, None, None) => Err(StoreError::StaleFence),
11642 _ => Err(StoreError::corrupt("run quarantine target lease")),
11643 }
11644}
11645
11646fn journal_expectation_matches_stored(
11647 stored: &StoredRun,
11648 expectation: &stateknot_core::JournalExpectation,
11649) -> bool {
11650 match expectation.head() {
11651 None => stored.journal_head().is_none(),
11652 Some(expected) => stored.journal_head() == Some(expected),
11653 }
11654}
11655
11656fn run_quarantine_cause_from_text(value: &str) -> Result<RunQuarantineCause, StoreError> {
11657 match value {
11658 "integrity_failure" => Ok(RunQuarantineCause::IntegrityFailure),
11659 "unsupported_schema" => Ok(RunQuarantineCause::UnsupportedSchema),
11660 "missing_artifact" => Ok(RunQuarantineCause::MissingArtifact),
11661 "cross_tenant_reference" => Ok(RunQuarantineCause::CrossTenantReference),
11662 "projection_mismatch" => Ok(RunQuarantineCause::ProjectionMismatch),
11663 "fencing_epoch_exhausted" => Ok(RunQuarantineCause::FencingEpochExhausted),
11664 "operator_policy" => Ok(RunQuarantineCause::OperatorPolicy),
11665 _ => Err(StoreError::corrupt("run quarantine cause")),
11666 }
11667}
11668
11669fn run_quarantine_reason(request: &RunQuarantineRequest) -> String {
11670 format!(
11671 "{}:{}",
11672 request.cause().as_str(),
11673 request.component().as_str()
11674 )
11675}
11676
11677fn materialize_run_quarantine(
11678 request: RunQuarantineRequest,
11679 quarantined_at: Timestamp,
11680) -> Result<RunQuarantine, StoreError> {
11681 if request
11682 .expectation()
11683 .head()
11684 .is_some_and(|head| head.recorded_at() > quarantined_at)
11685 {
11686 return Err(StoreError::DatabaseClockRegression);
11687 }
11688 let (domain, canonical) = if let Some(expected_fence) = request.expected_fence() {
11689 (
11690 RUN_QUARANTINE_DIGEST_DOMAIN_V2,
11691 serde_json_canonicalizer::to_vec(&RunQuarantineDigestWireV2 {
11692 schema_version: 2,
11693 tenant_id: request.tenant_id(),
11694 run_id: request.run_id(),
11695 quarantine_id: request.quarantine_id(),
11696 expectation: request.expectation(),
11697 expected_fence,
11698 cause: request.cause(),
11699 component: request.component().as_str(),
11700 evidence_digest: request.evidence_digest(),
11701 quarantined_at,
11702 })
11703 .map_err(|_| StoreError::encoding("run quarantine"))?,
11704 )
11705 } else {
11706 (
11707 RUN_QUARANTINE_DIGEST_DOMAIN_V1,
11708 serde_json_canonicalizer::to_vec(&RunQuarantineDigestWireV1 {
11709 schema_version: 1,
11710 tenant_id: request.tenant_id(),
11711 run_id: request.run_id(),
11712 quarantine_id: request.quarantine_id(),
11713 expectation: request.expectation(),
11714 cause: request.cause(),
11715 component: request.component().as_str(),
11716 evidence_digest: request.evidence_digest(),
11717 quarantined_at,
11718 })
11719 .map_err(|_| StoreError::encoding("run quarantine"))?,
11720 )
11721 };
11722 let mut preimage = Vec::with_capacity(domain.len() + canonical.len());
11723 preimage.extend_from_slice(domain);
11724 preimage.extend_from_slice(&canonical);
11725 Ok(RunQuarantine {
11726 request,
11727 quarantined_at,
11728 digest: Digest::sha256(preimage),
11729 })
11730}
11731
11732async fn insert_run_quarantine(
11733 transaction: &mut Transaction<'_, Postgres>,
11734 quarantine: &RunQuarantine,
11735) -> Result<(), StoreError> {
11736 let request = quarantine.request();
11737 let expected = request.expectation().head();
11738 let expected_sequence = expected
11739 .map(|head| {
11740 i64::try_from(head.sequence().get()).map_err(|_| StoreError::JournalSequenceExhausted)
11741 })
11742 .transpose()?;
11743 let expected_event_id = expected.map(|head| *head.event_id().as_uuid());
11744 let expected_recorded_at = expected
11745 .map(|head| to_database_time(head.recorded_at()))
11746 .transpose()?;
11747 let expected_digest = expected.map(|head| head.digest().as_bytes().to_vec());
11748 let expected_fence_attempt_id = request
11749 .expected_fence()
11750 .map(|fence| *fence.attempt_id().as_uuid());
11751 let expected_fence_epoch = request
11752 .expected_fence()
11753 .map(|fence| i64::try_from(fence.epoch().get()).map_err(|_| StoreError::StaleFence))
11754 .transpose()?;
11755 query(
11756 r"
11757INSERT INTO stateknot.run_quarantines (
11758 tenant_id,
11759 run_id,
11760 quarantine_id,
11761 quarantined_at,
11762 cause_kind,
11763 component,
11764 evidence_digest,
11765 expected_journal_sequence,
11766 expected_journal_event_id,
11767 expected_journal_recorded_at,
11768 expected_journal_digest,
11769 expected_fence_attempt_id,
11770 expected_fence_epoch,
11771 record_digest,
11772 created_at
11773)
11774VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $4)
11775",
11776 )
11777 .bind(request.tenant_id().as_str())
11778 .bind(*request.run_id().as_uuid())
11779 .bind(*request.quarantine_id().as_uuid())
11780 .bind(to_database_time(quarantine.quarantined_at())?)
11781 .bind(request.cause().as_str())
11782 .bind(request.component().as_str())
11783 .bind(request.evidence_digest().as_bytes())
11784 .bind(expected_sequence)
11785 .bind(expected_event_id)
11786 .bind(expected_recorded_at)
11787 .bind(expected_digest)
11788 .bind(expected_fence_attempt_id)
11789 .bind(expected_fence_epoch)
11790 .bind(quarantine.digest().as_bytes())
11791 .execute(&mut **transaction)
11792 .await
11793 .map_err(|source| StoreError::database("run quarantine evidence insert", source))?;
11794 Ok(())
11795}
11796
11797async fn commit_run_quarantine_projection(
11798 transaction: &mut Transaction<'_, Postgres>,
11799 quarantine: &RunQuarantine,
11800) -> Result<(), StoreError> {
11801 let request = quarantine.request();
11802 let expected_fence_attempt_id = request
11803 .expected_fence()
11804 .map(|fence| *fence.attempt_id().as_uuid());
11805 let expected_fence_epoch = request
11806 .expected_fence()
11807 .map(|fence| i64::try_from(fence.epoch().get()).map_err(|_| StoreError::StaleFence))
11808 .transpose()?;
11809 let updated = query(
11810 r"
11811UPDATE stateknot.runs
11812SET quarantined_at = $3,
11813 quarantine_reason = $4,
11814 lease_attempt_id = NULL,
11815 lease_acquired_at = NULL,
11816 lease_renewed_at = NULL,
11817 lease_expires_at = NULL,
11818 updated_at = GREATEST(updated_at, $3)
11819WHERE tenant_id = $1
11820 AND run_id = $2
11821 AND quarantined_at IS NULL
11822 AND (
11823 $5::uuid IS NULL
11824 OR (
11825 lease_attempt_id = $5
11826 AND fencing_epoch = $6
11827 AND lease_expires_at > clock_timestamp()
11828 )
11829 )
11830",
11831 )
11832 .bind(request.tenant_id().as_str())
11833 .bind(*request.run_id().as_uuid())
11834 .bind(to_database_time(quarantine.quarantined_at())?)
11835 .bind(run_quarantine_reason(request))
11836 .bind(expected_fence_attempt_id)
11837 .bind(expected_fence_epoch)
11838 .execute(&mut **transaction)
11839 .await
11840 .map_err(|source| StoreError::database("run quarantine projection update", source))?
11841 .rows_affected();
11842 if updated == 1 {
11843 return Ok(());
11844 }
11845 Err(if request.expected_fence().is_some() {
11846 StoreError::LeaseExpired
11847 } else {
11848 StoreError::RunQuarantineCommitConflict
11849 })
11850}
11851
11852fn decode_run_quarantine(row: &RunQuarantineRow) -> Result<RunQuarantine, StoreError> {
11853 let tenant_id = TenantId::try_from(row.tenant_id.clone())
11854 .map_err(|_| StoreError::corrupt("run quarantine tenant"))?;
11855 let run_id = RunId::from_uuid(row.run_id)
11856 .map_err(|_| StoreError::corrupt("run quarantine run identity"))?;
11857 let quarantine_id = QuarantineId::from_uuid(row.quarantine_id)
11858 .map_err(|_| StoreError::corrupt("run quarantine identity"))?;
11859 let expectation = match (
11860 row.expected_journal_sequence,
11861 row.expected_journal_event_id,
11862 row.expected_journal_recorded_at,
11863 row.expected_journal_digest.as_deref(),
11864 ) {
11865 (None, None, None, None) => stateknot_core::JournalExpectation::empty(),
11866 (Some(sequence), Some(event_id), Some(recorded_at), Some(digest)) => {
11867 stateknot_core::JournalExpectation::exact(JournalHead::new(
11868 tenant_id.clone(),
11869 run_id,
11870 positive_sequence(sequence)?,
11871 EventId::from_uuid(event_id)
11872 .map_err(|_| StoreError::corrupt("run quarantine event identity"))?,
11873 from_database_time(recorded_at)?,
11874 decode_digest(digest, "run quarantine expected journal digest")?,
11875 ))
11876 }
11877 _ => return Err(StoreError::corrupt("run quarantine journal shape")),
11878 };
11879 let cause = run_quarantine_cause_from_text(&row.cause_kind)?;
11880 let component = RunQuarantineComponent::new(row.component.clone())
11881 .map_err(|_| StoreError::corrupt("run quarantine component"))?;
11882 let expected_fence = match (row.expected_fence_attempt_id, row.expected_fence_epoch) {
11883 (None, None) => None,
11884 (Some(attempt_id), Some(epoch)) => Some(RunFence::new(
11885 tenant_id.clone(),
11886 run_id,
11887 AttemptId::from_uuid(attempt_id)
11888 .map_err(|_| StoreError::corrupt("run quarantine fence attempt"))?,
11889 FencingEpoch::new(
11890 u64::try_from(epoch)
11891 .map_err(|_| StoreError::corrupt("run quarantine fence epoch"))?,
11892 )
11893 .map_err(|_| StoreError::corrupt("run quarantine fence epoch"))?,
11894 )),
11895 _ => return Err(StoreError::corrupt("run quarantine fence shape")),
11896 };
11897 let mut request = RunQuarantineRequest::new(
11898 tenant_id,
11899 run_id,
11900 quarantine_id,
11901 expectation,
11902 cause,
11903 component,
11904 decode_digest(&row.evidence_digest, "run quarantine evidence digest")?,
11905 )
11906 .map_err(|_| StoreError::corrupt("run quarantine request"))?;
11907 if let Some(expected_fence) = expected_fence {
11908 request = request
11909 .with_expected_fence(expected_fence)
11910 .map_err(|_| StoreError::corrupt("run quarantine request fence"))?;
11911 }
11912 let quarantine = materialize_run_quarantine(request, from_database_time(row.quarantined_at)?)
11913 .map_err(|_| StoreError::corrupt("run quarantine record"))?;
11914 let reason = run_quarantine_reason(quarantine.request());
11915 let expected_fencing_epoch = quarantine
11916 .request()
11917 .expected_fence()
11918 .map(|fence| {
11919 i64::try_from(fence.epoch().get())
11920 .map_err(|_| StoreError::corrupt("run quarantine fence epoch projection"))
11921 })
11922 .transpose()?;
11923 if row.created_at != row.quarantined_at
11924 || decode_digest(&row.record_digest, "run quarantine record digest")? != quarantine.digest()
11925 || row.run_quarantined_at != Some(row.quarantined_at)
11926 || row.run_quarantine_reason.as_deref() != Some(reason.as_str())
11927 || row.run_lease_attempt_id.is_some()
11928 || row.run_lease_acquired_at.is_some()
11929 || row.run_lease_renewed_at.is_some()
11930 || row.run_lease_expires_at.is_some()
11931 || expected_fencing_epoch.is_some_and(|epoch| row.run_fencing_epoch != epoch)
11932 || row.run_updated_at < row.quarantined_at
11933 || row
11934 .run_scheduler_ready_at
11935 .is_some_and(|ready_at| ready_at > row.run_updated_at)
11936 {
11937 return Err(StoreError::corrupt("run quarantine projection"));
11938 }
11939 Ok(quarantine)
11940}
11941
11942async fn load_and_verify_wait_registration(
11943 transaction: &mut Transaction<'_, Postgres>,
11944 tenant_id: &TenantId,
11945 run_id: RunId,
11946 wait_id: Uuid,
11947) -> Result<DurableWait, StoreError> {
11948 let row = query_as::<_, WaitRegistrationRow>(SELECT_WAIT_REGISTRATION_BY_ID.as_str())
11949 .bind(tenant_id.as_str())
11950 .bind(*run_id.as_uuid())
11951 .bind(wait_id)
11952 .fetch_optional(&mut **transaction)
11953 .await
11954 .map_err(|source| StoreError::database("wait registration load", source))?
11955 .ok_or(StoreError::WaitRegistrationNotFound)?;
11956 let sequence = row.registration_sequence;
11957 let wait = decode_wait_registration(&row)?;
11958 verify_wait_registration_event(transaction, &wait, sequence).await?;
11959 if wait.tenant_id() != tenant_id
11960 || wait.run_id() != run_id
11961 || durable_wait_identity(&wait) != wait_id
11962 {
11963 return Err(StoreError::corrupt("wait registration event anchor"));
11964 }
11965 Ok(wait)
11966}
11967
11968async fn verify_wait_registration_event(
11969 transaction: &mut Transaction<'_, Postgres>,
11970 wait: &DurableWait,
11971 sequence: i64,
11972) -> Result<(), StoreError> {
11973 let event_row = query_as::<_, EventRow>(SELECT_EVENT_BY_SEQUENCE)
11974 .bind(wait.tenant_id().as_str())
11975 .bind(*wait.run_id().as_uuid())
11976 .bind(sequence)
11977 .fetch_optional(&mut **transaction)
11978 .await
11979 .map_err(|source| StoreError::database("wait registration event load", source))?
11980 .ok_or_else(|| StoreError::corrupt("wait registration event anchor"))?;
11981 let event = decode_event(event_row)?;
11982 if wait.journal() != &event.head() {
11983 return Err(StoreError::corrupt("wait registration event anchor"));
11984 }
11985 Ok(())
11986}
11987
11988async fn verify_current_wait_set(
11989 transaction: &mut Transaction<'_, Postgres>,
11990 stored: &StoredRun,
11991) -> Result<(), StoreError> {
11992 if stored.is_quarantined()
11993 && stored.lifecycle().status() == RunStatus::Waiting
11994 && stored.unresolved_wait_count() == 0
11995 {
11996 return Ok(());
11997 }
11998 let provenance = stored.lifecycle().provenance();
11999 let rows = query_as::<_, WaitRegistrationRow>(SELECT_OUTSTANDING_WAIT_REGISTRATIONS.as_str())
12000 .bind(provenance.tenant_id().as_str())
12001 .bind(*provenance.run_id().as_uuid())
12002 .fetch_all(&mut **transaction)
12003 .await
12004 .map_err(|source| StoreError::database("current wait-set load", source))?;
12005 let expected = stored.lifecycle().waits();
12006 if rows.len() != expected.map_or(0, RunWaits::len) {
12007 return Err(StoreError::corrupt("current durable wait set"));
12008 }
12009 let mut durable = BTreeMap::new();
12010 for row in rows {
12011 let sequence = row.registration_sequence;
12012 let wait = decode_wait_registration(&row)?;
12013 verify_wait_registration_event(transaction, &wait, sequence).await?;
12014 let marker = wait.marker();
12015 if durable.insert(run_wait_identity(&marker), marker).is_some() {
12016 return Err(StoreError::corrupt("current durable wait set"));
12017 }
12018 }
12019 if expected.is_some_and(|waits| {
12020 waits
12021 .iter()
12022 .any(|marker| durable.get(&run_wait_identity(marker)) != Some(marker))
12023 }) || !durable.is_empty() && expected.is_none()
12024 {
12025 return Err(StoreError::corrupt("current durable wait set"));
12026 }
12027 Ok(())
12028}
12029
12030async fn load_discovery_wait_owner<'owners>(
12031 transaction: &mut Transaction<'_, Postgres>,
12032 owners: &'owners mut BTreeMap<RunId, StoredRun>,
12033 tenant_id: &TenantId,
12034 run_id: RunId,
12035) -> Result<&'owners StoredRun, StoreError> {
12036 match owners.entry(run_id) {
12037 std::collections::btree_map::Entry::Occupied(entry) => Ok(entry.into_mut()),
12038 std::collections::btree_map::Entry::Vacant(entry) => {
12039 let row = query_as::<_, RunRow>(SELECT_RUN)
12040 .bind(tenant_id.as_str())
12041 .bind(*run_id.as_uuid())
12042 .fetch_optional(&mut **transaction)
12043 .await
12044 .map_err(|source| StoreError::database("wait discovery owner load", source))?
12045 .ok_or_else(|| StoreError::corrupt("wait discovery owner"))?;
12046 let owner = decode_run(row)?;
12047 if owner.lifecycle().provenance().tenant_id() != tenant_id
12048 || owner.lifecycle().provenance().run_id() != run_id
12049 || owner.lifecycle().status() != RunStatus::Waiting
12050 || owner.is_quarantined()
12051 {
12052 return Err(StoreError::corrupt("wait discovery owner"));
12053 }
12054 verify_current_wait_set(transaction, &owner).await?;
12055 Ok(entry.insert(owner))
12056 }
12057 }
12058}
12059
12060fn wait_discovery_key_after<Identity: Ord>(
12061 current: &(Timestamp, RunId, Identity),
12062 previous: &(Timestamp, RunId, Identity),
12063) -> bool {
12064 current > previous
12065}
12066
12067async fn load_interrupt_record_from_row(
12068 transaction: &mut Transaction<'_, Postgres>,
12069 registration: &WaitRegistrationRow,
12070) -> Result<InterruptRecord, StoreError> {
12071 let wait = decode_wait_registration(registration)?;
12072 let DurableWait::Interrupt { request } = wait else {
12073 return Err(StoreError::WaitRegistrationKindMismatch);
12074 };
12075 verify_wait_registration_event(
12076 transaction,
12077 &DurableWait::Interrupt {
12078 request: request.clone(),
12079 },
12080 registration.registration_sequence,
12081 )
12082 .await?;
12083 let row = query_as::<_, InterruptResolutionRow>(SELECT_INTERRUPT_RESOLUTION)
12084 .bind(request.intent().tenant_id().as_str())
12085 .bind(*request.intent().run_id().as_uuid())
12086 .bind(*request.marker().interrupt_id().as_uuid())
12087 .fetch_optional(&mut **transaction)
12088 .await
12089 .map_err(|source| StoreError::database("interrupt resolution load", source))?;
12090 let resolution = match (registration.status.as_str(), row) {
12091 ("outstanding", None) => None,
12092 ("abandoned", None) => return Err(StoreError::WaitWasAbandoned),
12093 ("resolved", Some(row)) => {
12094 let resolution = decode_interrupt_resolution(&row, &request)?;
12095 if registration
12096 .resolution_digest
12097 .as_deref()
12098 .map(|bytes| decode_digest(bytes, "interrupt projected resolution digest"))
12099 .transpose()?
12100 != Some(resolution.digest())
12101 || registration.terminal_sequence != Some(row.resolution_sequence)
12102 || registration.terminal_event_id != Some(row.resolution_event_id)
12103 || registration.terminal_recorded_at != Some(row.resolved_at)
12104 || registration.terminal_event_digest.as_deref()
12105 != Some(row.resolution_event_digest.as_slice())
12106 {
12107 return Err(StoreError::corrupt("interrupt terminal projection"));
12108 }
12109 verify_terminal_event(transaction, resolution.journal()).await?;
12110 Some(resolution)
12111 }
12112 _ => return Err(StoreError::InterruptResolutionCommitConflict),
12113 };
12114 InterruptRecord::restore(*request, resolution)
12115 .map_err(|_| StoreError::corrupt("interrupt history"))
12116}
12117
12118async fn load_timer_record_from_row(
12119 transaction: &mut Transaction<'_, Postgres>,
12120 registration: &WaitRegistrationRow,
12121) -> Result<DurableTimerRecord, StoreError> {
12122 let wait = decode_wait_registration(registration)?;
12123 let DurableWait::Timer { timer } = wait else {
12124 return Err(StoreError::WaitRegistrationKindMismatch);
12125 };
12126 verify_wait_registration_event(
12127 transaction,
12128 &DurableWait::Timer {
12129 timer: timer.clone(),
12130 },
12131 registration.registration_sequence,
12132 )
12133 .await?;
12134 let row = query_as::<_, TimerFiringRow>(SELECT_TIMER_FIRING)
12135 .bind(timer.intent().tenant_id().as_str())
12136 .bind(*timer.intent().run_id().as_uuid())
12137 .bind(*timer.marker().timer_id().as_uuid())
12138 .fetch_optional(&mut **transaction)
12139 .await
12140 .map_err(|source| StoreError::database("timer firing load", source))?;
12141 let firing = match (registration.status.as_str(), row) {
12142 ("outstanding", None) => None,
12143 ("abandoned", None) => return Err(StoreError::WaitWasAbandoned),
12144 ("fired", Some(row)) => {
12145 let firing = decode_timer_firing(&row, &timer)?;
12146 if registration
12147 .firing_digest
12148 .as_deref()
12149 .map(|bytes| decode_digest(bytes, "timer projected firing digest"))
12150 .transpose()?
12151 != Some(firing.digest())
12152 || registration.terminal_sequence != Some(row.firing_sequence)
12153 || registration.terminal_event_id != Some(row.firing_event_id)
12154 || registration.terminal_recorded_at != Some(row.fired_at)
12155 || registration.terminal_event_digest.as_deref()
12156 != Some(row.firing_event_digest.as_slice())
12157 {
12158 return Err(StoreError::corrupt("timer terminal projection"));
12159 }
12160 verify_terminal_event(transaction, firing.journal()).await?;
12161 Some(firing)
12162 }
12163 _ => return Err(StoreError::TimerFiringCommitConflict),
12164 };
12165 DurableTimerRecord::restore(*timer, firing)
12166 .map_err(|_| StoreError::corrupt("durable timer history"))
12167}
12168
12169async fn verify_terminal_event(
12170 transaction: &mut Transaction<'_, Postgres>,
12171 journal: &JournalHead,
12172) -> Result<(), StoreError> {
12173 let row = query_as::<_, EventRow>(SELECT_EVENT_BY_SEQUENCE)
12174 .bind(journal.tenant_id().as_str())
12175 .bind(*journal.run_id().as_uuid())
12176 .bind(
12177 i64::try_from(journal.sequence().get())
12178 .map_err(|_| StoreError::corrupt("wait terminal sequence"))?,
12179 )
12180 .fetch_optional(&mut **transaction)
12181 .await
12182 .map_err(|source| StoreError::database("wait terminal event load", source))?
12183 .ok_or_else(|| StoreError::corrupt("wait terminal event anchor"))?;
12184 let event = decode_event(row)?;
12185 if event.head() != *journal {
12186 return Err(StoreError::corrupt("wait terminal event anchor"));
12187 }
12188 Ok(())
12189}
12190
12191async fn load_locked_current_checkpoint(
12192 transaction: &mut Transaction<'_, Postgres>,
12193 stored: &StoredRun,
12194 tenant_id: &TenantId,
12195 run_id: RunId,
12196) -> Result<Option<Checkpoint>, StoreError> {
12197 let Some(pointer) = stored.checkpoint() else {
12198 return Ok(None);
12199 };
12200 let row = query_as::<_, CheckpointRow>(SELECT_CHECKPOINT_BY_ID)
12201 .bind(tenant_id.as_str())
12202 .bind(*run_id.as_uuid())
12203 .bind(*pointer.checkpoint_id().as_uuid())
12204 .fetch_optional(&mut **transaction)
12205 .await
12206 .map_err(|source| StoreError::database("locked checkpoint load", source))?
12207 .ok_or_else(|| StoreError::corrupt("locked checkpoint pointer"))?;
12208 let checkpoint = decode_checkpoint(row)?;
12209 if checkpoint.tenant_id() != tenant_id
12210 || checkpoint.run_id() != run_id
12211 || checkpoint.checkpoint_id() != pointer.checkpoint_id()
12212 || checkpoint.superstep() != pointer.superstep()
12213 || checkpoint.digest() != pointer.digest()
12214 {
12215 return Err(StoreError::corrupt("locked checkpoint projection"));
12216 }
12217 verify_checkpoint_anchor(transaction, &checkpoint).await?;
12218 Ok(Some(checkpoint))
12219}
12220
12221async fn verify_checkpoint_anchor(
12222 transaction: &mut Transaction<'_, Postgres>,
12223 checkpoint: &Checkpoint,
12224) -> Result<(), StoreError> {
12225 let sequence = i64::try_from(checkpoint.journal_head().sequence().get())
12226 .map_err(|_| StoreError::corrupt("checkpoint journal sequence"))?;
12227 let row = query_as::<_, EventRow>(SELECT_EVENT_BY_SEQUENCE)
12228 .bind(checkpoint.tenant_id().as_str())
12229 .bind(*checkpoint.run_id().as_uuid())
12230 .bind(sequence)
12231 .fetch_optional(&mut **transaction)
12232 .await
12233 .map_err(|source| StoreError::database("checkpoint anchor verification", source))?
12234 .ok_or_else(|| StoreError::corrupt("checkpoint journal anchor"))?;
12235 let event = decode_event(row)?;
12236 if event.head() != *checkpoint.journal_head() {
12237 return Err(StoreError::corrupt("checkpoint journal anchor"));
12238 }
12239 Ok(())
12240}
12241
12242fn decode_checkpoint_lineage(
12243 rows: Vec<CheckpointRow>,
12244 tenant_id: &TenantId,
12245 run_id: RunId,
12246 page_size: CheckpointLineagePageSize,
12247) -> Result<(Vec<Checkpoint>, Option<Checkpoint>), StoreError> {
12248 let mut checkpoints = rows
12249 .into_iter()
12250 .map(decode_checkpoint)
12251 .collect::<Result<Vec<_>, _>>()?;
12252 if checkpoints
12253 .iter()
12254 .any(|checkpoint| checkpoint.tenant_id() != tenant_id || checkpoint.run_id() != run_id)
12255 {
12256 return Err(StoreError::corrupt("checkpoint lineage scope"));
12257 }
12258 let lookahead = if checkpoints.len() > usize::from(page_size.get()) {
12259 checkpoints.pop()
12260 } else {
12261 None
12262 };
12263 Ok((checkpoints, lookahead))
12264}
12265
12266async fn verify_checkpoint_anchors(
12267 transaction: &mut Transaction<'_, Postgres>,
12268 checkpoints: &[Checkpoint],
12269) -> Result<(), StoreError> {
12270 let first = checkpoints
12271 .first()
12272 .ok_or_else(|| StoreError::corrupt("checkpoint lineage page"))?;
12273 let mut sequences = Vec::with_capacity(checkpoints.len());
12274 for checkpoint in checkpoints {
12275 sequences.push(
12276 i64::try_from(checkpoint.journal_head().sequence().get())
12277 .map_err(|_| StoreError::corrupt("checkpoint journal sequence"))?,
12278 );
12279 }
12280 let rows = query_as::<_, EventRow>(SELECT_EVENTS_BY_SEQUENCES)
12281 .bind(first.tenant_id().as_str())
12282 .bind(*first.run_id().as_uuid())
12283 .bind(&sequences)
12284 .fetch_all(&mut **transaction)
12285 .await
12286 .map_err(|source| StoreError::database("checkpoint lineage anchors", source))?;
12287 if rows.len() != checkpoints.len() {
12288 return Err(StoreError::corrupt("checkpoint lineage anchors"));
12289 }
12290
12291 let mut anchors = BTreeMap::new();
12292 for row in rows {
12293 let event = decode_event(row)?;
12294 if anchors.insert(event.sequence(), event).is_some() {
12295 return Err(StoreError::corrupt("checkpoint lineage anchors"));
12296 }
12297 }
12298 for checkpoint in checkpoints {
12299 let event = anchors
12300 .get(&checkpoint.journal_head().sequence())
12301 .ok_or_else(|| StoreError::corrupt("checkpoint lineage anchor"))?;
12302 if event.head() != *checkpoint.journal_head() {
12303 return Err(StoreError::corrupt("checkpoint lineage anchor"));
12304 }
12305 }
12306 Ok(())
12307}
12308
12309#[allow(clippy::too_many_lines)]
12310fn decode_run(row: RunRow) -> Result<StoredRun, StoreError> {
12311 let tenant_id =
12312 TenantId::try_from(row.tenant_id).map_err(|_| StoreError::corrupt("run tenant"))?;
12313 let run_id = RunId::from_uuid(row.run_id).map_err(|_| StoreError::corrupt("run identity"))?;
12314 let thread_id = stateknot_core::ThreadId::from_uuid(row.thread_id)
12315 .map_err(|_| StoreError::corrupt("run thread identity"))?;
12316 let invocation_id = InvocationId::from_uuid(row.invocation_id)
12317 .map_err(|_| StoreError::corrupt("run invocation identity"))?;
12318 let lifecycle = decode_lifecycle(&row.lifecycle_bytes)?;
12319 let provenance = lifecycle.provenance();
12320 if provenance.tenant_id() != &tenant_id
12321 || provenance.run_id() != run_id
12322 || provenance.thread_id() != thread_id
12323 || provenance.invocation_id() != invocation_id
12324 || lifecycle.revision().to_string() != row.lifecycle_revision
12325 || run_status_text(lifecycle.status()) != row.lifecycle_status
12326 || lifecycle.admitted_at() != from_database_time(row.admitted_at)?
12327 || lifecycle.changed_at() != from_database_time(row.changed_at)?
12328 {
12329 return Err(StoreError::corrupt("run projection"));
12330 }
12331
12332 let journal_head = match (
12333 row.journal_sequence,
12334 row.journal_event_id,
12335 row.journal_recorded_at,
12336 row.journal_digest,
12337 ) {
12338 (None, None, None, None) => None,
12339 (Some(sequence), Some(event_id), Some(recorded_at), Some(digest)) => {
12340 Some(JournalHead::new(
12341 tenant_id.clone(),
12342 run_id,
12343 positive_sequence(sequence)?,
12344 EventId::from_uuid(event_id)
12345 .map_err(|_| StoreError::corrupt("journal head event identity"))?,
12346 from_database_time(recorded_at)?,
12347 decode_digest(&digest, "journal head digest")?,
12348 ))
12349 }
12350 _ => return Err(StoreError::corrupt("journal head shape")),
12351 };
12352
12353 let checkpoint = match (
12354 row.checkpoint_id,
12355 row.checkpoint_superstep,
12356 row.checkpoint_digest,
12357 ) {
12358 (None, None, None) => None,
12359 (Some(checkpoint_id), Some(superstep), Some(digest)) => Some(CheckpointPointer {
12360 checkpoint_id: CheckpointId::from_uuid(checkpoint_id)
12361 .map_err(|_| StoreError::corrupt("checkpoint pointer identity"))?,
12362 superstep: nonnegative_superstep(superstep)?,
12363 digest: decode_digest(&digest, "checkpoint pointer digest")?,
12364 }),
12365 _ => return Err(StoreError::corrupt("checkpoint pointer shape")),
12366 };
12367
12368 let last_fencing_epoch = if row.fencing_epoch == 0 {
12369 None
12370 } else {
12371 let value =
12372 u64::try_from(row.fencing_epoch).map_err(|_| StoreError::corrupt("fencing epoch"))?;
12373 Some(FencingEpoch::new(value).map_err(|_| StoreError::corrupt("fencing epoch"))?)
12374 };
12375 let lease = match (
12376 row.lease_attempt_id,
12377 row.lease_acquired_at,
12378 row.lease_renewed_at,
12379 row.lease_expires_at,
12380 ) {
12381 (None, None, None, None) => None,
12382 (Some(attempt_id), Some(acquired_at), Some(renewed_at), Some(expires_at)) => {
12383 let epoch = last_fencing_epoch.ok_or_else(|| StoreError::corrupt("lease epoch"))?;
12384 Some(
12385 RunLease::restore(
12386 RunFence::new(
12387 tenant_id,
12388 run_id,
12389 AttemptId::from_uuid(attempt_id)
12390 .map_err(|_| StoreError::corrupt("lease attempt identity"))?,
12391 epoch,
12392 ),
12393 from_database_time(acquired_at)?,
12394 from_database_time(renewed_at)?,
12395 from_database_time(expires_at)?,
12396 )
12397 .map_err(|_| StoreError::corrupt("run lease"))?,
12398 )
12399 }
12400 _ => return Err(StoreError::corrupt("lease shape")),
12401 };
12402 let (scheduler_ready_at, scheduler_not_before) =
12403 decode_scheduler_readiness(row.scheduler_ready_at, row.scheduler_not_before, &lifecycle)?;
12404 if lease.is_some() && scheduler_not_before.is_some() {
12405 return Err(StoreError::corrupt(
12406 "run scheduler delayed retry lease shape",
12407 ));
12408 }
12409 let quarantined = row.quarantined_at.is_some();
12410 let expected_waits = wait_set_projection(&lifecycle)
12411 .map_err(|()| StoreError::corrupt("run wait-set projection"))?;
12412 let stored_wait_digest = row
12413 .wait_set_digest
12414 .as_deref()
12415 .map(|bytes| decode_digest(bytes, "run wait-set digest"))
12416 .transpose()?;
12417 let stored_next_timer_due_at = row.next_timer_due_at.map(from_database_time).transpose()?;
12418 let stored_next_interrupt_expiry_at = row
12419 .next_interrupt_expiry_at
12420 .map(from_database_time)
12421 .transpose()?;
12422 let legacy_quarantined_wait = quarantined
12423 && lifecycle.status() == RunStatus::Waiting
12424 && stored_wait_digest.is_none()
12425 && row.unresolved_wait_count == 0
12426 && stored_next_timer_due_at.is_none()
12427 && stored_next_interrupt_expiry_at.is_none();
12428 if !legacy_quarantined_wait
12429 && (stored_wait_digest != expected_waits.digest
12430 || row.unresolved_wait_count != expected_waits.count
12431 || stored_next_timer_due_at != expected_waits.next_timer_due_at
12432 || stored_next_interrupt_expiry_at != expected_waits.next_interrupt_expiry_at)
12433 {
12434 return Err(StoreError::corrupt("run wait-set projection"));
12435 }
12436 let unresolved_wait_count = u8::try_from(row.unresolved_wait_count)
12437 .map_err(|_| StoreError::corrupt("run wait-set count"))?;
12438
12439 Ok(StoredRun {
12440 lifecycle,
12441 journal_head,
12442 lease,
12443 last_fencing_epoch,
12444 checkpoint,
12445 scheduler_ready_at,
12446 scheduler_not_before,
12447 wait_set_digest: stored_wait_digest,
12448 unresolved_wait_count,
12449 next_timer_due_at: stored_next_timer_due_at,
12450 next_interrupt_expiry_at: stored_next_interrupt_expiry_at,
12451 quarantined,
12452 })
12453}
12454
12455fn wait_set_projection(lifecycle: &RunLifecycle) -> Result<WaitSetProjection, ()> {
12456 let Some(waits) = lifecycle.waits() else {
12457 return Ok(WaitSetProjection {
12458 digest: None,
12459 count: 0,
12460 next_timer_due_at: None,
12461 next_interrupt_expiry_at: None,
12462 });
12463 };
12464 let canonical = serde_json_canonicalizer::to_vec(waits).map_err(|_| ())?;
12465 let mut preimage = Vec::with_capacity(WAIT_SET_DIGEST_DOMAIN.len() + canonical.len());
12466 preimage.extend_from_slice(WAIT_SET_DIGEST_DOMAIN);
12467 preimage.extend_from_slice(&canonical);
12468 let next_timer_due_at = waits
12469 .iter()
12470 .filter_map(|wait| wait.as_timer().map(stateknot_core::RunTimer::due_at))
12471 .min();
12472 let next_interrupt_expiry_at = waits
12473 .iter()
12474 .filter_map(|wait| {
12475 wait.as_interrupt()
12476 .and_then(stateknot_core::RunInterrupt::expires_at)
12477 })
12478 .min();
12479 Ok(WaitSetProjection {
12480 digest: Some(Digest::sha256(preimage)),
12481 count: i16::try_from(waits.len()).map_err(|_| ())?,
12482 next_timer_due_at,
12483 next_interrupt_expiry_at,
12484 })
12485}
12486
12487fn decode_scheduler_readiness(
12488 scheduler_ready_at: Option<DateTime<Utc>>,
12489 scheduler_not_before: Option<DateTime<Utc>>,
12490 lifecycle: &RunLifecycle,
12491) -> Result<(Option<Timestamp>, Option<Timestamp>), StoreError> {
12492 let scheduler_ready_at = scheduler_ready_at.map(from_database_time).transpose()?;
12493 let scheduler_not_before = scheduler_not_before.map(from_database_time).transpose()?;
12494 if lifecycle_is_scheduler_runnable(lifecycle.status()) {
12495 let ready_at = scheduler_ready_at
12496 .ok_or_else(|| StoreError::corrupt("run scheduler readiness shape"))?;
12497 if ready_at < lifecycle.admitted_at() {
12498 return Err(StoreError::corrupt("run scheduler readiness time"));
12499 }
12500 if scheduler_not_before.is_some_and(|not_before| not_before < ready_at) {
12501 return Err(StoreError::corrupt("run scheduler delayed retry shape"));
12502 }
12503 } else if scheduler_ready_at.is_some() || scheduler_not_before.is_some() {
12504 return Err(StoreError::corrupt("run scheduler readiness shape"));
12505 }
12506 Ok((scheduler_ready_at, scheduler_not_before))
12507}
12508
12509fn encode_lifecycle(lifecycle: &RunLifecycle) -> Result<Vec<u8>, StoreError> {
12510 let value =
12511 serde_json::to_value(lifecycle).map_err(|_| StoreError::encoding("run lifecycle"))?;
12512 let bounded = BoundedJson::try_from_value_with_limits(value, JsonLimits::MAXIMUM)
12513 .map_err(|_| StoreError::encoding("run lifecycle"))?;
12514 let canonical =
12515 CanonicalJson::new(&bounded).map_err(|_| StoreError::encoding("run lifecycle"))?;
12516 Ok(canonical.as_bytes().to_vec())
12517}
12518
12519fn encode_durable_wait(wait: &DurableWait) -> Result<Vec<u8>, StoreError> {
12520 let bytes = serde_json_canonicalizer::to_vec(wait)
12521 .map_err(|_| StoreError::encoding("durable wait registration"))?;
12522 if bytes.is_empty() || bytes.len() > MAX_WAIT_REGISTRATION_BYTES {
12523 return Err(StoreError::InvalidWaitRegistrationBatch);
12524 }
12525 Ok(bytes)
12526}
12527
12528fn durable_wait_identity(wait: &DurableWait) -> Uuid {
12529 match wait {
12530 DurableWait::Interrupt { request } => *request.marker().interrupt_id().as_uuid(),
12531 DurableWait::Timer { timer } => *timer.marker().timer_id().as_uuid(),
12532 }
12533}
12534
12535fn run_wait_identity(wait: &stateknot_core::RunWait) -> Uuid {
12536 match wait {
12537 stateknot_core::RunWait::Interrupt { interrupt } => *interrupt.interrupt_id().as_uuid(),
12538 stateknot_core::RunWait::Timer { timer } => *timer.timer_id().as_uuid(),
12539 }
12540}
12541
12542fn durable_wait_digest(wait: &DurableWait) -> Digest {
12543 match wait {
12544 DurableWait::Interrupt { request } => request.digest(),
12545 DurableWait::Timer { timer } => timer.digest(),
12546 }
12547}
12548
12549fn durable_wait_kind_text(wait: &DurableWait) -> &'static str {
12550 match wait {
12551 DurableWait::Interrupt { .. } => "interrupt",
12552 DurableWait::Timer { .. } => "timer",
12553 }
12554}
12555
12556const fn wait_abandonment_reason_text(reason: WaitAbandonmentReason) -> &'static str {
12557 match reason {
12558 WaitAbandonmentReason::RunCancellation => "run_cancellation",
12559 WaitAbandonmentReason::RunFailure => "run_failure",
12560 }
12561}
12562
12563fn wait_abandonment_reason_from_text(value: &str) -> Result<WaitAbandonmentReason, StoreError> {
12564 match value {
12565 "run_cancellation" => Ok(WaitAbandonmentReason::RunCancellation),
12566 "run_failure" => Ok(WaitAbandonmentReason::RunFailure),
12567 _ => Err(StoreError::corrupt("wait abandonment reason")),
12568 }
12569}
12570
12571fn materialize_wait_abandonment(
12572 wait: DurableWait,
12573 reason: WaitAbandonmentReason,
12574 journal: JournalHead,
12575) -> Result<WaitAbandonment, StoreError> {
12576 if wait.tenant_id() != journal.tenant_id()
12577 || wait.run_id() != journal.run_id()
12578 || journal.sequence() <= wait.journal().sequence()
12579 || journal.recorded_at() < wait.journal().recorded_at()
12580 {
12581 return Err(StoreError::InvalidWaitAbandonment);
12582 }
12583 let canonical = serde_json_canonicalizer::to_vec(&WaitAbandonmentDigestWire {
12584 registration_digest: durable_wait_digest(&wait),
12585 reason,
12586 journal: &journal,
12587 })
12588 .map_err(|_| StoreError::encoding("wait abandonment"))?;
12589 let mut preimage = Vec::with_capacity(WAIT_ABANDONMENT_DIGEST_DOMAIN.len() + canonical.len());
12590 preimage.extend_from_slice(WAIT_ABANDONMENT_DIGEST_DOMAIN);
12591 preimage.extend_from_slice(&canonical);
12592 Ok(WaitAbandonment {
12593 wait,
12594 reason,
12595 journal,
12596 digest: Digest::sha256(preimage),
12597 })
12598}
12599
12600fn decode_wait_abandonment(
12601 row: &WaitAbandonmentRow,
12602 wait: DurableWait,
12603) -> Result<WaitAbandonment, StoreError> {
12604 let reason = wait_abandonment_reason_from_text(&row.reason_kind)?;
12605 let journal = JournalHead::new(
12606 wait.tenant_id().clone(),
12607 wait.run_id(),
12608 positive_sequence(row.abandonment_sequence)?,
12609 EventId::from_uuid(row.abandonment_event_id)
12610 .map_err(|_| StoreError::corrupt("wait abandonment event identity"))?,
12611 from_database_time(row.abandoned_at)?,
12612 decode_digest(
12613 &row.abandonment_event_digest,
12614 "wait abandonment event digest",
12615 )?,
12616 );
12617 let abandonment = materialize_wait_abandonment(wait, reason, journal)?;
12618 if row.tenant_id != abandonment.wait().tenant_id().as_str()
12619 || row.run_id != *abandonment.wait().run_id().as_uuid()
12620 || row.wait_id != durable_wait_identity(abandonment.wait())
12621 || row.wait_kind != durable_wait_kind_text(abandonment.wait())
12622 || decode_digest(
12623 &row.registration_digest,
12624 "wait abandonment registration digest",
12625 )? != durable_wait_digest(abandonment.wait())
12626 || row.created_at != row.abandoned_at
12627 || decode_digest(&row.abandonment_digest, "wait abandonment digest")?
12628 != abandonment.digest()
12629 {
12630 return Err(StoreError::corrupt("wait abandonment projection"));
12631 }
12632 Ok(abandonment)
12633}
12634
12635fn decode_wait_registration(row: &WaitRegistrationRow) -> Result<DurableWait, StoreError> {
12636 let wait = serde_json::from_slice::<DurableWait>(&row.record_bytes)
12637 .map_err(|_| StoreError::corrupt("durable wait registration bytes"))?;
12638 let canonical = serde_json_canonicalizer::to_vec(&wait)
12639 .map_err(|_| StoreError::corrupt("durable wait registration bytes"))?;
12640 if canonical != row.record_bytes
12641 || row.tenant_id != wait.tenant_id().as_str()
12642 || row.run_id != *wait.run_id().as_uuid()
12643 || row.wait_id != durable_wait_identity(&wait)
12644 || from_database_time(row.registered_at)? != wait.journal().recorded_at()
12645 || row.registration_sequence
12646 != i64::try_from(wait.journal().sequence().get())
12647 .map_err(|_| StoreError::corrupt("wait registration sequence"))?
12648 || row.registration_event_id != *wait.journal().event_id().as_uuid()
12649 || decode_digest(
12650 &row.registration_event_digest,
12651 "wait registration event digest",
12652 )? != wait.journal().digest()
12653 || row.created_at != row.registered_at
12654 || row.updated_at < row.created_at
12655 {
12656 return Err(StoreError::corrupt("durable wait registration projection"));
12657 }
12658
12659 match &wait {
12660 DurableWait::Interrupt { request } => {
12661 let marker = request.marker();
12662 if row.wait_kind != "interrupt"
12663 || row.interrupt_kind.as_deref() != Some(interrupt_kind_text(marker.kind()))
12664 || row.timer_kind.is_some()
12665 || row.due_at.is_some()
12666 || row.expires_at.map(from_database_time).transpose()? != marker.expires_at()
12667 || row
12668 .action_digest
12669 .as_deref()
12670 .map(|bytes| decode_digest(bytes, "interrupt action digest"))
12671 .transpose()?
12672 != Some(request.intent().action_digest())
12673 || decode_digest(&row.intent_digest, "interrupt registration intent digest")?
12674 != request.intent().intent_digest()
12675 || decode_digest(&row.record_digest, "interrupt request digest")?
12676 != request.digest()
12677 {
12678 return Err(StoreError::corrupt("interrupt registration projection"));
12679 }
12680 }
12681 DurableWait::Timer { timer } => {
12682 let marker = timer.marker();
12683 if row.wait_kind != "timer"
12684 || row.interrupt_kind.is_some()
12685 || row.timer_kind.as_deref() != Some(timer_kind_text(marker.kind()))
12686 || row.due_at.map(from_database_time).transpose()? != Some(marker.due_at())
12687 || row.expires_at.is_some()
12688 || row.action_digest.is_some()
12689 || decode_digest(&row.intent_digest, "timer registration intent digest")?
12690 != timer.intent().intent_digest()
12691 || decode_digest(&row.record_digest, "durable timer digest")? != timer.digest()
12692 {
12693 return Err(StoreError::corrupt("timer registration projection"));
12694 }
12695 }
12696 }
12697 validate_wait_terminal_projection(row, &wait)?;
12698 Ok(wait)
12699}
12700
12701fn validate_wait_terminal_projection(
12702 row: &WaitRegistrationRow,
12703 wait: &DurableWait,
12704) -> Result<(), StoreError> {
12705 let terminal_shape = (
12706 row.terminal_sequence,
12707 row.terminal_event_id,
12708 row.terminal_recorded_at,
12709 row.terminal_event_digest.as_deref(),
12710 );
12711 match row.status.as_str() {
12712 "outstanding" => {
12713 if terminal_shape != (None, None, None, None)
12714 || row.resolution_digest.is_some()
12715 || row.firing_digest.is_some()
12716 || row.abandonment_digest.is_some()
12717 || row.updated_at != row.registered_at
12718 {
12719 return Err(StoreError::corrupt("outstanding wait projection"));
12720 }
12721 }
12722 "resolved" | "fired" | "abandoned" => {
12723 let (Some(sequence), Some(event_id), Some(recorded_at), Some(event_digest)) =
12724 terminal_shape
12725 else {
12726 return Err(StoreError::corrupt("terminal wait projection"));
12727 };
12728 if sequence <= row.registration_sequence
12729 || stateknot_core::EventId::from_uuid(event_id).is_err()
12730 || recorded_at < row.registered_at
12731 || row.updated_at != recorded_at
12732 || decode_digest(event_digest, "terminal wait event digest").is_err()
12733 {
12734 return Err(StoreError::corrupt("terminal wait projection"));
12735 }
12736 let exact_kind = match (row.status.as_str(), wait) {
12737 ("resolved", DurableWait::Interrupt { request }) => {
12738 row.resolution_digest.is_some()
12739 && row.firing_digest.is_none()
12740 && row.abandonment_digest.is_none()
12741 && request.marker().expires_at().is_none_or(|expires_at| {
12742 from_database_time(recorded_at)
12743 .is_ok_and(|resolved_at| resolved_at < expires_at)
12744 })
12745 }
12746 ("fired", DurableWait::Timer { timer }) => {
12747 row.resolution_digest.is_none()
12748 && row.firing_digest.is_some()
12749 && row.abandonment_digest.is_none()
12750 && from_database_time(recorded_at)
12751 .is_ok_and(|fired_at| fired_at >= timer.marker().due_at())
12752 }
12753 ("abandoned", _) => {
12754 row.resolution_digest.is_none()
12755 && row.firing_digest.is_none()
12756 && row.abandonment_digest.is_some()
12757 }
12758 _ => false,
12759 };
12760 if !exact_kind {
12761 return Err(StoreError::corrupt("terminal wait projection"));
12762 }
12763 }
12764 _ => return Err(StoreError::corrupt("wait registration status")),
12765 }
12766 Ok(())
12767}
12768
12769fn encode_interrupt_resolution(resolution: &InterruptResolution) -> Result<Vec<u8>, StoreError> {
12770 let bytes = serde_json_canonicalizer::to_vec(resolution)
12771 .map_err(|_| StoreError::encoding("interrupt resolution"))?;
12772 if bytes.is_empty() || bytes.len() > MAX_INTERRUPT_RESOLUTION_BYTES {
12773 return Err(StoreError::InvalidInterruptResolution);
12774 }
12775 Ok(bytes)
12776}
12777
12778fn decode_interrupt_resolution(
12779 row: &InterruptResolutionRow,
12780 request: &InterruptRequest,
12781) -> Result<InterruptResolution, StoreError> {
12782 let resolution = serde_json::from_slice::<InterruptResolution>(&row.resolution_bytes)
12783 .map_err(|_| StoreError::corrupt("interrupt resolution bytes"))?;
12784 let canonical = serde_json_canonicalizer::to_vec(&resolution)
12785 .map_err(|_| StoreError::corrupt("interrupt resolution bytes"))?;
12786 let journal = resolution.journal();
12787 if canonical != row.resolution_bytes
12788 || row.tenant_id != request.intent().tenant_id().as_str()
12789 || row.run_id != *request.intent().run_id().as_uuid()
12790 || row.interrupt_id != *request.marker().interrupt_id().as_uuid()
12791 || decode_digest(&row.request_digest, "interrupt resolution request digest")?
12792 != request.digest()
12793 || row.resolution_sequence
12794 != i64::try_from(journal.sequence().get())
12795 .map_err(|_| StoreError::corrupt("interrupt resolution sequence"))?
12796 || row.resolution_event_id != *journal.event_id().as_uuid()
12797 || from_database_time(row.resolved_at)? != journal.recorded_at()
12798 || decode_digest(
12799 &row.resolution_event_digest,
12800 "interrupt resolution event digest",
12801 )? != journal.digest()
12802 || decode_digest(&row.intent_digest, "interrupt resolution intent digest")?
12803 != resolution.intent().intent_digest()
12804 || decode_digest(&row.resolution_digest, "interrupt resolution digest")?
12805 != resolution.digest()
12806 || row.created_at != row.resolved_at
12807 || resolution.intent().request() != &request.head()
12808 {
12809 return Err(StoreError::corrupt("interrupt resolution projection"));
12810 }
12811 Ok(resolution)
12812}
12813
12814fn encode_timer_firing(firing: &TimerFiring) -> Result<Vec<u8>, StoreError> {
12815 let bytes = serde_json_canonicalizer::to_vec(firing)
12816 .map_err(|_| StoreError::encoding("timer firing"))?;
12817 if bytes.is_empty() || bytes.len() > MAX_TIMER_FIRING_BYTES {
12818 return Err(StoreError::InvalidTimerFiring);
12819 }
12820 Ok(bytes)
12821}
12822
12823fn decode_timer_firing(
12824 row: &TimerFiringRow,
12825 timer: &DurableTimer,
12826) -> Result<TimerFiring, StoreError> {
12827 let firing = serde_json::from_slice::<TimerFiring>(&row.firing_bytes)
12828 .map_err(|_| StoreError::corrupt("timer firing bytes"))?;
12829 let canonical = serde_json_canonicalizer::to_vec(&firing)
12830 .map_err(|_| StoreError::corrupt("timer firing bytes"))?;
12831 let journal = firing.journal();
12832 if canonical != row.firing_bytes
12833 || row.tenant_id != timer.intent().tenant_id().as_str()
12834 || row.run_id != *timer.intent().run_id().as_uuid()
12835 || row.timer_id != *timer.marker().timer_id().as_uuid()
12836 || decode_digest(&row.timer_digest, "timer firing registration digest")? != timer.digest()
12837 || row.firing_sequence
12838 != i64::try_from(journal.sequence().get())
12839 .map_err(|_| StoreError::corrupt("timer firing sequence"))?
12840 || row.firing_event_id != *journal.event_id().as_uuid()
12841 || from_database_time(row.fired_at)? != journal.recorded_at()
12842 || decode_digest(&row.firing_event_digest, "timer firing event digest")? != journal.digest()
12843 || decode_digest(&row.intent_digest, "timer firing intent digest")?
12844 != firing.intent().intent_digest()
12845 || decode_digest(&row.firing_digest, "timer firing digest")? != firing.digest()
12846 || row.created_at != row.fired_at
12847 || firing.intent().timer() != &timer.head()
12848 {
12849 return Err(StoreError::corrupt("timer firing projection"));
12850 }
12851 Ok(firing)
12852}
12853
12854fn decode_lifecycle(bytes: &[u8]) -> Result<RunLifecycle, StoreError> {
12855 let bounded = BoundedJson::from_slice_with_limits(bytes, JsonLimits::MAXIMUM)
12856 .map_err(|_| StoreError::corrupt("run lifecycle bytes"))?;
12857 let canonical =
12858 CanonicalJson::new(&bounded).map_err(|_| StoreError::corrupt("run lifecycle bytes"))?;
12859 if canonical.as_bytes() != bytes {
12860 return Err(StoreError::corrupt("run lifecycle canonical bytes"));
12861 }
12862 serde_json::from_value(bounded.into_value())
12863 .map_err(|_| StoreError::corrupt("run lifecycle value"))
12864}
12865
12866fn encode_checkpoint(checkpoint: &Checkpoint) -> Result<Vec<u8>, StoreError> {
12867 let bytes = serde_json_canonicalizer::to_vec(checkpoint)
12868 .map_err(|_| StoreError::encoding("checkpoint"))?;
12869 if bytes.is_empty() || bytes.len() > MAX_CHECKPOINT_BYTES {
12870 return Err(StoreError::encoding("checkpoint size"));
12871 }
12872 Ok(bytes)
12873}
12874
12875#[allow(clippy::too_many_lines)]
12876fn decode_checkpoint(row: CheckpointRow) -> Result<Checkpoint, StoreError> {
12877 if row.checkpoint_bytes.is_empty() || row.checkpoint_bytes.len() > MAX_CHECKPOINT_BYTES {
12878 return Err(StoreError::corrupt("checkpoint byte length"));
12879 }
12880 let checkpoint = serde_json::from_slice::<Checkpoint>(&row.checkpoint_bytes)
12881 .map_err(|_| StoreError::corrupt("checkpoint value"))?;
12882 let canonical = serde_json_canonicalizer::to_vec(&checkpoint)
12883 .map_err(|_| StoreError::corrupt("checkpoint canonicalization"))?;
12884 if canonical != row.checkpoint_bytes {
12885 return Err(StoreError::corrupt("checkpoint canonical bytes"));
12886 }
12887
12888 let tenant_id =
12889 TenantId::try_from(row.tenant_id).map_err(|_| StoreError::corrupt("checkpoint tenant"))?;
12890 let run_id =
12891 RunId::from_uuid(row.run_id).map_err(|_| StoreError::corrupt("checkpoint run identity"))?;
12892 let checkpoint_id = CheckpointId::from_uuid(row.checkpoint_id)
12893 .map_err(|_| StoreError::corrupt("checkpoint identity"))?;
12894 let superstep = nonnegative_superstep(row.superstep)?;
12895 let journal_sequence = positive_sequence(row.journal_sequence)?;
12896 let journal_event_id = EventId::from_uuid(row.journal_event_id)
12897 .map_err(|_| StoreError::corrupt("checkpoint journal event identity"))?;
12898 let journal_recorded_at = from_database_time(row.journal_recorded_at)?;
12899 let journal_digest = decode_digest(&row.journal_digest, "checkpoint journal digest")?;
12900 let graph_definition_digest = decode_digest(
12901 &row.graph_definition_digest,
12902 "checkpoint graph definition digest",
12903 )?;
12904 let state_schema_digest =
12905 decode_digest(&row.state_schema_digest, "checkpoint state schema digest")?;
12906 let state_digest = decode_digest(&row.state_digest, "checkpoint state digest")?;
12907 let intent_digest = decode_digest(&row.intent_digest, "checkpoint intent digest")?;
12908 let checkpoint_digest = decode_digest(&row.checkpoint_digest, "checkpoint complete digest")?;
12909
12910 let parent_matches = match (
12911 checkpoint.parent(),
12912 row.parent_checkpoint_id,
12913 row.parent_superstep,
12914 row.parent_digest,
12915 ) {
12916 (None, None, None, None) => true,
12917 (Some(parent), Some(parent_id), Some(parent_superstep), Some(parent_digest)) => {
12918 CheckpointId::from_uuid(parent_id).ok() == Some(parent.checkpoint_id())
12919 && nonnegative_superstep(parent_superstep).ok() == Some(parent.superstep())
12920 && decode_digest(&parent_digest, "checkpoint parent digest").ok()
12921 == Some(parent.digest())
12922 }
12923 _ => false,
12924 };
12925
12926 let schema = checkpoint.graph().state_schema();
12927 if checkpoint.tenant_id() != &tenant_id
12928 || checkpoint.run_id() != run_id
12929 || checkpoint.checkpoint_id() != checkpoint_id
12930 || checkpoint.superstep() != superstep
12931 || !parent_matches
12932 || checkpoint.journal_head().sequence() != journal_sequence
12933 || checkpoint.journal_head().event_id() != journal_event_id
12934 || checkpoint.journal_head().recorded_at() != journal_recorded_at
12935 || checkpoint.journal_head().digest() != journal_digest
12936 || checkpoint.graph().definition_digest() != graph_definition_digest
12937 || schema.id().as_str() != row.state_schema_id
12938 || schema.version().to_string() != row.state_schema_version
12939 || schema.digest() != state_schema_digest
12940 || checkpoint.state().digest() != state_digest
12941 || checkpoint.intent_digest() != intent_digest
12942 || checkpoint.digest() != checkpoint_digest
12943 {
12944 return Err(StoreError::corrupt("checkpoint projection"));
12945 }
12946 Ok(checkpoint)
12947}
12948
12949fn encode_tool_invocation_intent(intent: &ToolInvocationIntent) -> Result<Vec<u8>, StoreError> {
12950 let bytes = serde_json_canonicalizer::to_vec(intent)
12951 .map_err(|_| StoreError::encoding("tool invocation intent"))?;
12952 if bytes.is_empty() || bytes.len() > MAX_TOOL_INVOCATION_INTENT_BYTES {
12953 return Err(StoreError::encoding("tool invocation intent size"));
12954 }
12955 Ok(bytes)
12956}
12957
12958fn encode_tool_invocation_record(invocation: &ToolInvocation) -> Result<Vec<u8>, StoreError> {
12959 let bytes = serde_json_canonicalizer::to_vec(invocation)
12960 .map_err(|_| StoreError::encoding("tool invocation record"))?;
12961 if bytes.is_empty() || bytes.len() > MAX_TOOL_INVOCATION_RECORD_BYTES {
12962 return Err(StoreError::encoding("tool invocation record size"));
12963 }
12964 Ok(bytes)
12965}
12966
12967#[allow(clippy::too_many_lines)]
12968fn decode_tool_invocation_intent(
12969 row: &ToolInvocationRow,
12970) -> Result<ToolInvocationIntent, StoreError> {
12971 if row.intent_bytes.is_empty() || row.intent_bytes.len() > MAX_TOOL_INVOCATION_INTENT_BYTES {
12972 return Err(StoreError::corrupt("tool invocation intent byte length"));
12973 }
12974 let intent = serde_json::from_slice::<ToolInvocationIntent>(&row.intent_bytes)
12975 .map_err(|_| StoreError::corrupt("tool invocation intent value"))?;
12976 let canonical = serde_json_canonicalizer::to_vec(&intent)
12977 .map_err(|_| StoreError::corrupt("tool invocation intent canonicalization"))?;
12978 if canonical != row.intent_bytes {
12979 return Err(StoreError::corrupt(
12980 "tool invocation intent canonical bytes",
12981 ));
12982 }
12983
12984 let tenant_id = TenantId::try_from(row.tenant_id.as_str())
12985 .map_err(|_| StoreError::corrupt("tool invocation tenant"))?;
12986 let run_id = RunId::from_uuid(row.run_id)
12987 .map_err(|_| StoreError::corrupt("tool invocation run identity"))?;
12988 let invocation_id = InvocationId::from_uuid(row.invocation_id)
12989 .map_err(|_| StoreError::corrupt("tool invocation identity"))?;
12990 let base_checkpoint_id = CheckpointId::from_uuid(row.base_checkpoint_id)
12991 .map_err(|_| StoreError::corrupt("tool invocation base checkpoint identity"))?;
12992 let base_superstep = nonnegative_superstep(row.base_superstep)?;
12993 let base_digest = decode_digest(
12994 &row.base_checkpoint_digest,
12995 "tool invocation base checkpoint digest",
12996 )?;
12997 let activation = intent.activation();
12998 if intent.tenant_id() != &tenant_id
12999 || intent.run_id() != run_id
13000 || intent.invocation_id() != invocation_id
13001 || activation.base_checkpoint().checkpoint_id() != base_checkpoint_id
13002 || activation.base_checkpoint().superstep() != base_superstep
13003 || activation.base_checkpoint().digest() != base_digest
13004 || activation.graph_namespace().as_str() != row.graph_namespace
13005 || activation.node_id().as_str() != row.node_id
13006 || activation.input_digest()
13007 != decode_digest(
13008 &row.activation_input_digest,
13009 "tool invocation activation input digest",
13010 )?
13011 || intent.intent_digest()
13012 != decode_digest(&row.intent_digest, "tool invocation intent digest")?
13013 {
13014 return Err(StoreError::corrupt("tool invocation intent projection"));
13015 }
13016 from_database_time(row.created_at)?;
13017 from_database_time(row.updated_at)?;
13018 Ok(intent)
13019}
13020
13021#[allow(clippy::too_many_lines)]
13022fn decode_tool_invocation_revision(
13023 row: ToolInvocationRevisionRow,
13024 intent: &ToolInvocationIntent,
13025) -> Result<ToolInvocation, StoreError> {
13026 if row.record_bytes.is_empty() || row.record_bytes.len() > MAX_TOOL_INVOCATION_RECORD_BYTES {
13027 return Err(StoreError::corrupt("tool invocation record byte length"));
13028 }
13029 let invocation = serde_json::from_slice::<ToolInvocation>(&row.record_bytes)
13030 .map_err(|_| StoreError::corrupt("tool invocation record value"))?;
13031 let canonical = serde_json_canonicalizer::to_vec(&invocation)
13032 .map_err(|_| StoreError::corrupt("tool invocation record canonicalization"))?;
13033 if canonical != row.record_bytes {
13034 return Err(StoreError::corrupt(
13035 "tool invocation record canonical bytes",
13036 ));
13037 }
13038
13039 let tenant_id = TenantId::try_from(row.tenant_id)
13040 .map_err(|_| StoreError::corrupt("tool invocation revision tenant"))?;
13041 let run_id = RunId::from_uuid(row.run_id)
13042 .map_err(|_| StoreError::corrupt("tool invocation revision run identity"))?;
13043 let invocation_id = InvocationId::from_uuid(row.invocation_id)
13044 .map_err(|_| StoreError::corrupt("tool invocation revision identity"))?;
13045 let revision = nonnegative_tool_invocation_revision(row.revision)?;
13046 let previous_matches = match (
13047 invocation.previous(),
13048 row.previous_revision,
13049 row.previous_digest,
13050 ) {
13051 (None, None, None) => true,
13052 (Some(previous), Some(previous_revision), Some(previous_digest)) => {
13053 nonnegative_tool_invocation_revision(previous_revision).ok()
13054 == Some(previous.revision())
13055 && decode_digest(&previous_digest, "tool invocation predecessor digest").ok()
13056 == Some(previous.digest())
13057 }
13058 _ => false,
13059 };
13060 let journal_sequence = positive_sequence(row.journal_sequence)?;
13061 let journal_event_id = EventId::from_uuid(row.journal_event_id)
13062 .map_err(|_| StoreError::corrupt("tool invocation journal event identity"))?;
13063 let journal_recorded_at = from_database_time(row.journal_recorded_at)?;
13064 let journal_digest = decode_digest(&row.journal_digest, "tool invocation journal digest")?;
13065 let attempt_id = row
13066 .attempt_id
13067 .map(AttemptId::from_uuid)
13068 .transpose()
13069 .map_err(|_| StoreError::corrupt("tool invocation attempt identity"))?;
13070 let started_attempt_id = row
13071 .started_attempt_id
13072 .map(AttemptId::from_uuid)
13073 .transpose()
13074 .map_err(|_| StoreError::corrupt("tool invocation started attempt identity"))?;
13075 let transition_kind = invocation.transition().map(ToolInvocationTransition::kind);
13076 let expected_started = match invocation.transition() {
13077 Some(ToolInvocationTransition::StartAttempt { attempt_id }) => Some(*attempt_id),
13078 _ => None,
13079 };
13080 let transition_digest = row
13081 .transition_digest
13082 .as_deref()
13083 .map(|bytes| decode_digest(bytes, "tool invocation transition digest"))
13084 .transpose()?;
13085
13086 if invocation.intent() != intent
13087 || invocation.intent().tenant_id() != &tenant_id
13088 || invocation.intent().run_id() != run_id
13089 || invocation.intent().invocation_id() != invocation_id
13090 || invocation.revision() != revision
13091 || !previous_matches
13092 || invocation.journal_head().sequence() != journal_sequence
13093 || invocation.journal_head().event_id() != journal_event_id
13094 || invocation.journal_head().recorded_at() != journal_recorded_at
13095 || invocation.journal_head().digest() != journal_digest
13096 || tool_invocation_status_text(invocation.status()) != row.status
13097 || invocation.attempt_id() != attempt_id
13098 || transition_kind.map(tool_invocation_transition_kind_text)
13099 != row.transition_kind.as_deref()
13100 || expected_started != started_attempt_id
13101 || invocation.transition_digest() != transition_digest
13102 || invocation.digest()
13103 != decode_digest(&row.record_digest, "tool invocation record digest")?
13104 || from_database_time(row.created_at)? != invocation.journal_head().recorded_at()
13105 {
13106 return Err(StoreError::corrupt("tool invocation revision projection"));
13107 }
13108 Ok(invocation)
13109}
13110
13111fn nonnegative_tool_invocation_revision(value: i64) -> Result<ToolInvocationRevision, StoreError> {
13112 let value =
13113 u64::try_from(value).map_err(|_| StoreError::corrupt("tool invocation revision"))?;
13114 ToolInvocationRevision::new(value).map_err(|_| StoreError::corrupt("tool invocation revision"))
13115}
13116
13117fn validate_tool_invocation_current_projection(
13118 row: &ToolInvocationRow,
13119 current: &ToolInvocation,
13120) -> Result<(), StoreError> {
13121 let current_revision = nonnegative_tool_invocation_revision(row.current_revision)?;
13122 let current_attempt = row
13123 .current_attempt_id
13124 .map(AttemptId::from_uuid)
13125 .transpose()
13126 .map_err(|_| StoreError::corrupt("tool invocation current attempt"))?;
13127 let current_digest = decode_digest(
13128 &row.current_record_digest,
13129 "tool invocation current record digest",
13130 )?;
13131 if current.revision() != current_revision
13132 || tool_invocation_status_text(current.status()) != row.current_status
13133 || current.attempt_id() != current_attempt
13134 || current.digest() != current_digest
13135 || from_database_time(row.updated_at)? != current.journal_head().recorded_at()
13136 {
13137 return Err(StoreError::corrupt("tool invocation current projection"));
13138 }
13139 Ok(())
13140}
13141
13142async fn load_tool_invocation_revision_row(
13143 transaction: &mut Transaction<'_, Postgres>,
13144 tenant_id: &TenantId,
13145 run_id: RunId,
13146 invocation_id: InvocationId,
13147 revision: ToolInvocationRevision,
13148) -> Result<ToolInvocationRevisionRow, StoreError> {
13149 let revision = i64::try_from(revision.get())
13150 .map_err(|_| StoreError::corrupt("tool invocation revision"))?;
13151 query_as::<_, ToolInvocationRevisionRow>(SELECT_TOOL_INVOCATION_REVISION)
13152 .bind(tenant_id.as_str())
13153 .bind(*run_id.as_uuid())
13154 .bind(*invocation_id.as_uuid())
13155 .bind(revision)
13156 .fetch_optional(&mut **transaction)
13157 .await
13158 .map_err(|source| StoreError::database("tool invocation revision load", source))?
13159 .ok_or(StoreError::ToolInvocationNotFound)
13160}
13161
13162async fn verify_tool_invocation_base_checkpoint(
13163 transaction: &mut Transaction<'_, Postgres>,
13164 intent: &ToolInvocationIntent,
13165) -> Result<(), StoreError> {
13166 let head = intent.activation().base_checkpoint();
13167 let row = query_as::<_, CheckpointRow>(SELECT_CHECKPOINT_BY_ID)
13168 .bind(intent.tenant_id().as_str())
13169 .bind(*intent.run_id().as_uuid())
13170 .bind(*head.checkpoint_id().as_uuid())
13171 .fetch_optional(&mut **transaction)
13172 .await
13173 .map_err(|source| StoreError::database("tool invocation base checkpoint", source))?
13174 .ok_or_else(|| StoreError::corrupt("tool invocation base checkpoint"))?;
13175 let checkpoint = decode_checkpoint(row)?;
13176 if checkpoint.head() != *head || !tool_invocation_activation_is_ready(&checkpoint, intent) {
13177 return Err(StoreError::corrupt("tool invocation base checkpoint"));
13178 }
13179 verify_checkpoint_anchor(transaction, &checkpoint).await
13180}
13181
13182fn tool_invocation_activation_is_ready(
13183 checkpoint: &Checkpoint,
13184 intent: &ToolInvocationIntent,
13185) -> bool {
13186 activation_is_canonical_ready_root(checkpoint, intent.activation())
13187}
13188
13189async fn ensure_no_unsettled_tool_invocations(
13190 transaction: &mut Transaction<'_, Postgres>,
13191 checkpoint: &Checkpoint,
13192) -> Result<(), StoreError> {
13193 let superstep = i64::try_from(checkpoint.superstep().get())
13194 .map_err(|_| StoreError::corrupt("checkpoint superstep"))?;
13195 let exists = query_scalar::<_, bool>(SELECT_UNSETTLED_TOOL_INVOCATION_EXISTS)
13196 .bind(checkpoint.tenant_id().as_str())
13197 .bind(*checkpoint.run_id().as_uuid())
13198 .bind(*checkpoint.checkpoint_id().as_uuid())
13199 .bind(superstep)
13200 .bind(checkpoint.digest().as_bytes())
13201 .fetch_one(&mut **transaction)
13202 .await
13203 .map_err(|source| StoreError::database("tool invocation barrier check", source))?;
13204 if exists {
13205 return Err(StoreError::CheckpointBlockedByToolInvocation);
13206 }
13207 Ok(())
13208}
13209
13210async fn verify_tool_invocation_anchor(
13211 transaction: &mut Transaction<'_, Postgres>,
13212 invocation: &ToolInvocation,
13213) -> Result<JournalEvent, StoreError> {
13214 let sequence = i64::try_from(invocation.journal_head().sequence().get())
13215 .map_err(|_| StoreError::corrupt("tool invocation journal sequence"))?;
13216 let row = query_as::<_, EventRow>(SELECT_EVENT_BY_SEQUENCE)
13217 .bind(invocation.intent().tenant_id().as_str())
13218 .bind(*invocation.intent().run_id().as_uuid())
13219 .bind(sequence)
13220 .fetch_optional(&mut **transaction)
13221 .await
13222 .map_err(|source| StoreError::database("tool invocation anchor", source))?
13223 .ok_or_else(|| StoreError::corrupt("tool invocation journal anchor"))?;
13224 let projection_digest = row
13225 .projection_digest
13226 .as_deref()
13227 .map(|bytes| decode_digest(bytes, "tool invocation projection digest"))
13228 .transpose()?;
13229 let event = decode_event(row)?;
13230 if event.head() != *invocation.journal_head() || projection_digest != Some(invocation.digest())
13231 {
13232 return Err(StoreError::corrupt("tool invocation journal anchor"));
13233 }
13234 Ok(event)
13235}
13236
13237fn encode_model_invocation_intent(intent: &ModelInvocationIntent) -> Result<Vec<u8>, StoreError> {
13238 let bytes = serde_json_canonicalizer::to_vec(intent)
13239 .map_err(|_| StoreError::encoding("model invocation intent"))?;
13240 if bytes.is_empty() || bytes.len() > MAX_MODEL_INVOCATION_INTENT_BYTES {
13241 return Err(StoreError::encoding("model invocation intent size"));
13242 }
13243 Ok(bytes)
13244}
13245
13246fn model_invocation_record_wire(invocation: &ModelInvocation) -> ModelInvocationRecordWire {
13247 ModelInvocationRecordWire {
13248 schema: MODEL_INVOCATION_RECORD_SCHEMA.to_owned(),
13249 intent_digest: invocation.intent().intent_digest(),
13250 revision: invocation.revision(),
13251 state: invocation.state().clone(),
13252 previous: invocation.previous().cloned(),
13253 journal_head: invocation.journal_head().clone(),
13254 transition: invocation.transition().cloned(),
13255 transition_digest: invocation.transition_digest(),
13256 digest: invocation.digest(),
13257 }
13258}
13259
13260fn encode_model_invocation_record(invocation: &ModelInvocation) -> Result<Vec<u8>, StoreError> {
13261 let bytes = serde_json_canonicalizer::to_vec(&model_invocation_record_wire(invocation))
13262 .map_err(|_| StoreError::encoding("model invocation record"))?;
13263 if bytes.is_empty() || bytes.len() > MAX_MODEL_INVOCATION_RECORD_BYTES {
13264 return Err(StoreError::encoding("model invocation record size"));
13265 }
13266 Ok(bytes)
13267}
13268
13269#[allow(clippy::too_many_lines)]
13270fn decode_model_invocation_intent(
13271 row: &ModelInvocationRow,
13272) -> Result<ModelInvocationIntent, StoreError> {
13273 if row.intent_bytes.is_empty() || row.intent_bytes.len() > MAX_MODEL_INVOCATION_INTENT_BYTES {
13274 return Err(StoreError::corrupt("model invocation intent byte length"));
13275 }
13276 let intent = serde_json::from_slice::<ModelInvocationIntent>(&row.intent_bytes)
13277 .map_err(|_| StoreError::corrupt("model invocation intent value"))?;
13278 let canonical = serde_json_canonicalizer::to_vec(&intent)
13279 .map_err(|_| StoreError::corrupt("model invocation intent canonicalization"))?;
13280 if canonical != row.intent_bytes {
13281 return Err(StoreError::corrupt(
13282 "model invocation intent canonical bytes",
13283 ));
13284 }
13285
13286 let tenant_id = TenantId::try_from(row.tenant_id.as_str())
13287 .map_err(|_| StoreError::corrupt("model invocation tenant"))?;
13288 let run_id = RunId::from_uuid(row.run_id)
13289 .map_err(|_| StoreError::corrupt("model invocation run identity"))?;
13290 let invocation_id = InvocationId::from_uuid(row.invocation_id)
13291 .map_err(|_| StoreError::corrupt("model invocation identity"))?;
13292 let base_checkpoint_id = CheckpointId::from_uuid(row.base_checkpoint_id)
13293 .map_err(|_| StoreError::corrupt("model invocation base checkpoint identity"))?;
13294 let base_superstep = nonnegative_superstep(row.base_superstep)?;
13295 let base_digest = decode_digest(
13296 &row.base_checkpoint_digest,
13297 "model invocation base checkpoint digest",
13298 )?;
13299 let activation = intent.activation();
13300 if intent.tenant_id() != &tenant_id
13301 || intent.run_id() != run_id
13302 || intent.invocation_id() != invocation_id
13303 || activation.base_checkpoint().checkpoint_id() != base_checkpoint_id
13304 || activation.base_checkpoint().superstep() != base_superstep
13305 || activation.base_checkpoint().digest() != base_digest
13306 || activation.graph_namespace().as_str() != row.graph_namespace
13307 || activation.node_id().as_str() != row.node_id
13308 || activation.input_digest()
13309 != decode_digest(
13310 &row.activation_input_digest,
13311 "model invocation activation input digest",
13312 )?
13313 || intent.intent_digest()
13314 != decode_digest(&row.intent_digest, "model invocation intent digest")?
13315 {
13316 return Err(StoreError::corrupt("model invocation intent projection"));
13317 }
13318 from_database_time(row.created_at)?;
13319 from_database_time(row.updated_at)?;
13320 Ok(intent)
13321}
13322
13323#[allow(clippy::too_many_lines)]
13324fn decode_model_invocation_revision(
13325 row: ModelInvocationRevisionRow,
13326 intent: &ModelInvocationIntent,
13327) -> Result<ModelInvocation, StoreError> {
13328 if row.record_bytes.is_empty() || row.record_bytes.len() > MAX_MODEL_INVOCATION_RECORD_BYTES {
13329 return Err(StoreError::corrupt("model invocation record byte length"));
13330 }
13331 let wire = serde_json::from_slice::<ModelInvocationRecordWire>(&row.record_bytes)
13332 .map_err(|_| StoreError::corrupt("model invocation record value"))?;
13333 let canonical = serde_json_canonicalizer::to_vec(&wire)
13334 .map_err(|_| StoreError::corrupt("model invocation record canonicalization"))?;
13335 if canonical != row.record_bytes {
13336 return Err(StoreError::corrupt(
13337 "model invocation record canonical bytes",
13338 ));
13339 }
13340 if wire.schema != MODEL_INVOCATION_RECORD_SCHEMA || wire.intent_digest != intent.intent_digest()
13341 {
13342 return Err(StoreError::corrupt("model invocation record format"));
13343 }
13344 let invocation = ModelInvocation::restore(
13345 intent.clone(),
13346 wire.revision,
13347 wire.state,
13348 wire.previous,
13349 wire.journal_head,
13350 wire.transition,
13351 wire.transition_digest,
13352 wire.digest,
13353 )
13354 .map_err(|_| StoreError::corrupt("model invocation record integrity"))?;
13355
13356 let tenant_id = TenantId::try_from(row.tenant_id)
13357 .map_err(|_| StoreError::corrupt("model invocation revision tenant"))?;
13358 let run_id = RunId::from_uuid(row.run_id)
13359 .map_err(|_| StoreError::corrupt("model invocation revision run identity"))?;
13360 let invocation_id = InvocationId::from_uuid(row.invocation_id)
13361 .map_err(|_| StoreError::corrupt("model invocation revision identity"))?;
13362 let revision = nonnegative_model_invocation_revision(row.revision)?;
13363 let previous_matches = match (
13364 invocation.previous(),
13365 row.previous_revision,
13366 row.previous_digest,
13367 ) {
13368 (None, None, None) => true,
13369 (Some(previous), Some(previous_revision), Some(previous_digest)) => {
13370 nonnegative_model_invocation_revision(previous_revision).ok()
13371 == Some(previous.revision())
13372 && decode_digest(&previous_digest, "model invocation predecessor digest").ok()
13373 == Some(previous.digest())
13374 }
13375 _ => false,
13376 };
13377 let journal_sequence = positive_sequence(row.journal_sequence)?;
13378 let journal_event_id = EventId::from_uuid(row.journal_event_id)
13379 .map_err(|_| StoreError::corrupt("model invocation journal event identity"))?;
13380 let journal_recorded_at = from_database_time(row.journal_recorded_at)?;
13381 let journal_digest = decode_digest(&row.journal_digest, "model invocation journal digest")?;
13382 let attempt_id = row
13383 .attempt_id
13384 .map(AttemptId::from_uuid)
13385 .transpose()
13386 .map_err(|_| StoreError::corrupt("model invocation attempt identity"))?;
13387 let started_attempt_id = row
13388 .started_attempt_id
13389 .map(AttemptId::from_uuid)
13390 .transpose()
13391 .map_err(|_| StoreError::corrupt("model invocation started attempt identity"))?;
13392 let transition_kind = invocation.transition().map(ModelInvocationTransition::kind);
13393 let expected_started = match invocation.transition() {
13394 Some(ModelInvocationTransition::StartAttempt { attempt_id }) => Some(*attempt_id),
13395 _ => None,
13396 };
13397 let transition_digest = row
13398 .transition_digest
13399 .as_deref()
13400 .map(|bytes| decode_digest(bytes, "model invocation transition digest"))
13401 .transpose()?;
13402
13403 if invocation.intent() != intent
13404 || invocation.intent().tenant_id() != &tenant_id
13405 || invocation.intent().run_id() != run_id
13406 || invocation.intent().invocation_id() != invocation_id
13407 || invocation.revision() != revision
13408 || !previous_matches
13409 || invocation.journal_head().sequence() != journal_sequence
13410 || invocation.journal_head().event_id() != journal_event_id
13411 || invocation.journal_head().recorded_at() != journal_recorded_at
13412 || invocation.journal_head().digest() != journal_digest
13413 || model_invocation_status_text(invocation.status()) != row.status
13414 || invocation.attempt_id() != attempt_id
13415 || transition_kind.map(model_invocation_transition_kind_text)
13416 != row.transition_kind.as_deref()
13417 || expected_started != started_attempt_id
13418 || invocation.transition_digest() != transition_digest
13419 || invocation.digest()
13420 != decode_digest(&row.record_digest, "model invocation record digest")?
13421 || from_database_time(row.created_at)? != invocation.journal_head().recorded_at()
13422 {
13423 return Err(StoreError::corrupt("model invocation revision projection"));
13424 }
13425 Ok(invocation)
13426}
13427
13428fn nonnegative_model_invocation_revision(
13429 value: i64,
13430) -> Result<ModelInvocationRevision, StoreError> {
13431 let value =
13432 u64::try_from(value).map_err(|_| StoreError::corrupt("model invocation revision"))?;
13433 ModelInvocationRevision::new(value)
13434 .map_err(|_| StoreError::corrupt("model invocation revision"))
13435}
13436
13437fn validate_model_invocation_current_projection(
13438 row: &ModelInvocationRow,
13439 current: &ModelInvocation,
13440) -> Result<(), StoreError> {
13441 let current_revision = nonnegative_model_invocation_revision(row.current_revision)?;
13442 let current_attempt = row
13443 .current_attempt_id
13444 .map(AttemptId::from_uuid)
13445 .transpose()
13446 .map_err(|_| StoreError::corrupt("model invocation current attempt"))?;
13447 let current_digest = decode_digest(
13448 &row.current_record_digest,
13449 "model invocation current record digest",
13450 )?;
13451 if current.revision() != current_revision
13452 || model_invocation_status_text(current.status()) != row.current_status
13453 || current.attempt_id() != current_attempt
13454 || current.digest() != current_digest
13455 || from_database_time(row.updated_at)? != current.journal_head().recorded_at()
13456 {
13457 return Err(StoreError::corrupt("model invocation current projection"));
13458 }
13459 Ok(())
13460}
13461
13462async fn load_model_invocation_revision_row(
13463 transaction: &mut Transaction<'_, Postgres>,
13464 tenant_id: &TenantId,
13465 run_id: RunId,
13466 invocation_id: InvocationId,
13467 revision: ModelInvocationRevision,
13468) -> Result<ModelInvocationRevisionRow, StoreError> {
13469 let revision = i64::try_from(revision.get())
13470 .map_err(|_| StoreError::corrupt("model invocation revision"))?;
13471 query_as::<_, ModelInvocationRevisionRow>(SELECT_MODEL_INVOCATION_REVISION)
13472 .bind(tenant_id.as_str())
13473 .bind(*run_id.as_uuid())
13474 .bind(*invocation_id.as_uuid())
13475 .bind(revision)
13476 .fetch_optional(&mut **transaction)
13477 .await
13478 .map_err(|source| StoreError::database("model invocation revision load", source))?
13479 .ok_or(StoreError::ModelInvocationNotFound)
13480}
13481
13482async fn verify_model_invocation_base_checkpoint(
13483 transaction: &mut Transaction<'_, Postgres>,
13484 intent: &ModelInvocationIntent,
13485) -> Result<(), StoreError> {
13486 let head = intent.activation().base_checkpoint();
13487 let row = query_as::<_, CheckpointRow>(SELECT_CHECKPOINT_BY_ID)
13488 .bind(intent.tenant_id().as_str())
13489 .bind(*intent.run_id().as_uuid())
13490 .bind(*head.checkpoint_id().as_uuid())
13491 .fetch_optional(&mut **transaction)
13492 .await
13493 .map_err(|source| StoreError::database("model invocation base checkpoint", source))?
13494 .ok_or_else(|| StoreError::corrupt("model invocation base checkpoint"))?;
13495 let checkpoint = decode_checkpoint(row)?;
13496 if checkpoint.head() != *head || !model_invocation_activation_is_ready(&checkpoint, intent) {
13497 return Err(StoreError::corrupt("model invocation base checkpoint"));
13498 }
13499 verify_checkpoint_anchor(transaction, &checkpoint).await
13500}
13501
13502fn model_invocation_activation_is_ready(
13503 checkpoint: &Checkpoint,
13504 intent: &ModelInvocationIntent,
13505) -> bool {
13506 activation_is_canonical_ready_root(checkpoint, intent.activation())
13507}
13508
13509async fn ensure_no_unsettled_model_invocations(
13510 transaction: &mut Transaction<'_, Postgres>,
13511 checkpoint: &Checkpoint,
13512) -> Result<(), StoreError> {
13513 let superstep = i64::try_from(checkpoint.superstep().get())
13514 .map_err(|_| StoreError::corrupt("checkpoint superstep"))?;
13515 let exists = query_scalar::<_, bool>(SELECT_UNSETTLED_MODEL_INVOCATION_EXISTS)
13516 .bind(checkpoint.tenant_id().as_str())
13517 .bind(*checkpoint.run_id().as_uuid())
13518 .bind(*checkpoint.checkpoint_id().as_uuid())
13519 .bind(superstep)
13520 .bind(checkpoint.digest().as_bytes())
13521 .fetch_one(&mut **transaction)
13522 .await
13523 .map_err(|source| StoreError::database("model invocation barrier check", source))?;
13524 if exists {
13525 return Err(StoreError::CheckpointBlockedByModelInvocation);
13526 }
13527 Ok(())
13528}
13529
13530async fn verify_model_invocation_anchor(
13531 transaction: &mut Transaction<'_, Postgres>,
13532 invocation: &ModelInvocation,
13533) -> Result<(), StoreError> {
13534 let sequence = i64::try_from(invocation.journal_head().sequence().get())
13535 .map_err(|_| StoreError::corrupt("model invocation journal sequence"))?;
13536 let row = query_as::<_, EventRow>(SELECT_EVENT_BY_SEQUENCE)
13537 .bind(invocation.intent().tenant_id().as_str())
13538 .bind(*invocation.intent().run_id().as_uuid())
13539 .bind(sequence)
13540 .fetch_optional(&mut **transaction)
13541 .await
13542 .map_err(|source| StoreError::database("model invocation anchor", source))?
13543 .ok_or_else(|| StoreError::corrupt("model invocation journal anchor"))?;
13544 let projection_digest = row
13545 .projection_digest
13546 .as_deref()
13547 .map(|bytes| decode_digest(bytes, "model invocation projection digest"))
13548 .transpose()?;
13549 let event = decode_event(row)?;
13550 if event.head() != *invocation.journal_head() || projection_digest != Some(invocation.digest())
13551 {
13552 return Err(StoreError::corrupt("model invocation journal anchor"));
13553 }
13554 Ok(())
13555}
13556
13557fn encode_graph_definition(graph: &CompiledGraph) -> Result<Vec<u8>, StoreError> {
13558 let bytes = serde_json_canonicalizer::to_vec(graph)
13559 .map_err(|_| StoreError::encoding("compiled graph definition"))?;
13560 if bytes.is_empty() || bytes.len() > MAX_COMPILED_GRAPH_BYTES {
13561 return Err(StoreError::encoding("compiled graph definition size"));
13562 }
13563 Ok(bytes)
13564}
13565
13566fn encode_artifact_ref(artifact: &ArtifactRef) -> Result<Vec<u8>, StoreError> {
13567 let bytes = serde_json_canonicalizer::to_vec(artifact)
13568 .map_err(|_| StoreError::encoding("artifact reference"))?;
13569 if !(2..=MAX_ARTIFACT_REF_BYTES).contains(&bytes.len()) {
13570 return Err(StoreError::encoding("artifact reference size"));
13571 }
13572 Ok(bytes)
13573}
13574
13575async fn load_artifact_record(
13576 transaction: &mut Transaction<'_, Postgres>,
13577 identity: &ArtifactIdentity,
13578) -> Result<Option<StoredArtifact>, StoreError> {
13579 let row = query_as::<_, ArtifactRow>(SELECT_ARTIFACT)
13580 .bind(identity.tenant_id().as_str())
13581 .bind(*identity.artifact_id().as_uuid())
13582 .fetch_optional(&mut **transaction)
13583 .await
13584 .map_err(|source| StoreError::database("artifact load", source))?;
13585 let Some(row) = row else {
13586 return Ok(None);
13587 };
13588 let stored = decode_artifact_row(row)?;
13589 if stored.artifact().identity() != identity {
13590 return Err(StoreError::corrupt("artifact identity projection"));
13591 }
13592
13593 let parent_rows = query_scalar::<_, Uuid>(SELECT_ARTIFACT_PARENTS)
13594 .bind(identity.tenant_id().as_str())
13595 .bind(*identity.artifact_id().as_uuid())
13596 .fetch_all(&mut **transaction)
13597 .await
13598 .map_err(|source| StoreError::database("artifact parent load", source))?;
13599 let mut parents = parent_rows
13600 .into_iter()
13601 .map(|parent| {
13602 ArtifactId::from_uuid(parent)
13603 .map_err(|_| StoreError::corrupt("artifact parent identifier"))
13604 })
13605 .collect::<Result<Vec<_>, _>>()?;
13606 parents.sort_unstable();
13607 if parents.as_slice() != stored.artifact().parents().as_slice() {
13608 return Err(StoreError::corrupt("artifact parent projection"));
13609 }
13610 Ok(Some(stored))
13611}
13612
13613fn decode_artifact_row(row: ArtifactRow) -> Result<StoredArtifact, StoreError> {
13614 if !(2..=MAX_ARTIFACT_REF_BYTES).contains(&row.artifact_ref_bytes.len()) {
13615 return Err(StoreError::corrupt("artifact reference byte length"));
13616 }
13617 let artifact = serde_json::from_slice::<ArtifactRef>(&row.artifact_ref_bytes)
13618 .map_err(|_| StoreError::corrupt("artifact reference bytes"))?;
13619 let canonical = serde_json_canonicalizer::to_vec(&artifact)
13620 .map_err(|_| StoreError::corrupt("artifact reference canonicalization"))?;
13621 if canonical != row.artifact_ref_bytes
13622 || Digest::sha256(&canonical)
13623 != decode_digest(&row.artifact_ref_digest, "artifact reference digest")?
13624 {
13625 return Err(StoreError::corrupt("artifact reference integrity"));
13626 }
13627
13628 let tenant_id =
13629 TenantId::try_from(row.tenant_id).map_err(|_| StoreError::corrupt("artifact tenant"))?;
13630 let artifact_id = ArtifactId::from_uuid(row.artifact_id)
13631 .map_err(|_| StoreError::corrupt("artifact identifier"))?;
13632 let identity = ArtifactIdentity::new(tenant_id, artifact_id);
13633 let representation = artifact.representation();
13634 let content_byte_length = u64::try_from(row.content_byte_length)
13635 .map_err(|_| StoreError::corrupt("artifact content byte length"))?;
13636 let provenance = artifact.provenance();
13637 if artifact.identity() != &identity
13638 || representation.byte_length().get() != content_byte_length
13639 || representation.digest() != decode_digest(&row.content_digest, "artifact content digest")?
13640 || *provenance.run_id().as_uuid() != row.provenance_run_id
13641 || *provenance.event_id().as_uuid() != row.provenance_event_id
13642 {
13643 return Err(StoreError::corrupt("artifact redundant projection"));
13644 }
13645
13646 let registration_key = decode_digest(
13647 &row.registration_key_digest,
13648 "artifact registration key digest",
13649 )?;
13650 let locator = ArtifactStorageLocator::new(
13651 row.storage_namespace,
13652 row.object_key,
13653 row.object_version,
13654 row.object_etag,
13655 )
13656 .map_err(|_| StoreError::corrupt("artifact storage locator"))?;
13657 Ok(StoredArtifact {
13658 registration: ArtifactRegistration::new(registration_key, artifact, locator),
13659 registered_at: from_database_time(row.registered_at)?,
13660 })
13661}
13662
13663fn map_artifact_insert_error(source: sqlx_core::Error) -> StoreError {
13664 if has_database_constraint(&source, "artifacts_provenance_event_fk") {
13665 StoreError::ArtifactProvenanceNotFound
13666 } else {
13667 StoreError::database("artifact insert", source)
13668 }
13669}
13670
13671fn decode_graph_definition(row: GraphDefinitionRow) -> Result<StoredGraphDefinition, StoreError> {
13672 if row.definition_bytes.is_empty() || row.definition_bytes.len() > MAX_COMPILED_GRAPH_BYTES {
13673 return Err(StoreError::corrupt("compiled graph byte length"));
13674 }
13675 let graph = serde_json::from_slice::<CompiledGraph>(&row.definition_bytes)
13676 .map_err(|_| StoreError::corrupt("compiled graph definition"))?;
13677 let canonical = serde_json_canonicalizer::to_vec(&graph)
13678 .map_err(|_| StoreError::corrupt("compiled graph canonicalization"))?;
13679 if canonical != row.definition_bytes {
13680 return Err(StoreError::corrupt("compiled graph canonical bytes"));
13681 }
13682 let tenant_id = TenantId::try_from(row.tenant_id)
13683 .map_err(|_| StoreError::corrupt("compiled graph tenant"))?;
13684 let identity = graph.identity();
13685 let owner = identity.owner();
13686 if owner.issuer().as_str() != row.owner_issuer
13687 || owner.subject().as_str() != row.owner_subject
13688 || identity.name().as_str() != row.graph_name
13689 || identity.version().to_string() != row.graph_version
13690 || graph.definition_digest()
13691 != decode_digest(&row.definition_digest, "compiled graph definition digest")?
13692 {
13693 return Err(StoreError::corrupt("compiled graph projection"));
13694 }
13695 Ok(StoredGraphDefinition {
13696 tenant_id,
13697 graph,
13698 registered_at: from_database_time(row.registered_at)?,
13699 })
13700}
13701
13702async fn load_graph_definition_row(
13703 transaction: &mut Transaction<'_, Postgres>,
13704 tenant_id: &TenantId,
13705 reference: &GraphReference,
13706) -> Result<Option<GraphDefinitionRow>, StoreError> {
13707 let identity = reference.identity();
13708 let owner = identity.owner();
13709 query_as::<_, GraphDefinitionRow>(SELECT_GRAPH_DEFINITION)
13710 .bind(tenant_id.as_str())
13711 .bind(owner.issuer().as_str())
13712 .bind(owner.subject().as_str())
13713 .bind(identity.name().as_str())
13714 .bind(identity.version().to_string())
13715 .fetch_optional(&mut **transaction)
13716 .await
13717 .map_err(|source| StoreError::database("graph definition load", source))
13718}
13719
13720fn encode_agent_admission(admission: &AgentAdmission) -> Result<Vec<u8>, StoreError> {
13721 let bytes = admission
13722 .canonical_bytes()
13723 .map_err(|_| StoreError::encoding("agent admission"))?;
13724 if bytes.is_empty() || bytes.len() > MAX_AGENT_ADMISSION_BYTES {
13725 return Err(StoreError::encoding("agent admission size"));
13726 }
13727 Ok(bytes)
13728}
13729
13730#[allow(clippy::too_many_lines)]
13731fn decode_agent_admission(row: &AgentAdmissionRow) -> Result<AgentAdmission, StoreError> {
13732 if row.admission_bytes.is_empty() || row.admission_bytes.len() > MAX_AGENT_ADMISSION_BYTES {
13733 return Err(StoreError::corrupt("agent admission byte length"));
13734 }
13735 let admission = serde_json::from_slice::<AgentAdmission>(&row.admission_bytes)
13736 .map_err(|_| StoreError::corrupt("agent admission bytes"))?;
13737 let canonical = admission
13738 .canonical_bytes()
13739 .map_err(|_| StoreError::corrupt("agent admission canonicalization"))?;
13740 if canonical != row.admission_bytes {
13741 return Err(StoreError::corrupt("agent admission canonical bytes"));
13742 }
13743
13744 let intent = admission.intent();
13745 let provenance = intent.provenance();
13746 let agent = intent.descriptor().metadata().identity();
13747 let agent_owner = agent.owner();
13748 let graph = intent.graph();
13749 let graph_identity = graph.identity();
13750 let graph_owner = graph_identity.owner();
13751 let policy = intent.authority().policy();
13752 let policy_owner = policy.owner();
13753 let admitted_at = from_database_time(row.admitted_at)?;
13754
13755 if row.tenant_id != provenance.tenant_id().as_str()
13756 || row.run_id != *provenance.run_id().as_uuid()
13757 || row.agent_owner_issuer != agent_owner.issuer().as_str()
13758 || row.agent_owner_subject != agent_owner.subject().as_str()
13759 || row.agent_name != agent.name().as_str()
13760 || row.agent_version != agent.version().to_string()
13761 || row.graph_owner_issuer != graph_owner.issuer().as_str()
13762 || row.graph_owner_subject != graph_owner.subject().as_str()
13763 || row.graph_name != graph_identity.name().as_str()
13764 || row.graph_version != graph_identity.version().to_string()
13765 || decode_digest(
13766 &row.graph_definition_digest,
13767 "agent admission graph definition digest",
13768 )? != graph.definition_digest()
13769 || row.policy_owner_issuer != policy_owner.issuer().as_str()
13770 || row.policy_owner_subject != policy_owner.subject().as_str()
13771 || row.policy_name != policy.name().as_str()
13772 || row.policy_version != policy.version().to_string()
13773 || decode_digest(&row.policy_digest, "agent admission policy digest")?
13774 != intent.authority().policy_digest()
13775 || decode_digest(&row.intent_digest, "agent admission intent digest")?
13776 != intent.intent_digest()
13777 || decode_digest(&row.admission_digest, "agent admission digest")? != admission.digest()
13778 || admitted_at != admission.admitted_at()
13779 || row.journal_sequence != 1
13780 || row.checkpoint_superstep != 0
13781 || from_database_time(row.journal_recorded_at)? != admission.admitted_at()
13782 || from_database_time(row.created_at)? != admission.admitted_at()
13783 {
13784 return Err(StoreError::corrupt("agent admission projection"));
13785 }
13786 decode_digest(&row.journal_digest, "agent admission journal digest")?;
13787 decode_digest(&row.checkpoint_digest, "agent admission checkpoint digest")?;
13788 Ok(admission)
13789}
13790
13791async fn load_agent_admission_row(
13792 transaction: &mut Transaction<'_, Postgres>,
13793 tenant_id: &TenantId,
13794 run_id: RunId,
13795) -> Result<Option<AgentAdmissionRow>, StoreError> {
13796 query_as::<_, AgentAdmissionRow>(SELECT_AGENT_ADMISSION)
13797 .bind(tenant_id.as_str())
13798 .bind(*run_id.as_uuid())
13799 .fetch_optional(&mut **transaction)
13800 .await
13801 .map_err(|source| StoreError::database("agent admission load", source))
13802}
13803
13804fn initial_active_agent_lifecycle(admission: &AgentAdmission) -> Result<RunLifecycle, StoreError> {
13805 RunLifecycle::admitted(
13806 admission.intent().provenance().clone(),
13807 admission.admitted_at(),
13808 )
13809 .apply(RunTransition::Start {
13810 started_at: admission.admitted_at(),
13811 })
13812 .map_err(|_| StoreError::corrupt("agent admission initial lifecycle"))
13813}
13814
13815#[allow(clippy::too_many_lines)]
13816async fn verify_stored_agent_admission(
13817 transaction: &mut Transaction<'_, Postgres>,
13818 run: StoredRun,
13819 row: AgentAdmissionRow,
13820) -> Result<StoredAgentAdmission, StoreError> {
13821 let admission = decode_agent_admission(&row)?;
13822 let intent = admission.intent();
13823 let provenance = intent.provenance();
13824 if run.lifecycle().provenance() != provenance
13825 || run.lifecycle().admitted_at() != admission.admitted_at()
13826 || run.lifecycle().revision() <= RunRevision::ZERO
13827 {
13828 return Err(StoreError::corrupt("agent admission run projection"));
13829 }
13830
13831 let graph_row = load_graph_definition_row(transaction, provenance.tenant_id(), intent.graph())
13832 .await?
13833 .ok_or_else(|| StoreError::corrupt("agent admission graph reference"))?;
13834 let graph = decode_graph_definition(graph_row)?;
13835 if graph.tenant_id() != provenance.tenant_id() || graph.graph().reference() != *intent.graph() {
13836 return Err(StoreError::corrupt("agent admission graph reference"));
13837 }
13838
13839 let event_row = query_as::<_, EventRow>(SELECT_EVENT_BY_SEQUENCE)
13840 .bind(provenance.tenant_id().as_str())
13841 .bind(*provenance.run_id().as_uuid())
13842 .bind(row.journal_sequence)
13843 .fetch_optional(&mut **transaction)
13844 .await
13845 .map_err(|source| StoreError::database("agent admission event load", source))?
13846 .ok_or_else(|| StoreError::corrupt("agent admission event anchor"))?;
13847 let committed_projection = event_row
13848 .projection_digest
13849 .as_deref()
13850 .map(|bytes| decode_digest(bytes, "agent admission event projection digest"))
13851 .transpose()?;
13852 let event = decode_event(event_row)?;
13853
13854 let checkpoint_row = query_as::<_, CheckpointRow>(SELECT_CHECKPOINT_BY_ID)
13855 .bind(provenance.tenant_id().as_str())
13856 .bind(*provenance.run_id().as_uuid())
13857 .bind(row.checkpoint_id)
13858 .fetch_optional(&mut **transaction)
13859 .await
13860 .map_err(|source| StoreError::database("agent admission checkpoint load", source))?
13861 .ok_or_else(|| StoreError::corrupt("agent admission checkpoint anchor"))?;
13862 let checkpoint = decode_checkpoint(checkpoint_row)?;
13863 let initial_lifecycle = initial_active_agent_lifecycle(&admission)?;
13864 let expected_projection = agent_admission_projection_digest(
13865 &admission,
13866 event.intent_digest(),
13867 &checkpoint.write_intent(),
13868 &initial_lifecycle,
13869 )?;
13870
13871 if event.tenant_id() != provenance.tenant_id()
13872 || event.run_id() != provenance.run_id()
13873 || event.sequence() != JournalSequence::FIRST
13874 || event.event_id().as_uuid() != &row.journal_event_id
13875 || event.recorded_at() != admission.admitted_at()
13876 || event.digest() != decode_digest(&row.journal_digest, "agent admission journal digest")?
13877 || event.previous_digest().is_some()
13878 || event.source().worker_fence().is_some()
13879 || event.payload().kind().as_str() != AgentAdmission::JOURNAL_EVENT_KIND
13880 || committed_projection != Some(expected_projection)
13881 || checkpoint.tenant_id() != provenance.tenant_id()
13882 || checkpoint.run_id() != provenance.run_id()
13883 || checkpoint.checkpoint_id().as_uuid() != &row.checkpoint_id
13884 || checkpoint.superstep() != Superstep::INITIAL
13885 || checkpoint.parent().is_some()
13886 || checkpoint.graph() != intent.graph()
13887 || checkpoint.journal_head() != &event.head()
13888 || checkpoint.digest()
13889 != decode_digest(&row.checkpoint_digest, "agent admission checkpoint digest")?
13890 {
13891 return Err(StoreError::corrupt("agent admission atomic anchors"));
13892 }
13893
13894 let current_journal = run
13895 .journal_head()
13896 .ok_or_else(|| StoreError::corrupt("agent admission current journal"))?;
13897 let current_checkpoint = run
13898 .checkpoint()
13899 .ok_or_else(|| StoreError::corrupt("agent admission current checkpoint"))?;
13900 if current_journal.sequence() < event.sequence()
13901 || current_journal.recorded_at() < event.recorded_at()
13902 || current_checkpoint.superstep() < checkpoint.superstep()
13903 {
13904 return Err(StoreError::corrupt("agent admission current heads"));
13905 }
13906
13907 Ok(StoredAgentAdmission {
13908 admission,
13909 run,
13910 event,
13911 checkpoint,
13912 })
13913}
13914
13915fn validate_agent_admission_commit_input(
13916 intent: &AgentAdmissionIntent,
13917 append: &JournalAppend,
13918 checkpoint: &CheckpointWrite,
13919) -> Result<(), StoreError> {
13920 let provenance = intent.provenance();
13921 if append.worker_fence().is_some()
13922 || append.expectation().head().is_some()
13923 || append.intent().tenant_id() != provenance.tenant_id()
13924 || append.intent().run_id() != provenance.run_id()
13925 || append.intent().payload().kind().as_str() != AgentAdmission::JOURNAL_EVENT_KIND
13926 || checkpoint.tenant_id() != provenance.tenant_id()
13927 || checkpoint.run_id() != provenance.run_id()
13928 || checkpoint.superstep() != Superstep::INITIAL
13929 || checkpoint.parent().is_some()
13930 || checkpoint.graph() != intent.graph()
13931 {
13932 return Err(StoreError::InvalidAgentAdmissionCommit);
13933 }
13934 Ok(())
13935}
13936
13937fn validate_agent_initial_checkpoint<V>(
13938 graph: &CompiledGraph,
13939 checkpoint: &CheckpointWrite,
13940 schemas: &V,
13941) -> Result<(), StoreError>
13942where
13943 V: GraphSchemaValidator + ?Sized,
13944{
13945 if checkpoint.graph() != &graph.reference()
13946 || checkpoint.superstep() != Superstep::INITIAL
13947 || checkpoint.parent().is_some()
13948 || checkpoint.ready_nodes() != graph.entry_nodes()
13949 {
13950 return Err(StoreError::InvalidAgentAdmissionCommit);
13951 }
13952 match catch_unwind(AssertUnwindSafe(|| {
13953 schemas.validate(graph.state_schema(), checkpoint.state().data())
13954 })) {
13955 Err(_) | Ok(Err(GraphSchemaValidationError::Unavailable)) => {
13956 Err(StoreError::AgentAdmissionSchemaUnavailable)
13957 }
13958 Ok(Err(GraphSchemaValidationError::Rejected)) => {
13959 Err(StoreError::AgentAdmissionStateRejected)
13960 }
13961 Ok(Err(_)) => Err(StoreError::AgentAdmissionStateRejected),
13962 Ok(Ok(())) => Ok(()),
13963 }
13964}
13965
13966async fn lock_agent_admission_key(
13967 transaction: &mut Transaction<'_, Postgres>,
13968 tenant_id: &TenantId,
13969 run_id: RunId,
13970) -> Result<(), StoreError> {
13971 query(
13972 r"
13973SELECT pg_advisory_xact_lock(
13974 hashtextextended('stateknot:agent-admission:' || $1 || ':' || $2::text, 0)
13975)
13976",
13977 )
13978 .bind(tenant_id.as_str())
13979 .bind(*run_id.as_uuid())
13980 .execute(&mut **transaction)
13981 .await
13982 .map_err(|source| StoreError::database("agent admission serialization lock", source))?;
13983 Ok(())
13984}
13985
13986async fn load_locked_agent_admission(
13987 transaction: &mut Transaction<'_, Postgres>,
13988 tenant_id: &TenantId,
13989 run_id: RunId,
13990) -> Result<Option<StoredAgentAdmission>, StoreError> {
13991 lock_agent_admission_key(transaction, tenant_id, run_id).await?;
13992 let run_row = query_as::<_, RunRow>(SELECT_RUN_FOR_UPDATE)
13993 .bind(tenant_id.as_str())
13994 .bind(*run_id.as_uuid())
13995 .fetch_optional(&mut **transaction)
13996 .await
13997 .map_err(|source| StoreError::database("agent admission run lookup", source))?;
13998 let Some(run_row) = run_row else {
13999 return Ok(None);
14000 };
14001 let run = decode_run(run_row)?;
14002 verify_current_wait_set(transaction, &run).await?;
14003 let admission_row = load_agent_admission_row(transaction, tenant_id, run_id)
14004 .await?
14005 .ok_or(StoreError::AgentAdmissionConflict)?;
14006 verify_stored_agent_admission(transaction, run, admission_row)
14007 .await
14008 .map(Some)
14009}
14010
14011fn agent_submission_digest(
14012 intent: &AgentAdmissionIntent,
14013 initial_state: &CheckpointState,
14014 initial_ready_nodes: &ReadyNodes,
14015) -> Result<Digest, StoreError> {
14016 #[derive(Serialize)]
14017 struct Wire<'a> {
14018 tenant_id: &'a TenantId,
14019 descriptor: &'a AgentDescriptor,
14020 request: &'a AgentRequest,
14021 budget_layers: &'a [AgentAdmissionBudgetLayer],
14022 budget: &'a ResolvedBudget,
14023 graph: &'a GraphReference,
14024 authority: &'a AgentAdmissionAuthority,
14025 initial_state: &'a CheckpointState,
14026 initial_ready_nodes: &'a ReadyNodes,
14027 }
14028
14029 let canonical = serde_json_canonicalizer::to_vec(&Wire {
14030 tenant_id: intent.provenance().tenant_id(),
14031 descriptor: intent.descriptor(),
14032 request: intent.request(),
14033 budget_layers: intent.budget_layers(),
14034 budget: intent.budget(),
14035 graph: intent.graph(),
14036 authority: intent.authority(),
14037 initial_state,
14038 initial_ready_nodes,
14039 })
14040 .map_err(|_| StoreError::encoding("agent submission"))?;
14041 let mut preimage = Vec::with_capacity(AGENT_SUBMISSION_DIGEST_DOMAIN.len() + canonical.len());
14042 preimage.extend_from_slice(AGENT_SUBMISSION_DIGEST_DOMAIN);
14043 preimage.extend_from_slice(&canonical);
14044 Ok(Digest::sha256(preimage))
14045}
14046
14047async fn lock_agent_submission_key(
14048 transaction: &mut Transaction<'_, Postgres>,
14049 tenant_id: &TenantId,
14050 key_digest: Digest,
14051) -> Result<(), StoreError> {
14052 query(
14053 r"
14054SELECT pg_advisory_xact_lock(
14055 hashtextextended(
14056 'stateknot:agent-submission:' || $1 || ':' || encode($2::bytea, 'hex'),
14057 0
14058 )
14059)
14060",
14061 )
14062 .bind(tenant_id.as_str())
14063 .bind(key_digest.as_bytes())
14064 .execute(&mut **transaction)
14065 .await
14066 .map_err(|source| StoreError::database("agent submission serialization lock", source))?;
14067 Ok(())
14068}
14069
14070async fn load_agent_submission_row(
14071 transaction: &mut Transaction<'_, Postgres>,
14072 tenant_id: &TenantId,
14073 key_digest: Digest,
14074) -> Result<Option<AgentSubmissionRow>, StoreError> {
14075 query_as::<_, AgentSubmissionRow>(SELECT_AGENT_SUBMISSION)
14076 .bind(tenant_id.as_str())
14077 .bind(key_digest.as_bytes())
14078 .fetch_optional(&mut **transaction)
14079 .await
14080 .map_err(|source| StoreError::database("agent submission load", source))
14081}
14082
14083fn verify_agent_submission(
14084 row: &AgentSubmissionRow,
14085 key_digest: Digest,
14086 admission: StoredAgentAdmission,
14087) -> Result<StoredAgentSubmission, StoreError> {
14088 let durable_key_digest = decode_digest(&row.key_digest, "agent submission key digest")?;
14089 let durable_submission_digest =
14090 decode_digest(&row.submission_digest, "agent submission digest")?;
14091 let durable_admission_digest =
14092 decode_digest(&row.admission_digest, "agent submission admission digest")?;
14093 let created_at = from_database_time(row.created_at)?;
14094 let intent = admission.admission().intent();
14095 let expected_submission_digest = agent_submission_digest(
14096 intent,
14097 admission.checkpoint().state(),
14098 admission.checkpoint().ready_nodes(),
14099 )?;
14100
14101 if row.tenant_id != intent.provenance().tenant_id().as_str()
14102 || row.run_id != *intent.provenance().run_id().as_uuid()
14103 || durable_key_digest != key_digest
14104 || durable_submission_digest != expected_submission_digest
14105 || durable_admission_digest != admission.admission().digest()
14106 || created_at != admission.admission().admitted_at()
14107 {
14108 return Err(StoreError::corrupt("agent submission projection"));
14109 }
14110
14111 Ok(StoredAgentSubmission {
14112 key_digest,
14113 submission_digest: durable_submission_digest,
14114 admission,
14115 created_at,
14116 })
14117}
14118
14119async fn load_locked_agent_submission(
14120 transaction: &mut Transaction<'_, Postgres>,
14121 tenant_id: &TenantId,
14122 key_digest: Digest,
14123) -> Result<Option<StoredAgentSubmission>, StoreError> {
14124 lock_agent_submission_key(transaction, tenant_id, key_digest).await?;
14125 let row = query_as::<_, AgentSubmissionRow>(SELECT_AGENT_SUBMISSION)
14129 .bind(tenant_id.as_str())
14130 .bind(key_digest.as_bytes())
14131 .fetch_optional(&mut **transaction)
14132 .await
14133 .map_err(|source| StoreError::database("agent submission locked load", source))?;
14134 let Some(row) = row else {
14135 return Ok(None);
14136 };
14137 let run_id = RunId::from_uuid(row.run_id)
14138 .map_err(|_| StoreError::corrupt("agent submission run identity"))?;
14139 let admission = load_locked_agent_admission(transaction, tenant_id, run_id)
14140 .await?
14141 .ok_or_else(|| StoreError::corrupt("agent submission admission reference"))?;
14142 verify_agent_submission(&row, key_digest, admission).map(Some)
14143}
14144
14145async fn insert_agent_submission(
14146 transaction: &mut Transaction<'_, Postgres>,
14147 key_digest: Digest,
14148 submission_digest: Digest,
14149 admission: &StoredAgentAdmission,
14150) -> Result<(), StoreError> {
14151 let snapshot = admission.admission();
14152 let provenance = snapshot.intent().provenance();
14153 let inserted = query(
14154 r"
14155INSERT INTO stateknot.agent_submission_keys (
14156 tenant_id,
14157 key_digest,
14158 submission_digest,
14159 run_id,
14160 admission_digest,
14161 created_at
14162)
14163VALUES ($1, $2, $3, $4, $5, $6)
14164",
14165 )
14166 .bind(provenance.tenant_id().as_str())
14167 .bind(key_digest.as_bytes())
14168 .bind(submission_digest.as_bytes())
14169 .bind(*provenance.run_id().as_uuid())
14170 .bind(snapshot.digest().as_bytes())
14171 .bind(to_database_time(snapshot.admitted_at())?)
14172 .execute(&mut **transaction)
14173 .await;
14174 match inserted {
14175 Ok(result) if result.rows_affected() == 1 => Ok(()),
14176 Ok(_) => Err(StoreError::corrupt("agent submission insert row count")),
14177 Err(source)
14178 if has_database_constraint(&source, "agent_submission_keys_pkey")
14179 || has_database_constraint(&source, "agent_submission_keys_run_unique") =>
14180 {
14181 Err(StoreError::AgentSubmissionConflict)
14182 }
14183 Err(source) => Err(StoreError::database("agent submission insert", source)),
14184 }
14185}
14186
14187fn validate_agent_admission_retry(
14188 stored: &StoredAgentAdmission,
14189 intent: &AgentAdmissionIntent,
14190 append: &JournalAppend,
14191 checkpoint: &CheckpointWrite,
14192) -> Result<(), StoreError> {
14193 if stored.admission().intent() != intent
14194 || !stored.event().matches_intent(append.intent())
14195 || !stored.checkpoint().matches_write(checkpoint)
14196 {
14197 return Err(StoreError::AgentAdmissionConflict);
14198 }
14199 Ok(())
14200}
14201
14202async fn insert_agent_pending_run(
14203 transaction: &mut Transaction<'_, Postgres>,
14204 lifecycle: &RunLifecycle,
14205) -> Result<bool, StoreError> {
14206 let provenance = lifecycle.provenance();
14207 let lifecycle_bytes = encode_lifecycle(lifecycle)?;
14208 let observed_at = to_database_time(lifecycle.admitted_at())?;
14209 let inserted = query(
14210 r"
14211INSERT INTO stateknot.runs (
14212 tenant_id,
14213 run_id,
14214 thread_id,
14215 invocation_id,
14216 lifecycle_bytes,
14217 lifecycle_revision,
14218 lifecycle_status,
14219 admitted_at,
14220 changed_at,
14221 scheduler_ready_at
14222)
14223VALUES ($1, $2, $3, $4, $5, $6::numeric, $7, $8, $8, $8)
14224ON CONFLICT (tenant_id, run_id) DO NOTHING
14225",
14226 )
14227 .bind(provenance.tenant_id().as_str())
14228 .bind(*provenance.run_id().as_uuid())
14229 .bind(*provenance.thread_id().as_uuid())
14230 .bind(*provenance.invocation_id().as_uuid())
14231 .bind(lifecycle_bytes)
14232 .bind(lifecycle.revision().to_string())
14233 .bind(run_status_text(lifecycle.status()))
14234 .bind(observed_at)
14235 .execute(&mut **transaction)
14236 .await
14237 .map_err(|source| StoreError::database("agent admission run insert", source))?
14238 .rows_affected();
14239 Ok(inserted == 1)
14240}
14241
14242#[allow(clippy::too_many_lines)]
14243async fn insert_agent_admission(
14244 transaction: &mut Transaction<'_, Postgres>,
14245 admission: &AgentAdmission,
14246 event: &JournalEvent,
14247 checkpoint: &Checkpoint,
14248) -> Result<(), StoreError> {
14249 let intent = admission.intent();
14250 let provenance = intent.provenance();
14251 let agent = intent.descriptor().metadata().identity();
14252 let agent_owner = agent.owner();
14253 let graph = intent.graph();
14254 let graph_identity = graph.identity();
14255 let graph_owner = graph_identity.owner();
14256 let policy = intent.authority().policy();
14257 let policy_owner = policy.owner();
14258 let admission_bytes = encode_agent_admission(admission)?;
14259 let journal_sequence =
14260 i64::try_from(event.sequence().get()).map_err(|_| StoreError::JournalSequenceExhausted)?;
14261 let checkpoint_superstep = i64::try_from(checkpoint.superstep().get())
14262 .map_err(|_| StoreError::encoding("agent admission checkpoint superstep"))?;
14263 let admitted_at = to_database_time(admission.admitted_at())?;
14264
14265 let inserted = query(
14266 r"
14267INSERT INTO stateknot.agent_admissions (
14268 tenant_id,
14269 run_id,
14270 agent_owner_issuer,
14271 agent_owner_subject,
14272 agent_name,
14273 agent_version,
14274 graph_owner_issuer,
14275 graph_owner_subject,
14276 graph_name,
14277 graph_version,
14278 graph_definition_digest,
14279 policy_owner_issuer,
14280 policy_owner_subject,
14281 policy_name,
14282 policy_version,
14283 policy_digest,
14284 intent_digest,
14285 admission_digest,
14286 admitted_at,
14287 journal_sequence,
14288 journal_event_id,
14289 journal_recorded_at,
14290 journal_digest,
14291 checkpoint_id,
14292 checkpoint_superstep,
14293 checkpoint_digest,
14294 admission_bytes,
14295 created_at
14296)
14297VALUES (
14298 $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14,
14299 $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28
14300)
14301",
14302 )
14303 .bind(provenance.tenant_id().as_str())
14304 .bind(*provenance.run_id().as_uuid())
14305 .bind(agent_owner.issuer().as_str())
14306 .bind(agent_owner.subject().as_str())
14307 .bind(agent.name().as_str())
14308 .bind(agent.version().to_string())
14309 .bind(graph_owner.issuer().as_str())
14310 .bind(graph_owner.subject().as_str())
14311 .bind(graph_identity.name().as_str())
14312 .bind(graph_identity.version().to_string())
14313 .bind(graph.definition_digest().as_bytes())
14314 .bind(policy_owner.issuer().as_str())
14315 .bind(policy_owner.subject().as_str())
14316 .bind(policy.name().as_str())
14317 .bind(policy.version().to_string())
14318 .bind(intent.authority().policy_digest().as_bytes())
14319 .bind(intent.intent_digest().as_bytes())
14320 .bind(admission.digest().as_bytes())
14321 .bind(admitted_at)
14322 .bind(journal_sequence)
14323 .bind(*event.event_id().as_uuid())
14324 .bind(to_database_time(event.recorded_at())?)
14325 .bind(event.digest().as_bytes())
14326 .bind(*checkpoint.checkpoint_id().as_uuid())
14327 .bind(checkpoint_superstep)
14328 .bind(checkpoint.digest().as_bytes())
14329 .bind(admission_bytes)
14330 .bind(admitted_at)
14331 .execute(&mut **transaction)
14332 .await
14333 .map_err(|source| StoreError::database("agent admission insert", source))?
14334 .rows_affected();
14335 if inserted != 1 {
14336 return Err(StoreError::corrupt("agent admission insert row count"));
14337 }
14338 Ok(())
14339}
14340
14341enum NewAgentAdmissionOutcome {
14342 Committed(StoredAgentAdmission),
14343 Idempotent(StoredAgentAdmission),
14344}
14345
14346async fn commit_new_agent_admission(
14347 transaction: &mut Transaction<'_, Postgres>,
14348 intent: AgentAdmissionIntent,
14349 append: JournalAppend,
14350 checkpoint_write: CheckpointWrite,
14351) -> Result<NewAgentAdmissionOutcome, StoreError> {
14352 let tenant_id = intent.provenance().tenant_id().clone();
14353 let run_id = intent.provenance().run_id();
14354 let admitted_at = database_now(transaction, "agent admission clock").await?;
14355 let admission = AgentAdmission::commit(intent, admitted_at)
14356 .map_err(|_| StoreError::AgentAdmissionRejected)?;
14357 let pending = RunLifecycle::admitted(admission.intent().provenance().clone(), admitted_at);
14358 let active = pending
14359 .clone()
14360 .apply(RunTransition::Start {
14361 started_at: admitted_at,
14362 })
14363 .map_err(|_| StoreError::InvalidAgentAdmissionCommit)?;
14364
14365 if !insert_agent_pending_run(transaction, &pending).await? {
14366 let stored = load_locked_agent_admission(transaction, &tenant_id, run_id)
14367 .await?
14368 .ok_or(StoreError::AgentAdmissionConflict)?;
14369 validate_agent_admission_retry(&stored, admission.intent(), &append, &checkpoint_write)?;
14370 return Ok(NewAgentAdmissionOutcome::Idempotent(stored));
14371 }
14372
14373 let event = JournalEvent::commit(append, admitted_at)
14374 .map_err(|error| map_event_commit_error(&error))?;
14375 let checkpoint = Checkpoint::commit(checkpoint_write, event.head())
14376 .map_err(|_| StoreError::InvalidAgentAdmissionCommit)?;
14377 let projection_digest = agent_admission_projection_digest(
14378 &admission,
14379 event.intent_digest(),
14380 &checkpoint.write_intent(),
14381 &active,
14382 )?;
14383 let prepared = prepared_projection(&active)?;
14384
14385 insert_event(transaction, &event, projection_digest).await?;
14386 insert_checkpoint(transaction, &checkpoint, event.source()).await?;
14387 update_checkpoint_pointer(transaction, &checkpoint, event.source()).await?;
14388 update_run_head(transaction, &event, Some(&prepared)).await?;
14389 insert_agent_admission(transaction, &admission, &event, &checkpoint).await?;
14390
14391 let run_row = fetch_locked_run_row(transaction, &tenant_id, run_id).await?;
14392 let run = decode_run(run_row)?;
14393 verify_current_wait_set(transaction, &run).await?;
14394 let admission_row = load_agent_admission_row(transaction, &tenant_id, run_id)
14395 .await?
14396 .ok_or_else(|| StoreError::corrupt("agent admission committed row"))?;
14397 let stored = verify_stored_agent_admission(transaction, run, admission_row).await?;
14398 Ok(NewAgentAdmissionOutcome::Committed(stored))
14399}
14400
14401fn decode_scheduler_fairness_policy(
14402 row: SchedulerFairnessShardRow,
14403) -> Result<StoredSchedulerFairnessPolicy, StoreError> {
14404 let shard_id = SchedulerShardId::try_from(row.shard_id)
14405 .map_err(|_| StoreError::corrupt("scheduler fairness shard identity"))?;
14406 let cycle_length = u16::try_from(row.cycle_length)
14407 .map_err(|_| StoreError::corrupt("scheduler fairness cycle length"))?;
14408 let registration =
14409 SchedulerFairnessPolicyRegistration::new(shard_id, row.policy_bytes, cycle_length)
14410 .map_err(|_| StoreError::corrupt("scheduler fairness policy shape"))?;
14411 let digest = decode_digest(&row.policy_digest, "scheduler fairness policy digest")?;
14412 if registration.policy_digest() != digest {
14413 return Err(StoreError::corrupt("scheduler fairness policy checksum"));
14414 }
14415 let next_slot = u16::try_from(row.next_slot)
14416 .map_err(|_| StoreError::corrupt("scheduler fairness cursor"))?;
14417 if next_slot >= cycle_length || row.next_sequence < 0 {
14418 return Err(StoreError::corrupt("scheduler fairness cursor projection"));
14419 }
14420 let registered_at = from_database_time(row.registered_at)?;
14421 let updated_at = from_database_time(row.updated_at)?;
14422 if updated_at < registered_at {
14423 return Err(StoreError::corrupt("scheduler fairness clock projection"));
14424 }
14425 Ok(StoredSchedulerFairnessPolicy {
14426 registration,
14427 registered_at,
14428 })
14429}
14430
14431async fn load_scheduler_fairness_reservation_row(
14432 transaction: &mut Transaction<'_, Postgres>,
14433 reservation_id: SchedulerReservationId,
14434) -> Result<Option<SchedulerFairnessReservationRow>, StoreError> {
14435 query_as::<_, SchedulerFairnessReservationRow>(SELECT_SCHEDULER_FAIRNESS_RESERVATION)
14436 .bind(*reservation_id.as_uuid())
14437 .fetch_optional(&mut **transaction)
14438 .await
14439 .map_err(|source| StoreError::database("scheduler fairness reservation lookup", source))
14440}
14441
14442async fn load_valid_scheduler_fairness_reservation(
14443 transaction: &mut Transaction<'_, Postgres>,
14444 shard_id: &SchedulerShardId,
14445 policy_digest: Digest,
14446 reservation_id: SchedulerReservationId,
14447) -> Result<Option<SchedulerFairnessReservation>, StoreError> {
14448 let Some(row) = load_scheduler_fairness_reservation_row(transaction, reservation_id).await?
14449 else {
14450 return Ok(None);
14451 };
14452 let reservation = decode_scheduler_fairness_reservation(row)?;
14453 validate_scheduler_fairness_reservation(&reservation, shard_id, policy_digest, reservation_id)?;
14454 Ok(Some(reservation))
14455}
14456
14457async fn insert_scheduler_fairness_reservation(
14458 transaction: &mut Transaction<'_, Postgres>,
14459 shard_id: &SchedulerShardId,
14460 policy_digest: Digest,
14461 reservation_id: SchedulerReservationId,
14462 shard_row: SchedulerFairnessShardRow,
14463) -> Result<SchedulerFairnessReservation, StoreError> {
14464 let stored = decode_scheduler_fairness_policy(shard_row.clone())?;
14465 if stored.registration().policy_digest() != policy_digest {
14466 return Err(StoreError::SchedulerFairnessPolicyConflict);
14467 }
14468 let sequence = u64::try_from(shard_row.next_sequence)
14469 .map_err(|_| StoreError::corrupt("scheduler fairness sequence"))?;
14470 if shard_row.next_sequence == i64::MAX {
14471 return Err(StoreError::SchedulerFairnessSequenceExhausted);
14472 }
14473 let slot = u16::try_from(shard_row.next_slot)
14474 .map_err(|_| StoreError::corrupt("scheduler fairness cursor"))?;
14475 let cycle_length = stored.registration().cycle_length();
14476 if slot >= cycle_length {
14477 return Err(StoreError::corrupt("scheduler fairness cursor"));
14478 }
14479 let reserved_at = database_now(transaction, "scheduler fairness reservation clock").await?;
14480 let reservation = SchedulerFairnessReservation {
14481 shard_id: shard_id.clone(),
14482 reservation_id,
14483 policy_digest,
14484 sequence,
14485 slot,
14486 reserved_at,
14487 };
14488
14489 let insert = query(
14490 r"
14491INSERT INTO stateknot.scheduler_fairness_reservations (
14492 shard_id,
14493 reservation_id,
14494 policy_digest,
14495 sequence,
14496 slot,
14497 reserved_at
14498)
14499VALUES ($1, $2, $3, $4, $5, $6)
14500",
14501 )
14502 .bind(shard_id.as_str())
14503 .bind(*reservation_id.as_uuid())
14504 .bind(policy_digest.as_bytes())
14505 .bind(shard_row.next_sequence)
14506 .bind(shard_row.next_slot)
14507 .bind(to_database_time(reserved_at)?)
14508 .execute(&mut **transaction)
14509 .await;
14510 if let Err(source) = insert {
14511 if has_database_error_code(&source, "23505") {
14512 return Err(StoreError::SchedulerFairnessReservationConflict);
14513 }
14514 return Err(StoreError::database(
14515 "scheduler fairness reservation insert",
14516 source,
14517 ));
14518 }
14519
14520 let next_slot = if slot + 1 == cycle_length {
14521 0
14522 } else {
14523 slot + 1
14524 };
14525 let updated = query(
14526 r"
14527UPDATE stateknot.scheduler_fairness_shards
14528SET next_slot = $1,
14529 next_sequence = next_sequence + 1,
14530 updated_at = $2
14531WHERE shard_id = $3
14532 AND policy_digest = $4
14533 AND next_slot = $5
14534 AND next_sequence = $6
14535",
14536 )
14537 .bind(i32::from(next_slot))
14538 .bind(to_database_time(reserved_at)?)
14539 .bind(shard_id.as_str())
14540 .bind(policy_digest.as_bytes())
14541 .bind(shard_row.next_slot)
14542 .bind(shard_row.next_sequence)
14543 .execute(&mut **transaction)
14544 .await
14545 .map_err(|source| StoreError::database("scheduler fairness cursor advance", source))?
14546 .rows_affected();
14547 if updated != 1 {
14548 return Err(StoreError::SchedulerFairnessPolicyConflict);
14549 }
14550 Ok(reservation)
14551}
14552
14553fn decode_scheduler_fairness_reservation(
14554 row: SchedulerFairnessReservationRow,
14555) -> Result<SchedulerFairnessReservation, StoreError> {
14556 let shard_id = SchedulerShardId::try_from(row.shard_id)
14557 .map_err(|_| StoreError::corrupt("scheduler fairness reservation shard"))?;
14558 let reservation_id = SchedulerReservationId::from_uuid(row.reservation_id)
14559 .map_err(|_| StoreError::corrupt("scheduler fairness reservation identity"))?;
14560 let policy_digest = decode_digest(
14561 &row.policy_digest,
14562 "scheduler fairness reservation policy digest",
14563 )?;
14564 let shard_policy_digest = decode_digest(
14565 &row.shard_policy_digest,
14566 "scheduler fairness shard policy digest",
14567 )?;
14568 let sequence = u64::try_from(row.sequence)
14569 .map_err(|_| StoreError::corrupt("scheduler fairness reservation sequence"))?;
14570 let slot = u16::try_from(row.slot)
14571 .map_err(|_| StoreError::corrupt("scheduler fairness reservation slot"))?;
14572 let cycle_length = u16::try_from(row.cycle_length)
14573 .map_err(|_| StoreError::corrupt("scheduler fairness reservation cycle"))?;
14574 if cycle_length == 0
14575 || cycle_length > SchedulerFairnessPolicyRegistration::MAX_CYCLE_LENGTH
14576 || slot >= cycle_length
14577 || policy_digest != shard_policy_digest
14578 {
14579 return Err(StoreError::corrupt(
14580 "scheduler fairness reservation policy projection",
14581 ));
14582 }
14583 Ok(SchedulerFairnessReservation {
14584 shard_id,
14585 reservation_id,
14586 policy_digest,
14587 sequence,
14588 slot,
14589 reserved_at: from_database_time(row.reserved_at)?,
14590 })
14591}
14592
14593fn validate_scheduler_fairness_reservation(
14594 reservation: &SchedulerFairnessReservation,
14595 shard_id: &SchedulerShardId,
14596 policy_digest: Digest,
14597 reservation_id: SchedulerReservationId,
14598) -> Result<(), StoreError> {
14599 if reservation.shard_id() != shard_id
14600 || reservation.policy_digest() != policy_digest
14601 || reservation.reservation_id() != reservation_id
14602 {
14603 return Err(StoreError::SchedulerFairnessReservationConflict);
14604 }
14605 Ok(())
14606}
14607
14608fn encode_outbox_destination_config(config: &JournalPayload) -> Result<Vec<u8>, StoreError> {
14609 let canonical = config
14610 .canonical_json()
14611 .map_err(|_| StoreError::encoding("outbox destination config"))?;
14612 let bytes = canonical.as_bytes().to_vec();
14613 if bytes.is_empty() || bytes.len() > MAX_OUTBOX_DESTINATION_BYTES {
14614 return Err(StoreError::encoding("outbox destination config size"));
14615 }
14616 Ok(bytes)
14617}
14618
14619fn decode_outbox_destination(
14620 row: OutboxDestinationRow,
14621) -> Result<StoredOutboxDestination, StoreError> {
14622 if row.config_bytes.is_empty() || row.config_bytes.len() > MAX_OUTBOX_DESTINATION_BYTES {
14623 return Err(StoreError::corrupt("outbox destination byte length"));
14624 }
14625 let config = serde_json::from_slice::<JournalPayload>(&row.config_bytes)
14626 .map_err(|_| StoreError::corrupt("outbox destination config"))?;
14627 let canonical = config
14628 .canonical_json()
14629 .map_err(|_| StoreError::corrupt("outbox destination canonicalization"))?;
14630 if canonical.as_bytes() != row.config_bytes {
14631 return Err(StoreError::corrupt("outbox destination canonical bytes"));
14632 }
14633 let tenant_id = TenantId::try_from(row.tenant_id)
14634 .map_err(|_| StoreError::corrupt("outbox destination tenant"))?;
14635 let destination_id = DestinationId::from_uuid(row.destination_id)
14636 .map_err(|_| StoreError::corrupt("outbox destination identity"))?;
14637 let snapshot_digest = decode_digest(&row.snapshot_digest, "outbox destination digest")?;
14638 if config.digest() != snapshot_digest
14639 || config.kind().as_str() != row.config_kind
14640 || config.schema().id().as_str() != row.schema_id
14641 || config.schema().version().to_string() != row.schema_version
14642 || config.schema().digest()
14643 != decode_digest(&row.schema_digest, "outbox destination schema digest")?
14644 {
14645 return Err(StoreError::corrupt("outbox destination projection"));
14646 }
14647 Ok(StoredOutboxDestination {
14648 destination: OutboxDestinationRef::new(tenant_id, destination_id, snapshot_digest),
14649 config,
14650 created_at: from_database_time(row.created_at)?,
14651 })
14652}
14653
14654async fn load_outbox_destination_row(
14655 transaction: &mut Transaction<'_, Postgres>,
14656 destination: &OutboxDestinationRef,
14657) -> Result<Option<OutboxDestinationRow>, StoreError> {
14658 query_as::<_, OutboxDestinationRow>(SELECT_OUTBOX_DESTINATION)
14659 .bind(destination.tenant_id().as_str())
14660 .bind(*destination.destination_id().as_uuid())
14661 .bind(destination.snapshot_digest().as_bytes())
14662 .fetch_optional(&mut **transaction)
14663 .await
14664 .map_err(|source| StoreError::database("outbox destination load", source))
14665}
14666
14667async fn load_and_decode_outbox_destination(
14668 transaction: &mut Transaction<'_, Postgres>,
14669 destination: &OutboxDestinationRef,
14670) -> Result<StoredOutboxDestination, StoreError> {
14671 let row = load_outbox_destination_row(transaction, destination)
14672 .await?
14673 .ok_or(StoreError::OutboxDestinationNotFound)?;
14674 let stored = decode_outbox_destination(row)?;
14675 if stored.destination() != destination {
14676 return Err(StoreError::corrupt("outbox destination binding"));
14677 }
14678 Ok(stored)
14679}
14680
14681fn encode_outbox_delivery(delivery: &OutboxDelivery) -> Result<Vec<u8>, StoreError> {
14682 let bytes = serde_json_canonicalizer::to_vec(delivery)
14683 .map_err(|_| StoreError::encoding("outbox delivery"))?;
14684 if bytes.is_empty() || bytes.len() > MAX_OUTBOX_DELIVERY_BYTES {
14685 return Err(StoreError::encoding("outbox delivery size"));
14686 }
14687 Ok(bytes)
14688}
14689
14690#[allow(clippy::too_many_lines)]
14691fn decode_outbox_delivery(row: &OutboxDeliveryRow) -> Result<OutboxDelivery, StoreError> {
14692 if row.delivery_bytes.is_empty() || row.delivery_bytes.len() > MAX_OUTBOX_DELIVERY_BYTES {
14693 return Err(StoreError::corrupt("outbox delivery byte length"));
14694 }
14695 let delivery = serde_json::from_slice::<OutboxDelivery>(&row.delivery_bytes)
14696 .map_err(|_| StoreError::corrupt("outbox delivery value"))?;
14697 let canonical = serde_json_canonicalizer::to_vec(&delivery)
14698 .map_err(|_| StoreError::corrupt("outbox delivery canonicalization"))?;
14699 if canonical != row.delivery_bytes {
14700 return Err(StoreError::corrupt("outbox delivery canonical bytes"));
14701 }
14702 let intent = delivery.intent();
14703 let origin = delivery.origin();
14704 let destination = intent.destination();
14705 let origin_sequence = i64::try_from(origin.sequence().get())
14706 .map_err(|_| StoreError::corrupt("outbox origin sequence"))?;
14707 if intent.tenant_id().as_str() != row.tenant_id
14708 || *intent.run_id().as_uuid() != row.run_id
14709 || *intent.delivery_id().as_uuid() != row.delivery_id
14710 || origin_sequence != row.origin_sequence
14711 || *origin.event_id().as_uuid() != row.origin_event_id
14712 || origin.recorded_at() != from_database_time(row.origin_recorded_at)?
14713 || origin.digest() != decode_digest(&row.origin_digest, "outbox origin digest")?
14714 || *destination.destination_id().as_uuid() != row.destination_id
14715 || destination.snapshot_digest()
14716 != decode_digest(
14717 &row.destination_snapshot_digest,
14718 "outbox destination snapshot digest",
14719 )?
14720 || intent.intent_digest() != decode_digest(&row.intent_digest, "outbox intent digest")?
14721 || intent.expires_at() != from_database_time(row.expires_at)?
14722 || delivery.digest() != decode_digest(&row.delivery_digest, "outbox delivery digest")?
14723 || origin.recorded_at() != from_database_time(row.created_at)?
14724 || from_database_time(row.updated_at)? < origin.recorded_at()
14725 {
14726 return Err(StoreError::corrupt("outbox delivery projection"));
14727 }
14728 Ok(delivery)
14729}
14730
14731async fn load_outbox_delivery_row(
14732 transaction: &mut Transaction<'_, Postgres>,
14733 tenant_id: &TenantId,
14734 run_id: RunId,
14735 delivery_id: DeliveryId,
14736 for_update: bool,
14737) -> Result<Option<OutboxDeliveryRow>, StoreError> {
14738 let statement = if for_update {
14739 SELECT_OUTBOX_DELIVERY_FOR_UPDATE.as_str()
14740 } else {
14741 SELECT_OUTBOX_DELIVERY.as_str()
14742 };
14743 query_as::<_, OutboxDeliveryRow>(statement)
14744 .bind(tenant_id.as_str())
14745 .bind(*run_id.as_uuid())
14746 .bind(*delivery_id.as_uuid())
14747 .fetch_optional(&mut **transaction)
14748 .await
14749 .map_err(|source| StoreError::database("outbox delivery load", source))
14750}
14751
14752fn encode_outbox_attempt_start(start: &OutboxAttemptStart) -> Result<Vec<u8>, StoreError> {
14753 let bytes = serde_json_canonicalizer::to_vec(start)
14754 .map_err(|_| StoreError::encoding("outbox attempt start"))?;
14755 if bytes.is_empty() || bytes.len() > MAX_OUTBOX_ATTEMPT_START_BYTES {
14756 return Err(StoreError::encoding("outbox attempt start size"));
14757 }
14758 Ok(bytes)
14759}
14760
14761fn decode_outbox_attempt_start(
14762 row: &OutboxAttemptStartRow,
14763) -> Result<OutboxAttemptStart, StoreError> {
14764 if row.start_bytes.is_empty() || row.start_bytes.len() > MAX_OUTBOX_ATTEMPT_START_BYTES {
14765 return Err(StoreError::corrupt("outbox attempt start byte length"));
14766 }
14767 let start = serde_json::from_slice::<OutboxAttemptStart>(&row.start_bytes)
14768 .map_err(|_| StoreError::corrupt("outbox attempt start value"))?;
14769 let canonical = serde_json_canonicalizer::to_vec(&start)
14770 .map_err(|_| StoreError::corrupt("outbox attempt start canonicalization"))?;
14771 if canonical != row.start_bytes {
14772 return Err(StoreError::corrupt("outbox attempt start canonical bytes"));
14773 }
14774 let delivery = start.delivery();
14775 let fence = start.fence();
14776 let epoch = i64::try_from(fence.epoch().get())
14777 .map_err(|_| StoreError::corrupt("outbox attempt epoch"))?;
14778 if delivery.tenant_id().as_str() != row.tenant_id
14779 || *delivery.run_id().as_uuid() != row.run_id
14780 || *delivery.delivery_id().as_uuid() != row.delivery_id
14781 || delivery.expires_at() != from_database_time(row.delivery_expires_at)?
14782 || delivery.digest() != decode_digest(&row.delivery_digest, "outbox delivery digest")?
14783 || epoch != row.epoch
14784 || *fence.attempt_id().as_uuid() != row.attempt_id
14785 || start.started_at() != from_database_time(row.started_at)?
14786 || start.expires_at() != from_database_time(row.expires_at)?
14787 || start.digest() != decode_digest(&row.start_digest, "outbox attempt start digest")?
14788 || start.started_at() != from_database_time(row.created_at)?
14789 {
14790 return Err(StoreError::corrupt("outbox attempt start projection"));
14791 }
14792 Ok(start)
14793}
14794
14795fn encode_outbox_attempt_completion(
14796 completion: &OutboxAttemptCompletion,
14797) -> Result<Vec<u8>, StoreError> {
14798 let bytes = serde_json_canonicalizer::to_vec(completion)
14799 .map_err(|_| StoreError::encoding("outbox attempt completion"))?;
14800 if bytes.is_empty() || bytes.len() > MAX_OUTBOX_ATTEMPT_COMPLETION_BYTES {
14801 return Err(StoreError::encoding("outbox attempt completion size"));
14802 }
14803 Ok(bytes)
14804}
14805
14806#[allow(clippy::too_many_lines)]
14807fn decode_outbox_attempt_completion(
14808 row: &OutboxAttemptCompletionRow,
14809) -> Result<OutboxAttemptCompletion, StoreError> {
14810 if row.completion_bytes.is_empty()
14811 || row.completion_bytes.len() > MAX_OUTBOX_ATTEMPT_COMPLETION_BYTES
14812 {
14813 return Err(StoreError::corrupt("outbox attempt completion byte length"));
14814 }
14815 let completion = serde_json::from_slice::<OutboxAttemptCompletion>(&row.completion_bytes)
14816 .map_err(|_| StoreError::corrupt("outbox attempt completion value"))?;
14817 let canonical = serde_json_canonicalizer::to_vec(&completion)
14818 .map_err(|_| StoreError::corrupt("outbox attempt completion canonicalization"))?;
14819 if canonical != row.completion_bytes {
14820 return Err(StoreError::corrupt(
14821 "outbox attempt completion canonical bytes",
14822 ));
14823 }
14824 let start = completion.start();
14825 let fence = start.fence();
14826 let delivery = start.delivery();
14827 let epoch = i64::try_from(fence.epoch().get())
14828 .map_err(|_| StoreError::corrupt("outbox completion epoch"))?;
14829 let (outcome_kind, retry_kind, retry_delay) = outbox_completion_projection(&completion)?;
14830 if delivery.tenant_id().as_str() != row.tenant_id
14831 || *delivery.run_id().as_uuid() != row.run_id
14832 || *delivery.delivery_id().as_uuid() != row.delivery_id
14833 || epoch != row.epoch
14834 || *fence.attempt_id().as_uuid() != row.attempt_id
14835 || start.started_at() != from_database_time(row.started_at)?
14836 || start.expires_at() != from_database_time(row.attempt_expires_at)?
14837 || start.digest() != decode_digest(&row.start_digest, "outbox completion start digest")?
14838 || outcome_kind != row.outcome_kind
14839 || retry_kind != row.retry_advice_kind.as_deref()
14840 || retry_delay != row.retry_delay_millis
14841 || completion.completed_at() != from_database_time(row.completed_at)?
14842 || completion.digest() != decode_digest(&row.completion_digest, "outbox completion digest")?
14843 || completion.completed_at() != from_database_time(row.created_at)?
14844 {
14845 return Err(StoreError::corrupt("outbox attempt completion projection"));
14846 }
14847 Ok(completion)
14848}
14849
14850fn outbox_completion_projection(
14851 completion: &OutboxAttemptCompletion,
14852) -> Result<(&'static str, Option<&'static str>, Option<i64>), StoreError> {
14853 match completion.outcome() {
14854 OutboxAttemptOutcome::Acknowledged { .. } => Ok(("acknowledged", None, None)),
14855 OutboxAttemptOutcome::Failed { failure } => match failure.retry_advice() {
14856 RetryAdvice::Never => Ok(("failed", Some("never"), None)),
14857 RetryAdvice::SafeAfter { delay } => {
14858 Ok(("failed", Some("safe_after"), Some(delay.as_i64())))
14859 }
14860 RetryAdvice::ReconcileFirst => Err(StoreError::InvalidOutboxTransition),
14861 },
14862 _ => Err(StoreError::InvalidOutboxTransition),
14863 }
14864}
14865
14866async fn load_outbox_attempt_completion_row(
14867 transaction: &mut Transaction<'_, Postgres>,
14868 tenant_id: &TenantId,
14869 run_id: RunId,
14870 delivery_id: DeliveryId,
14871 epoch: i64,
14872) -> Result<Option<OutboxAttemptCompletionRow>, StoreError> {
14873 query_as::<_, OutboxAttemptCompletionRow>(SELECT_OUTBOX_ATTEMPT_COMPLETION)
14874 .bind(tenant_id.as_str())
14875 .bind(*run_id.as_uuid())
14876 .bind(*delivery_id.as_uuid())
14877 .bind(epoch)
14878 .fetch_optional(&mut **transaction)
14879 .await
14880 .map_err(|source| StoreError::database("outbox attempt completion load", source))
14881}
14882
14883async fn load_and_verify_outbox_attempts(
14884 transaction: &mut Transaction<'_, Postgres>,
14885 delivery: &OutboxDelivery,
14886) -> Result<Vec<OutboxAttempt>, StoreError> {
14887 let rows = query_as::<_, OutboxAttemptStartRow>(SELECT_OUTBOX_ATTEMPT_HISTORY.as_str())
14888 .bind(delivery.intent().tenant_id().as_str())
14889 .bind(*delivery.intent().run_id().as_uuid())
14890 .bind(*delivery.intent().delivery_id().as_uuid())
14891 .bind(0_i64)
14892 .bind(i64::try_from(MAX_OUTBOX_ATTEMPTS + 1).unwrap_or(i64::MAX))
14893 .fetch_all(&mut **transaction)
14894 .await
14895 .map_err(|source| StoreError::database("outbox attempt history load", source))?;
14896 if rows.len() > MAX_OUTBOX_ATTEMPTS {
14897 return Err(StoreError::corrupt("outbox attempt history bound"));
14898 }
14899 let completion_rows =
14900 query_as::<_, OutboxAttemptCompletionRow>(SELECT_OUTBOX_ATTEMPT_COMPLETION_HISTORY)
14901 .bind(delivery.intent().tenant_id().as_str())
14902 .bind(*delivery.intent().run_id().as_uuid())
14903 .bind(*delivery.intent().delivery_id().as_uuid())
14904 .bind(i64::try_from(MAX_OUTBOX_ATTEMPTS + 1).unwrap_or(i64::MAX))
14905 .fetch_all(&mut **transaction)
14906 .await
14907 .map_err(|source| StoreError::database("outbox completion history load", source))?;
14908 if completion_rows.len() > MAX_OUTBOX_ATTEMPTS {
14909 return Err(StoreError::corrupt("outbox completion history bound"));
14910 }
14911 let mut completions = BTreeMap::new();
14912 for row in completion_rows {
14913 let epoch = row.epoch;
14914 if completions.insert(epoch, row).is_some() {
14915 return Err(StoreError::corrupt("outbox completion history identity"));
14916 }
14917 }
14918 let mut verifier = OutboxAttemptHistoryVerifier::new(delivery);
14919 let mut attempts = Vec::with_capacity(rows.len());
14920 for row in rows {
14921 let start = decode_outbox_attempt_start(&row)?;
14922 if start.delivery() != &delivery.head() {
14923 return Err(StoreError::corrupt("outbox attempt delivery binding"));
14924 }
14925 let epoch = i64::try_from(start.fence().epoch().get())
14926 .map_err(|_| StoreError::corrupt("outbox attempt epoch"))?;
14927 let completion = completions
14928 .remove(&epoch)
14929 .map(|row| decode_outbox_attempt_completion(&row))
14930 .transpose()?;
14931 let attempt = OutboxAttempt::restore(start, completion)
14932 .map_err(|_| StoreError::corrupt("outbox attempt join"))?;
14933 verifier
14934 .verify_next(&attempt)
14935 .map_err(|_| StoreError::corrupt("outbox attempt history"))?;
14936 attempts.push(attempt);
14937 }
14938 if !completions.is_empty() {
14939 return Err(StoreError::corrupt("outbox completion without start"));
14940 }
14941 Ok(attempts)
14942}
14943
14944async fn verify_outbox_projection(
14945 transaction: &mut Transaction<'_, Postgres>,
14946 row: &OutboxDeliveryRow,
14947 delivery: &OutboxDelivery,
14948) -> Result<(), StoreError> {
14949 verify_outbox_delivery_anchor(transaction, delivery).await?;
14950 let destination =
14951 load_and_decode_outbox_destination(transaction, delivery.intent().destination()).await?;
14952 if destination.destination() != delivery.intent().destination() {
14953 return Err(StoreError::corrupt("outbox delivery destination binding"));
14954 }
14955 let attempts = load_and_verify_outbox_attempts(transaction, delivery).await?;
14956 verify_outbox_projection_records(row, delivery, &attempts)
14957}
14958
14959async fn verify_outbox_delivery_anchor(
14960 transaction: &mut Transaction<'_, Postgres>,
14961 delivery: &OutboxDelivery,
14962) -> Result<(), StoreError> {
14963 let sequence = i64::try_from(delivery.origin().sequence().get())
14964 .map_err(|_| StoreError::corrupt("outbox origin sequence"))?;
14965 let row = query_as::<_, EventRow>(SELECT_EVENT_BY_SEQUENCE)
14966 .bind(delivery.intent().tenant_id().as_str())
14967 .bind(*delivery.intent().run_id().as_uuid())
14968 .bind(sequence)
14969 .fetch_optional(&mut **transaction)
14970 .await
14971 .map_err(|source| StoreError::database("outbox journal anchor load", source))?
14972 .ok_or_else(|| StoreError::corrupt("outbox journal anchor"))?;
14973 let event = decode_event(row)?;
14974 if event.head() != *delivery.origin() {
14975 return Err(StoreError::corrupt("outbox journal anchor"));
14976 }
14977 Ok(())
14978}
14979
14980fn verify_outbox_projection_records(
14981 row: &OutboxDeliveryRow,
14982 delivery: &OutboxDelivery,
14983 attempts: &[OutboxAttempt],
14984) -> Result<(), StoreError> {
14985 if usize::try_from(row.attempt_count).ok() != Some(attempts.len())
14986 || row.attempt_count < 0
14987 || usize::try_from(row.attempt_count)
14988 .ok()
14989 .is_none_or(|count| count > MAX_OUTBOX_ATTEMPTS)
14990 {
14991 return Err(StoreError::corrupt("outbox attempt count projection"));
14992 }
14993 if row.status == "expired" {
14994 return verify_expired_outbox_projection(row, delivery, attempts);
14995 }
14996 let Some(last) = attempts.last() else {
14997 return verify_initial_outbox_projection(row, delivery);
14998 };
14999 let updated_at = from_database_time(row.updated_at)?;
15000 let next_attempt_at = row.next_attempt_at.map(from_database_time).transpose()?;
15001 let terminal_at = row.terminal_at.map(from_database_time).transpose()?;
15002 let last_completion_digest = row
15003 .last_completion_digest
15004 .as_deref()
15005 .map(|bytes| decode_digest(bytes, "outbox last completion digest"))
15006 .transpose()?;
15007
15008 let start = last.start();
15009 verify_outbox_current_attempt_projection(row, start)?;
15010
15011 let Some(completion) = last.completion() else {
15012 if attempts.len() == MAX_OUTBOX_ATTEMPTS
15013 && start.expires_at() < delivery.intent().expires_at()
15014 && row.status == "dead_letter"
15015 && next_attempt_at.is_none()
15016 && last_completion_digest.is_none()
15017 && terminal_at == Some(start.expires_at())
15018 && updated_at == start.expires_at()
15019 {
15020 return Ok(());
15021 }
15022 if row.status != "delivering"
15023 || next_attempt_at != Some(start.expires_at())
15024 || last_completion_digest.is_some()
15025 || terminal_at.is_some()
15026 || updated_at != start.started_at()
15027 {
15028 return Err(StoreError::corrupt("outbox delivering projection"));
15029 }
15030 return Ok(());
15031 };
15032
15033 if last_completion_digest != Some(completion.digest())
15034 || updated_at != completion.completed_at()
15035 {
15036 return Err(StoreError::corrupt("outbox completion projection"));
15037 }
15038 match completion.outcome() {
15039 OutboxAttemptOutcome::Acknowledged { .. } => {
15040 if row.status != "acknowledged"
15041 || next_attempt_at.is_some()
15042 || terminal_at != Some(completion.completed_at())
15043 {
15044 return Err(StoreError::corrupt("outbox acknowledgement projection"));
15045 }
15046 }
15047 OutboxAttemptOutcome::Failed { failure } => match failure.retry_advice() {
15048 RetryAdvice::Never => {
15049 if row.status != "dead_letter"
15050 || next_attempt_at.is_some()
15051 || terminal_at != Some(completion.completed_at())
15052 {
15053 return Err(StoreError::corrupt("outbox dead-letter projection"));
15054 }
15055 }
15056 RetryAdvice::SafeAfter { .. } if attempts.len() == MAX_OUTBOX_ATTEMPTS => {
15057 if row.status != "dead_letter"
15058 || next_attempt_at.is_some()
15059 || terminal_at != Some(completion.completed_at())
15060 {
15061 return Err(StoreError::corrupt("outbox attempt-limit projection"));
15062 }
15063 }
15064 RetryAdvice::SafeAfter { delay } => {
15065 let retry_at = add_duration(
15066 completion.completed_at(),
15067 Duration::from_millis(
15068 u64::try_from(delay.as_i64())
15069 .map_err(|_| StoreError::corrupt("outbox retry delay"))?,
15070 ),
15071 )?;
15072 if row.status != "retry_scheduled"
15073 || next_attempt_at != Some(retry_at)
15074 || terminal_at.is_some()
15075 {
15076 return Err(StoreError::corrupt("outbox retry projection"));
15077 }
15078 }
15079 RetryAdvice::ReconcileFirst => {
15080 return Err(StoreError::corrupt("outbox retry advice"));
15081 }
15082 },
15083 _ => return Err(StoreError::corrupt("outbox attempt outcome")),
15084 }
15085 Ok(())
15086}
15087
15088fn verify_initial_outbox_projection(
15089 row: &OutboxDeliveryRow,
15090 delivery: &OutboxDelivery,
15091) -> Result<(), StoreError> {
15092 let next_attempt_at = row.next_attempt_at.map(from_database_time).transpose()?;
15093 if row.status != "pending"
15094 || row.current_attempt_id.is_some()
15095 || row.current_epoch.is_some()
15096 || row.current_attempt_started_at.is_some()
15097 || row.current_attempt_expires_at.is_some()
15098 || next_attempt_at != Some(delivery.origin().recorded_at())
15099 || row.last_completion_digest.is_some()
15100 || row.terminal_at.is_some()
15101 || from_database_time(row.updated_at)? != delivery.origin().recorded_at()
15102 {
15103 return Err(StoreError::corrupt("outbox initial projection"));
15104 }
15105 Ok(())
15106}
15107
15108fn verify_expired_outbox_projection(
15109 row: &OutboxDeliveryRow,
15110 delivery: &OutboxDelivery,
15111 attempts: &[OutboxAttempt],
15112) -> Result<(), StoreError> {
15113 let mut verifier = OutboxAttemptHistoryVerifier::new(delivery);
15114 for attempt in attempts {
15115 verifier
15116 .verify_next(attempt)
15117 .map_err(|_| StoreError::corrupt("outbox expired history"))?;
15118 }
15119 let next_attempt_at = row.next_attempt_at.map(from_database_time).transpose()?;
15120 let terminal_at = row.terminal_at.map(from_database_time).transpose()?;
15121 let updated_at = from_database_time(row.updated_at)?;
15122 let last_completion_digest = row
15123 .last_completion_digest
15124 .as_deref()
15125 .map(|bytes| decode_digest(bytes, "outbox last completion digest"))
15126 .transpose()?;
15127 if verifier
15128 .status_at(delivery.intent().expires_at())
15129 .map_err(|_| StoreError::corrupt("outbox expiry projection"))?
15130 != OutboxDeliveryStatus::Expired
15131 || next_attempt_at.is_some()
15132 || terminal_at != Some(delivery.intent().expires_at())
15133 || updated_at != delivery.intent().expires_at()
15134 || last_completion_digest
15135 != attempts
15136 .last()
15137 .and_then(OutboxAttempt::completion)
15138 .map(OutboxAttemptCompletion::digest)
15139 {
15140 return Err(StoreError::corrupt("outbox expiry projection"));
15141 }
15142 if let Some(last) = attempts.last() {
15143 verify_outbox_current_attempt_projection(row, last.start())?;
15144 } else if row.current_attempt_id.is_some()
15145 || row.current_epoch.is_some()
15146 || row.current_attempt_started_at.is_some()
15147 || row.current_attempt_expires_at.is_some()
15148 {
15149 return Err(StoreError::corrupt("outbox expired current attempt"));
15150 }
15151 Ok(())
15152}
15153
15154fn verify_outbox_current_attempt_projection(
15155 row: &OutboxDeliveryRow,
15156 start: &OutboxAttemptStart,
15157) -> Result<(), StoreError> {
15158 let epoch = i64::try_from(start.fence().epoch().get())
15159 .map_err(|_| StoreError::corrupt("outbox current epoch"))?;
15160 if row.current_attempt_id != Some(*start.fence().attempt_id().as_uuid())
15161 || row.current_epoch != Some(epoch)
15162 || row
15163 .current_attempt_started_at
15164 .map(from_database_time)
15165 .transpose()?
15166 != Some(start.started_at())
15167 || row
15168 .current_attempt_expires_at
15169 .map(from_database_time)
15170 .transpose()?
15171 != Some(start.expires_at())
15172 {
15173 return Err(StoreError::corrupt("outbox current attempt projection"));
15174 }
15175 Ok(())
15176}
15177
15178fn outbox_outcomes_equal(left: &OutboxAttemptOutcome, right: &OutboxAttemptOutcome) -> bool {
15179 match (
15180 serde_json_canonicalizer::to_vec(left),
15181 serde_json_canonicalizer::to_vec(right),
15182 ) {
15183 (Ok(left), Ok(right)) => left == right,
15184 _ => false,
15185 }
15186}
15187
15188fn outbox_attempts_equal(left: &OutboxAttempt, right: &OutboxAttempt) -> bool {
15189 match (
15190 serde_json_canonicalizer::to_vec(left),
15191 serde_json_canonicalizer::to_vec(right),
15192 ) {
15193 (Ok(left), Ok(right)) => left == right,
15194 _ => false,
15195 }
15196}
15197
15198fn encode_node_attempt(attempt: &NodeAttempt) -> Result<Vec<u8>, StoreError> {
15199 serde_json_canonicalizer::to_vec(attempt).map_err(|_| StoreError::encoding("node attempt"))
15200}
15201
15202fn encode_node_attempt_start(start: &NodeAttemptStart) -> Result<Vec<u8>, StoreError> {
15203 let bytes = serde_json_canonicalizer::to_vec(start)
15204 .map_err(|_| StoreError::encoding("node attempt start"))?;
15205 if bytes.is_empty() || bytes.len() > MAX_NODE_ATTEMPT_START_BYTES {
15206 return Err(StoreError::encoding("node attempt start size"));
15207 }
15208 Ok(bytes)
15209}
15210
15211#[allow(clippy::too_many_lines)]
15212fn decode_node_attempt_start(row: &NodeAttemptStartRow) -> Result<NodeAttemptStart, StoreError> {
15213 if row.start_bytes.is_empty() || row.start_bytes.len() > MAX_NODE_ATTEMPT_START_BYTES {
15214 return Err(StoreError::corrupt("node attempt start byte length"));
15215 }
15216 let start = serde_json::from_slice::<NodeAttemptStart>(&row.start_bytes)
15217 .map_err(|_| StoreError::corrupt("node attempt start value"))?;
15218 let canonical = serde_json_canonicalizer::to_vec(&start)
15219 .map_err(|_| StoreError::corrupt("node attempt start canonicalization"))?;
15220 if canonical != row.start_bytes {
15221 return Err(StoreError::corrupt("node attempt start canonical bytes"));
15222 }
15223
15224 let activation = start.activation();
15225 let base = activation.base_checkpoint();
15226 let base_journal = base.journal_head();
15227 let journal = start.journal_head();
15228 let base_superstep = i64::try_from(base.superstep().get())
15229 .map_err(|_| StoreError::corrupt("node attempt base superstep"))?;
15230 let base_journal_sequence = i64::try_from(base_journal.sequence().get())
15231 .map_err(|_| StoreError::corrupt("node attempt base journal sequence"))?;
15232 let fence_epoch = i64::try_from(start.fence().epoch().get())
15233 .map_err(|_| StoreError::corrupt("node attempt fence epoch"))?;
15234 let journal_sequence = i64::try_from(journal.sequence().get())
15235 .map_err(|_| StoreError::corrupt("node attempt journal sequence"))?;
15236
15237 if activation.tenant_id().as_str() != row.tenant_id
15238 || *activation.run_id().as_uuid() != row.run_id
15239 || *base.checkpoint_id().as_uuid() != row.base_checkpoint_id
15240 || base_superstep != row.base_superstep
15241 || base.digest()
15242 != decode_digest(
15243 &row.base_checkpoint_digest,
15244 "node attempt base checkpoint digest",
15245 )?
15246 || base_journal_sequence != row.base_journal_sequence
15247 || *base_journal.event_id().as_uuid() != row.base_journal_event_id
15248 || base_journal.recorded_at() != from_database_time(row.base_journal_recorded_at)?
15249 || base_journal.digest()
15250 != decode_digest(&row.base_journal_digest, "node attempt base journal digest")?
15251 || activation.graph_namespace().as_str() != row.graph_namespace
15252 || activation.node_id().as_str() != row.node_id
15253 || activation.input_digest()
15254 != decode_digest(&row.activation_input_digest, "node attempt input digest")?
15255 || start.activation_digest()
15256 != decode_digest(&row.activation_digest, "node attempt activation digest")?
15257 || *start.attempt_id().as_uuid() != row.attempt_id
15258 || *start.fence().attempt_id().as_uuid() != row.fence_attempt_id
15259 || fence_epoch != row.fence_epoch
15260 || journal_sequence != row.journal_sequence
15261 || *journal.event_id().as_uuid() != row.journal_event_id
15262 || journal.recorded_at() != from_database_time(row.journal_recorded_at)?
15263 || journal.digest() != decode_digest(&row.journal_digest, "node attempt journal digest")?
15264 || start.digest() != decode_digest(&row.start_digest, "node attempt start digest")?
15265 || journal.recorded_at() != from_database_time(row.created_at)?
15266 {
15267 return Err(StoreError::corrupt("node attempt start projection"));
15268 }
15269 Ok(start)
15270}
15271
15272fn encode_node_attempt_completion(
15273 completion: &NodeAttemptCompletion,
15274) -> Result<Vec<u8>, StoreError> {
15275 let bytes = serde_json_canonicalizer::to_vec(completion)
15276 .map_err(|_| StoreError::encoding("node attempt completion"))?;
15277 if bytes.is_empty() || bytes.len() > MAX_NODE_ATTEMPT_COMPLETION_BYTES {
15278 return Err(StoreError::encoding("node attempt completion size"));
15279 }
15280 Ok(bytes)
15281}
15282
15283fn node_attempt_status_text(status: NodeAttemptStatus) -> Result<&'static str, StoreError> {
15284 match status {
15285 NodeAttemptStatus::Executing => Ok("executing"),
15286 NodeAttemptStatus::Succeeded => Ok("succeeded"),
15287 NodeAttemptStatus::Failed => Ok("failed"),
15288 _ => Err(StoreError::encoding("node attempt status")),
15289 }
15290}
15291
15292fn node_attempt_retry_projection(
15293 completion: &NodeAttemptCompletion,
15294) -> Result<(Option<&'static str>, Option<DateTime<Utc>>), StoreError> {
15295 let Some(failure) = completion.outcome().failure() else {
15296 return Ok((None, None));
15297 };
15298 match failure.retry_advice() {
15299 RetryAdvice::Never => Ok((Some("never"), None)),
15300 RetryAdvice::SafeAfter { delay } => {
15301 let delay_micros = delay
15302 .as_i64()
15303 .checked_mul(1_000)
15304 .ok_or_else(|| StoreError::encoding("node attempt retry time"))?;
15305 let eligible_micros = completion
15306 .journal_head()
15307 .recorded_at()
15308 .unix_micros()
15309 .checked_add(delay_micros)
15310 .ok_or_else(|| StoreError::encoding("node attempt retry time"))?;
15311 let eligible = Timestamp::from_unix_micros(eligible_micros)
15312 .map_err(|_| StoreError::encoding("node attempt retry time"))?;
15313 Ok((Some("safe_after"), Some(to_database_time(eligible)?)))
15314 }
15315 RetryAdvice::ReconcileFirst => Err(StoreError::InvalidNodeAttemptTransition),
15316 }
15317}
15318
15319#[allow(clippy::too_many_lines)]
15320fn decode_node_attempt_completion(
15321 row: &NodeAttemptCompletionRow,
15322 expected_start: &NodeAttemptStart,
15323) -> Result<NodeAttemptCompletion, StoreError> {
15324 if row.completion_bytes.is_empty()
15325 || row.completion_bytes.len() > MAX_NODE_ATTEMPT_COMPLETION_BYTES
15326 {
15327 return Err(StoreError::corrupt("node attempt completion byte length"));
15328 }
15329 let completion = serde_json::from_slice::<NodeAttemptCompletion>(&row.completion_bytes)
15330 .map_err(|_| StoreError::corrupt("node attempt completion value"))?;
15331 let canonical = serde_json_canonicalizer::to_vec(&completion)
15332 .map_err(|_| StoreError::corrupt("node attempt completion canonicalization"))?;
15333 if canonical != row.completion_bytes || completion.start() != &expected_start.head() {
15334 return Err(StoreError::corrupt(
15335 "node attempt completion canonical bytes",
15336 ));
15337 }
15338
15339 let start = completion.start();
15340 let activation = start.activation();
15341 let base = activation.base_checkpoint();
15342 let start_journal = start.journal_head();
15343 let journal = completion.journal_head();
15344 let base_superstep = i64::try_from(base.superstep().get())
15345 .map_err(|_| StoreError::corrupt("node completion base superstep"))?;
15346 let fence_epoch = i64::try_from(start.fence().epoch().get())
15347 .map_err(|_| StoreError::corrupt("node completion fence epoch"))?;
15348 let start_sequence = i64::try_from(start_journal.sequence().get())
15349 .map_err(|_| StoreError::corrupt("node completion start sequence"))?;
15350 let journal_sequence = i64::try_from(journal.sequence().get())
15351 .map_err(|_| StoreError::corrupt("node completion journal sequence"))?;
15352 let (result_intent_digest, result_record_digest, failure_id) = match completion.outcome() {
15353 NodeAttemptOutcome::Succeeded { result } => {
15354 (Some(result.intent_digest()), Some(result.digest()), None)
15355 }
15356 NodeAttemptOutcome::Failed { failure } => (None, None, Some(*failure.id().as_uuid())),
15357 _ => return Err(StoreError::corrupt("node attempt completion outcome")),
15358 };
15359 let (retry_kind, retry_not_before) = node_attempt_retry_projection(&completion)
15360 .map_err(|_| StoreError::corrupt("node completion retry projection"))?;
15361
15362 if activation.tenant_id().as_str() != row.tenant_id
15363 || *activation.run_id().as_uuid() != row.run_id
15364 || *start.attempt_id().as_uuid() != row.attempt_id
15365 || *base.checkpoint_id().as_uuid() != row.base_checkpoint_id
15366 || base_superstep != row.base_superstep
15367 || base.digest()
15368 != decode_digest(&row.base_checkpoint_digest, "node completion base digest")?
15369 || activation.graph_namespace().as_str() != row.graph_namespace
15370 || activation.node_id().as_str() != row.node_id
15371 || activation.input_digest()
15372 != decode_digest(&row.activation_input_digest, "node completion input digest")?
15373 || expected_start.activation_digest()
15374 != decode_digest(&row.activation_digest, "node completion activation digest")?
15375 || *start.fence().attempt_id().as_uuid() != row.fence_attempt_id
15376 || fence_epoch != row.fence_epoch
15377 || start_sequence != row.start_journal_sequence
15378 || *start_journal.event_id().as_uuid() != row.start_journal_event_id
15379 || start_journal.recorded_at() != from_database_time(row.start_journal_recorded_at)?
15380 || start_journal.digest()
15381 != decode_digest(
15382 &row.start_journal_digest,
15383 "node completion start journal digest",
15384 )?
15385 || start.digest() != decode_digest(&row.start_digest, "node completion start digest")?
15386 || node_attempt_status_text(completion.status())
15387 .map_err(|_| StoreError::corrupt("node completion status"))?
15388 != row.status
15389 || journal_sequence != row.journal_sequence
15390 || *journal.event_id().as_uuid() != row.journal_event_id
15391 || journal.recorded_at() != from_database_time(row.journal_recorded_at)?
15392 || journal.digest() != decode_digest(&row.journal_digest, "node completion journal digest")?
15393 || decode_optional_digest(
15394 row.result_intent_digest.as_deref(),
15395 "node result intent digest",
15396 )? != result_intent_digest
15397 || decode_optional_digest(
15398 row.result_record_digest.as_deref(),
15399 "node result record digest",
15400 )? != result_record_digest
15401 || row.failure_id != failure_id
15402 || row.retry_kind.as_deref() != retry_kind
15403 || row.retry_not_before != retry_not_before
15404 || completion.digest() != decode_digest(&row.completion_digest, "node completion digest")?
15405 || journal.recorded_at() != from_database_time(row.created_at)?
15406 {
15407 return Err(StoreError::corrupt("node attempt completion projection"));
15408 }
15409 Ok(completion)
15410}
15411
15412fn decode_optional_digest(
15413 bytes: Option<&[u8]>,
15414 record: &'static str,
15415) -> Result<Option<Digest>, StoreError> {
15416 bytes.map(|bytes| decode_digest(bytes, record)).transpose()
15417}
15418
15419async fn load_node_attempt_record(
15420 transaction: &mut Transaction<'_, Postgres>,
15421 tenant_id: &TenantId,
15422 run_id: &RunId,
15423 attempt_id: AttemptId,
15424) -> Result<Option<NodeAttempt>, StoreError> {
15425 let Some(start_row) = query_as::<_, NodeAttemptStartRow>(SELECT_NODE_ATTEMPT_BY_ID)
15426 .bind(tenant_id.as_str())
15427 .bind(*run_id.as_uuid())
15428 .bind(*attempt_id.as_uuid())
15429 .fetch_optional(&mut **transaction)
15430 .await
15431 .map_err(|source| StoreError::database("node attempt lookup", source))?
15432 else {
15433 return Ok(None);
15434 };
15435 load_node_attempt_from_start_row(transaction, start_row)
15436 .await
15437 .map(Some)
15438}
15439
15440async fn load_node_attempt_from_start_row(
15441 transaction: &mut Transaction<'_, Postgres>,
15442 start_row: NodeAttemptStartRow,
15443) -> Result<NodeAttempt, StoreError> {
15444 let start = decode_node_attempt_start(&start_row)?;
15445 let completion_row = query_as::<_, NodeAttemptCompletionRow>(SELECT_NODE_ATTEMPT_COMPLETION)
15446 .bind(start.activation().tenant_id().as_str())
15447 .bind(*start.activation().run_id().as_uuid())
15448 .bind(*start.attempt_id().as_uuid())
15449 .fetch_optional(&mut **transaction)
15450 .await
15451 .map_err(|source| StoreError::database("node attempt completion lookup", source))?;
15452 let completion = completion_row
15453 .map(|row| decode_node_attempt_completion(&row, &start))
15454 .transpose()?;
15455 NodeAttempt::restore(start, completion).map_err(|_| StoreError::corrupt("node attempt join"))
15456}
15457
15458async fn load_locked_node_attempt(
15459 transaction: &mut Transaction<'_, Postgres>,
15460 expected: &NodeAttemptStartHead,
15461) -> Result<NodeAttempt, StoreError> {
15462 let activation = expected.activation();
15463 let row = query_as::<_, NodeAttemptStartRow>(SELECT_NODE_ATTEMPT_BY_ID)
15466 .bind(activation.tenant_id().as_str())
15467 .bind(*activation.run_id().as_uuid())
15468 .bind(*expected.attempt_id().as_uuid())
15469 .fetch_optional(&mut **transaction)
15470 .await
15471 .map_err(|source| StoreError::database("run-locked node attempt load", source))?
15472 .ok_or(StoreError::NodeAttemptNotFound)?;
15473 let attempt = load_node_attempt_from_start_row(transaction, row).await?;
15474 if attempt.start().head() != *expected {
15475 return Err(StoreError::StaleNodeAttemptStart);
15476 }
15477 Ok(attempt)
15478}
15479
15480async fn load_latest_locked_node_attempt(
15481 transaction: &mut Transaction<'_, Postgres>,
15482 activation: &NodeActivation,
15483) -> Result<Option<NodeAttempt>, StoreError> {
15484 let base = activation.base_checkpoint();
15485 let base_superstep = i64::try_from(base.superstep().get())
15486 .map_err(|_| StoreError::InvalidNodeAttemptTransition)?;
15487 let row = query_as::<_, NodeAttemptStartRow>(SELECT_LATEST_NODE_ATTEMPT)
15489 .bind(activation.tenant_id().as_str())
15490 .bind(*activation.run_id().as_uuid())
15491 .bind(*base.checkpoint_id().as_uuid())
15492 .bind(base_superstep)
15493 .bind(base.digest().as_bytes())
15494 .bind(activation.graph_namespace().as_str())
15495 .bind(activation.node_id().as_str())
15496 .bind(activation.input_digest().as_bytes())
15497 .fetch_optional(&mut **transaction)
15498 .await
15499 .map_err(|source| StoreError::database("run-locked latest node attempt load", source))?;
15500 match row {
15501 Some(row) => load_node_attempt_from_start_row(transaction, row)
15502 .await
15503 .map(Some),
15504 None => Ok(None),
15505 }
15506}
15507
15508async fn count_node_attempts(
15509 transaction: &mut Transaction<'_, Postgres>,
15510 activation: &NodeActivation,
15511) -> Result<usize, StoreError> {
15512 let base = activation.base_checkpoint();
15513 let base_superstep = i64::try_from(base.superstep().get())
15514 .map_err(|_| StoreError::InvalidNodeAttemptTransition)?;
15515 let count = query_scalar::<_, i64>(SELECT_NODE_ATTEMPT_COUNT)
15516 .bind(activation.tenant_id().as_str())
15517 .bind(*activation.run_id().as_uuid())
15518 .bind(*base.checkpoint_id().as_uuid())
15519 .bind(base_superstep)
15520 .bind(base.digest().as_bytes())
15521 .bind(activation.graph_namespace().as_str())
15522 .bind(activation.node_id().as_str())
15523 .bind(activation.input_digest().as_bytes())
15524 .fetch_one(&mut **transaction)
15525 .await
15526 .map_err(|source| StoreError::database("node attempt history count", source))?;
15527 usize::try_from(count).map_err(|_| StoreError::corrupt("node attempt history count"))
15528}
15529
15530async fn verify_node_attempt_anchor(
15531 transaction: &mut Transaction<'_, Postgres>,
15532 journal: &JournalHead,
15533 fence: &RunFence,
15534 projection_digest: Digest,
15535 operation: &'static str,
15536) -> Result<JournalEvent, StoreError> {
15537 let sequence = i64::try_from(journal.sequence().get())
15538 .map_err(|_| StoreError::corrupt("node attempt journal sequence"))?;
15539 let row = query_as::<_, EventRow>(SELECT_EVENT_BY_SEQUENCE)
15540 .bind(journal.tenant_id().as_str())
15541 .bind(*journal.run_id().as_uuid())
15542 .bind(sequence)
15543 .fetch_optional(&mut **transaction)
15544 .await
15545 .map_err(|source| StoreError::database(operation, source))?
15546 .ok_or_else(|| StoreError::corrupt("node attempt journal anchor"))?;
15547 let durable_projection = row
15548 .projection_digest
15549 .as_deref()
15550 .map(|bytes| decode_digest(bytes, "node attempt projection digest"))
15551 .transpose()?;
15552 let event = decode_event(row)?;
15553 if event.head() != *journal
15554 || event.source().worker_fence() != Some(fence)
15555 || durable_projection != Some(projection_digest)
15556 {
15557 return Err(StoreError::corrupt("node attempt journal anchor"));
15558 }
15559 Ok(event)
15560}
15561
15562async fn verify_node_attempt_base_checkpoint(
15563 transaction: &mut Transaction<'_, Postgres>,
15564 start: &NodeAttemptStart,
15565) -> Result<(), StoreError> {
15566 let activation = start.activation();
15567 let base = activation.base_checkpoint();
15568 let row = query_as::<_, CheckpointRow>(SELECT_CHECKPOINT_BY_ID)
15569 .bind(activation.tenant_id().as_str())
15570 .bind(*activation.run_id().as_uuid())
15571 .bind(*base.checkpoint_id().as_uuid())
15572 .fetch_optional(&mut **transaction)
15573 .await
15574 .map_err(|source| StoreError::database("node attempt base checkpoint", source))?
15575 .ok_or_else(|| StoreError::corrupt("node attempt base checkpoint"))?;
15576 let checkpoint = decode_checkpoint(row)?;
15577 if checkpoint.head() != *base || !node_attempt_activation_is_ready(&checkpoint, activation) {
15578 return Err(StoreError::corrupt("node attempt base checkpoint"));
15579 }
15580 verify_checkpoint_anchor(transaction, &checkpoint).await
15581}
15582
15583async fn verify_node_attempt(
15584 transaction: &mut Transaction<'_, Postgres>,
15585 attempt: &NodeAttempt,
15586) -> Result<JournalEvent, StoreError> {
15587 let start = attempt.start();
15588 let start_event = verify_node_attempt_start(transaction, start).await?;
15589
15590 let Some(completion) = attempt.completion() else {
15591 return Ok(start_event);
15592 };
15593 let event = verify_node_attempt_anchor(
15594 transaction,
15595 completion.journal_head(),
15596 start.fence(),
15597 completion.digest(),
15598 "node attempt completion anchor",
15599 )
15600 .await?;
15601
15602 if let NodeAttemptOutcome::Succeeded { result } = completion.outcome() {
15603 let row = load_pending_node_result_row(transaction, result.activation())
15604 .await?
15605 .ok_or_else(|| StoreError::corrupt("node attempt successful result"))?;
15606 if row.node_attempt_id != Some(*start.attempt_id().as_uuid()) {
15607 return Err(StoreError::corrupt("node attempt successful result owner"));
15608 }
15609 let durable_result = decode_pending_node_result(&row)?;
15610 if durable_result.head() != **result {
15611 return Err(StoreError::corrupt("node attempt successful result"));
15612 }
15613 verify_pending_node_result_base_checkpoint(transaction, &durable_result).await?;
15614 verify_pending_node_result_bindings(transaction, &durable_result).await?;
15615 }
15616 Ok(event)
15617}
15618
15619async fn verify_node_attempt_start(
15623 transaction: &mut Transaction<'_, Postgres>,
15624 start: &NodeAttemptStart,
15625) -> Result<JournalEvent, StoreError> {
15626 verify_node_attempt_base_checkpoint(transaction, start).await?;
15627 verify_node_attempt_anchor(
15628 transaction,
15629 start.journal_head(),
15630 start.fence(),
15631 start.digest(),
15632 "node attempt start anchor",
15633 )
15634 .await
15635}
15636
15637fn encode_pending_node_result(result: &PendingNodeResult) -> Result<Vec<u8>, StoreError> {
15638 let bytes = serde_json_canonicalizer::to_vec(result)
15639 .map_err(|_| StoreError::encoding("pending node result"))?;
15640 if bytes.is_empty() || bytes.len() > MAX_PENDING_NODE_RESULT_BYTES {
15641 return Err(StoreError::encoding("pending node result size"));
15642 }
15643 Ok(bytes)
15644}
15645
15646#[allow(clippy::too_many_lines)]
15647fn decode_pending_node_result(row: &PendingNodeResultRow) -> Result<PendingNodeResult, StoreError> {
15648 if row.result_bytes.is_empty() || row.result_bytes.len() > MAX_PENDING_NODE_RESULT_BYTES {
15649 return Err(StoreError::corrupt("pending node result byte length"));
15650 }
15651 let result = serde_json::from_slice::<PendingNodeResult>(&row.result_bytes)
15652 .map_err(|_| StoreError::corrupt("pending node result value"))?;
15653 let canonical = serde_json_canonicalizer::to_vec(&result)
15654 .map_err(|_| StoreError::corrupt("pending node result canonicalization"))?;
15655 if canonical != row.result_bytes {
15656 return Err(StoreError::corrupt("pending node result canonical bytes"));
15657 }
15658
15659 let activation = result.intent().activation();
15660 let base = activation.base_checkpoint();
15661 let base_journal = base.journal_head();
15662 let journal = result.journal_head();
15663 let fence_epoch = i64::try_from(result.fence().epoch().get())
15664 .map_err(|_| StoreError::corrupt("pending node result fence epoch"))?;
15665 let base_superstep = i64::try_from(base.superstep().get())
15666 .map_err(|_| StoreError::corrupt("pending node result base superstep"))?;
15667 let base_journal_sequence = i64::try_from(base_journal.sequence().get())
15668 .map_err(|_| StoreError::corrupt("pending node result base journal sequence"))?;
15669 let journal_sequence = i64::try_from(journal.sequence().get())
15670 .map_err(|_| StoreError::corrupt("pending node result journal sequence"))?;
15671
15672 if activation.tenant_id().as_str() != row.tenant_id
15673 || *activation.run_id().as_uuid() != row.run_id
15674 || *base.checkpoint_id().as_uuid() != row.base_checkpoint_id
15675 || base_superstep != row.base_superstep
15676 || base.digest()
15677 != decode_digest(
15678 &row.base_checkpoint_digest,
15679 "pending node result base checkpoint digest",
15680 )?
15681 || base_journal_sequence != row.base_journal_sequence
15682 || *base_journal.event_id().as_uuid() != row.base_journal_event_id
15683 || base_journal.recorded_at() != from_database_time(row.base_journal_recorded_at)?
15684 || base_journal.digest()
15685 != decode_digest(
15686 &row.base_journal_digest,
15687 "pending node result base journal digest",
15688 )?
15689 || activation.graph_namespace().as_str() != row.graph_namespace
15690 || activation.node_id().as_str() != row.node_id
15691 || activation.input_digest()
15692 != decode_digest(
15693 &row.activation_input_digest,
15694 "pending node result activation input digest",
15695 )?
15696 || result.intent().intent_digest()
15697 != decode_digest(&row.intent_digest, "pending node result intent digest")?
15698 || pending_node_result_control_kind_text(result.intent().control().kind())
15699 != row.control_kind
15700 || *result.fence().attempt_id().as_uuid() != row.fence_attempt_id
15701 || fence_epoch != row.fence_epoch
15702 || journal_sequence != row.journal_sequence
15703 || *journal.event_id().as_uuid() != row.journal_event_id
15704 || journal.recorded_at() != from_database_time(row.journal_recorded_at)?
15705 || journal.digest()
15706 != decode_digest(&row.journal_digest, "pending node result journal digest")?
15707 || result.digest()
15708 != decode_digest(&row.record_digest, "pending node result record digest")?
15709 || journal.recorded_at() != from_database_time(row.created_at)?
15710 {
15711 return Err(StoreError::corrupt("pending node result projection"));
15712 }
15713 Ok(result)
15714}
15715
15716async fn load_pending_node_result_row(
15717 transaction: &mut Transaction<'_, Postgres>,
15718 activation: &NodeActivation,
15719) -> Result<Option<PendingNodeResultRow>, StoreError> {
15720 query_as::<_, PendingNodeResultRow>(SELECT_PENDING_NODE_RESULT)
15721 .bind(activation.tenant_id().as_str())
15722 .bind(*activation.run_id().as_uuid())
15723 .bind(*activation.base_checkpoint().checkpoint_id().as_uuid())
15724 .bind(activation.graph_namespace().as_str())
15725 .bind(activation.node_id().as_str())
15726 .fetch_optional(&mut **transaction)
15727 .await
15728 .map_err(|source| StoreError::database("pending node result lookup", source))
15729}
15730
15731async fn load_pending_node_result_head_row(
15732 transaction: &mut Transaction<'_, Postgres>,
15733 head: &PendingNodeResultHead,
15734) -> Result<PendingNodeResultHeadRow, StoreError> {
15735 let activation = head.activation();
15736 query_as::<_, PendingNodeResultHeadRow>(SELECT_PENDING_NODE_RESULT_HEAD)
15737 .bind(activation.tenant_id().as_str())
15738 .bind(*activation.run_id().as_uuid())
15739 .bind(*activation.base_checkpoint().checkpoint_id().as_uuid())
15740 .bind(activation.graph_namespace().as_str())
15741 .bind(activation.node_id().as_str())
15742 .fetch_optional(&mut **transaction)
15743 .await
15744 .map_err(|source| StoreError::database("pending node result cursor lookup", source))?
15745 .ok_or(StoreError::InvalidPendingNodeResultCursor)
15746}
15747
15748fn decode_pending_node_result_head(
15749 row: PendingNodeResultHeadRow,
15750 base: &CheckpointHead,
15751) -> Result<PendingNodeResultHead, StoreError> {
15752 let base_superstep = nonnegative_superstep(row.base_superstep)?;
15753 if row.tenant_id != base.tenant_id().as_str()
15754 || row.run_id != *base.run_id().as_uuid()
15755 || row.base_checkpoint_id != *base.checkpoint_id().as_uuid()
15756 || base_superstep != base.superstep()
15757 || decode_digest(
15758 &row.base_checkpoint_digest,
15759 "pending result compact base digest",
15760 )? != base.digest()
15761 {
15762 return Err(StoreError::corrupt("pending result compact base"));
15763 }
15764 let graph_namespace = GraphNamespace::new(row.graph_namespace)
15765 .map_err(|_| StoreError::corrupt("pending result compact namespace"))?;
15766 let node_id =
15767 NodeId::new(row.node_id).map_err(|_| StoreError::corrupt("pending result compact node"))?;
15768 let activation = NodeActivation::new(
15769 base.clone(),
15770 graph_namespace,
15771 node_id,
15772 decode_digest(
15773 &row.activation_input_digest,
15774 "pending result compact input digest",
15775 )?,
15776 );
15777 let fence_epoch = u64::try_from(row.fence_epoch)
15778 .ok()
15779 .and_then(|value| FencingEpoch::new(value).ok())
15780 .ok_or_else(|| StoreError::corrupt("pending result compact fence epoch"))?;
15781 let fence = RunFence::new(
15782 base.tenant_id().clone(),
15783 base.run_id(),
15784 AttemptId::from_uuid(row.fence_attempt_id)
15785 .map_err(|_| StoreError::corrupt("pending result compact attempt"))?,
15786 fence_epoch,
15787 );
15788 let journal = JournalHead::new(
15789 base.tenant_id().clone(),
15790 base.run_id(),
15791 positive_sequence(row.journal_sequence)?,
15792 EventId::from_uuid(row.journal_event_id)
15793 .map_err(|_| StoreError::corrupt("pending result compact event"))?,
15794 from_database_time(row.journal_recorded_at)?,
15795 decode_digest(&row.journal_digest, "pending result compact journal digest")?,
15796 );
15797 PendingNodeResultHead::new(
15798 activation,
15799 decode_digest(&row.intent_digest, "pending result compact intent digest")?,
15800 fence,
15801 journal,
15802 decode_digest(&row.record_digest, "pending result compact record digest")?,
15803 )
15804 .map_err(|_| StoreError::corrupt("pending result compact head"))
15805}
15806
15807async fn load_locked_barrier_result_heads(
15808 transaction: &mut Transaction<'_, Postgres>,
15809 base: &CheckpointHead,
15810) -> Result<Vec<PendingNodeResultHead>, StoreError> {
15811 let query_limit = i64::try_from(BarrierResultHeads::MAX_LEN + 1)
15812 .map_err(|_| StoreError::InvalidCheckpointBarrier)?;
15813 let rows =
15814 query_as::<_, PendingNodeResultHeadRow>(SELECT_PENDING_NODE_RESULT_HEADS_FOR_BARRIER)
15815 .bind(base.tenant_id().as_str())
15816 .bind(*base.run_id().as_uuid())
15817 .bind(*base.checkpoint_id().as_uuid())
15818 .bind(query_limit)
15819 .fetch_all(&mut **transaction)
15820 .await
15821 .map_err(|source| StoreError::database("checkpoint barrier result heads", source))?;
15822 rows.into_iter()
15823 .map(|row| decode_pending_node_result_head(row, base))
15824 .collect()
15825}
15826
15827fn validate_complete_barrier_result_heads(
15828 durable: &[PendingNodeResultHead],
15829 expected: &BarrierResultHeads,
15830) -> Result<(), StoreError> {
15831 if durable == expected.as_slice() {
15832 return Ok(());
15833 }
15834 if durable.len() < expected.len()
15835 && durable
15836 .iter()
15837 .all(|head| expected.iter().any(|expected| expected == head))
15838 {
15839 return Err(StoreError::CheckpointBarrierIncomplete);
15840 }
15841 Err(StoreError::CheckpointBarrierResultConflict)
15842}
15843
15844async fn load_barrier_consumption_rows(
15845 transaction: &mut Transaction<'_, Postgres>,
15846 base: &CheckpointHead,
15847) -> Result<Vec<PendingNodeResultConsumptionRow>, StoreError> {
15848 query_as::<_, PendingNodeResultConsumptionRow>(SELECT_PENDING_NODE_RESULT_CONSUMPTIONS_BY_BASE)
15849 .bind(base.tenant_id().as_str())
15850 .bind(*base.run_id().as_uuid())
15851 .bind(*base.checkpoint_id().as_uuid())
15852 .fetch_all(&mut **transaction)
15853 .await
15854 .map_err(|source| StoreError::database("checkpoint barrier consumptions", source))
15855}
15856
15857async fn verify_barrier_consumptions(
15858 transaction: &mut Transaction<'_, Postgres>,
15859 barrier: &CheckpointBarrier,
15860 successor: &Checkpoint,
15861) -> Result<(), StoreError> {
15862 let base = barrier.base_checkpoint();
15863 let rows = load_barrier_consumption_rows(transaction, base).await?;
15864 if rows.len() != barrier.result_heads().len() {
15865 return Err(StoreError::CheckpointBarrierCommitConflict);
15866 }
15867 for (row, result) in rows.iter().zip(barrier.result_heads().iter()) {
15868 let activation = result.activation();
15869 if row.tenant_id != base.tenant_id().as_str()
15870 || row.run_id != *base.run_id().as_uuid()
15871 || row.base_checkpoint_id != *base.checkpoint_id().as_uuid()
15872 || nonnegative_superstep(row.base_superstep)? != base.superstep()
15873 || decode_digest(
15874 &row.base_checkpoint_digest,
15875 "checkpoint barrier consumption base digest",
15876 )? != base.digest()
15877 || row.graph_namespace != activation.graph_namespace().as_str()
15878 || row.node_id != activation.node_id().as_str()
15879 || decode_digest(
15880 &row.result_record_digest,
15881 "checkpoint barrier consumption result digest",
15882 )? != result.digest()
15883 || row.successor_checkpoint_id != *successor.checkpoint_id().as_uuid()
15884 || nonnegative_superstep(row.successor_superstep)? != successor.superstep()
15885 || decode_digest(
15886 &row.successor_checkpoint_digest,
15887 "checkpoint barrier consumption successor digest",
15888 )? != successor.digest()
15889 || positive_sequence(row.successor_journal_sequence)?
15890 != successor.journal_head().sequence()
15891 || row.successor_journal_event_id != *successor.journal_head().event_id().as_uuid()
15892 || from_database_time(row.successor_journal_recorded_at)?
15893 != successor.journal_head().recorded_at()
15894 || decode_digest(
15895 &row.successor_journal_digest,
15896 "checkpoint barrier consumption journal digest",
15897 )? != successor.journal_head().digest()
15898 || from_database_time(row.created_at)? != successor.journal_head().recorded_at()
15899 {
15900 return Err(StoreError::CheckpointBarrierCommitConflict);
15901 }
15902 }
15903 Ok(())
15904}
15905
15906fn pending_result_cursor_matches_base(
15907 cursor: &PendingNodeResultPageCursor,
15908 base: &CheckpointHead,
15909) -> bool {
15910 let snapshot = cursor.snapshot_journal_head();
15911 let after = cursor.after();
15912 cursor.base_checkpoint() == base
15913 && snapshot.tenant_id() == base.tenant_id()
15914 && snapshot.run_id() == base.run_id()
15915 && snapshot.sequence() >= after.journal_head().sequence()
15916 && snapshot.recorded_at() >= after.journal_head().recorded_at()
15917 && after.activation().base_checkpoint() == base
15918 && after.activation().graph_namespace().is_root()
15919}
15920
15921fn pending_node_result_activation_is_ready(
15922 checkpoint: &Checkpoint,
15923 intent: &PendingNodeResultIntent,
15924) -> bool {
15925 activation_is_canonical_ready_root(checkpoint, intent.activation())
15926}
15927
15928fn node_attempt_activation_is_ready(checkpoint: &Checkpoint, activation: &NodeActivation) -> bool {
15929 activation_is_canonical_ready_root(checkpoint, activation)
15930}
15931
15932fn activation_is_canonical_ready_root(
15933 checkpoint: &Checkpoint,
15934 activation: &NodeActivation,
15935) -> bool {
15936 NodeActivation::for_ready_root(checkpoint, activation.node_id().clone())
15937 .is_ok_and(|expected| expected == *activation)
15938}
15939
15940async fn reject_reused_node_worker_attempt(
15941 transaction: &mut Transaction<'_, Postgres>,
15942 activation: &NodeActivation,
15943 fence: &RunFence,
15944) -> Result<(), StoreError> {
15945 let base = activation.base_checkpoint();
15946 let base_superstep = i64::try_from(base.superstep().get())
15947 .map_err(|_| StoreError::InvalidNodeAttemptTransition)?;
15948 let fence_epoch =
15949 i64::try_from(fence.epoch().get()).map_err(|_| StoreError::InvalidNodeAttemptTransition)?;
15950 let reused: bool = query_scalar(
15951 r"
15952SELECT EXISTS (
15953 SELECT 1
15954 FROM stateknot.node_attempts
15955 WHERE tenant_id = $1
15956 AND run_id = $2
15957 AND base_checkpoint_id = $3
15958 AND base_superstep = $4
15959 AND base_checkpoint_digest = $5
15960 AND graph_namespace = $6
15961 AND node_id = $7
15962 AND activation_input_digest = $8
15963 AND fence_attempt_id = $9
15964 AND fence_epoch <> $10
15965)
15966",
15967 )
15968 .bind(activation.tenant_id().as_str())
15969 .bind(*activation.run_id().as_uuid())
15970 .bind(*base.checkpoint_id().as_uuid())
15971 .bind(base_superstep)
15972 .bind(base.digest().as_bytes())
15973 .bind(activation.graph_namespace().as_str())
15974 .bind(activation.node_id().as_str())
15975 .bind(activation.input_digest().as_bytes())
15976 .bind(*fence.attempt_id().as_uuid())
15977 .bind(fence_epoch)
15978 .fetch_one(&mut **transaction)
15979 .await
15980 .map_err(|source| StoreError::database("node attempt worker history", source))?;
15981 if reused {
15982 return Err(StoreError::InvalidNodeAttemptTransition);
15983 }
15984 Ok(())
15985}
15986
15987async fn verify_pending_node_result_base_checkpoint(
15988 transaction: &mut Transaction<'_, Postgres>,
15989 result: &PendingNodeResult,
15990) -> Result<(), StoreError> {
15991 let activation = result.intent().activation();
15992 let base = activation.base_checkpoint();
15993 let row = query_as::<_, CheckpointRow>(SELECT_CHECKPOINT_BY_ID)
15994 .bind(activation.tenant_id().as_str())
15995 .bind(*activation.run_id().as_uuid())
15996 .bind(*base.checkpoint_id().as_uuid())
15997 .fetch_optional(&mut **transaction)
15998 .await
15999 .map_err(|source| StoreError::database("pending node result base checkpoint", source))?
16000 .ok_or_else(|| StoreError::corrupt("pending node result base checkpoint"))?;
16001 let checkpoint = decode_checkpoint(row)?;
16002 if checkpoint.head() != *base
16003 || !pending_node_result_activation_is_ready(&checkpoint, result.intent())
16004 {
16005 return Err(StoreError::corrupt("pending node result base checkpoint"));
16006 }
16007 verify_checkpoint_anchor(transaction, &checkpoint).await
16008}
16009
16010async fn verify_pending_node_result_anchor(
16011 transaction: &mut Transaction<'_, Postgres>,
16012 result: &PendingNodeResult,
16013) -> Result<JournalEvent, StoreError> {
16014 let sequence = i64::try_from(result.journal_head().sequence().get())
16015 .map_err(|_| StoreError::corrupt("pending node result journal sequence"))?;
16016 let row = query_as::<_, EventRow>(SELECT_EVENT_BY_SEQUENCE)
16017 .bind(result.intent().tenant_id().as_str())
16018 .bind(*result.intent().run_id().as_uuid())
16019 .bind(sequence)
16020 .fetch_optional(&mut **transaction)
16021 .await
16022 .map_err(|source| StoreError::database("pending node result anchor", source))?
16023 .ok_or_else(|| StoreError::corrupt("pending node result journal anchor"))?;
16024 let projection_digest = row
16025 .projection_digest
16026 .as_deref()
16027 .map(|bytes| decode_digest(bytes, "pending node result projection digest"))
16028 .transpose()?;
16029 let event = decode_event(row)?;
16030 if event.head() != *result.journal_head()
16031 || event.source().worker_fence() != Some(result.fence())
16032 || projection_digest != Some(result.digest())
16033 {
16034 return Err(StoreError::corrupt("pending node result journal anchor"));
16035 }
16036 Ok(event)
16037}
16038
16039async fn verify_pending_node_result(
16040 transaction: &mut Transaction<'_, Postgres>,
16041 result: &PendingNodeResult,
16042) -> Result<JournalEvent, StoreError> {
16043 verify_pending_node_result_base_checkpoint(transaction, result).await?;
16044 let row = load_pending_node_result_row(transaction, result.intent().activation())
16045 .await?
16046 .ok_or_else(|| StoreError::corrupt("pending node result owner row"))?;
16047 let durable = decode_pending_node_result(&row)?;
16048 if encode_pending_node_result(&durable)? != encode_pending_node_result(result)? {
16049 return Err(StoreError::corrupt("pending node result owner row"));
16050 }
16051 let event = if let Some(owner) = row.node_attempt_id {
16052 let attempt_id = AttemptId::from_uuid(owner)
16053 .map_err(|_| StoreError::corrupt("pending node result attempt owner"))?;
16054 let attempt = load_node_attempt_record(
16055 transaction,
16056 result.intent().tenant_id(),
16057 &result.intent().run_id(),
16058 attempt_id,
16059 )
16060 .await?
16061 .ok_or_else(|| StoreError::corrupt("pending node result attempt owner"))?;
16062 let completion = attempt
16063 .completion()
16064 .ok_or_else(|| StoreError::corrupt("pending node result attempt completion"))?;
16065 if attempt.start().attempt_id() != attempt_id
16066 || completion.outcome().result() != Some(&result.head())
16067 {
16068 return Err(StoreError::corrupt(
16069 "pending node result attempt completion",
16070 ));
16071 }
16072 verify_node_attempt_base_checkpoint(transaction, attempt.start()).await?;
16073 verify_node_attempt_anchor(
16074 transaction,
16075 attempt.start().journal_head(),
16076 attempt.start().fence(),
16077 attempt.start().digest(),
16078 "pending result node start anchor",
16079 )
16080 .await?;
16081 verify_node_attempt_anchor(
16082 transaction,
16083 completion.journal_head(),
16084 attempt.start().fence(),
16085 completion.digest(),
16086 "pending result node completion anchor",
16087 )
16088 .await?
16089 } else {
16090 verify_pending_node_result_anchor(transaction, result).await?
16091 };
16092 verify_pending_node_result_bindings(transaction, result).await?;
16093 Ok(event)
16094}
16095
16096async fn verify_pending_node_result_bindings(
16097 transaction: &mut Transaction<'_, Postgres>,
16098 result: &PendingNodeResult,
16099) -> Result<(), StoreError> {
16100 child_joins::verify_result(transaction, result, true).await?;
16101 let activation = result.intent().activation();
16102 let tool_rows =
16103 query_as::<_, PendingNodeResultBindingRow>(SELECT_PENDING_NODE_RESULT_TOOL_BINDINGS)
16104 .bind(activation.tenant_id().as_str())
16105 .bind(*activation.run_id().as_uuid())
16106 .bind(*activation.base_checkpoint().checkpoint_id().as_uuid())
16107 .bind(activation.graph_namespace().as_str())
16108 .bind(activation.node_id().as_str())
16109 .fetch_all(&mut **transaction)
16110 .await
16111 .map_err(|source| StoreError::database("pending node result tool bindings", source))?;
16112 let model_rows =
16113 query_as::<_, PendingNodeResultBindingRow>(SELECT_PENDING_NODE_RESULT_MODEL_BINDINGS)
16114 .bind(activation.tenant_id().as_str())
16115 .bind(*activation.run_id().as_uuid())
16116 .bind(*activation.base_checkpoint().checkpoint_id().as_uuid())
16117 .bind(activation.graph_namespace().as_str())
16118 .bind(activation.node_id().as_str())
16119 .fetch_all(&mut **transaction)
16120 .await
16121 .map_err(|source| StoreError::database("pending node result model bindings", source))?;
16122
16123 let tool_bindings = result
16124 .intent()
16125 .bindings()
16126 .iter()
16127 .filter(|binding| binding.kind() == NodeInvocationBindingKind::Tool)
16128 .collect::<Vec<_>>();
16129 let model_bindings = result
16130 .intent()
16131 .bindings()
16132 .iter()
16133 .filter(|binding| binding.kind() == NodeInvocationBindingKind::Model)
16134 .collect::<Vec<_>>();
16135 verify_pending_node_result_binding_rows(result, &tool_bindings, tool_rows)?;
16136 verify_pending_node_result_binding_rows(result, &model_bindings, model_rows)?;
16137
16138 let mut anchors = BTreeMap::new();
16139 verify_pending_tool_invocations(transaction, result, &tool_bindings, &mut anchors).await?;
16140 verify_pending_model_invocations(transaction, result, &model_bindings, &mut anchors).await?;
16141 verify_pending_invocation_anchors(transaction, result, anchors).await
16142}
16143
16144fn verify_pending_node_result_binding_rows(
16145 result: &PendingNodeResult,
16146 expected: &[&NodeInvocationBinding],
16147 rows: Vec<PendingNodeResultBindingRow>,
16148) -> Result<(), StoreError> {
16149 if rows.len() != expected.len() {
16150 return Err(StoreError::corrupt("pending node result binding count"));
16151 }
16152 let mut rows_by_id = BTreeMap::new();
16153 for row in rows {
16154 let id = row.invocation_id;
16155 if rows_by_id.insert(id, row).is_some() {
16156 return Err(StoreError::corrupt("pending node result binding identity"));
16157 }
16158 }
16159 for binding in expected {
16160 let row = rows_by_id
16161 .remove(binding.invocation_id().as_uuid())
16162 .ok_or_else(|| StoreError::corrupt("pending node result binding identity"))?;
16163 if !pending_node_result_binding_row_matches(&row, result, binding)? {
16164 return Err(StoreError::corrupt(
16165 "pending node result binding projection",
16166 ));
16167 }
16168 }
16169 if !rows_by_id.is_empty() {
16170 return Err(StoreError::corrupt("pending node result binding identity"));
16171 }
16172 Ok(())
16173}
16174
16175#[allow(clippy::too_many_lines)]
16176fn pending_node_result_binding_row_matches(
16177 row: &PendingNodeResultBindingRow,
16178 result: &PendingNodeResult,
16179 binding: &NodeInvocationBinding,
16180) -> Result<bool, StoreError> {
16181 let activation = result.intent().activation();
16182 let base = activation.base_checkpoint();
16183 let result_journal = result.journal_head();
16184 let invocation_journal = binding.journal_head();
16185 let base_superstep = i64::try_from(base.superstep().get())
16186 .map_err(|_| StoreError::corrupt("pending node result binding base superstep"))?;
16187 let result_journal_sequence = i64::try_from(result_journal.sequence().get())
16188 .map_err(|_| StoreError::corrupt("pending node result binding result sequence"))?;
16189 let invocation_journal_sequence = i64::try_from(invocation_journal.sequence().get())
16190 .map_err(|_| StoreError::corrupt("pending node result binding invocation sequence"))?;
16191 let (revision, record_digest, status) = match binding {
16192 NodeInvocationBinding::Tool { head, .. } => (
16193 i64::try_from(head.revision().get())
16194 .map_err(|_| StoreError::corrupt("pending tool binding revision"))?,
16195 head.digest(),
16196 tool_invocation_status_text(head.status()),
16197 ),
16198 NodeInvocationBinding::Model { head, .. } => (
16199 i64::try_from(head.revision().get())
16200 .map_err(|_| StoreError::corrupt("pending model binding revision"))?,
16201 head.digest(),
16202 model_invocation_status_text(head.status()),
16203 ),
16204 };
16205 Ok(row.tenant_id == activation.tenant_id().as_str()
16206 && row.run_id == *activation.run_id().as_uuid()
16207 && row.base_checkpoint_id == *base.checkpoint_id().as_uuid()
16208 && row.base_superstep == base_superstep
16209 && decode_digest(
16210 &row.base_checkpoint_digest,
16211 "pending node result binding base digest",
16212 )? == base.digest()
16213 && row.graph_namespace == activation.graph_namespace().as_str()
16214 && row.node_id == activation.node_id().as_str()
16215 && decode_digest(
16216 &row.activation_input_digest,
16217 "pending node result binding input digest",
16218 )? == activation.input_digest()
16219 && decode_digest(
16220 &row.result_record_digest,
16221 "pending node result binding result digest",
16222 )? == result.digest()
16223 && row.result_journal_sequence == result_journal_sequence
16224 && from_database_time(row.result_journal_recorded_at)? == result_journal.recorded_at()
16225 && decode_digest(
16226 &row.result_journal_digest,
16227 "pending node result binding result journal digest",
16228 )? == result_journal.digest()
16229 && row.invocation_id == *binding.invocation_id().as_uuid()
16230 && row.invocation_revision == revision
16231 && decode_digest(
16232 &row.invocation_record_digest,
16233 "pending node result binding invocation digest",
16234 )? == record_digest
16235 && row.invocation_status == status
16236 && row.invocation_journal_sequence == invocation_journal_sequence
16237 && from_database_time(row.invocation_journal_recorded_at)?
16238 == invocation_journal.recorded_at()
16239 && decode_digest(
16240 &row.invocation_journal_digest,
16241 "pending node result binding invocation journal digest",
16242 )? == invocation_journal.digest())
16243}
16244
16245async fn verify_pending_tool_invocations(
16246 transaction: &mut Transaction<'_, Postgres>,
16247 result: &PendingNodeResult,
16248 bindings: &[&NodeInvocationBinding],
16249 anchors: &mut BTreeMap<JournalSequence, (JournalHead, Digest)>,
16250) -> Result<(), StoreError> {
16251 for chunk in bindings.chunks(PENDING_TOOL_BINDING_BATCH_SIZE) {
16252 verify_pending_tool_invocation_chunk(transaction, result, chunk, anchors).await?;
16253 }
16254 Ok(())
16255}
16256
16257async fn verify_pending_tool_invocation_chunk(
16258 transaction: &mut Transaction<'_, Postgres>,
16259 result: &PendingNodeResult,
16260 bindings: &[&NodeInvocationBinding],
16261 anchors: &mut BTreeMap<JournalSequence, (JournalHead, Digest)>,
16262) -> Result<(), StoreError> {
16263 let ids = bindings
16264 .iter()
16265 .map(|binding| *binding.invocation_id().as_uuid())
16266 .collect::<Vec<_>>();
16267 let revisions = bindings
16268 .iter()
16269 .map(|binding| {
16270 let head = binding
16271 .tool_head()
16272 .ok_or_else(|| StoreError::corrupt("pending tool binding kind"))?;
16273 i64::try_from(head.revision().get())
16274 .map_err(|_| StoreError::corrupt("pending tool binding revision"))
16275 })
16276 .collect::<Result<Vec<_>, _>>()?;
16277 let intent_rows = query_as::<_, ToolInvocationRow>(SELECT_TOOL_INVOCATIONS_BY_IDS)
16278 .bind(result.intent().tenant_id().as_str())
16279 .bind(*result.intent().run_id().as_uuid())
16280 .bind(&ids)
16281 .fetch_all(&mut **transaction)
16282 .await
16283 .map_err(|source| StoreError::database("pending tool binding intents", source))?;
16284 let revision_rows =
16285 query_as::<_, ToolInvocationRevisionRow>(SELECT_TOOL_INVOCATION_REVISIONS_BY_HEADS)
16286 .bind(result.intent().tenant_id().as_str())
16287 .bind(*result.intent().run_id().as_uuid())
16288 .bind(&ids)
16289 .bind(&revisions)
16290 .fetch_all(&mut **transaction)
16291 .await
16292 .map_err(|source| StoreError::database("pending tool binding revisions", source))?;
16293 let mut intents = unique_tool_invocation_rows(intent_rows)?;
16294 let mut records = unique_tool_invocation_revision_rows(revision_rows)?;
16295 if intents.len() != bindings.len() || records.len() != bindings.len() {
16296 return Err(StoreError::corrupt("pending tool binding records"));
16297 }
16298 for binding in bindings {
16299 let head = binding
16300 .tool_head()
16301 .ok_or_else(|| StoreError::corrupt("pending tool binding kind"))?;
16302 let intent_row = intents
16303 .remove(head.invocation_id().as_uuid())
16304 .ok_or_else(|| StoreError::corrupt("pending tool binding intent"))?;
16305 let intent = decode_tool_invocation_intent(&intent_row)?;
16306 let revision = i64::try_from(head.revision().get())
16307 .map_err(|_| StoreError::corrupt("pending tool binding revision"))?;
16308 let revision_row = records
16309 .remove(&(*head.invocation_id().as_uuid(), revision))
16310 .ok_or_else(|| StoreError::corrupt("pending tool binding revision"))?;
16311 let invocation = decode_tool_invocation_revision(revision_row, &intent)?;
16312 validate_tool_invocation_current_projection(&intent_row, &invocation)?;
16313 if intent.activation() != result.intent().activation() || invocation.head() != *head {
16314 return Err(StoreError::corrupt("pending tool binding record"));
16315 }
16316 insert_pending_invocation_anchor(anchors, invocation.journal_head(), invocation.digest())?;
16317 }
16318 if !intents.is_empty() || !records.is_empty() {
16319 return Err(StoreError::corrupt("pending tool binding records"));
16320 }
16321 Ok(())
16322}
16323
16324async fn verify_pending_model_invocations(
16325 transaction: &mut Transaction<'_, Postgres>,
16326 result: &PendingNodeResult,
16327 bindings: &[&NodeInvocationBinding],
16328 anchors: &mut BTreeMap<JournalSequence, (JournalHead, Digest)>,
16329) -> Result<(), StoreError> {
16330 for chunk in bindings.chunks(PENDING_MODEL_BINDING_BATCH_SIZE) {
16331 verify_pending_model_invocation_chunk(transaction, result, chunk, anchors).await?;
16332 }
16333 Ok(())
16334}
16335
16336async fn verify_pending_model_invocation_chunk(
16337 transaction: &mut Transaction<'_, Postgres>,
16338 result: &PendingNodeResult,
16339 bindings: &[&NodeInvocationBinding],
16340 anchors: &mut BTreeMap<JournalSequence, (JournalHead, Digest)>,
16341) -> Result<(), StoreError> {
16342 let ids = bindings
16343 .iter()
16344 .map(|binding| *binding.invocation_id().as_uuid())
16345 .collect::<Vec<_>>();
16346 let revisions = bindings
16347 .iter()
16348 .map(|binding| {
16349 let head = binding
16350 .model_head()
16351 .ok_or_else(|| StoreError::corrupt("pending model binding kind"))?;
16352 i64::try_from(head.revision().get())
16353 .map_err(|_| StoreError::corrupt("pending model binding revision"))
16354 })
16355 .collect::<Result<Vec<_>, _>>()?;
16356 let intent_rows = query_as::<_, ModelInvocationRow>(SELECT_MODEL_INVOCATIONS_BY_IDS)
16357 .bind(result.intent().tenant_id().as_str())
16358 .bind(*result.intent().run_id().as_uuid())
16359 .bind(&ids)
16360 .fetch_all(&mut **transaction)
16361 .await
16362 .map_err(|source| StoreError::database("pending model binding intents", source))?;
16363 let revision_rows =
16364 query_as::<_, ModelInvocationRevisionRow>(SELECT_MODEL_INVOCATION_REVISIONS_BY_HEADS)
16365 .bind(result.intent().tenant_id().as_str())
16366 .bind(*result.intent().run_id().as_uuid())
16367 .bind(&ids)
16368 .bind(&revisions)
16369 .fetch_all(&mut **transaction)
16370 .await
16371 .map_err(|source| StoreError::database("pending model binding revisions", source))?;
16372 let mut intents = unique_model_invocation_rows(intent_rows)?;
16373 let mut records = unique_model_invocation_revision_rows(revision_rows)?;
16374 if intents.len() != bindings.len() || records.len() != bindings.len() {
16375 return Err(StoreError::corrupt("pending model binding records"));
16376 }
16377 for binding in bindings {
16378 let head = binding
16379 .model_head()
16380 .ok_or_else(|| StoreError::corrupt("pending model binding kind"))?;
16381 let intent_row = intents
16382 .remove(head.invocation_id().as_uuid())
16383 .ok_or_else(|| StoreError::corrupt("pending model binding intent"))?;
16384 let intent = decode_model_invocation_intent(&intent_row)?;
16385 let revision = i64::try_from(head.revision().get())
16386 .map_err(|_| StoreError::corrupt("pending model binding revision"))?;
16387 let revision_row = records
16388 .remove(&(*head.invocation_id().as_uuid(), revision))
16389 .ok_or_else(|| StoreError::corrupt("pending model binding revision"))?;
16390 let invocation = decode_model_invocation_revision(revision_row, &intent)?;
16391 validate_model_invocation_current_projection(&intent_row, &invocation)?;
16392 if intent.activation() != result.intent().activation() || invocation.head() != *head {
16393 return Err(StoreError::corrupt("pending model binding record"));
16394 }
16395 insert_pending_invocation_anchor(anchors, invocation.journal_head(), invocation.digest())?;
16396 }
16397 if !intents.is_empty() || !records.is_empty() {
16398 return Err(StoreError::corrupt("pending model binding records"));
16399 }
16400 Ok(())
16401}
16402
16403fn unique_tool_invocation_rows(
16404 rows: Vec<ToolInvocationRow>,
16405) -> Result<BTreeMap<Uuid, ToolInvocationRow>, StoreError> {
16406 let mut values = BTreeMap::new();
16407 for row in rows {
16408 let id = row.invocation_id;
16409 if values.insert(id, row).is_some() {
16410 return Err(StoreError::corrupt("pending tool binding intents"));
16411 }
16412 }
16413 Ok(values)
16414}
16415
16416fn unique_tool_invocation_revision_rows(
16417 rows: Vec<ToolInvocationRevisionRow>,
16418) -> Result<BTreeMap<(Uuid, i64), ToolInvocationRevisionRow>, StoreError> {
16419 let mut values = BTreeMap::new();
16420 for row in rows {
16421 let key = (row.invocation_id, row.revision);
16422 if values.insert(key, row).is_some() {
16423 return Err(StoreError::corrupt("pending tool binding revisions"));
16424 }
16425 }
16426 Ok(values)
16427}
16428
16429fn unique_model_invocation_rows(
16430 rows: Vec<ModelInvocationRow>,
16431) -> Result<BTreeMap<Uuid, ModelInvocationRow>, StoreError> {
16432 let mut values = BTreeMap::new();
16433 for row in rows {
16434 let id = row.invocation_id;
16435 if values.insert(id, row).is_some() {
16436 return Err(StoreError::corrupt("pending model binding intents"));
16437 }
16438 }
16439 Ok(values)
16440}
16441
16442fn unique_model_invocation_revision_rows(
16443 rows: Vec<ModelInvocationRevisionRow>,
16444) -> Result<BTreeMap<(Uuid, i64), ModelInvocationRevisionRow>, StoreError> {
16445 let mut values = BTreeMap::new();
16446 for row in rows {
16447 let key = (row.invocation_id, row.revision);
16448 if values.insert(key, row).is_some() {
16449 return Err(StoreError::corrupt("pending model binding revisions"));
16450 }
16451 }
16452 Ok(values)
16453}
16454
16455fn insert_pending_invocation_anchor(
16456 anchors: &mut BTreeMap<JournalSequence, (JournalHead, Digest)>,
16457 head: &JournalHead,
16458 digest: Digest,
16459) -> Result<(), StoreError> {
16460 if anchors
16461 .insert(head.sequence(), (head.clone(), digest))
16462 .is_some()
16463 {
16464 return Err(StoreError::corrupt(
16465 "pending node result invocation journal identity",
16466 ));
16467 }
16468 Ok(())
16469}
16470
16471async fn verify_pending_invocation_anchors(
16472 transaction: &mut Transaction<'_, Postgres>,
16473 result: &PendingNodeResult,
16474 expected: BTreeMap<JournalSequence, (JournalHead, Digest)>,
16475) -> Result<(), StoreError> {
16476 if expected.is_empty() {
16477 return Ok(());
16478 }
16479 let expected = expected.into_iter().collect::<Vec<_>>();
16480 for expected in expected.chunks(PENDING_INVOCATION_ANCHOR_BATCH_SIZE) {
16481 let sequences = expected
16482 .iter()
16483 .map(|(sequence, _)| {
16484 i64::try_from(sequence.get())
16485 .map_err(|_| StoreError::corrupt("pending invocation journal sequence"))
16486 })
16487 .collect::<Result<Vec<_>, _>>()?;
16488 let rows = query_as::<_, EventRow>(SELECT_EVENTS_BY_SEQUENCES)
16489 .bind(result.intent().tenant_id().as_str())
16490 .bind(*result.intent().run_id().as_uuid())
16491 .bind(&sequences)
16492 .fetch_all(&mut **transaction)
16493 .await
16494 .map_err(|source| StoreError::database("pending invocation journal anchors", source))?;
16495 if rows.len() != expected.len() {
16496 return Err(StoreError::corrupt("pending invocation journal anchors"));
16497 }
16498 let mut actual = BTreeMap::new();
16499 for row in rows {
16500 let projection = row
16501 .projection_digest
16502 .as_deref()
16503 .map(|bytes| decode_digest(bytes, "pending invocation projection digest"))
16504 .transpose()?;
16505 let event = decode_event(row)?;
16506 if actual
16507 .insert(event.sequence(), (event, projection))
16508 .is_some()
16509 {
16510 return Err(StoreError::corrupt("pending invocation journal anchors"));
16511 }
16512 }
16513 for (sequence, (head, digest)) in expected {
16514 let (event, projection) = actual
16515 .remove(sequence)
16516 .ok_or_else(|| StoreError::corrupt("pending invocation journal anchor"))?;
16517 if event.head() != *head || projection != Some(*digest) {
16518 return Err(StoreError::corrupt("pending invocation journal anchor"));
16519 }
16520 }
16521 if !actual.is_empty() {
16522 return Err(StoreError::corrupt("pending invocation journal anchors"));
16523 }
16524 }
16525 Ok(())
16526}
16527
16528fn projection_digest(projection: &RunProjection) -> Result<Digest, StoreError> {
16529 let wire = match projection {
16530 RunProjection::Unchanged => ProjectionDigestWire::Unchanged,
16531 RunProjection::Transition {
16532 expected_revision,
16533 transition,
16534 } => ProjectionDigestWire::Transition {
16535 expected_revision,
16536 transition,
16537 },
16538 };
16539 let canonical = serde_json_canonicalizer::to_vec(&wire)
16540 .map_err(|_| StoreError::encoding("run projection intent"))?;
16541 let mut preimage = Vec::with_capacity(PROJECTION_DIGEST_DOMAIN.len() + canonical.len());
16542 preimage.extend_from_slice(PROJECTION_DIGEST_DOMAIN);
16543 preimage.extend_from_slice(&canonical);
16544 Ok(Digest::sha256(preimage))
16545}
16546
16547fn agent_admission_projection_digest(
16548 admission: &AgentAdmission,
16549 event_intent_digest: Digest,
16550 checkpoint: &CheckpointWrite,
16551 lifecycle: &RunLifecycle,
16552) -> Result<Digest, StoreError> {
16553 let lifecycle_digest = Digest::sha256(encode_lifecycle(lifecycle)?);
16554 let canonical = serde_json_canonicalizer::to_vec(&AgentAdmissionProjectionDigestWire {
16555 admission: admission.digest(),
16556 event_intent: event_intent_digest,
16557 checkpoint_intent: checkpoint.intent_digest(),
16558 lifecycle: lifecycle_digest,
16559 })
16560 .map_err(|_| StoreError::encoding("agent admission projection intent"))?;
16561 let mut preimage =
16562 Vec::with_capacity(AGENT_ADMISSION_PROJECTION_DIGEST_DOMAIN.len() + canonical.len());
16563 preimage.extend_from_slice(AGENT_ADMISSION_PROJECTION_DIGEST_DOMAIN);
16564 preimage.extend_from_slice(&canonical);
16565 Ok(Digest::sha256(preimage))
16566}
16567
16568fn barrier_projection_digest(
16569 projection: &RunProjection,
16570 barrier_intent_digest: Digest,
16571) -> Result<Digest, StoreError> {
16572 let wire = BarrierProjectionDigestWire {
16573 run_projection_digest: projection_digest(projection)?,
16574 barrier_intent_digest,
16575 };
16576 let canonical = serde_json_canonicalizer::to_vec(&wire)
16577 .map_err(|_| StoreError::encoding("checkpoint barrier projection intent"))?;
16578 let mut preimage = Vec::with_capacity(BARRIER_PROJECTION_DIGEST_DOMAIN.len() + canonical.len());
16579 preimage.extend_from_slice(BARRIER_PROJECTION_DIGEST_DOMAIN);
16580 preimage.extend_from_slice(&canonical);
16581 Ok(Digest::sha256(preimage))
16582}
16583
16584fn wait_registration_projection_digest(
16585 expected_revision: RunRevision,
16586 checkpoint_write: &CheckpointWrite,
16587 intents: &[WaitRegistrationIntent],
16588) -> Result<Digest, StoreError> {
16589 let registrations = intents
16590 .iter()
16591 .map(wait_registration_projection_item)
16592 .collect::<Vec<_>>();
16593 let wire = WaitRegistrationProjectionDigestWire {
16594 expected_revision: &expected_revision,
16595 checkpoint_intent_digest: checkpoint_write.intent_digest(),
16596 registrations: ®istrations,
16597 };
16598 let canonical = serde_json_canonicalizer::to_vec(&wire)
16599 .map_err(|_| StoreError::encoding("wait registration projection intent"))?;
16600 let mut preimage =
16601 Vec::with_capacity(WAIT_REGISTRATION_PROJECTION_DIGEST_DOMAIN.len() + canonical.len());
16602 preimage.extend_from_slice(WAIT_REGISTRATION_PROJECTION_DIGEST_DOMAIN);
16603 preimage.extend_from_slice(&canonical);
16604 Ok(Digest::sha256(preimage))
16605}
16606
16607fn wait_barrier_projection_digest(
16608 expected_revision: RunRevision,
16609 barrier_intent_digest: Digest,
16610 intents: &[WaitRegistrationIntent],
16611) -> Result<Digest, StoreError> {
16612 let registrations = intents
16613 .iter()
16614 .map(wait_registration_projection_item)
16615 .collect::<Vec<_>>();
16616 let wire = WaitBarrierProjectionDigestWire {
16617 expected_revision: &expected_revision,
16618 barrier_intent_digest,
16619 registrations: ®istrations,
16620 };
16621 let canonical = serde_json_canonicalizer::to_vec(&wire)
16622 .map_err(|_| StoreError::encoding("wait barrier projection intent"))?;
16623 let mut preimage =
16624 Vec::with_capacity(WAIT_BARRIER_PROJECTION_DIGEST_DOMAIN.len() + canonical.len());
16625 preimage.extend_from_slice(WAIT_BARRIER_PROJECTION_DIGEST_DOMAIN);
16626 preimage.extend_from_slice(&canonical);
16627 Ok(Digest::sha256(preimage))
16628}
16629
16630fn wait_terminal_projection_digest(
16631 domain: &[u8],
16632 expected_revision: RunRevision,
16633 intent_digest: Digest,
16634) -> Result<Digest, StoreError> {
16635 let canonical = serde_json_canonicalizer::to_vec(&WaitTerminalProjectionDigestWire {
16636 expected_revision: &expected_revision,
16637 intent_digest,
16638 })
16639 .map_err(|_| StoreError::encoding("wait terminal projection intent"))?;
16640 let mut preimage = Vec::with_capacity(domain.len() + canonical.len());
16641 preimage.extend_from_slice(domain);
16642 preimage.extend_from_slice(&canonical);
16643 Ok(Digest::sha256(preimage))
16644}
16645
16646fn wait_abandonment_projection_digest(
16647 projection: &RunProjection,
16648 waits: &[DurableWait],
16649) -> Result<Digest, StoreError> {
16650 let mut registrations = waits
16651 .iter()
16652 .map(|wait| WaitAbandonmentProjectionItem {
16653 wait_kind: durable_wait_kind_text(wait),
16654 wait_id: durable_wait_identity(wait).to_string(),
16655 registration_digest: durable_wait_digest(wait),
16656 })
16657 .collect::<Vec<_>>();
16658 registrations.sort_unstable_by(|left, right| left.wait_id.cmp(&right.wait_id));
16659 let wire = WaitAbandonmentProjectionDigestWire {
16660 run_projection_digest: projection_digest(projection)?,
16661 registrations: ®istrations,
16662 };
16663 let canonical = serde_json_canonicalizer::to_vec(&wire)
16664 .map_err(|_| StoreError::encoding("wait abandonment projection intent"))?;
16665 let mut preimage =
16666 Vec::with_capacity(WAIT_ABANDONMENT_PROJECTION_DIGEST_DOMAIN.len() + canonical.len());
16667 preimage.extend_from_slice(WAIT_ABANDONMENT_PROJECTION_DIGEST_DOMAIN);
16668 preimage.extend_from_slice(&canonical);
16669 Ok(Digest::sha256(preimage))
16670}
16671
16672fn wait_abandonment_transition_reason(
16673 transition: &RunTransition,
16674) -> Result<WaitAbandonmentReason, StoreError> {
16675 match transition {
16676 RunTransition::RequestCancellation { .. } => Ok(WaitAbandonmentReason::RunCancellation),
16677 RunTransition::Fail { .. } => Ok(WaitAbandonmentReason::RunFailure),
16678 _ => Err(StoreError::InvalidWaitAbandonment),
16679 }
16680}
16681
16682fn wait_registration_projection_item(
16683 intent: &WaitRegistrationIntent,
16684) -> WaitRegistrationProjectionItem {
16685 match intent {
16686 WaitRegistrationIntent::Interrupt { request } => WaitRegistrationProjectionItem {
16687 wait_kind: "interrupt",
16688 wait_id: request.interrupt_id().to_string(),
16689 intent_digest: request.intent_digest(),
16690 },
16691 WaitRegistrationIntent::Timer { timer } => WaitRegistrationProjectionItem {
16692 wait_kind: "timer",
16693 wait_id: timer.timer_id().to_string(),
16694 intent_digest: timer.intent_digest(),
16695 },
16696 }
16697}
16698
16699fn validate_wait_registration_batch(
16700 append: &JournalAppend,
16701 intents: &[WaitRegistrationIntent],
16702) -> Result<(), StoreError> {
16703 if intents.is_empty() || intents.len() > RunWaits::MAX_LEN {
16704 return Err(StoreError::InvalidWaitRegistrationBatch);
16705 }
16706 let tenant_id = append.intent().tenant_id();
16707 let run_id = append.intent().run_id();
16708 let event_id = append.intent().event_id();
16709 let mut identities = BTreeMap::new();
16710 for intent in intents {
16711 let item = wait_registration_projection_item(intent);
16712 if intent.tenant_id() != tenant_id
16713 || intent.run_id() != run_id
16714 || intent.registration_event_id() != event_id
16715 || identities
16716 .insert(item.wait_id, (item.wait_kind, item.intent_digest))
16717 .is_some()
16718 {
16719 return Err(StoreError::InvalidWaitRegistrationBatch);
16720 }
16721 }
16722 Ok(())
16723}
16724
16725fn materialize_wait_registrations(
16726 intents: Vec<WaitRegistrationIntent>,
16727 event: &JournalEvent,
16728) -> Result<Vec<DurableWait>, StoreError> {
16729 intents
16730 .into_iter()
16731 .map(|intent| {
16732 intent
16733 .commit(event.head())
16734 .map_err(|_| StoreError::InvalidWaitRegistrationBatch)
16735 })
16736 .collect()
16737}
16738
16739fn prepare_durable_wait_projection(
16740 stored: &StoredRun,
16741 tenant_id: &TenantId,
16742 run_id: RunId,
16743 expected_revision: RunRevision,
16744 transition: RunTransition,
16745 recorded_at: Timestamp,
16746) -> Result<PreparedProjection, StoreError> {
16747 let current = stored.lifecycle();
16748 if current.revision() != expected_revision {
16749 return Err(StoreError::StaleLifecycleRevision);
16750 }
16751 if current.provenance().tenant_id() != tenant_id || current.provenance().run_id() != run_id {
16752 return Err(StoreError::InvalidLifecycleTransition);
16753 }
16754 let lifecycle = current
16755 .clone()
16756 .apply(transition)
16757 .map_err(|_| StoreError::InvalidLifecycleTransition)?;
16758 if lifecycle.changed_at() > recorded_at {
16759 return Err(StoreError::LifecycleObservationAfterCommit);
16760 }
16761 prepared_projection(&lifecycle)
16762}
16763
16764fn prepared_projection(lifecycle: &RunLifecycle) -> Result<PreparedProjection, StoreError> {
16765 let wait_projection = wait_set_projection(lifecycle)
16766 .map_err(|()| StoreError::encoding("run wait-set projection"))?;
16767 Ok(PreparedProjection {
16768 lifecycle_bytes: encode_lifecycle(lifecycle)?,
16769 revision: lifecycle.revision().to_string(),
16770 status: run_status_text(lifecycle.status()),
16771 changed_at: to_database_time(lifecycle.changed_at())?,
16772 wait_set_digest: wait_projection.digest,
16773 unresolved_wait_count: wait_projection.count,
16774 next_timer_due_at: wait_projection
16775 .next_timer_due_at
16776 .map(to_database_time)
16777 .transpose()?,
16778 next_interrupt_expiry_at: wait_projection
16779 .next_interrupt_expiry_at
16780 .map(to_database_time)
16781 .transpose()?,
16782 })
16783}
16784
16785fn decode_event(row: EventRow) -> Result<JournalEvent, StoreError> {
16786 if let Some(bytes) = row.projection_digest.as_deref() {
16787 decode_digest(bytes, "journal projection digest")?;
16788 }
16789 let tenant_id = TenantId::try_from(row.tenant_id)
16790 .map_err(|_| StoreError::corrupt("journal event tenant"))?;
16791 let run_id = RunId::from_uuid(row.run_id)
16792 .map_err(|_| StoreError::corrupt("journal event run identity"))?;
16793 let sequence = positive_sequence(row.sequence)?;
16794 let event_id = EventId::from_uuid(row.event_id)
16795 .map_err(|_| StoreError::corrupt("journal event identity"))?;
16796 let recorded_at = from_database_time(row.recorded_at)?;
16797 let payload: stateknot_core::JournalPayload = serde_json::from_slice(&row.payload_bytes)
16798 .map_err(|_| StoreError::corrupt("journal payload value"))?;
16799 let canonical = payload
16800 .canonical_json()
16801 .map_err(|_| StoreError::corrupt("journal payload canonicalization"))?;
16802 if canonical.as_bytes() != row.payload_bytes
16803 || payload.kind().as_str() != row.event_kind
16804 || payload.schema().id().as_str() != row.schema_id
16805 || payload.schema().version().to_string() != row.schema_version
16806 || payload.schema().digest() != decode_digest(&row.schema_digest, "schema digest")?
16807 || payload.digest() != decode_digest(&row.payload_digest, "payload digest")?
16808 {
16809 return Err(StoreError::corrupt("journal payload projection"));
16810 }
16811
16812 let source = match (
16813 row.source_kind.as_str(),
16814 row.worker_attempt_id,
16815 row.worker_epoch,
16816 ) {
16817 ("control_plane", None, None) => JournalEventSource::control_plane(),
16818 ("worker", Some(attempt_id), Some(epoch)) => {
16819 let attempt_id = AttemptId::from_uuid(attempt_id)
16820 .map_err(|_| StoreError::corrupt("journal worker attempt"))?;
16821 let epoch = u64::try_from(epoch)
16822 .ok()
16823 .and_then(|value| FencingEpoch::new(value).ok())
16824 .ok_or_else(|| StoreError::corrupt("journal worker epoch"))?;
16825 JournalEventSource::worker(RunFence::new(tenant_id.clone(), run_id, attempt_id, epoch))
16826 }
16827 _ => return Err(StoreError::corrupt("journal source shape")),
16828 };
16829 let intent = match source {
16830 JournalEventSource::ControlPlane => {
16831 JournalEventIntent::control_plane(tenant_id, run_id, event_id, payload)
16832 }
16833 JournalEventSource::Worker { fence } => {
16834 JournalEventIntent::worker(tenant_id, run_id, event_id, fence, payload)
16835 }
16836 }
16837 .map_err(|_| StoreError::corrupt("journal intent"))?;
16838 if intent.intent_digest() != decode_digest(&row.intent_digest, "intent digest")? {
16839 return Err(StoreError::corrupt("journal intent digest"));
16840 }
16841
16842 let previous_digest = row
16843 .previous_digest
16844 .as_deref()
16845 .map(|bytes| decode_digest(bytes, "previous event digest"))
16846 .transpose()?;
16847 let digest = decode_digest(&row.event_digest, "event digest")?;
16848 JournalEvent::restore(intent, sequence, recorded_at, previous_digest, digest)
16849 .map_err(|_| StoreError::corrupt("journal event"))
16850}
16851
16852fn prepare_projection(
16853 stored: &StoredRun,
16854 append: &JournalAppend,
16855 projection: RunProjection,
16856 recorded_at: Timestamp,
16857) -> Result<Option<PreparedProjection>, StoreError> {
16858 let RunProjection::Transition {
16859 expected_revision,
16860 transition,
16861 } = projection
16862 else {
16863 return Ok(None);
16864 };
16865 let current = stored.lifecycle();
16866 if current.revision() != expected_revision {
16867 return Err(StoreError::StaleLifecycleRevision);
16868 }
16869 if current.provenance().tenant_id() != append.intent().tenant_id()
16870 || current.provenance().run_id() != append.intent().run_id()
16871 {
16872 return Err(StoreError::InvalidLifecycleTransition);
16873 }
16874 if matches!(
16875 transition.kind(),
16876 RunTransitionKind::Wait
16877 | RunTransitionKind::ResolveInterrupt
16878 | RunTransitionKind::FireTimer
16879 ) || (current.status() == RunStatus::Waiting
16880 && matches!(
16881 transition.kind(),
16882 RunTransitionKind::RequestCancellation | RunTransitionKind::Fail
16883 ))
16884 {
16885 return Err(StoreError::DurableWaitMutationRequired);
16886 }
16887 let lifecycle = current
16888 .clone()
16889 .apply(transition)
16890 .map_err(|_| StoreError::InvalidLifecycleTransition)?;
16891 if lifecycle.changed_at() > recorded_at {
16892 return Err(StoreError::LifecycleObservationAfterCommit);
16893 }
16894
16895 prepared_projection(&lifecycle).map(Some)
16896}
16897
16898fn validate_outbox_batch(
16899 append: &JournalAppend,
16900 intents: &[OutboxDeliveryIntent],
16901) -> Result<(), StoreError> {
16902 if intents.is_empty() || intents.len() > MAX_OUTBOX_DELIVERIES_PER_EVENT {
16903 return Err(StoreError::InvalidOutboxBatch);
16904 }
16905 let mut identities = BTreeMap::new();
16906 for intent in intents {
16907 if intent.tenant_id() != append.intent().tenant_id()
16908 || intent.run_id() != append.intent().run_id()
16909 || intent.origin_event_id() != append.intent().event_id()
16910 || identities
16911 .insert(intent.delivery_id(), intent.intent_digest())
16912 .is_some()
16913 {
16914 return Err(StoreError::InvalidOutboxBatch);
16915 }
16916 }
16917 Ok(())
16918}
16919
16920fn materialize_outbox_deliveries(
16921 intents: Vec<OutboxDeliveryIntent>,
16922 event: &JournalEvent,
16923) -> Result<Vec<OutboxDelivery>, StoreError> {
16924 intents
16925 .into_iter()
16926 .map(|intent| {
16927 OutboxDelivery::commit(intent, event.head())
16928 .map_err(|_| StoreError::OutboxEnqueueConflict)
16929 })
16930 .collect()
16931}
16932
16933async fn reap_outbox_terminals(
16934 transaction: &mut Transaction<'_, Postgres>,
16935 tenant_id: &TenantId,
16936 observed_at: Timestamp,
16937) -> Result<(), StoreError> {
16938 let observed_at = to_database_time(observed_at)?;
16939 query(
16940 r"
16941WITH candidates AS (
16942 SELECT tenant_id, run_id, delivery_id
16943 FROM stateknot.outbox_deliveries
16944 WHERE tenant_id = $1
16945 AND status = 'delivering'
16946 AND attempt_count = 64
16947 AND last_completion_digest IS NULL
16948 AND current_attempt_expires_at <= $2
16949 AND current_attempt_expires_at < expires_at
16950 ORDER BY current_attempt_expires_at ASC, delivery_id ASC
16951 FOR UPDATE SKIP LOCKED
16952 LIMIT $3
16953)
16954UPDATE stateknot.outbox_deliveries AS delivery
16955SET status = 'dead_letter',
16956 next_attempt_at = NULL,
16957 terminal_at = delivery.current_attempt_expires_at,
16958 updated_at = delivery.current_attempt_expires_at
16959FROM candidates
16960WHERE delivery.tenant_id = candidates.tenant_id
16961 AND delivery.run_id = candidates.run_id
16962 AND delivery.delivery_id = candidates.delivery_id
16963",
16964 )
16965 .bind(tenant_id.as_str())
16966 .bind(observed_at)
16967 .bind(OUTBOX_TERMINAL_REAP_BATCH_SIZE)
16968 .execute(&mut **transaction)
16969 .await
16970 .map_err(|source| StoreError::database("outbox attempt-limit reap", source))?;
16971
16972 query(
16973 r"
16974WITH candidates AS (
16975 SELECT tenant_id, run_id, delivery_id
16976 FROM stateknot.outbox_deliveries
16977 WHERE tenant_id = $1
16978 AND status IN ('pending', 'delivering', 'retry_scheduled')
16979 AND expires_at <= $2
16980 ORDER BY expires_at ASC, delivery_id ASC
16981 FOR UPDATE SKIP LOCKED
16982 LIMIT $3
16983)
16984UPDATE stateknot.outbox_deliveries AS delivery
16985SET status = 'expired',
16986 next_attempt_at = NULL,
16987 terminal_at = delivery.expires_at,
16988 updated_at = delivery.expires_at
16989FROM candidates
16990WHERE delivery.tenant_id = candidates.tenant_id
16991 AND delivery.run_id = candidates.run_id
16992 AND delivery.delivery_id = candidates.delivery_id
16993",
16994 )
16995 .bind(tenant_id.as_str())
16996 .bind(observed_at)
16997 .bind(OUTBOX_TERMINAL_REAP_BATCH_SIZE)
16998 .execute(&mut **transaction)
16999 .await
17000 .map_err(|source| StoreError::database("outbox delivery-expiry reap", source))?;
17001 Ok(())
17002}
17003
17004async fn load_idempotent_outbox_claim(
17005 transaction: &mut Transaction<'_, Postgres>,
17006 tenant_id: &TenantId,
17007 attempt_id: AttemptId,
17008) -> Result<Option<OutboxClaim>, StoreError> {
17009 let Some(start_row) =
17010 query_as::<_, OutboxAttemptStartRow>(SELECT_OUTBOX_ATTEMPT_BY_ID.as_str())
17011 .bind(tenant_id.as_str())
17012 .bind(*attempt_id.as_uuid())
17013 .fetch_optional(&mut **transaction)
17014 .await
17015 .map_err(|source| StoreError::database("outbox claim idempotency lookup", source))?
17016 else {
17017 return Ok(None);
17018 };
17019 let start = decode_outbox_attempt_start(&start_row)?;
17020 let delivery_row = load_outbox_delivery_row(
17021 transaction,
17022 tenant_id,
17023 start.delivery().run_id(),
17024 start.delivery().delivery_id(),
17025 true,
17026 )
17027 .await?
17028 .ok_or_else(|| StoreError::corrupt("outbox claim delivery"))?;
17029 let delivery = decode_outbox_delivery(&delivery_row)?;
17030 if delivery.head() != *start.delivery() {
17031 return Err(StoreError::corrupt("outbox claim start binding"));
17032 }
17033 verify_outbox_projection(transaction, &delivery_row, &delivery).await?;
17034 let current_observation = database_now(transaction, "idempotent outbox claim clock").await?;
17035 let epoch = i64::try_from(start.fence().epoch().get())
17036 .map_err(|_| StoreError::InvalidOutboxTransition)?;
17037 if delivery_row.status != "delivering"
17038 || delivery_row.current_attempt_id != Some(*start.fence().attempt_id().as_uuid())
17039 || delivery_row.current_epoch != Some(epoch)
17040 {
17041 return Err(StoreError::StaleOutboxFence);
17042 }
17043 if current_observation >= start.expires_at() {
17044 return Err(StoreError::OutboxAttemptExpired);
17045 }
17046 let destination =
17047 load_and_decode_outbox_destination(transaction, delivery.intent().destination()).await?;
17048 Ok(Some(OutboxClaim {
17049 destination,
17050 delivery,
17051 start,
17052 }))
17053}
17054
17055async fn insert_outbox_delivery(
17056 transaction: &mut Transaction<'_, Postgres>,
17057 delivery: &OutboxDelivery,
17058 source: &JournalEventSource,
17059) -> Result<(), StoreError> {
17060 let intent = delivery.intent();
17061 let origin = delivery.origin();
17062 let destination = intent.destination();
17063 let bytes = encode_outbox_delivery(delivery)?;
17064 let sequence =
17065 i64::try_from(origin.sequence().get()).map_err(|_| StoreError::JournalSequenceExhausted)?;
17066 let (source_kind, worker_attempt_id, worker_epoch) = match source {
17067 JournalEventSource::ControlPlane => ("control_plane", None, None),
17068 JournalEventSource::Worker { fence } => (
17069 "worker",
17070 Some(*fence.attempt_id().as_uuid()),
17071 Some(i64::try_from(fence.epoch().get()).map_err(|_| StoreError::StaleFence)?),
17072 ),
17073 };
17074 let inserted = query(
17075 r"
17076INSERT INTO stateknot.outbox_deliveries (
17077 tenant_id,
17078 run_id,
17079 delivery_id,
17080 origin_sequence,
17081 origin_event_id,
17082 origin_recorded_at,
17083 origin_digest,
17084 destination_id,
17085 destination_snapshot_digest,
17086 intent_digest,
17087 expires_at,
17088 delivery_digest,
17089 delivery_bytes,
17090 status,
17091 attempt_count,
17092 current_attempt_id,
17093 current_epoch,
17094 current_attempt_started_at,
17095 current_attempt_expires_at,
17096 next_attempt_at,
17097 last_completion_digest,
17098 terminal_at,
17099 created_at,
17100 updated_at
17101)
17102SELECT
17103 $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13,
17104 'pending', 0, NULL, NULL, NULL, NULL, $6, NULL, NULL, $6, $6
17105FROM stateknot.runs AS current_run
17106WHERE current_run.tenant_id = $1
17107 AND current_run.run_id = $2
17108 AND (
17109 $14 = 'control_plane'
17110 OR (
17111 $14 = 'worker'
17112 AND current_run.lease_attempt_id = $15
17113 AND current_run.fencing_epoch = $16
17114 AND current_run.lease_expires_at > clock_timestamp()
17115 )
17116 )
17117",
17118 )
17119 .bind(intent.tenant_id().as_str())
17120 .bind(*intent.run_id().as_uuid())
17121 .bind(*intent.delivery_id().as_uuid())
17122 .bind(sequence)
17123 .bind(*origin.event_id().as_uuid())
17124 .bind(to_database_time(origin.recorded_at())?)
17125 .bind(origin.digest().as_bytes())
17126 .bind(*destination.destination_id().as_uuid())
17127 .bind(destination.snapshot_digest().as_bytes())
17128 .bind(intent.intent_digest().as_bytes())
17129 .bind(to_database_time(intent.expires_at())?)
17130 .bind(delivery.digest().as_bytes())
17131 .bind(&bytes)
17132 .bind(source_kind)
17133 .bind(worker_attempt_id)
17134 .bind(worker_epoch)
17135 .execute(&mut **transaction)
17136 .await
17137 .map_err(|source| {
17138 if has_database_constraint(&source, "outbox_deliveries_pkey") {
17139 StoreError::OutboxDeliveryIdConflict
17140 } else if has_database_constraint(&source, "outbox_deliveries_destination_fk") {
17141 StoreError::OutboxDestinationNotFound
17142 } else {
17143 StoreError::database("outbox delivery insert", source)
17144 }
17145 })?
17146 .rows_affected();
17147 if inserted != 1 {
17148 return Err(if source_kind == "worker" {
17149 StoreError::LeaseExpired
17150 } else {
17151 StoreError::corrupt("outbox delivery insert row count")
17152 });
17153 }
17154 Ok(())
17155}
17156
17157async fn insert_outbox_attempt_claim(
17158 transaction: &mut Transaction<'_, Postgres>,
17159 delivery: &OutboxDelivery,
17160 start: &OutboxAttemptStart,
17161) -> Result<(), StoreError> {
17162 let origin = delivery.origin();
17163 let sequence =
17164 i64::try_from(origin.sequence().get()).map_err(|_| StoreError::InvalidOutboxTransition)?;
17165 let epoch = i64::try_from(start.fence().epoch().get())
17166 .map_err(|_| StoreError::InvalidOutboxTransition)?;
17167 query(
17168 r"
17169INSERT INTO stateknot.run_attempt_claims (
17170 tenant_id,
17171 run_id,
17172 attempt_id,
17173 claim_kind,
17174 invocation_id,
17175 invocation_revision,
17176 journal_sequence,
17177 journal_event_id,
17178 journal_recorded_at,
17179 journal_digest,
17180 claimed_at,
17181 activation_digest,
17182 delivery_id,
17183 delivery_epoch
17184)
17185VALUES ($1, $2, $3, 'outbox_attempt', NULL, NULL, $4, $5, $6, $7, $8, NULL, $9, $10)
17186",
17187 )
17188 .bind(start.delivery().tenant_id().as_str())
17189 .bind(*start.delivery().run_id().as_uuid())
17190 .bind(*start.fence().attempt_id().as_uuid())
17191 .bind(sequence)
17192 .bind(*origin.event_id().as_uuid())
17193 .bind(to_database_time(origin.recorded_at())?)
17194 .bind(origin.digest().as_bytes())
17195 .bind(to_database_time(start.started_at())?)
17196 .bind(*start.delivery().delivery_id().as_uuid())
17197 .bind(epoch)
17198 .execute(&mut **transaction)
17199 .await
17200 .map_err(|source| {
17201 if has_database_error_code(&source, "23505") {
17202 StoreError::OutboxAttemptIdConflict
17203 } else {
17204 StoreError::database("outbox attempt claim insert", source)
17205 }
17206 })?;
17207 Ok(())
17208}
17209
17210async fn insert_outbox_attempt_start(
17211 transaction: &mut Transaction<'_, Postgres>,
17212 delivery: &OutboxDelivery,
17213 start: &OutboxAttemptStart,
17214) -> Result<(), StoreError> {
17215 let bytes = encode_outbox_attempt_start(start)?;
17216 let epoch = i64::try_from(start.fence().epoch().get())
17217 .map_err(|_| StoreError::InvalidOutboxTransition)?;
17218 query(
17219 r"
17220INSERT INTO stateknot.outbox_attempts (
17221 tenant_id,
17222 run_id,
17223 delivery_id,
17224 delivery_expires_at,
17225 delivery_digest,
17226 epoch,
17227 attempt_id,
17228 started_at,
17229 expires_at,
17230 start_digest,
17231 start_bytes,
17232 created_at
17233)
17234VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $8)
17235",
17236 )
17237 .bind(start.delivery().tenant_id().as_str())
17238 .bind(*start.delivery().run_id().as_uuid())
17239 .bind(*start.delivery().delivery_id().as_uuid())
17240 .bind(to_database_time(delivery.intent().expires_at())?)
17241 .bind(delivery.digest().as_bytes())
17242 .bind(epoch)
17243 .bind(*start.fence().attempt_id().as_uuid())
17244 .bind(to_database_time(start.started_at())?)
17245 .bind(to_database_time(start.expires_at())?)
17246 .bind(start.digest().as_bytes())
17247 .bind(&bytes)
17248 .execute(&mut **transaction)
17249 .await
17250 .map_err(|source| {
17251 if has_database_error_code(&source, "23505") {
17252 StoreError::OutboxAttemptIdConflict
17253 } else {
17254 StoreError::database("outbox attempt start insert", source)
17255 }
17256 })?;
17257 Ok(())
17258}
17259
17260async fn update_outbox_delivery_claim(
17261 transaction: &mut Transaction<'_, Postgres>,
17262 delivery: &OutboxDelivery,
17263 start: &OutboxAttemptStart,
17264 previous_count: i64,
17265) -> Result<(), StoreError> {
17266 let epoch = i64::try_from(start.fence().epoch().get())
17267 .map_err(|_| StoreError::InvalidOutboxTransition)?;
17268 let updated = query(
17269 r"
17270UPDATE stateknot.outbox_deliveries
17271SET status = 'delivering',
17272 attempt_count = $4,
17273 current_attempt_id = $5,
17274 current_epoch = $4,
17275 current_attempt_started_at = $6,
17276 current_attempt_expires_at = $7,
17277 next_attempt_at = $7,
17278 last_completion_digest = NULL,
17279 terminal_at = NULL,
17280 updated_at = $6
17281WHERE tenant_id = $1
17282 AND run_id = $2
17283 AND delivery_id = $3
17284 AND delivery_digest = $8
17285 AND attempt_count = $9
17286 AND status IN ('pending', 'delivering', 'retry_scheduled')
17287 AND next_attempt_at <= $6
17288 AND expires_at > $6
17289",
17290 )
17291 .bind(start.delivery().tenant_id().as_str())
17292 .bind(*start.delivery().run_id().as_uuid())
17293 .bind(*start.delivery().delivery_id().as_uuid())
17294 .bind(epoch)
17295 .bind(*start.fence().attempt_id().as_uuid())
17296 .bind(to_database_time(start.started_at())?)
17297 .bind(to_database_time(start.expires_at())?)
17298 .bind(delivery.digest().as_bytes())
17299 .bind(previous_count)
17300 .execute(&mut **transaction)
17301 .await
17302 .map_err(|source| StoreError::database("outbox delivery claim update", source))?
17303 .rows_affected();
17304 if updated != 1 {
17305 return Err(StoreError::InvalidOutboxTransition);
17306 }
17307 Ok(())
17308}
17309
17310async fn insert_outbox_attempt_completion(
17311 transaction: &mut Transaction<'_, Postgres>,
17312 completion: &OutboxAttemptCompletion,
17313) -> Result<(), StoreError> {
17314 let start = completion.start();
17315 let fence = start.fence();
17316 let delivery = start.delivery();
17317 let epoch =
17318 i64::try_from(fence.epoch().get()).map_err(|_| StoreError::InvalidOutboxTransition)?;
17319 let bytes = encode_outbox_attempt_completion(completion)?;
17320 let (outcome_kind, retry_kind, retry_delay) = outbox_completion_projection(completion)?;
17321 query(
17322 r"
17323INSERT INTO stateknot.outbox_attempt_completions (
17324 tenant_id,
17325 run_id,
17326 delivery_id,
17327 epoch,
17328 attempt_id,
17329 started_at,
17330 attempt_expires_at,
17331 start_digest,
17332 outcome_kind,
17333 retry_advice_kind,
17334 retry_delay_millis,
17335 completed_at,
17336 completion_digest,
17337 completion_bytes,
17338 created_at
17339)
17340VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $12)
17341",
17342 )
17343 .bind(delivery.tenant_id().as_str())
17344 .bind(*delivery.run_id().as_uuid())
17345 .bind(*delivery.delivery_id().as_uuid())
17346 .bind(epoch)
17347 .bind(*fence.attempt_id().as_uuid())
17348 .bind(to_database_time(start.started_at())?)
17349 .bind(to_database_time(start.expires_at())?)
17350 .bind(start.digest().as_bytes())
17351 .bind(outcome_kind)
17352 .bind(retry_kind)
17353 .bind(retry_delay)
17354 .bind(to_database_time(completion.completed_at())?)
17355 .bind(completion.digest().as_bytes())
17356 .bind(&bytes)
17357 .execute(&mut **transaction)
17358 .await
17359 .map_err(|source| {
17360 if has_database_error_code(&source, "23505") {
17361 StoreError::OutboxCompletionConflict
17362 } else {
17363 StoreError::database("outbox attempt completion insert", source)
17364 }
17365 })?;
17366 Ok(())
17367}
17368
17369async fn update_outbox_delivery_completion(
17370 transaction: &mut Transaction<'_, Postgres>,
17371 delivery: &OutboxDelivery,
17372 start: &OutboxAttemptStart,
17373 completion: &OutboxAttemptCompletion,
17374 attempt_count: i64,
17375) -> Result<(), StoreError> {
17376 let (status, next_attempt_at, terminal_at) = match completion.outcome() {
17377 OutboxAttemptOutcome::Acknowledged { .. } => (
17378 "acknowledged",
17379 None,
17380 Some(to_database_time(completion.completed_at())?),
17381 ),
17382 OutboxAttemptOutcome::Failed { failure } => match failure.retry_advice() {
17383 RetryAdvice::Never => (
17384 "dead_letter",
17385 None,
17386 Some(to_database_time(completion.completed_at())?),
17387 ),
17388 RetryAdvice::SafeAfter { .. }
17389 if usize::try_from(attempt_count).ok() == Some(MAX_OUTBOX_ATTEMPTS) =>
17390 {
17391 (
17392 "dead_letter",
17393 None,
17394 Some(to_database_time(completion.completed_at())?),
17395 )
17396 }
17397 RetryAdvice::SafeAfter { delay } => {
17398 let duration = Duration::from_millis(
17399 u64::try_from(delay.as_i64())
17400 .map_err(|_| StoreError::InvalidOutboxTransition)?,
17401 );
17402 let retry_at = add_duration(completion.completed_at(), duration)?;
17403 ("retry_scheduled", Some(to_database_time(retry_at)?), None)
17404 }
17405 RetryAdvice::ReconcileFirst => return Err(StoreError::InvalidOutboxTransition),
17406 },
17407 _ => return Err(StoreError::InvalidOutboxTransition),
17408 };
17409 let epoch = i64::try_from(start.fence().epoch().get())
17410 .map_err(|_| StoreError::InvalidOutboxTransition)?;
17411 let updated = query(
17412 r"
17413UPDATE stateknot.outbox_deliveries
17414SET status = $6,
17415 next_attempt_at = $7,
17416 last_completion_digest = $8,
17417 terminal_at = $9,
17418 updated_at = $10
17419WHERE tenant_id = $1
17420 AND run_id = $2
17421 AND delivery_id = $3
17422 AND delivery_digest = $4
17423 AND attempt_count = $5
17424 AND status = 'delivering'
17425 AND current_attempt_id = $11
17426 AND current_epoch = $12
17427 AND current_attempt_expires_at > $10
17428 AND last_completion_digest IS NULL
17429",
17430 )
17431 .bind(start.delivery().tenant_id().as_str())
17432 .bind(*start.delivery().run_id().as_uuid())
17433 .bind(*start.delivery().delivery_id().as_uuid())
17434 .bind(delivery.digest().as_bytes())
17435 .bind(attempt_count)
17436 .bind(status)
17437 .bind(next_attempt_at)
17438 .bind(completion.digest().as_bytes())
17439 .bind(terminal_at)
17440 .bind(to_database_time(completion.completed_at())?)
17441 .bind(*start.fence().attempt_id().as_uuid())
17442 .bind(epoch)
17443 .execute(&mut **transaction)
17444 .await
17445 .map_err(|source| StoreError::database("outbox delivery completion update", source))?
17446 .rows_affected();
17447 if updated != 1 {
17448 return Err(StoreError::StaleOutboxFence);
17449 }
17450 Ok(())
17451}
17452
17453async fn insert_event(
17454 transaction: &mut Transaction<'_, Postgres>,
17455 event: &JournalEvent,
17456 projection_digest: Digest,
17457) -> Result<(), StoreError> {
17458 let (source_kind, worker_attempt_id, worker_epoch, worker_write) = match event.source() {
17459 JournalEventSource::ControlPlane => ("control_plane", None, None, false),
17460 JournalEventSource::Worker { fence } => (
17461 "worker",
17462 Some(*fence.attempt_id().as_uuid()),
17463 Some(
17464 i64::try_from(fence.epoch().get())
17465 .map_err(|_| StoreError::encoding("journal worker epoch"))?,
17466 ),
17467 true,
17468 ),
17469 };
17470 let payload = event
17471 .payload()
17472 .canonical_json()
17473 .map_err(|_| StoreError::encoding("journal payload"))?;
17474 let schema = event.payload().schema();
17475 let sequence =
17476 i64::try_from(event.sequence().get()).map_err(|_| StoreError::JournalSequenceExhausted)?;
17477
17478 let inserted = query(
17479 r"
17480INSERT INTO stateknot.run_events (
17481 tenant_id,
17482 run_id,
17483 sequence,
17484 event_id,
17485 recorded_at,
17486 source_kind,
17487 worker_attempt_id,
17488 worker_epoch,
17489 event_kind,
17490 schema_id,
17491 schema_version,
17492 schema_digest,
17493 payload_bytes,
17494 payload_digest,
17495 intent_digest,
17496 projection_digest,
17497 previous_digest,
17498 event_digest
17499)
17500SELECT
17501 $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18
17502FROM stateknot.runs AS lease_run
17503WHERE lease_run.tenant_id = $1
17504 AND lease_run.run_id = $2
17505 AND (
17506 $6 = 'control_plane'
17507 OR (
17508 $6 = 'worker'
17509 AND lease_run.lease_attempt_id = $7
17510 AND lease_run.fencing_epoch = $8
17511 AND lease_run.lease_expires_at > clock_timestamp()
17512 )
17513 )
17514",
17515 )
17516 .bind(event.tenant_id().as_str())
17517 .bind(*event.run_id().as_uuid())
17518 .bind(sequence)
17519 .bind(*event.event_id().as_uuid())
17520 .bind(to_database_time(event.recorded_at())?)
17521 .bind(source_kind)
17522 .bind(worker_attempt_id)
17523 .bind(worker_epoch)
17524 .bind(event.payload().kind().as_str())
17525 .bind(schema.id().as_str())
17526 .bind(schema.version().to_string())
17527 .bind(schema.digest().as_bytes())
17528 .bind(payload.as_bytes())
17529 .bind(event.payload_digest().as_bytes())
17530 .bind(event.intent_digest().as_bytes())
17531 .bind(projection_digest.as_bytes())
17532 .bind(
17533 event
17534 .previous_digest()
17535 .map(|digest| digest.as_bytes().to_vec()),
17536 )
17537 .bind(event.digest().as_bytes())
17538 .execute(&mut **transaction)
17539 .await
17540 .map_err(|source| StoreError::database("journal event insert", source))?
17541 .rows_affected();
17542 if inserted != 1 {
17543 if worker_write {
17544 return Err(StoreError::LeaseExpired);
17545 }
17546 return Err(StoreError::corrupt("journal insert row count"));
17547 }
17548 Ok(())
17549}
17550
17551async fn insert_wait_registration(
17552 transaction: &mut Transaction<'_, Postgres>,
17553 wait: &DurableWait,
17554) -> Result<(), StoreError> {
17555 let record_bytes = encode_durable_wait(wait)?;
17556 let sequence = i64::try_from(wait.journal().sequence().get())
17557 .map_err(|_| StoreError::JournalSequenceExhausted)?;
17558 let registered_at = to_database_time(wait.journal().recorded_at())?;
17559 let (
17560 wait_id,
17561 wait_kind,
17562 interrupt_kind,
17563 timer_kind,
17564 due_at,
17565 expires_at,
17566 action_digest,
17567 intent_digest,
17568 record_digest,
17569 ) = match wait {
17570 DurableWait::Interrupt { request } => (
17571 *request.marker().interrupt_id().as_uuid(),
17572 "interrupt",
17573 Some(interrupt_kind_text(request.marker().kind())),
17574 None,
17575 None,
17576 request
17577 .marker()
17578 .expires_at()
17579 .map(to_database_time)
17580 .transpose()?,
17581 Some(request.intent().action_digest()),
17582 request.intent().intent_digest(),
17583 request.digest(),
17584 ),
17585 DurableWait::Timer { timer } => (
17586 *timer.marker().timer_id().as_uuid(),
17587 "timer",
17588 None,
17589 Some(timer_kind_text(timer.marker().kind())),
17590 Some(to_database_time(timer.marker().due_at())?),
17591 None,
17592 None,
17593 timer.intent().intent_digest(),
17594 timer.digest(),
17595 ),
17596 };
17597 let result = query(
17598 r"
17599INSERT INTO stateknot.run_wait_registrations (
17600 tenant_id,
17601 run_id,
17602 wait_id,
17603 wait_kind,
17604 interrupt_kind,
17605 timer_kind,
17606 registered_at,
17607 due_at,
17608 expires_at,
17609 action_digest,
17610 registration_sequence,
17611 registration_event_id,
17612 registration_event_digest,
17613 intent_digest,
17614 record_digest,
17615 record_bytes,
17616 status,
17617 created_at,
17618 updated_at
17619)
17620VALUES (
17621 $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16,
17622 'outstanding', $7, $7
17623)
17624",
17625 )
17626 .bind(wait.tenant_id().as_str())
17627 .bind(*wait.run_id().as_uuid())
17628 .bind(wait_id)
17629 .bind(wait_kind)
17630 .bind(interrupt_kind)
17631 .bind(timer_kind)
17632 .bind(registered_at)
17633 .bind(due_at)
17634 .bind(expires_at)
17635 .bind(action_digest.as_ref().map(Digest::as_bytes))
17636 .bind(sequence)
17637 .bind(*wait.journal().event_id().as_uuid())
17638 .bind(wait.journal().digest().as_bytes())
17639 .bind(intent_digest.as_bytes())
17640 .bind(record_digest.as_bytes())
17641 .bind(&record_bytes)
17642 .execute(&mut **transaction)
17643 .await;
17644 match result {
17645 Ok(result) if result.rows_affected() == 1 => Ok(()),
17646 Ok(_) => Err(StoreError::corrupt("wait registration insert row count")),
17647 Err(source) if has_database_error_code(&source, "23505") => {
17648 Err(StoreError::WaitRegistrationCommitConflict)
17649 }
17650 Err(source) => Err(StoreError::database("wait registration insert", source)),
17651 }
17652}
17653
17654async fn verify_wait_registration_set(
17655 transaction: &mut Transaction<'_, Postgres>,
17656 event: &JournalEvent,
17657 expected: &[DurableWait],
17658) -> Result<(), StoreError> {
17659 let sequence =
17660 i64::try_from(event.sequence().get()).map_err(|_| StoreError::JournalSequenceExhausted)?;
17661 let rows = query_as::<_, WaitRegistrationRow>(SELECT_WAIT_REGISTRATIONS_BY_ORIGIN.as_str())
17662 .bind(event.tenant_id().as_str())
17663 .bind(*event.run_id().as_uuid())
17664 .bind(sequence)
17665 .fetch_all(&mut **transaction)
17666 .await
17667 .map_err(|source| StoreError::database("wait registration set load", source))?;
17668 let mut durable = BTreeMap::new();
17669 for row in rows {
17670 let wait = decode_wait_registration(&row)?;
17671 if wait.journal() != &event.head()
17672 || durable.insert(durable_wait_identity(&wait), wait).is_some()
17673 {
17674 return Err(StoreError::corrupt("wait registration set"));
17675 }
17676 }
17677 if durable.len() != expected.len()
17678 || expected
17679 .iter()
17680 .any(|wait| durable.get(&durable_wait_identity(wait)) != Some(wait))
17681 {
17682 return Err(StoreError::WaitRegistrationCommitConflict);
17683 }
17684 Ok(())
17685}
17686
17687async fn insert_interrupt_resolution(
17688 transaction: &mut Transaction<'_, Postgres>,
17689 request: &InterruptRequest,
17690 resolution: &InterruptResolution,
17691) -> Result<(), StoreError> {
17692 let bytes = encode_interrupt_resolution(resolution)?;
17693 let journal = resolution.journal();
17694 let result = query(
17695 r"
17696INSERT INTO stateknot.interrupt_resolutions (
17697 tenant_id,
17698 run_id,
17699 interrupt_id,
17700 request_digest,
17701 resolution_sequence,
17702 resolution_event_id,
17703 resolved_at,
17704 resolution_event_digest,
17705 intent_digest,
17706 resolution_digest,
17707 resolution_bytes,
17708 created_at
17709)
17710VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $7)
17711",
17712 )
17713 .bind(request.intent().tenant_id().as_str())
17714 .bind(*request.intent().run_id().as_uuid())
17715 .bind(*request.marker().interrupt_id().as_uuid())
17716 .bind(request.digest().as_bytes())
17717 .bind(
17718 i64::try_from(journal.sequence().get())
17719 .map_err(|_| StoreError::JournalSequenceExhausted)?,
17720 )
17721 .bind(*journal.event_id().as_uuid())
17722 .bind(to_database_time(journal.recorded_at())?)
17723 .bind(journal.digest().as_bytes())
17724 .bind(resolution.intent().intent_digest().as_bytes())
17725 .bind(resolution.digest().as_bytes())
17726 .bind(&bytes)
17727 .execute(&mut **transaction)
17728 .await;
17729 match result {
17730 Ok(result) if result.rows_affected() == 1 => Ok(()),
17731 Ok(_) => Err(StoreError::corrupt("interrupt resolution insert row count")),
17732 Err(source) if has_database_error_code(&source, "23505") => {
17733 Err(StoreError::InterruptResolutionCommitConflict)
17734 }
17735 Err(source) => Err(StoreError::database("interrupt resolution insert", source)),
17736 }
17737}
17738
17739async fn project_interrupt_resolution(
17740 transaction: &mut Transaction<'_, Postgres>,
17741 request: &InterruptRequest,
17742 resolution: &InterruptResolution,
17743) -> Result<(), StoreError> {
17744 let journal = resolution.journal();
17745 let updated = query(
17746 r"
17747UPDATE stateknot.run_wait_registrations
17748SET status = 'resolved',
17749 terminal_sequence = $5,
17750 terminal_event_id = $6,
17751 terminal_recorded_at = $7,
17752 terminal_event_digest = $8,
17753 resolution_digest = $9,
17754 updated_at = $7
17755WHERE tenant_id = $1
17756 AND run_id = $2
17757 AND wait_id = $3
17758 AND wait_kind = 'interrupt'
17759 AND record_digest = $4
17760 AND status = 'outstanding'
17761",
17762 )
17763 .bind(request.intent().tenant_id().as_str())
17764 .bind(*request.intent().run_id().as_uuid())
17765 .bind(*request.marker().interrupt_id().as_uuid())
17766 .bind(request.digest().as_bytes())
17767 .bind(
17768 i64::try_from(journal.sequence().get())
17769 .map_err(|_| StoreError::JournalSequenceExhausted)?,
17770 )
17771 .bind(*journal.event_id().as_uuid())
17772 .bind(to_database_time(journal.recorded_at())?)
17773 .bind(journal.digest().as_bytes())
17774 .bind(resolution.digest().as_bytes())
17775 .execute(&mut **transaction)
17776 .await
17777 .map_err(|source| StoreError::database("interrupt resolution projection", source))?
17778 .rows_affected();
17779 if updated != 1 {
17780 return Err(StoreError::InterruptResolutionCommitConflict);
17781 }
17782 Ok(())
17783}
17784
17785async fn insert_timer_firing(
17786 transaction: &mut Transaction<'_, Postgres>,
17787 timer: &DurableTimer,
17788 firing: &TimerFiring,
17789) -> Result<(), StoreError> {
17790 let bytes = encode_timer_firing(firing)?;
17791 let journal = firing.journal();
17792 let result = query(
17793 r"
17794INSERT INTO stateknot.timer_firings (
17795 tenant_id,
17796 run_id,
17797 timer_id,
17798 timer_digest,
17799 firing_sequence,
17800 firing_event_id,
17801 fired_at,
17802 firing_event_digest,
17803 intent_digest,
17804 firing_digest,
17805 firing_bytes,
17806 created_at
17807)
17808VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $7)
17809",
17810 )
17811 .bind(timer.intent().tenant_id().as_str())
17812 .bind(*timer.intent().run_id().as_uuid())
17813 .bind(*timer.marker().timer_id().as_uuid())
17814 .bind(timer.digest().as_bytes())
17815 .bind(
17816 i64::try_from(journal.sequence().get())
17817 .map_err(|_| StoreError::JournalSequenceExhausted)?,
17818 )
17819 .bind(*journal.event_id().as_uuid())
17820 .bind(to_database_time(journal.recorded_at())?)
17821 .bind(journal.digest().as_bytes())
17822 .bind(firing.intent().intent_digest().as_bytes())
17823 .bind(firing.digest().as_bytes())
17824 .bind(&bytes)
17825 .execute(&mut **transaction)
17826 .await;
17827 match result {
17828 Ok(result) if result.rows_affected() == 1 => Ok(()),
17829 Ok(_) => Err(StoreError::corrupt("timer firing insert row count")),
17830 Err(source) if has_database_error_code(&source, "23505") => {
17831 Err(StoreError::TimerFiringCommitConflict)
17832 }
17833 Err(source) => Err(StoreError::database("timer firing insert", source)),
17834 }
17835}
17836
17837async fn project_timer_firing(
17838 transaction: &mut Transaction<'_, Postgres>,
17839 timer: &DurableTimer,
17840 firing: &TimerFiring,
17841) -> Result<(), StoreError> {
17842 let journal = firing.journal();
17843 let updated = query(
17844 r"
17845UPDATE stateknot.run_wait_registrations
17846SET status = 'fired',
17847 terminal_sequence = $5,
17848 terminal_event_id = $6,
17849 terminal_recorded_at = $7,
17850 terminal_event_digest = $8,
17851 firing_digest = $9,
17852 updated_at = $7
17853WHERE tenant_id = $1
17854 AND run_id = $2
17855 AND wait_id = $3
17856 AND wait_kind = 'timer'
17857 AND record_digest = $4
17858 AND status = 'outstanding'
17859",
17860 )
17861 .bind(timer.intent().tenant_id().as_str())
17862 .bind(*timer.intent().run_id().as_uuid())
17863 .bind(*timer.marker().timer_id().as_uuid())
17864 .bind(timer.digest().as_bytes())
17865 .bind(
17866 i64::try_from(journal.sequence().get())
17867 .map_err(|_| StoreError::JournalSequenceExhausted)?,
17868 )
17869 .bind(*journal.event_id().as_uuid())
17870 .bind(to_database_time(journal.recorded_at())?)
17871 .bind(journal.digest().as_bytes())
17872 .bind(firing.digest().as_bytes())
17873 .execute(&mut **transaction)
17874 .await
17875 .map_err(|source| StoreError::database("timer firing projection", source))?
17876 .rows_affected();
17877 if updated != 1 {
17878 return Err(StoreError::TimerFiringCommitConflict);
17879 }
17880 Ok(())
17881}
17882
17883async fn insert_wait_abandonment(
17884 transaction: &mut Transaction<'_, Postgres>,
17885 abandonment: &WaitAbandonment,
17886) -> Result<(), StoreError> {
17887 let result = query(
17888 r"
17889INSERT INTO stateknot.wait_abandonments (
17890 tenant_id,
17891 run_id,
17892 wait_id,
17893 wait_kind,
17894 registration_digest,
17895 reason_kind,
17896 abandonment_sequence,
17897 abandonment_event_id,
17898 abandoned_at,
17899 abandonment_event_digest,
17900 abandonment_digest,
17901 created_at
17902)
17903VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $9)
17904",
17905 )
17906 .bind(abandonment.wait().tenant_id().as_str())
17907 .bind(*abandonment.wait().run_id().as_uuid())
17908 .bind(durable_wait_identity(abandonment.wait()))
17909 .bind(durable_wait_kind_text(abandonment.wait()))
17910 .bind(durable_wait_digest(abandonment.wait()).as_bytes())
17911 .bind(wait_abandonment_reason_text(abandonment.reason()))
17912 .bind(
17913 i64::try_from(abandonment.journal().sequence().get())
17914 .map_err(|_| StoreError::JournalSequenceExhausted)?,
17915 )
17916 .bind(*abandonment.journal().event_id().as_uuid())
17917 .bind(to_database_time(abandonment.journal().recorded_at())?)
17918 .bind(abandonment.journal().digest().as_bytes())
17919 .bind(abandonment.digest().as_bytes())
17920 .execute(&mut **transaction)
17921 .await;
17922 match result {
17923 Ok(result) if result.rows_affected() == 1 => Ok(()),
17924 Ok(_) => Err(StoreError::corrupt("wait abandonment insert row count")),
17925 Err(source) if has_database_error_code(&source, "23505") => {
17926 Err(StoreError::WaitAbandonmentCommitConflict)
17927 }
17928 Err(source) => Err(StoreError::database("wait abandonment insert", source)),
17929 }
17930}
17931
17932async fn project_wait_abandonment(
17933 transaction: &mut Transaction<'_, Postgres>,
17934 abandonment: &WaitAbandonment,
17935) -> Result<(), StoreError> {
17936 let updated = query(
17937 r"
17938UPDATE stateknot.run_wait_registrations
17939SET status = 'abandoned',
17940 terminal_sequence = $5,
17941 terminal_event_id = $6,
17942 terminal_recorded_at = $7,
17943 terminal_event_digest = $8,
17944 abandonment_digest = $9,
17945 updated_at = $7
17946WHERE tenant_id = $1
17947 AND run_id = $2
17948 AND wait_id = $3
17949 AND record_digest = $4
17950 AND status = 'outstanding'
17951",
17952 )
17953 .bind(abandonment.wait().tenant_id().as_str())
17954 .bind(*abandonment.wait().run_id().as_uuid())
17955 .bind(durable_wait_identity(abandonment.wait()))
17956 .bind(durable_wait_digest(abandonment.wait()).as_bytes())
17957 .bind(
17958 i64::try_from(abandonment.journal().sequence().get())
17959 .map_err(|_| StoreError::JournalSequenceExhausted)?,
17960 )
17961 .bind(*abandonment.journal().event_id().as_uuid())
17962 .bind(to_database_time(abandonment.journal().recorded_at())?)
17963 .bind(abandonment.journal().digest().as_bytes())
17964 .bind(abandonment.digest().as_bytes())
17965 .execute(&mut **transaction)
17966 .await
17967 .map_err(|source| StoreError::database("wait abandonment projection", source))?
17968 .rows_affected();
17969 if updated != 1 {
17970 return Err(StoreError::WaitAbandonmentCommitConflict);
17971 }
17972 Ok(())
17973}
17974
17975async fn load_wait_abandonment_set(
17976 transaction: &mut Transaction<'_, Postgres>,
17977 event: &JournalEvent,
17978) -> Result<Vec<WaitAbandonment>, StoreError> {
17979 let values = load_wait_abandonment_set_or_empty(transaction, event).await?;
17980 if values.is_empty() {
17981 return Err(StoreError::WaitAbandonmentCommitConflict);
17982 }
17983 Ok(values)
17984}
17985
17986async fn load_wait_abandonment_set_or_empty(
17987 transaction: &mut Transaction<'_, Postgres>,
17988 event: &JournalEvent,
17989) -> Result<Vec<WaitAbandonment>, StoreError> {
17990 let sequence =
17991 i64::try_from(event.sequence().get()).map_err(|_| StoreError::JournalSequenceExhausted)?;
17992 let rows = query_as::<_, WaitAbandonmentRow>(SELECT_WAIT_ABANDONMENTS_BY_EVENT.as_str())
17993 .bind(event.tenant_id().as_str())
17994 .bind(*event.run_id().as_uuid())
17995 .bind(sequence)
17996 .fetch_all(&mut **transaction)
17997 .await
17998 .map_err(|source| StoreError::database("wait abandonment set load", source))?;
17999 if rows.len() > RunWaits::MAX_LEN {
18000 return Err(StoreError::WaitAbandonmentCommitConflict);
18001 }
18002 let mut abandonments = Vec::with_capacity(rows.len());
18003 for row in rows {
18004 let registration =
18005 query_as::<_, WaitRegistrationRow>(SELECT_WAIT_REGISTRATION_BY_ID.as_str())
18006 .bind(event.tenant_id().as_str())
18007 .bind(*event.run_id().as_uuid())
18008 .bind(row.wait_id)
18009 .fetch_optional(&mut **transaction)
18010 .await
18011 .map_err(|source| StoreError::database("abandoned registration load", source))?
18012 .ok_or(StoreError::WaitAbandonmentCommitConflict)?;
18013 let wait = decode_wait_registration(®istration)?;
18014 let abandonment = decode_wait_abandonment(&row, wait)?;
18015 if registration.status != "abandoned"
18016 || registration.abandonment_digest.as_deref() != Some(row.abandonment_digest.as_slice())
18017 || registration.terminal_sequence != Some(row.abandonment_sequence)
18018 || registration.terminal_event_id != Some(row.abandonment_event_id)
18019 || registration.terminal_recorded_at != Some(row.abandoned_at)
18020 || registration.terminal_event_digest.as_deref()
18021 != Some(row.abandonment_event_digest.as_slice())
18022 || abandonment.journal() != &event.head()
18023 {
18024 return Err(StoreError::corrupt("wait abandonment terminal projection"));
18025 }
18026 verify_wait_registration_event(
18027 transaction,
18028 abandonment.wait(),
18029 registration.registration_sequence,
18030 )
18031 .await?;
18032 abandonments.push(abandonment);
18033 }
18034 Ok(abandonments)
18035}
18036
18037async fn load_wait_abandonment_by_id(
18038 transaction: &mut Transaction<'_, Postgres>,
18039 tenant_id: &TenantId,
18040 run_id: RunId,
18041 wait_id: Uuid,
18042) -> Result<WaitAbandonment, StoreError> {
18043 let registration = query_as::<_, WaitRegistrationRow>(SELECT_WAIT_REGISTRATION_BY_ID.as_str())
18044 .bind(tenant_id.as_str())
18045 .bind(*run_id.as_uuid())
18046 .bind(wait_id)
18047 .fetch_optional(&mut **transaction)
18048 .await
18049 .map_err(|source| StoreError::database("abandoned registration load", source))?
18050 .ok_or(StoreError::WaitRegistrationNotFound)?;
18051 let wait = decode_wait_registration(®istration)?;
18052 verify_wait_registration_event(transaction, &wait, registration.registration_sequence).await?;
18053
18054 let row = query_as::<_, WaitAbandonmentRow>(SELECT_WAIT_ABANDONMENT_BY_ID.as_str())
18055 .bind(tenant_id.as_str())
18056 .bind(*run_id.as_uuid())
18057 .bind(wait_id)
18058 .fetch_optional(&mut **transaction)
18059 .await
18060 .map_err(|source| StoreError::database("wait abandonment load", source))?
18061 .ok_or(StoreError::WaitAbandonmentNotFound)?;
18062 let abandonment = decode_wait_abandonment(&row, wait)?;
18063 if registration.status != "abandoned"
18064 || registration.abandonment_digest.as_deref() != Some(row.abandonment_digest.as_slice())
18065 || registration.terminal_sequence != Some(row.abandonment_sequence)
18066 || registration.terminal_event_id != Some(row.abandonment_event_id)
18067 || registration.terminal_recorded_at != Some(row.abandoned_at)
18068 || registration.terminal_event_digest.as_deref()
18069 != Some(row.abandonment_event_digest.as_slice())
18070 {
18071 return Err(StoreError::corrupt("wait abandonment terminal projection"));
18072 }
18073 verify_terminal_event(transaction, abandonment.journal()).await?;
18074 Ok(abandonment)
18075}
18076
18077async fn insert_tool_invocation_intent(
18078 transaction: &mut Transaction<'_, Postgres>,
18079 invocation: &ToolInvocation,
18080 fence: &RunFence,
18081) -> Result<(), StoreError> {
18082 let intent = invocation.intent();
18083 let activation = intent.activation();
18084 let base = activation.base_checkpoint();
18085 let intent_bytes = encode_tool_invocation_intent(intent)?;
18086 let base_superstep = i64::try_from(base.superstep().get())
18087 .map_err(|_| StoreError::encoding("tool invocation base superstep"))?;
18088 let current_revision = i64::try_from(invocation.revision().get())
18089 .map_err(|_| StoreError::encoding("tool invocation revision"))?;
18090 let fence_epoch = i64::try_from(fence.epoch().get()).map_err(|_| StoreError::StaleFence)?;
18091 let created_at = to_database_time(invocation.journal_head().recorded_at())?;
18092 let inserted = query(
18093 r"
18094INSERT INTO stateknot.tool_invocations (
18095 tenant_id,
18096 run_id,
18097 invocation_id,
18098 base_checkpoint_id,
18099 base_superstep,
18100 base_checkpoint_digest,
18101 graph_namespace,
18102 node_id,
18103 activation_input_digest,
18104 intent_digest,
18105 intent_bytes,
18106 current_revision,
18107 current_status,
18108 current_attempt_id,
18109 current_record_digest,
18110 created_at,
18111 updated_at
18112)
18113SELECT
18114 $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $16
18115FROM stateknot.runs AS current_run
18116WHERE current_run.tenant_id = $1
18117 AND current_run.run_id = $2
18118 AND current_run.checkpoint_id = $4
18119 AND current_run.checkpoint_superstep = $5
18120 AND current_run.checkpoint_digest = $6
18121 AND current_run.lease_attempt_id = $17
18122 AND current_run.fencing_epoch = $18
18123 AND current_run.lease_expires_at > clock_timestamp()
18124",
18125 )
18126 .bind(intent.tenant_id().as_str())
18127 .bind(*intent.run_id().as_uuid())
18128 .bind(*intent.invocation_id().as_uuid())
18129 .bind(*base.checkpoint_id().as_uuid())
18130 .bind(base_superstep)
18131 .bind(base.digest().as_bytes())
18132 .bind(activation.graph_namespace().as_str())
18133 .bind(activation.node_id().as_str())
18134 .bind(activation.input_digest().as_bytes())
18135 .bind(intent.intent_digest().as_bytes())
18136 .bind(intent_bytes)
18137 .bind(current_revision)
18138 .bind(tool_invocation_status_text(invocation.status()))
18139 .bind(invocation.attempt_id().map(|attempt| *attempt.as_uuid()))
18140 .bind(invocation.digest().as_bytes())
18141 .bind(created_at)
18142 .bind(*fence.attempt_id().as_uuid())
18143 .bind(fence_epoch)
18144 .execute(&mut **transaction)
18145 .await
18146 .map_err(|source| StoreError::database("tool invocation intent insert", source))?
18147 .rows_affected();
18148 if inserted != 1 {
18149 return Err(StoreError::LeaseExpired);
18150 }
18151 Ok(())
18152}
18153
18154#[allow(clippy::too_many_arguments)]
18155async fn insert_invocation_attempt_claim(
18156 transaction: &mut Transaction<'_, Postgres>,
18157 claim_kind: InvocationAttemptKind,
18158 tenant_id: &TenantId,
18159 run_id: RunId,
18160 invocation_id: InvocationId,
18161 revision: i64,
18162 attempt_id: AttemptId,
18163 journal_head: &JournalHead,
18164 fence: &RunFence,
18165) -> Result<(), StoreError> {
18166 let journal_sequence = i64::try_from(journal_head.sequence().get())
18167 .map_err(|_| StoreError::JournalSequenceExhausted)?;
18168 let fence_epoch = i64::try_from(fence.epoch().get()).map_err(|_| StoreError::StaleFence)?;
18169 let claimed_at = to_database_time(journal_head.recorded_at())?;
18170 let result = query(
18171 r"
18172INSERT INTO stateknot.run_attempt_claims (
18173 tenant_id,
18174 run_id,
18175 attempt_id,
18176 claim_kind,
18177 invocation_id,
18178 invocation_revision,
18179 journal_sequence,
18180 journal_event_id,
18181 journal_recorded_at,
18182 journal_digest,
18183 claimed_at
18184)
18185SELECT $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $9
18186FROM stateknot.runs AS current_run
18187WHERE current_run.tenant_id = $1
18188 AND current_run.run_id = $2
18189 AND current_run.lease_attempt_id = $11
18190 AND current_run.fencing_epoch = $12
18191 AND current_run.lease_expires_at > clock_timestamp()
18192",
18193 )
18194 .bind(tenant_id.as_str())
18195 .bind(*run_id.as_uuid())
18196 .bind(*attempt_id.as_uuid())
18197 .bind(claim_kind.as_str())
18198 .bind(*invocation_id.as_uuid())
18199 .bind(revision)
18200 .bind(journal_sequence)
18201 .bind(*journal_head.event_id().as_uuid())
18202 .bind(claimed_at)
18203 .bind(journal_head.digest().as_bytes())
18204 .bind(*fence.attempt_id().as_uuid())
18205 .bind(fence_epoch)
18206 .execute(&mut **transaction)
18207 .await;
18208 let inserted = match result {
18209 Ok(result) => result.rows_affected(),
18210 Err(source)
18211 if has_database_constraint(&source, "run_attempt_claims_pkey")
18212 || has_database_constraint(
18213 &source,
18214 "run_attempt_claims_logical_revision_unique",
18215 )
18216 || has_database_constraint(&source, "run_attempt_claims_anchor_unique") =>
18217 {
18218 return Err(claim_kind.conflict());
18219 }
18220 Err(source) => {
18221 return Err(StoreError::database(
18222 "invocation attempt claim insert",
18223 source,
18224 ));
18225 }
18226 };
18227 if inserted != 1 {
18228 return Err(StoreError::LeaseExpired);
18229 }
18230 Ok(())
18231}
18232
18233async fn insert_node_attempt_claim(
18234 transaction: &mut Transaction<'_, Postgres>,
18235 start: &NodeAttemptStart,
18236) -> Result<(), StoreError> {
18237 let journal = start.journal_head();
18238 let journal_sequence = i64::try_from(journal.sequence().get())
18239 .map_err(|_| StoreError::JournalSequenceExhausted)?;
18240 let fence_epoch =
18241 i64::try_from(start.fence().epoch().get()).map_err(|_| StoreError::StaleFence)?;
18242 let claimed_at = to_database_time(journal.recorded_at())?;
18243 let result = query(
18244 r"
18245INSERT INTO stateknot.run_attempt_claims (
18246 tenant_id,
18247 run_id,
18248 attempt_id,
18249 claim_kind,
18250 invocation_id,
18251 invocation_revision,
18252 activation_digest,
18253 journal_sequence,
18254 journal_event_id,
18255 journal_recorded_at,
18256 journal_digest,
18257 claimed_at
18258)
18259SELECT $1, $2, $3, 'node_attempt', NULL, NULL, $4, $5, $6, $7, $8, $7
18260FROM stateknot.runs AS current_run
18261WHERE current_run.tenant_id = $1
18262 AND current_run.run_id = $2
18263 AND current_run.lease_attempt_id = $9
18264 AND current_run.fencing_epoch = $10
18265 AND current_run.lease_expires_at > clock_timestamp()
18266",
18267 )
18268 .bind(start.activation().tenant_id().as_str())
18269 .bind(*start.activation().run_id().as_uuid())
18270 .bind(*start.attempt_id().as_uuid())
18271 .bind(start.activation_digest().as_bytes())
18272 .bind(journal_sequence)
18273 .bind(*journal.event_id().as_uuid())
18274 .bind(claimed_at)
18275 .bind(journal.digest().as_bytes())
18276 .bind(*start.fence().attempt_id().as_uuid())
18277 .bind(fence_epoch)
18278 .execute(&mut **transaction)
18279 .await;
18280 let inserted = match result {
18281 Ok(result) => result.rows_affected(),
18282 Err(source) if has_database_constraint(&source, "run_attempt_claims_pkey") => {
18283 return Err(StoreError::NodeAttemptIdConflict);
18284 }
18285 Err(source)
18286 if has_database_constraint(&source, "run_attempt_claims_anchor_unique")
18287 || has_database_constraint(&source, "run_attempt_claims_node_exact_unique") =>
18288 {
18289 return Err(StoreError::NodeAttemptCommitConflict);
18290 }
18291 Err(source) => {
18292 return Err(StoreError::database("node attempt claim insert", source));
18293 }
18294 };
18295 if inserted != 1 {
18296 return Err(StoreError::LeaseExpired);
18297 }
18298 Ok(())
18299}
18300
18301async fn insert_node_attempt_start(
18302 transaction: &mut Transaction<'_, Postgres>,
18303 start: &NodeAttemptStart,
18304) -> Result<(), StoreError> {
18305 let activation = start.activation();
18306 let base = activation.base_checkpoint();
18307 let base_journal = base.journal_head();
18308 let journal = start.journal_head();
18309 let base_superstep = i64::try_from(base.superstep().get())
18310 .map_err(|_| StoreError::encoding("node attempt base superstep"))?;
18311 let base_sequence = i64::try_from(base_journal.sequence().get())
18312 .map_err(|_| StoreError::JournalSequenceExhausted)?;
18313 let fence_epoch =
18314 i64::try_from(start.fence().epoch().get()).map_err(|_| StoreError::StaleFence)?;
18315 let journal_sequence = i64::try_from(journal.sequence().get())
18316 .map_err(|_| StoreError::JournalSequenceExhausted)?;
18317 let created_at = to_database_time(journal.recorded_at())?;
18318 let start_bytes = encode_node_attempt_start(start)?;
18319 let result = query(
18320 r"
18321INSERT INTO stateknot.node_attempts (
18322 tenant_id,
18323 run_id,
18324 base_checkpoint_id,
18325 base_superstep,
18326 base_checkpoint_digest,
18327 base_journal_sequence,
18328 base_journal_event_id,
18329 base_journal_recorded_at,
18330 base_journal_digest,
18331 graph_namespace,
18332 node_id,
18333 activation_input_digest,
18334 activation_digest,
18335 attempt_id,
18336 fence_attempt_id,
18337 fence_epoch,
18338 journal_sequence,
18339 journal_event_id,
18340 journal_recorded_at,
18341 journal_digest,
18342 start_digest,
18343 start_bytes,
18344 created_at
18345)
18346SELECT
18347 $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12,
18348 $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23
18349FROM stateknot.runs AS current_run
18350WHERE current_run.tenant_id = $1
18351 AND current_run.run_id = $2
18352 AND current_run.checkpoint_id = $3
18353 AND current_run.checkpoint_superstep = $4
18354 AND current_run.checkpoint_digest = $5
18355 AND current_run.lease_attempt_id = $15
18356 AND current_run.fencing_epoch = $16
18357 AND current_run.lease_expires_at > clock_timestamp()
18358",
18359 )
18360 .bind(activation.tenant_id().as_str())
18361 .bind(*activation.run_id().as_uuid())
18362 .bind(*base.checkpoint_id().as_uuid())
18363 .bind(base_superstep)
18364 .bind(base.digest().as_bytes())
18365 .bind(base_sequence)
18366 .bind(*base_journal.event_id().as_uuid())
18367 .bind(to_database_time(base_journal.recorded_at())?)
18368 .bind(base_journal.digest().as_bytes())
18369 .bind(activation.graph_namespace().as_str())
18370 .bind(activation.node_id().as_str())
18371 .bind(activation.input_digest().as_bytes())
18372 .bind(start.activation_digest().as_bytes())
18373 .bind(*start.attempt_id().as_uuid())
18374 .bind(*start.fence().attempt_id().as_uuid())
18375 .bind(fence_epoch)
18376 .bind(journal_sequence)
18377 .bind(*journal.event_id().as_uuid())
18378 .bind(created_at)
18379 .bind(journal.digest().as_bytes())
18380 .bind(start.digest().as_bytes())
18381 .bind(start_bytes)
18382 .bind(created_at)
18383 .execute(&mut **transaction)
18384 .await;
18385 let inserted = match result {
18386 Ok(result) => result.rows_affected(),
18387 Err(source) if has_database_constraint(&source, "node_attempts_pkey") => {
18388 return Err(StoreError::NodeAttemptIdConflict);
18389 }
18390 Err(source) if has_database_constraint(&source, "node_attempts_start_anchor_unique") => {
18391 return Err(StoreError::NodeAttemptCommitConflict);
18392 }
18393 Err(source) => return Err(StoreError::database("node attempt start insert", source)),
18394 };
18395 if inserted != 1 {
18396 return Err(StoreError::LeaseExpired);
18397 }
18398 Ok(())
18399}
18400
18401#[allow(clippy::too_many_lines)]
18402async fn insert_node_attempt_completion(
18403 transaction: &mut Transaction<'_, Postgres>,
18404 durable_start: &NodeAttemptStart,
18405 completion: &NodeAttemptCompletion,
18406) -> Result<(), StoreError> {
18407 let start = completion.start();
18408 if start != &durable_start.head() {
18409 return Err(StoreError::InvalidNodeAttemptTransition);
18410 }
18411 let activation = start.activation();
18412 let base = activation.base_checkpoint();
18413 let start_journal = start.journal_head();
18414 let journal = completion.journal_head();
18415 let base_superstep = i64::try_from(base.superstep().get())
18416 .map_err(|_| StoreError::encoding("node completion base superstep"))?;
18417 let fence_epoch =
18418 i64::try_from(start.fence().epoch().get()).map_err(|_| StoreError::StaleFence)?;
18419 let start_sequence = i64::try_from(start_journal.sequence().get())
18420 .map_err(|_| StoreError::JournalSequenceExhausted)?;
18421 let journal_sequence = i64::try_from(journal.sequence().get())
18422 .map_err(|_| StoreError::JournalSequenceExhausted)?;
18423 let created_at = to_database_time(journal.recorded_at())?;
18424 let completion_bytes = encode_node_attempt_completion(completion)?;
18425 let (result_intent_digest, result_record_digest, failure_id) = match completion.outcome() {
18426 NodeAttemptOutcome::Succeeded { result } => (
18427 Some(result.intent_digest().as_bytes().to_vec()),
18428 Some(result.digest().as_bytes().to_vec()),
18429 None,
18430 ),
18431 NodeAttemptOutcome::Failed { failure } => (None, None, Some(*failure.id().as_uuid())),
18432 _ => return Err(StoreError::InvalidNodeAttemptTransition),
18433 };
18434 let (retry_kind, retry_not_before) = node_attempt_retry_projection(completion)?;
18435 let result = query(
18436 r"
18437INSERT INTO stateknot.node_attempt_completions (
18438 tenant_id,
18439 run_id,
18440 attempt_id,
18441 base_checkpoint_id,
18442 base_superstep,
18443 base_checkpoint_digest,
18444 graph_namespace,
18445 node_id,
18446 activation_input_digest,
18447 activation_digest,
18448 fence_attempt_id,
18449 fence_epoch,
18450 start_journal_sequence,
18451 start_journal_event_id,
18452 start_journal_recorded_at,
18453 start_journal_digest,
18454 start_digest,
18455 status,
18456 journal_sequence,
18457 journal_event_id,
18458 journal_recorded_at,
18459 journal_digest,
18460 result_intent_digest,
18461 result_record_digest,
18462 failure_id,
18463 retry_kind,
18464 retry_not_before,
18465 completion_digest,
18466 completion_bytes,
18467 created_at
18468)
18469SELECT
18470 $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12,
18471 $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24,
18472 $25, $26, $27, $28, $29, $30
18473FROM stateknot.runs AS current_run
18474WHERE current_run.tenant_id = $1
18475 AND current_run.run_id = $2
18476 AND current_run.lease_attempt_id = $11
18477 AND current_run.fencing_epoch = $12
18478 AND current_run.lease_expires_at > clock_timestamp()
18479",
18480 )
18481 .bind(activation.tenant_id().as_str())
18482 .bind(*activation.run_id().as_uuid())
18483 .bind(*start.attempt_id().as_uuid())
18484 .bind(*base.checkpoint_id().as_uuid())
18485 .bind(base_superstep)
18486 .bind(base.digest().as_bytes())
18487 .bind(activation.graph_namespace().as_str())
18488 .bind(activation.node_id().as_str())
18489 .bind(activation.input_digest().as_bytes())
18490 .bind(durable_start.activation_digest().as_bytes())
18491 .bind(*start.fence().attempt_id().as_uuid())
18492 .bind(fence_epoch)
18493 .bind(start_sequence)
18494 .bind(*start_journal.event_id().as_uuid())
18495 .bind(to_database_time(start_journal.recorded_at())?)
18496 .bind(start_journal.digest().as_bytes())
18497 .bind(start.digest().as_bytes())
18498 .bind(node_attempt_status_text(completion.status())?)
18499 .bind(journal_sequence)
18500 .bind(*journal.event_id().as_uuid())
18501 .bind(created_at)
18502 .bind(journal.digest().as_bytes())
18503 .bind(result_intent_digest)
18504 .bind(result_record_digest)
18505 .bind(failure_id)
18506 .bind(retry_kind)
18507 .bind(retry_not_before)
18508 .bind(completion.digest().as_bytes())
18509 .bind(completion_bytes)
18510 .bind(created_at)
18511 .execute(&mut **transaction)
18512 .await;
18513 let inserted = match result {
18514 Ok(result) => result.rows_affected(),
18515 Err(source) if has_database_constraint(&source, "node_attempt_completions_pkey") => {
18516 return Err(StoreError::InvalidNodeAttemptTransition);
18517 }
18518 Err(source)
18519 if has_database_constraint(&source, "node_attempt_completions_anchor_unique") =>
18520 {
18521 return Err(StoreError::NodeAttemptCommitConflict);
18522 }
18523 Err(source) => {
18524 return Err(StoreError::database(
18525 "node attempt completion insert",
18526 source,
18527 ));
18528 }
18529 };
18530 if inserted != 1 {
18531 return Err(StoreError::LeaseExpired);
18532 }
18533 Ok(())
18534}
18535
18536async fn insert_initial_tool_invocation_revision(
18537 transaction: &mut Transaction<'_, Postgres>,
18538 invocation: &ToolInvocation,
18539 fence: &RunFence,
18540) -> Result<(), StoreError> {
18541 insert_tool_invocation_revision(transaction, invocation, None, fence).await
18542}
18543
18544async fn insert_successor_tool_invocation_revision(
18545 transaction: &mut Transaction<'_, Postgres>,
18546 invocation: &ToolInvocation,
18547 expected: &ToolInvocationHead,
18548 fence: &RunFence,
18549) -> Result<(), StoreError> {
18550 insert_tool_invocation_revision(transaction, invocation, Some(expected), fence).await
18551}
18552
18553#[allow(clippy::too_many_lines)]
18554async fn insert_tool_invocation_revision(
18555 transaction: &mut Transaction<'_, Postgres>,
18556 invocation: &ToolInvocation,
18557 expected: Option<&ToolInvocationHead>,
18558 fence: &RunFence,
18559) -> Result<(), StoreError> {
18560 let intent = invocation.intent();
18561 let record_bytes = encode_tool_invocation_record(invocation)?;
18562 let revision = i64::try_from(invocation.revision().get())
18563 .map_err(|_| StoreError::encoding("tool invocation revision"))?;
18564 let (previous_revision, previous_digest) =
18565 invocation.previous().map_or((None, None), |previous| {
18566 (
18567 i64::try_from(previous.revision().get()).ok(),
18568 Some(previous.digest().as_bytes().to_vec()),
18569 )
18570 });
18571 if invocation.previous().is_some() && previous_revision.is_none() {
18572 return Err(StoreError::encoding("tool invocation predecessor revision"));
18573 }
18574 let journal_sequence = i64::try_from(invocation.journal_head().sequence().get())
18575 .map_err(|_| StoreError::JournalSequenceExhausted)?;
18576 let transition_kind = invocation
18577 .transition()
18578 .map(ToolInvocationTransition::kind)
18579 .map(tool_invocation_transition_kind_text);
18580 let started_attempt = match invocation.transition() {
18581 Some(ToolInvocationTransition::StartAttempt { attempt_id }) => Some(*attempt_id),
18582 _ => None,
18583 };
18584 let (expected_revision, expected_digest) = expected.map_or((None, None), |head| {
18585 (
18586 i64::try_from(head.revision().get()).ok(),
18587 Some(head.digest().as_bytes().to_vec()),
18588 )
18589 });
18590 if expected.is_some() && expected_revision.is_none() {
18591 return Err(StoreError::encoding("tool invocation expected revision"));
18592 }
18593 let fence_epoch = i64::try_from(fence.epoch().get()).map_err(|_| StoreError::StaleFence)?;
18594 let created_at = to_database_time(invocation.journal_head().recorded_at())?;
18595
18596 if let Some(attempt_id) = started_attempt {
18597 insert_invocation_attempt_claim(
18598 transaction,
18599 InvocationAttemptKind::Tool,
18600 intent.tenant_id(),
18601 intent.run_id(),
18602 intent.invocation_id(),
18603 revision,
18604 attempt_id,
18605 invocation.journal_head(),
18606 fence,
18607 )
18608 .await?;
18609 }
18610
18611 let result = query(
18612 r"
18613INSERT INTO stateknot.tool_invocation_revisions (
18614 tenant_id,
18615 run_id,
18616 invocation_id,
18617 revision,
18618 previous_revision,
18619 previous_digest,
18620 journal_sequence,
18621 journal_event_id,
18622 journal_recorded_at,
18623 journal_digest,
18624 status,
18625 attempt_id,
18626 transition_kind,
18627 started_attempt_id,
18628 transition_digest,
18629 record_digest,
18630 record_bytes,
18631 created_at
18632)
18633SELECT
18634 $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18
18635FROM stateknot.runs AS current_run
18636JOIN stateknot.tool_invocations AS current_invocation
18637 ON current_invocation.tenant_id = current_run.tenant_id
18638 AND current_invocation.run_id = current_run.run_id
18639 AND current_invocation.invocation_id = $3
18640WHERE current_run.tenant_id = $1
18641 AND current_run.run_id = $2
18642 AND (
18643 (
18644 $19::bigint IS NULL
18645 AND current_invocation.current_revision = $4
18646 AND current_invocation.current_record_digest = $16
18647 )
18648 OR
18649 (
18650 $19::bigint IS NOT NULL
18651 AND current_invocation.current_revision = $19
18652 AND current_invocation.current_record_digest = $20
18653 )
18654 )
18655 AND current_run.lease_attempt_id = $21
18656 AND current_run.fencing_epoch = $22
18657 AND current_run.lease_expires_at > clock_timestamp()
18658",
18659 )
18660 .bind(intent.tenant_id().as_str())
18661 .bind(*intent.run_id().as_uuid())
18662 .bind(*intent.invocation_id().as_uuid())
18663 .bind(revision)
18664 .bind(previous_revision)
18665 .bind(previous_digest)
18666 .bind(journal_sequence)
18667 .bind(*invocation.journal_head().event_id().as_uuid())
18668 .bind(to_database_time(invocation.journal_head().recorded_at())?)
18669 .bind(invocation.journal_head().digest().as_bytes())
18670 .bind(tool_invocation_status_text(invocation.status()))
18671 .bind(invocation.attempt_id().map(|attempt| *attempt.as_uuid()))
18672 .bind(transition_kind)
18673 .bind(started_attempt.map(|attempt| *attempt.as_uuid()))
18674 .bind(
18675 invocation
18676 .transition_digest()
18677 .map(|digest| digest.as_bytes().to_vec()),
18678 )
18679 .bind(invocation.digest().as_bytes())
18680 .bind(record_bytes)
18681 .bind(created_at)
18682 .bind(expected_revision)
18683 .bind(expected_digest)
18684 .bind(*fence.attempt_id().as_uuid())
18685 .bind(fence_epoch)
18686 .execute(&mut **transaction)
18687 .await;
18688 let inserted = match result {
18689 Ok(result) => result.rows_affected(),
18690 Err(source)
18691 if has_database_constraint(
18692 &source,
18693 "tool_invocation_revisions_started_attempt_unique",
18694 ) =>
18695 {
18696 return Err(StoreError::InvalidToolInvocationTransition);
18697 }
18698 Err(source) => {
18699 return Err(StoreError::database(
18700 "tool invocation revision insert",
18701 source,
18702 ));
18703 }
18704 };
18705 if inserted != 1 {
18706 return Err(StoreError::LeaseExpired);
18707 }
18708 Ok(())
18709}
18710
18711async fn update_tool_invocation_current(
18712 transaction: &mut Transaction<'_, Postgres>,
18713 invocation: &ToolInvocation,
18714 expected: &ToolInvocationHead,
18715 fence: &RunFence,
18716) -> Result<(), StoreError> {
18717 let revision = i64::try_from(invocation.revision().get())
18718 .map_err(|_| StoreError::encoding("tool invocation revision"))?;
18719 let expected_revision = i64::try_from(expected.revision().get())
18720 .map_err(|_| StoreError::StaleToolInvocationHead)?;
18721 let fence_epoch = i64::try_from(fence.epoch().get()).map_err(|_| StoreError::StaleFence)?;
18722 let updated = query(
18723 r"
18724UPDATE stateknot.tool_invocations AS current_invocation
18725SET current_revision = $4,
18726 current_status = $5,
18727 current_attempt_id = $6,
18728 current_record_digest = $7,
18729 updated_at = $8
18730FROM stateknot.runs AS current_run
18731WHERE current_invocation.tenant_id = $1
18732 AND current_invocation.run_id = $2
18733 AND current_invocation.invocation_id = $3
18734 AND current_invocation.current_revision = $9
18735 AND current_invocation.current_record_digest = $10
18736 AND current_run.tenant_id = current_invocation.tenant_id
18737 AND current_run.run_id = current_invocation.run_id
18738 AND current_run.lease_attempt_id = $11
18739 AND current_run.fencing_epoch = $12
18740 AND current_run.lease_expires_at > clock_timestamp()
18741",
18742 )
18743 .bind(invocation.intent().tenant_id().as_str())
18744 .bind(*invocation.intent().run_id().as_uuid())
18745 .bind(*invocation.intent().invocation_id().as_uuid())
18746 .bind(revision)
18747 .bind(tool_invocation_status_text(invocation.status()))
18748 .bind(invocation.attempt_id().map(|attempt| *attempt.as_uuid()))
18749 .bind(invocation.digest().as_bytes())
18750 .bind(to_database_time(invocation.journal_head().recorded_at())?)
18751 .bind(expected_revision)
18752 .bind(expected.digest().as_bytes())
18753 .bind(*fence.attempt_id().as_uuid())
18754 .bind(fence_epoch)
18755 .execute(&mut **transaction)
18756 .await
18757 .map_err(|source| StoreError::database("tool invocation current update", source))?
18758 .rows_affected();
18759 if updated != 1 {
18760 return Err(StoreError::LeaseExpired);
18761 }
18762 Ok(())
18763}
18764
18765async fn insert_model_invocation_intent(
18766 transaction: &mut Transaction<'_, Postgres>,
18767 invocation: &ModelInvocation,
18768 fence: &RunFence,
18769) -> Result<(), StoreError> {
18770 let intent = invocation.intent();
18771 let activation = intent.activation();
18772 let base = activation.base_checkpoint();
18773 let intent_bytes = encode_model_invocation_intent(intent)?;
18774 let base_superstep = i64::try_from(base.superstep().get())
18775 .map_err(|_| StoreError::encoding("model invocation base superstep"))?;
18776 let current_revision = i64::try_from(invocation.revision().get())
18777 .map_err(|_| StoreError::encoding("model invocation revision"))?;
18778 let fence_epoch = i64::try_from(fence.epoch().get()).map_err(|_| StoreError::StaleFence)?;
18779 let created_at = to_database_time(invocation.journal_head().recorded_at())?;
18780 let inserted = query(
18781 r"
18782INSERT INTO stateknot.model_invocations (
18783 tenant_id,
18784 run_id,
18785 invocation_id,
18786 base_checkpoint_id,
18787 base_superstep,
18788 base_checkpoint_digest,
18789 graph_namespace,
18790 node_id,
18791 activation_input_digest,
18792 intent_digest,
18793 intent_bytes,
18794 current_revision,
18795 current_status,
18796 current_attempt_id,
18797 current_record_digest,
18798 created_at,
18799 updated_at
18800)
18801SELECT
18802 $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $16
18803FROM stateknot.runs AS current_run
18804WHERE current_run.tenant_id = $1
18805 AND current_run.run_id = $2
18806 AND current_run.checkpoint_id = $4
18807 AND current_run.checkpoint_superstep = $5
18808 AND current_run.checkpoint_digest = $6
18809 AND current_run.lease_attempt_id = $17
18810 AND current_run.fencing_epoch = $18
18811 AND current_run.lease_expires_at > clock_timestamp()
18812",
18813 )
18814 .bind(intent.tenant_id().as_str())
18815 .bind(*intent.run_id().as_uuid())
18816 .bind(*intent.invocation_id().as_uuid())
18817 .bind(*base.checkpoint_id().as_uuid())
18818 .bind(base_superstep)
18819 .bind(base.digest().as_bytes())
18820 .bind(activation.graph_namespace().as_str())
18821 .bind(activation.node_id().as_str())
18822 .bind(activation.input_digest().as_bytes())
18823 .bind(intent.intent_digest().as_bytes())
18824 .bind(intent_bytes)
18825 .bind(current_revision)
18826 .bind(model_invocation_status_text(invocation.status()))
18827 .bind(invocation.attempt_id().map(|attempt| *attempt.as_uuid()))
18828 .bind(invocation.digest().as_bytes())
18829 .bind(created_at)
18830 .bind(*fence.attempt_id().as_uuid())
18831 .bind(fence_epoch)
18832 .execute(&mut **transaction)
18833 .await
18834 .map_err(|source| StoreError::database("model invocation intent insert", source))?
18835 .rows_affected();
18836 if inserted != 1 {
18837 return Err(StoreError::LeaseExpired);
18838 }
18839 Ok(())
18840}
18841
18842async fn insert_initial_model_invocation_revision(
18843 transaction: &mut Transaction<'_, Postgres>,
18844 invocation: &ModelInvocation,
18845 fence: &RunFence,
18846) -> Result<(), StoreError> {
18847 insert_model_invocation_revision(transaction, invocation, None, fence).await
18848}
18849
18850async fn insert_successor_model_invocation_revision(
18851 transaction: &mut Transaction<'_, Postgres>,
18852 invocation: &ModelInvocation,
18853 expected: &ModelInvocationHead,
18854 fence: &RunFence,
18855) -> Result<(), StoreError> {
18856 insert_model_invocation_revision(transaction, invocation, Some(expected), fence).await
18857}
18858
18859#[allow(clippy::too_many_lines)]
18860async fn insert_model_invocation_revision(
18861 transaction: &mut Transaction<'_, Postgres>,
18862 invocation: &ModelInvocation,
18863 expected: Option<&ModelInvocationHead>,
18864 fence: &RunFence,
18865) -> Result<(), StoreError> {
18866 let intent = invocation.intent();
18867 let record_bytes = encode_model_invocation_record(invocation)?;
18868 let revision = i64::try_from(invocation.revision().get())
18869 .map_err(|_| StoreError::encoding("model invocation revision"))?;
18870 let (previous_revision, previous_digest) =
18871 invocation.previous().map_or((None, None), |previous| {
18872 (
18873 i64::try_from(previous.revision().get()).ok(),
18874 Some(previous.digest().as_bytes().to_vec()),
18875 )
18876 });
18877 if invocation.previous().is_some() && previous_revision.is_none() {
18878 return Err(StoreError::encoding(
18879 "model invocation predecessor revision",
18880 ));
18881 }
18882 let journal_sequence = i64::try_from(invocation.journal_head().sequence().get())
18883 .map_err(|_| StoreError::JournalSequenceExhausted)?;
18884 let transition_kind = invocation
18885 .transition()
18886 .map(ModelInvocationTransition::kind)
18887 .map(model_invocation_transition_kind_text);
18888 let started_attempt = match invocation.transition() {
18889 Some(ModelInvocationTransition::StartAttempt { attempt_id }) => Some(*attempt_id),
18890 _ => None,
18891 };
18892 let (expected_revision, expected_digest) = expected.map_or((None, None), |head| {
18893 (
18894 i64::try_from(head.revision().get()).ok(),
18895 Some(head.digest().as_bytes().to_vec()),
18896 )
18897 });
18898 if expected.is_some() && expected_revision.is_none() {
18899 return Err(StoreError::encoding("model invocation expected revision"));
18900 }
18901 let fence_epoch = i64::try_from(fence.epoch().get()).map_err(|_| StoreError::StaleFence)?;
18902 let created_at = to_database_time(invocation.journal_head().recorded_at())?;
18903
18904 if let Some(attempt_id) = started_attempt {
18905 insert_invocation_attempt_claim(
18906 transaction,
18907 InvocationAttemptKind::Model,
18908 intent.tenant_id(),
18909 intent.run_id(),
18910 intent.invocation_id(),
18911 revision,
18912 attempt_id,
18913 invocation.journal_head(),
18914 fence,
18915 )
18916 .await?;
18917 }
18918
18919 let result = query(
18920 r"
18921INSERT INTO stateknot.model_invocation_revisions (
18922 tenant_id,
18923 run_id,
18924 invocation_id,
18925 revision,
18926 previous_revision,
18927 previous_digest,
18928 journal_sequence,
18929 journal_event_id,
18930 journal_recorded_at,
18931 journal_digest,
18932 status,
18933 attempt_id,
18934 transition_kind,
18935 started_attempt_id,
18936 transition_digest,
18937 record_digest,
18938 record_bytes,
18939 created_at
18940)
18941SELECT
18942 $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18
18943FROM stateknot.runs AS current_run
18944JOIN stateknot.model_invocations AS current_invocation
18945 ON current_invocation.tenant_id = current_run.tenant_id
18946 AND current_invocation.run_id = current_run.run_id
18947 AND current_invocation.invocation_id = $3
18948WHERE current_run.tenant_id = $1
18949 AND current_run.run_id = $2
18950 AND (
18951 (
18952 $19::bigint IS NULL
18953 AND current_invocation.current_revision = $4
18954 AND current_invocation.current_record_digest = $16
18955 )
18956 OR
18957 (
18958 $19::bigint IS NOT NULL
18959 AND current_invocation.current_revision = $19
18960 AND current_invocation.current_record_digest = $20
18961 )
18962 )
18963 AND current_run.lease_attempt_id = $21
18964 AND current_run.fencing_epoch = $22
18965 AND current_run.lease_expires_at > clock_timestamp()
18966",
18967 )
18968 .bind(intent.tenant_id().as_str())
18969 .bind(*intent.run_id().as_uuid())
18970 .bind(*intent.invocation_id().as_uuid())
18971 .bind(revision)
18972 .bind(previous_revision)
18973 .bind(previous_digest)
18974 .bind(journal_sequence)
18975 .bind(*invocation.journal_head().event_id().as_uuid())
18976 .bind(to_database_time(invocation.journal_head().recorded_at())?)
18977 .bind(invocation.journal_head().digest().as_bytes())
18978 .bind(model_invocation_status_text(invocation.status()))
18979 .bind(invocation.attempt_id().map(|attempt| *attempt.as_uuid()))
18980 .bind(transition_kind)
18981 .bind(started_attempt.map(|attempt| *attempt.as_uuid()))
18982 .bind(
18983 invocation
18984 .transition_digest()
18985 .map(|digest| digest.as_bytes().to_vec()),
18986 )
18987 .bind(invocation.digest().as_bytes())
18988 .bind(record_bytes)
18989 .bind(created_at)
18990 .bind(expected_revision)
18991 .bind(expected_digest)
18992 .bind(*fence.attempt_id().as_uuid())
18993 .bind(fence_epoch)
18994 .execute(&mut **transaction)
18995 .await;
18996 let inserted = match result {
18997 Ok(result) => result.rows_affected(),
18998 Err(source)
18999 if has_database_constraint(
19000 &source,
19001 "model_invocation_revisions_started_attempt_unique",
19002 ) =>
19003 {
19004 return Err(StoreError::InvalidModelInvocationTransition);
19005 }
19006 Err(source) => {
19007 return Err(StoreError::database(
19008 "model invocation revision insert",
19009 source,
19010 ));
19011 }
19012 };
19013 if inserted != 1 {
19014 return Err(StoreError::LeaseExpired);
19015 }
19016 Ok(())
19017}
19018
19019async fn update_model_invocation_current(
19020 transaction: &mut Transaction<'_, Postgres>,
19021 invocation: &ModelInvocation,
19022 expected: &ModelInvocationHead,
19023 fence: &RunFence,
19024) -> Result<(), StoreError> {
19025 let revision = i64::try_from(invocation.revision().get())
19026 .map_err(|_| StoreError::encoding("model invocation revision"))?;
19027 let expected_revision = i64::try_from(expected.revision().get())
19028 .map_err(|_| StoreError::StaleModelInvocationHead)?;
19029 let fence_epoch = i64::try_from(fence.epoch().get()).map_err(|_| StoreError::StaleFence)?;
19030 let updated = query(
19031 r"
19032UPDATE stateknot.model_invocations AS current_invocation
19033SET current_revision = $4,
19034 current_status = $5,
19035 current_attempt_id = $6,
19036 current_record_digest = $7,
19037 updated_at = $8
19038FROM stateknot.runs AS current_run
19039WHERE current_invocation.tenant_id = $1
19040 AND current_invocation.run_id = $2
19041 AND current_invocation.invocation_id = $3
19042 AND current_invocation.current_revision = $9
19043 AND current_invocation.current_record_digest = $10
19044 AND current_run.tenant_id = current_invocation.tenant_id
19045 AND current_run.run_id = current_invocation.run_id
19046 AND current_run.lease_attempt_id = $11
19047 AND current_run.fencing_epoch = $12
19048 AND current_run.lease_expires_at > clock_timestamp()
19049",
19050 )
19051 .bind(invocation.intent().tenant_id().as_str())
19052 .bind(*invocation.intent().run_id().as_uuid())
19053 .bind(*invocation.intent().invocation_id().as_uuid())
19054 .bind(revision)
19055 .bind(model_invocation_status_text(invocation.status()))
19056 .bind(invocation.attempt_id().map(|attempt| *attempt.as_uuid()))
19057 .bind(invocation.digest().as_bytes())
19058 .bind(to_database_time(invocation.journal_head().recorded_at())?)
19059 .bind(expected_revision)
19060 .bind(expected.digest().as_bytes())
19061 .bind(*fence.attempt_id().as_uuid())
19062 .bind(fence_epoch)
19063 .execute(&mut **transaction)
19064 .await
19065 .map_err(|source| StoreError::database("model invocation current update", source))?
19066 .rows_affected();
19067 if updated != 1 {
19068 return Err(StoreError::LeaseExpired);
19069 }
19070 Ok(())
19071}
19072
19073async fn insert_pending_node_result(
19074 transaction: &mut Transaction<'_, Postgres>,
19075 result: &PendingNodeResult,
19076 node_attempt_id: AttemptId,
19077 fence: &RunFence,
19078) -> Result<(), StoreError> {
19079 child_joins::verify_result(transaction, result, false).await?;
19080 let intent = result.intent();
19081 let activation = intent.activation();
19082 let base = activation.base_checkpoint();
19083 let base_journal = base.journal_head();
19084 let journal = result.journal_head();
19085 let result_bytes = encode_pending_node_result(result)?;
19086 let base_superstep = i64::try_from(base.superstep().get())
19087 .map_err(|_| StoreError::encoding("pending node result base superstep"))?;
19088 let base_journal_sequence = i64::try_from(base_journal.sequence().get())
19089 .map_err(|_| StoreError::JournalSequenceExhausted)?;
19090 let fence_epoch = i64::try_from(fence.epoch().get()).map_err(|_| StoreError::StaleFence)?;
19091 let journal_sequence = i64::try_from(journal.sequence().get())
19092 .map_err(|_| StoreError::JournalSequenceExhausted)?;
19093 let created_at = to_database_time(journal.recorded_at())?;
19094 let inserted = query(
19095 r"
19096INSERT INTO stateknot.pending_node_results (
19097 tenant_id,
19098 run_id,
19099 base_checkpoint_id,
19100 base_superstep,
19101 base_checkpoint_digest,
19102 base_journal_sequence,
19103 base_journal_event_id,
19104 base_journal_recorded_at,
19105 base_journal_digest,
19106 graph_namespace,
19107 node_id,
19108 activation_input_digest,
19109 node_attempt_id,
19110 intent_digest,
19111 control_kind,
19112 fence_attempt_id,
19113 fence_epoch,
19114 journal_sequence,
19115 journal_event_id,
19116 journal_recorded_at,
19117 journal_digest,
19118 record_digest,
19119 result_bytes,
19120 created_at
19121)
19122SELECT
19123 $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12,
19124 $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24
19125FROM stateknot.runs AS current_run
19126WHERE current_run.tenant_id = $1
19127 AND current_run.run_id = $2
19128 AND current_run.checkpoint_id = $3
19129 AND current_run.checkpoint_superstep = $4
19130 AND current_run.checkpoint_digest = $5
19131 AND current_run.lease_attempt_id = $16
19132 AND current_run.fencing_epoch = $17
19133 AND current_run.lease_expires_at > clock_timestamp()
19134",
19135 )
19136 .bind(intent.tenant_id().as_str())
19137 .bind(*intent.run_id().as_uuid())
19138 .bind(*base.checkpoint_id().as_uuid())
19139 .bind(base_superstep)
19140 .bind(base.digest().as_bytes())
19141 .bind(base_journal_sequence)
19142 .bind(*base_journal.event_id().as_uuid())
19143 .bind(to_database_time(base_journal.recorded_at())?)
19144 .bind(base_journal.digest().as_bytes())
19145 .bind(activation.graph_namespace().as_str())
19146 .bind(activation.node_id().as_str())
19147 .bind(activation.input_digest().as_bytes())
19148 .bind(*node_attempt_id.as_uuid())
19149 .bind(intent.intent_digest().as_bytes())
19150 .bind(pending_node_result_control_kind_text(
19151 intent.control().kind(),
19152 ))
19153 .bind(*fence.attempt_id().as_uuid())
19154 .bind(fence_epoch)
19155 .bind(journal_sequence)
19156 .bind(*journal.event_id().as_uuid())
19157 .bind(created_at)
19158 .bind(journal.digest().as_bytes())
19159 .bind(result.digest().as_bytes())
19160 .bind(result_bytes)
19161 .bind(created_at)
19162 .execute(&mut **transaction)
19163 .await
19164 .map_err(|source| StoreError::database("pending node result insert", source))?
19165 .rows_affected();
19166 if inserted != 1 {
19167 return Err(StoreError::LeaseExpired);
19168 }
19169 Ok(())
19170}
19171
19172async fn insert_pending_node_result_bindings(
19173 transaction: &mut Transaction<'_, Postgres>,
19174 result: &PendingNodeResult,
19175 fence: &RunFence,
19176) -> Result<(), StoreError> {
19177 insert_pending_node_result_binding_kind(
19178 transaction,
19179 result,
19180 fence,
19181 NodeInvocationBindingKind::Model,
19182 )
19183 .await?;
19184 insert_pending_node_result_binding_kind(
19185 transaction,
19186 result,
19187 fence,
19188 NodeInvocationBindingKind::Tool,
19189 )
19190 .await
19191}
19192
19193struct PendingBindingValues {
19194 invocation_ids: Vec<Uuid>,
19195 revisions: Vec<i64>,
19196 record_digests: Vec<Vec<u8>>,
19197 statuses: Vec<String>,
19198 journal_sequences: Vec<i64>,
19199 journal_recorded_at: Vec<DateTime<Utc>>,
19200 journal_digests: Vec<Vec<u8>>,
19201}
19202
19203fn pending_binding_values(
19204 result: &PendingNodeResult,
19205 kind: NodeInvocationBindingKind,
19206) -> Result<PendingBindingValues, StoreError> {
19207 let mut values = PendingBindingValues {
19208 invocation_ids: Vec::new(),
19209 revisions: Vec::new(),
19210 record_digests: Vec::new(),
19211 statuses: Vec::new(),
19212 journal_sequences: Vec::new(),
19213 journal_recorded_at: Vec::new(),
19214 journal_digests: Vec::new(),
19215 };
19216 for binding in result
19217 .intent()
19218 .bindings()
19219 .iter()
19220 .filter(|binding| binding.kind() == kind)
19221 {
19222 let (revision, digest, status) = match binding {
19223 NodeInvocationBinding::Model { head, .. } => (
19224 head.revision().get(),
19225 head.digest(),
19226 model_invocation_status_text(head.status()),
19227 ),
19228 NodeInvocationBinding::Tool { head, .. } => (
19229 head.revision().get(),
19230 head.digest(),
19231 tool_invocation_status_text(head.status()),
19232 ),
19233 };
19234 values
19235 .invocation_ids
19236 .push(*binding.invocation_id().as_uuid());
19237 values.revisions.push(
19238 i64::try_from(revision)
19239 .map_err(|_| StoreError::encoding("pending node result binding revision"))?,
19240 );
19241 values.record_digests.push(digest.as_bytes().to_vec());
19242 values.statuses.push(status.to_owned());
19243 values.journal_sequences.push(
19244 i64::try_from(binding.journal_head().sequence().get())
19245 .map_err(|_| StoreError::JournalSequenceExhausted)?,
19246 );
19247 values
19248 .journal_recorded_at
19249 .push(to_database_time(binding.journal_head().recorded_at())?);
19250 values
19251 .journal_digests
19252 .push(binding.journal_head().digest().as_bytes().to_vec());
19253 }
19254 Ok(values)
19255}
19256
19257#[allow(clippy::too_many_lines)]
19258async fn insert_pending_node_result_binding_kind(
19259 transaction: &mut Transaction<'_, Postgres>,
19260 result: &PendingNodeResult,
19261 fence: &RunFence,
19262 kind: NodeInvocationBindingKind,
19263) -> Result<(), StoreError> {
19264 let values = pending_binding_values(result, kind)?;
19265 if values.invocation_ids.is_empty() {
19266 return Ok(());
19267 }
19268 let (table, operation) = match kind {
19269 NodeInvocationBindingKind::Model => (
19270 "stateknot.pending_node_result_model_bindings",
19271 "pending node result model binding insert",
19272 ),
19273 NodeInvocationBindingKind::Tool => (
19274 "stateknot.pending_node_result_tool_bindings",
19275 "pending node result tool binding insert",
19276 ),
19277 };
19278 let statement = format!(
19279 r"
19280INSERT INTO {table} (
19281 tenant_id,
19282 run_id,
19283 base_checkpoint_id,
19284 base_superstep,
19285 base_checkpoint_digest,
19286 graph_namespace,
19287 node_id,
19288 activation_input_digest,
19289 result_record_digest,
19290 result_journal_sequence,
19291 result_journal_recorded_at,
19292 result_journal_digest,
19293 invocation_id,
19294 invocation_revision,
19295 invocation_record_digest,
19296 invocation_status,
19297 invocation_journal_sequence,
19298 invocation_journal_recorded_at,
19299 invocation_journal_digest
19300)
19301SELECT
19302 $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12,
19303 binding.invocation_id,
19304 binding.invocation_revision,
19305 binding.invocation_record_digest,
19306 binding.invocation_status,
19307 binding.invocation_journal_sequence,
19308 binding.invocation_journal_recorded_at,
19309 binding.invocation_journal_digest
19310FROM UNNEST(
19311 $13::uuid[],
19312 $14::bigint[],
19313 $15::bytea[],
19314 $16::text[],
19315 $17::bigint[],
19316 $18::timestamptz[],
19317 $19::bytea[]
19318) AS binding (
19319 invocation_id,
19320 invocation_revision,
19321 invocation_record_digest,
19322 invocation_status,
19323 invocation_journal_sequence,
19324 invocation_journal_recorded_at,
19325 invocation_journal_digest
19326)
19327CROSS JOIN stateknot.runs AS current_run
19328WHERE current_run.tenant_id = $1
19329 AND current_run.run_id = $2
19330 AND current_run.checkpoint_id = $3
19331 AND current_run.checkpoint_superstep = $4
19332 AND current_run.checkpoint_digest = $5
19333 AND current_run.lease_attempt_id = $20
19334 AND current_run.fencing_epoch = $21
19335 AND current_run.lease_expires_at > clock_timestamp()
19336"
19337 );
19338 let activation = result.intent().activation();
19339 let base = activation.base_checkpoint();
19340 let base_superstep = i64::try_from(base.superstep().get())
19341 .map_err(|_| StoreError::encoding("pending node result binding base superstep"))?;
19342 let result_sequence = i64::try_from(result.journal_head().sequence().get())
19343 .map_err(|_| StoreError::JournalSequenceExhausted)?;
19344 let fence_epoch = i64::try_from(fence.epoch().get()).map_err(|_| StoreError::StaleFence)?;
19345 let inserted = query(&statement)
19346 .bind(activation.tenant_id().as_str())
19347 .bind(*activation.run_id().as_uuid())
19348 .bind(*base.checkpoint_id().as_uuid())
19349 .bind(base_superstep)
19350 .bind(base.digest().as_bytes())
19351 .bind(activation.graph_namespace().as_str())
19352 .bind(activation.node_id().as_str())
19353 .bind(activation.input_digest().as_bytes())
19354 .bind(result.digest().as_bytes())
19355 .bind(result_sequence)
19356 .bind(to_database_time(result.journal_head().recorded_at())?)
19357 .bind(result.journal_head().digest().as_bytes())
19358 .bind(&values.invocation_ids)
19359 .bind(&values.revisions)
19360 .bind(&values.record_digests)
19361 .bind(&values.statuses)
19362 .bind(&values.journal_sequences)
19363 .bind(&values.journal_recorded_at)
19364 .bind(&values.journal_digests)
19365 .bind(*fence.attempt_id().as_uuid())
19366 .bind(fence_epoch)
19367 .execute(&mut **transaction)
19368 .await;
19369 let inserted = match inserted {
19370 Ok(result) => result.rows_affected(),
19371 Err(source) if is_invalid_pending_binding_constraint(&source) => {
19372 return Err(StoreError::InvalidPendingNodeResultBinding);
19373 }
19374 Err(source) => return Err(StoreError::database(operation, source)),
19375 };
19376 if inserted
19377 != u64::try_from(values.invocation_ids.len())
19378 .map_err(|_| StoreError::encoding("pending node result binding count"))?
19379 {
19380 return Err(StoreError::LeaseExpired);
19381 }
19382 Ok(())
19383}
19384
19385#[allow(clippy::too_many_lines)]
19386async fn insert_barrier_consumptions(
19387 transaction: &mut Transaction<'_, Postgres>,
19388 barrier: &CheckpointBarrier,
19389 successor: &Checkpoint,
19390 source: &JournalEventSource,
19391) -> Result<(), StoreError> {
19392 let base = barrier.base_checkpoint();
19393 let base_superstep =
19394 i64::try_from(base.superstep().get()).map_err(|_| StoreError::InvalidCheckpointBarrier)?;
19395 let successor_superstep = i64::try_from(successor.superstep().get())
19396 .map_err(|_| StoreError::encoding("checkpoint barrier successor superstep"))?;
19397 let successor_sequence = i64::try_from(successor.journal_head().sequence().get())
19398 .map_err(|_| StoreError::JournalSequenceExhausted)?;
19399 let graph_namespaces = barrier
19400 .result_heads()
19401 .iter()
19402 .map(|head| head.activation().graph_namespace().as_str().to_owned())
19403 .collect::<Vec<_>>();
19404 let node_ids = barrier
19405 .result_heads()
19406 .iter()
19407 .map(|head| head.activation().node_id().as_str().to_owned())
19408 .collect::<Vec<_>>();
19409 let result_digests = barrier
19410 .result_heads()
19411 .iter()
19412 .map(|head| head.digest().as_bytes().to_vec())
19413 .collect::<Vec<_>>();
19414 let (worker_attempt_id, worker_epoch, worker_write) = match source {
19415 JournalEventSource::ControlPlane => (None, None, false),
19416 JournalEventSource::Worker { fence } => (
19417 Some(*fence.attempt_id().as_uuid()),
19418 Some(i64::try_from(fence.epoch().get()).map_err(|_| StoreError::StaleFence)?),
19419 true,
19420 ),
19421 };
19422
19423 let inserted = query(
19424 r"
19425INSERT INTO stateknot.pending_node_result_consumptions (
19426 tenant_id,
19427 run_id,
19428 base_checkpoint_id,
19429 base_superstep,
19430 base_checkpoint_digest,
19431 graph_namespace,
19432 node_id,
19433 result_record_digest,
19434 successor_checkpoint_id,
19435 successor_superstep,
19436 successor_checkpoint_digest,
19437 successor_journal_sequence,
19438 successor_journal_event_id,
19439 successor_journal_recorded_at,
19440 successor_journal_digest,
19441 created_at
19442)
19443SELECT
19444 $1, $2, $3, $4, $5,
19445 expected.graph_namespace,
19446 expected.node_id,
19447 expected.result_record_digest,
19448 $6, $7, $8, $9, $10, $11, $12, $11
19449FROM UNNEST(
19450 $13::text[],
19451 $14::text[],
19452 $15::bytea[]
19453) AS expected (graph_namespace, node_id, result_record_digest)
19454JOIN stateknot.pending_node_results AS pending
19455 ON pending.tenant_id = $1
19456 AND pending.run_id = $2
19457 AND pending.base_checkpoint_id = $3
19458 AND pending.base_superstep = $4
19459 AND pending.base_checkpoint_digest = $5
19460 AND pending.graph_namespace = expected.graph_namespace
19461 AND pending.node_id = expected.node_id
19462 AND pending.record_digest = expected.result_record_digest
19463CROSS JOIN stateknot.runs AS current_run
19464WHERE current_run.tenant_id = $1
19465 AND current_run.run_id = $2
19466 AND current_run.checkpoint_id = $3
19467 AND current_run.checkpoint_superstep = $4
19468 AND current_run.checkpoint_digest = $5
19469 AND (
19470 $16::uuid IS NULL
19471 OR (
19472 current_run.lease_attempt_id = $16
19473 AND current_run.fencing_epoch = $17
19474 AND current_run.lease_expires_at > clock_timestamp()
19475 )
19476 )
19477",
19478 )
19479 .bind(base.tenant_id().as_str())
19480 .bind(*base.run_id().as_uuid())
19481 .bind(*base.checkpoint_id().as_uuid())
19482 .bind(base_superstep)
19483 .bind(base.digest().as_bytes())
19484 .bind(*successor.checkpoint_id().as_uuid())
19485 .bind(successor_superstep)
19486 .bind(successor.digest().as_bytes())
19487 .bind(successor_sequence)
19488 .bind(*successor.journal_head().event_id().as_uuid())
19489 .bind(to_database_time(successor.journal_head().recorded_at())?)
19490 .bind(successor.journal_head().digest().as_bytes())
19491 .bind(&graph_namespaces)
19492 .bind(&node_ids)
19493 .bind(&result_digests)
19494 .bind(worker_attempt_id)
19495 .bind(worker_epoch)
19496 .execute(&mut **transaction)
19497 .await;
19498 let inserted = match inserted {
19499 Ok(result) => result.rows_affected(),
19500 Err(source)
19501 if has_database_constraint(&source, "pending_node_result_consumptions_result_fk")
19502 || has_database_constraint(&source, "pending_node_result_consumptions_pkey") =>
19503 {
19504 return Err(StoreError::CheckpointBarrierResultConflict);
19505 }
19506 Err(source)
19507 if has_database_constraint(
19508 &source,
19509 "pending_node_result_consumptions_successor_fk",
19510 ) =>
19511 {
19512 return Err(StoreError::CheckpointBarrierCommitConflict);
19513 }
19514 Err(source) => {
19515 return Err(StoreError::database(
19516 "checkpoint barrier consumption insert",
19517 source,
19518 ));
19519 }
19520 };
19521 let expected_count = u64::try_from(barrier.result_heads().len())
19522 .map_err(|_| StoreError::InvalidCheckpointBarrier)?;
19523 if inserted != expected_count {
19524 if worker_write {
19525 return Err(StoreError::LeaseExpired);
19526 }
19527 return Err(StoreError::CheckpointBarrierResultConflict);
19528 }
19529 Ok(())
19530}
19531
19532#[allow(clippy::too_many_lines)]
19533async fn insert_checkpoint(
19534 transaction: &mut Transaction<'_, Postgres>,
19535 checkpoint: &Checkpoint,
19536 source: &JournalEventSource,
19537) -> Result<(), StoreError> {
19538 let checkpoint_bytes = encode_checkpoint(checkpoint)?;
19539 let superstep = i64::try_from(checkpoint.superstep().get())
19540 .map_err(|_| StoreError::encoding("checkpoint superstep"))?;
19541 let journal_sequence = i64::try_from(checkpoint.journal_head().sequence().get())
19542 .map_err(|_| StoreError::JournalSequenceExhausted)?;
19543 let (parent_id, parent_superstep, parent_digest) =
19544 checkpoint.parent().map_or((None, None, None), |parent| {
19545 (
19546 Some(*parent.checkpoint_id().as_uuid()),
19547 i64::try_from(parent.superstep().get()).ok(),
19548 Some(parent.digest().as_bytes().to_vec()),
19549 )
19550 });
19551 if checkpoint.parent().is_some() && parent_superstep.is_none() {
19552 return Err(StoreError::encoding("checkpoint parent superstep"));
19553 }
19554 let (worker_attempt_id, worker_epoch, worker_write) = match source {
19555 JournalEventSource::ControlPlane => (None, None, false),
19556 JournalEventSource::Worker { fence } => (
19557 Some(*fence.attempt_id().as_uuid()),
19558 Some(i64::try_from(fence.epoch().get()).map_err(|_| StoreError::StaleFence)?),
19559 true,
19560 ),
19561 };
19562 let schema = checkpoint.graph().state_schema();
19563
19564 let inserted = query(
19565 r"
19566INSERT INTO stateknot.run_checkpoints (
19567 tenant_id,
19568 run_id,
19569 checkpoint_id,
19570 superstep,
19571 parent_checkpoint_id,
19572 parent_superstep,
19573 parent_digest,
19574 journal_sequence,
19575 journal_event_id,
19576 journal_recorded_at,
19577 journal_digest,
19578 graph_definition_digest,
19579 state_schema_id,
19580 state_schema_version,
19581 state_schema_digest,
19582 state_digest,
19583 intent_digest,
19584 checkpoint_digest,
19585 checkpoint_bytes
19586)
19587SELECT
19588 $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19
19589FROM stateknot.runs AS current_run
19590WHERE current_run.tenant_id = $1
19591 AND current_run.run_id = $2
19592 AND (
19593 ($5::uuid IS NULL AND current_run.checkpoint_id IS NULL)
19594 OR (
19595 current_run.checkpoint_id = $5
19596 AND current_run.checkpoint_superstep = $6
19597 AND current_run.checkpoint_digest = $7
19598 )
19599 )
19600 AND (
19601 $20::uuid IS NULL
19602 OR (
19603 current_run.lease_attempt_id = $20
19604 AND current_run.fencing_epoch = $21
19605 AND current_run.lease_expires_at > clock_timestamp()
19606 )
19607 )
19608",
19609 )
19610 .bind(checkpoint.tenant_id().as_str())
19611 .bind(*checkpoint.run_id().as_uuid())
19612 .bind(*checkpoint.checkpoint_id().as_uuid())
19613 .bind(superstep)
19614 .bind(parent_id)
19615 .bind(parent_superstep)
19616 .bind(parent_digest)
19617 .bind(journal_sequence)
19618 .bind(*checkpoint.journal_head().event_id().as_uuid())
19619 .bind(to_database_time(checkpoint.journal_head().recorded_at())?)
19620 .bind(checkpoint.journal_head().digest().as_bytes())
19621 .bind(checkpoint.graph().definition_digest().as_bytes())
19622 .bind(schema.id().as_str())
19623 .bind(schema.version().to_string())
19624 .bind(schema.digest().as_bytes())
19625 .bind(checkpoint.state().digest().as_bytes())
19626 .bind(checkpoint.intent_digest().as_bytes())
19627 .bind(checkpoint.digest().as_bytes())
19628 .bind(checkpoint_bytes)
19629 .bind(worker_attempt_id)
19630 .bind(worker_epoch)
19631 .execute(&mut **transaction)
19632 .await
19633 .map_err(|source| StoreError::database("checkpoint insert", source))?
19634 .rows_affected();
19635 if inserted != 1 {
19636 if worker_write {
19637 return Err(StoreError::LeaseExpired);
19638 }
19639 return Err(StoreError::StaleCheckpointHead);
19640 }
19641 Ok(())
19642}
19643
19644async fn update_checkpoint_pointer(
19645 transaction: &mut Transaction<'_, Postgres>,
19646 checkpoint: &Checkpoint,
19647 source: &JournalEventSource,
19648) -> Result<(), StoreError> {
19649 let superstep = i64::try_from(checkpoint.superstep().get())
19650 .map_err(|_| StoreError::encoding("checkpoint superstep"))?;
19651 let (parent_id, parent_superstep, parent_digest) =
19652 checkpoint.parent().map_or((None, None, None), |parent| {
19653 (
19654 Some(*parent.checkpoint_id().as_uuid()),
19655 i64::try_from(parent.superstep().get()).ok(),
19656 Some(parent.digest().as_bytes().to_vec()),
19657 )
19658 });
19659 if checkpoint.parent().is_some() && parent_superstep.is_none() {
19660 return Err(StoreError::encoding("checkpoint parent superstep"));
19661 }
19662 let (worker_attempt_id, worker_epoch, worker_write) = match source {
19663 JournalEventSource::ControlPlane => (None, None, false),
19664 JournalEventSource::Worker { fence } => (
19665 Some(*fence.attempt_id().as_uuid()),
19666 Some(i64::try_from(fence.epoch().get()).map_err(|_| StoreError::StaleFence)?),
19667 true,
19668 ),
19669 };
19670
19671 let updated = query(
19672 r"
19673UPDATE stateknot.runs
19674SET checkpoint_id = $3,
19675 checkpoint_superstep = $4,
19676 checkpoint_digest = $5
19677WHERE tenant_id = $1
19678 AND run_id = $2
19679 AND (
19680 ($6::uuid IS NULL AND checkpoint_id IS NULL)
19681 OR (
19682 checkpoint_id = $6
19683 AND checkpoint_superstep = $7
19684 AND checkpoint_digest = $8
19685 )
19686 )
19687 AND (
19688 $9::uuid IS NULL
19689 OR (
19690 lease_attempt_id = $9
19691 AND fencing_epoch = $10
19692 AND lease_expires_at > clock_timestamp()
19693 )
19694 )
19695",
19696 )
19697 .bind(checkpoint.tenant_id().as_str())
19698 .bind(*checkpoint.run_id().as_uuid())
19699 .bind(*checkpoint.checkpoint_id().as_uuid())
19700 .bind(superstep)
19701 .bind(checkpoint.digest().as_bytes())
19702 .bind(parent_id)
19703 .bind(parent_superstep)
19704 .bind(parent_digest)
19705 .bind(worker_attempt_id)
19706 .bind(worker_epoch)
19707 .execute(&mut **transaction)
19708 .await
19709 .map_err(|source| StoreError::database("checkpoint pointer update", source))?
19710 .rows_affected();
19711 if updated != 1 {
19712 if worker_write {
19713 return Err(StoreError::LeaseExpired);
19714 }
19715 return Err(StoreError::StaleCheckpointHead);
19716 }
19717 Ok(())
19718}
19719
19720#[allow(clippy::too_many_lines)]
19721async fn update_run_head(
19722 transaction: &mut Transaction<'_, Postgres>,
19723 event: &JournalEvent,
19724 projection: Option<&PreparedProjection>,
19725) -> Result<(), StoreError> {
19726 if let Some(projection) = projection {
19727 Box::pin(child_runs::validate_terminal_accounting(
19728 transaction,
19729 event,
19730 projection,
19731 ))
19732 .await?;
19733 }
19734 let sequence =
19735 i64::try_from(event.sequence().get()).map_err(|_| StoreError::JournalSequenceExhausted)?;
19736 let recorded_at = to_database_time(event.recorded_at())?;
19737 let (worker_attempt_id, worker_epoch, worker_write) = match event.source() {
19738 JournalEventSource::ControlPlane => (None, None, false),
19739 JournalEventSource::Worker { fence } => (
19740 Some(*fence.attempt_id().as_uuid()),
19741 Some(i64::try_from(fence.epoch().get()).map_err(|_| StoreError::StaleFence)?),
19742 true,
19743 ),
19744 };
19745 let updated = if let Some(projection) = projection {
19746 query(
19747 r"
19748UPDATE stateknot.runs
19749SET journal_sequence = $3,
19750 journal_event_id = $4,
19751 journal_recorded_at = $5,
19752 journal_digest = $6,
19753 lifecycle_bytes = $7,
19754 lifecycle_revision = $8::numeric,
19755 lifecycle_status = $9,
19756 changed_at = $10,
19757 scheduler_ready_at = CASE
19758 WHEN $9 IN ('pending', 'active', 'cancellation_requested') THEN $5
19759 ELSE NULL
19760 END,
19761 scheduler_not_before = NULL,
19762 lease_attempt_id = CASE
19763 WHEN $9 IN ('pending', 'active', 'cancellation_requested')
19764 THEN lease_attempt_id
19765 ELSE NULL
19766 END,
19767 lease_acquired_at = CASE
19768 WHEN $9 IN ('pending', 'active', 'cancellation_requested')
19769 THEN lease_acquired_at
19770 ELSE NULL
19771 END,
19772 lease_renewed_at = CASE
19773 WHEN $9 IN ('pending', 'active', 'cancellation_requested')
19774 THEN lease_renewed_at
19775 ELSE NULL
19776 END,
19777 lease_expires_at = CASE
19778 WHEN $9 IN ('pending', 'active', 'cancellation_requested')
19779 THEN lease_expires_at
19780 ELSE NULL
19781 END,
19782 wait_set_digest = $11,
19783 unresolved_wait_count = $12,
19784 next_timer_due_at = $13,
19785 next_interrupt_expiry_at = $14,
19786 updated_at = $5
19787WHERE tenant_id = $1
19788 AND run_id = $2
19789 AND (
19790 $15::uuid IS NULL
19791 OR (
19792 lease_attempt_id = $15
19793 AND fencing_epoch = $16
19794 AND lease_expires_at > clock_timestamp()
19795 )
19796 )
19797",
19798 )
19799 .bind(event.tenant_id().as_str())
19800 .bind(*event.run_id().as_uuid())
19801 .bind(sequence)
19802 .bind(*event.event_id().as_uuid())
19803 .bind(recorded_at)
19804 .bind(event.digest().as_bytes())
19805 .bind(&projection.lifecycle_bytes)
19806 .bind(&projection.revision)
19807 .bind(projection.status)
19808 .bind(projection.changed_at)
19809 .bind(
19810 projection
19811 .wait_set_digest
19812 .as_ref()
19813 .map(stateknot_core::Digest::as_bytes),
19814 )
19815 .bind(projection.unresolved_wait_count)
19816 .bind(projection.next_timer_due_at)
19817 .bind(projection.next_interrupt_expiry_at)
19818 .bind(worker_attempt_id)
19819 .bind(worker_epoch)
19820 .execute(&mut **transaction)
19821 .await
19822 } else {
19823 query(
19824 r"
19825UPDATE stateknot.runs
19826SET journal_sequence = $3,
19827 journal_event_id = $4,
19828 journal_recorded_at = $5,
19829 journal_digest = $6,
19830 updated_at = $5
19831WHERE tenant_id = $1
19832 AND run_id = $2
19833 AND (
19834 $7::uuid IS NULL
19835 OR (
19836 lease_attempt_id = $7
19837 AND fencing_epoch = $8
19838 AND lease_expires_at > clock_timestamp()
19839 )
19840 )
19841",
19842 )
19843 .bind(event.tenant_id().as_str())
19844 .bind(*event.run_id().as_uuid())
19845 .bind(sequence)
19846 .bind(*event.event_id().as_uuid())
19847 .bind(recorded_at)
19848 .bind(event.digest().as_bytes())
19849 .bind(worker_attempt_id)
19850 .bind(worker_epoch)
19851 .execute(&mut **transaction)
19852 .await
19853 }
19854 .map_err(|source| StoreError::database("run head update", source))?
19855 .rows_affected();
19856 if updated != 1 {
19857 if worker_write {
19858 return Err(StoreError::LeaseExpired);
19859 }
19860 return Err(StoreError::corrupt("run head row count"));
19861 }
19862 Ok(())
19863}
19864
19865fn authorize_worker<'a>(
19866 stored: &'a StoredRun,
19867 fence: &RunFence,
19868 observed_at: Timestamp,
19869) -> Result<&'a RunLease, StoreError> {
19870 let lease = stored.lease().ok_or(StoreError::NoActiveLease)?;
19871 match lease.validate_write(fence, observed_at) {
19872 Ok(()) => Ok(lease),
19873 Err(RunLeaseValidationError::Expired { .. }) => Err(StoreError::LeaseExpired),
19874 Err(
19875 RunLeaseValidationError::ObservationBeforeAcquisition { .. }
19876 | RunLeaseValidationError::ObservationBeforeRenewal { .. },
19877 ) => Err(StoreError::DatabaseClockRegression),
19878 Err(_) => Err(StoreError::StaleFence),
19879 }
19880}
19881
19882fn validate_runnable(stored: &StoredRun) -> Result<(), StoreError> {
19883 if stored.is_quarantined() {
19884 return Err(StoreError::RunQuarantined);
19885 }
19886 if !lifecycle_is_scheduler_runnable(stored.lifecycle().status()) {
19887 return Err(StoreError::RunNotRunnable);
19888 }
19889 Ok(())
19890}
19891
19892fn scheduler_available_at(stored: &StoredRun) -> Result<Timestamp, StoreError> {
19893 let ready_at = stored
19894 .scheduler_ready_at()
19895 .ok_or_else(|| StoreError::corrupt("run scheduler readiness shape"))?;
19896 let after_retry = stored
19897 .scheduler_not_before()
19898 .map_or(ready_at, |not_before| ready_at.max(not_before));
19899 Ok(stored
19900 .lease()
19901 .map_or(after_retry, |lease| after_retry.max(lease.expires_at())))
19902}
19903
19904const fn lifecycle_is_scheduler_runnable(status: RunStatus) -> bool {
19905 matches!(
19906 status,
19907 RunStatus::Pending | RunStatus::Active | RunStatus::CancellationRequested
19908 )
19909}
19910
19911fn validate_tool_invocation_transition_lifecycle(
19912 stored: &StoredRun,
19913 transition: ToolInvocationTransitionKind,
19914) -> Result<(), StoreError> {
19915 let status = stored.lifecycle().status();
19916 let allowed = match transition {
19917 ToolInvocationTransitionKind::StartAttempt => status == RunStatus::Active,
19918 ToolInvocationTransitionKind::RecordResult
19919 | ToolInvocationTransitionKind::RecordError
19920 | ToolInvocationTransitionKind::ReconcileResult
19921 | ToolInvocationTransitionKind::ReconcileError => matches!(
19922 status,
19923 RunStatus::Active | RunStatus::Waiting | RunStatus::CancellationRequested
19924 ),
19925 };
19926 if !allowed {
19927 return Err(StoreError::RunNotRunnable);
19928 }
19929 Ok(())
19930}
19931
19932fn validate_model_invocation_transition_lifecycle(
19933 stored: &StoredRun,
19934 transition: ModelInvocationTransitionKind,
19935) -> Result<(), StoreError> {
19936 let status = stored.lifecycle().status();
19937 let allowed = match transition {
19938 ModelInvocationTransitionKind::StartAttempt => status == RunStatus::Active,
19939 ModelInvocationTransitionKind::RecordResponse
19940 | ModelInvocationTransitionKind::RecordError => matches!(
19941 status,
19942 RunStatus::Active | RunStatus::Waiting | RunStatus::CancellationRequested
19943 ),
19944 };
19945 if !allowed {
19946 return Err(StoreError::RunNotRunnable);
19947 }
19948 Ok(())
19949}
19950
19951fn validate_node_attempt_completion_lifecycle(stored: &StoredRun) -> Result<(), StoreError> {
19952 if matches!(
19953 stored.lifecycle().status(),
19954 RunStatus::Active | RunStatus::Waiting | RunStatus::CancellationRequested
19955 ) {
19956 Ok(())
19957 } else {
19958 Err(StoreError::RunNotRunnable)
19959 }
19960}
19961
19962fn positive_sequence(value: i64) -> Result<JournalSequence, StoreError> {
19963 let value = u64::try_from(value).map_err(|_| StoreError::corrupt("journal sequence"))?;
19964 JournalSequence::new(value).map_err(|_| StoreError::corrupt("journal sequence"))
19965}
19966
19967fn nonnegative_superstep(value: i64) -> Result<Superstep, StoreError> {
19968 let value = u64::try_from(value).map_err(|_| StoreError::corrupt("checkpoint superstep"))?;
19969 Superstep::new(value).map_err(|_| StoreError::corrupt("checkpoint superstep"))
19970}
19971
19972fn decode_digest(bytes: &[u8], record: &'static str) -> Result<Digest, StoreError> {
19973 let bytes: [u8; Digest::SHA256_LEN] =
19974 bytes.try_into().map_err(|_| StoreError::corrupt(record))?;
19975 Ok(Digest::from_sha256(bytes))
19976}
19977
19978fn from_database_time(value: DateTime<Utc>) -> Result<Timestamp, StoreError> {
19979 Timestamp::from_unix_micros(value.timestamp_micros())
19980 .map_err(|_| StoreError::corrupt("timestamp"))
19981}
19982
19983fn to_database_time(value: Timestamp) -> Result<DateTime<Utc>, StoreError> {
19984 DateTime::from_timestamp_micros(value.unix_micros())
19985 .ok_or_else(|| StoreError::encoding("timestamp"))
19986}
19987
19988fn add_duration(value: Timestamp, duration: Duration) -> Result<Timestamp, StoreError> {
19989 let microseconds =
19990 i64::try_from(duration.as_micros()).map_err(|_| StoreError::encoding("lease duration"))?;
19991 let value = value
19992 .unix_micros()
19993 .checked_add(microseconds)
19994 .ok_or_else(|| StoreError::encoding("lease expiry"))?;
19995 Timestamp::from_unix_micros(value).map_err(|_| StoreError::encoding("lease expiry"))
19996}
19997
19998const fn run_status_text(status: RunStatus) -> &'static str {
19999 match status {
20000 RunStatus::Pending => "pending",
20001 RunStatus::Active => "active",
20002 RunStatus::Waiting => "waiting",
20003 RunStatus::CancellationRequested => "cancellation_requested",
20004 RunStatus::Succeeded => "succeeded",
20005 RunStatus::Failed => "failed",
20006 RunStatus::Cancelled => "cancelled",
20007 }
20008}
20009
20010const fn interrupt_kind_text(kind: RunInterruptKind) -> &'static str {
20011 match kind {
20012 RunInterruptKind::Approval => "approval",
20013 RunInterruptKind::Input => "input",
20014 RunInterruptKind::Authentication => "authentication",
20015 RunInterruptKind::ExternalSignal => "external_signal",
20016 RunInterruptKind::Reconciliation => "reconciliation",
20017 }
20018}
20019
20020const fn timer_kind_text(kind: RunTimerKind) -> &'static str {
20021 match kind {
20022 RunTimerKind::Sleep => "sleep",
20023 RunTimerKind::RetryBackoff => "retry_backoff",
20024 }
20025}
20026
20027const fn tool_invocation_status_text(status: ToolInvocationStatus) -> &'static str {
20028 match status {
20029 ToolInvocationStatus::Prepared => "prepared",
20030 ToolInvocationStatus::Executing => "executing",
20031 ToolInvocationStatus::Committed => "committed",
20032 ToolInvocationStatus::Failed => "failed",
20033 ToolInvocationStatus::Unknown => "unknown",
20034 }
20035}
20036
20037const fn tool_invocation_transition_kind_text(kind: ToolInvocationTransitionKind) -> &'static str {
20038 match kind {
20039 ToolInvocationTransitionKind::StartAttempt => "start_attempt",
20040 ToolInvocationTransitionKind::RecordResult => "record_result",
20041 ToolInvocationTransitionKind::RecordError => "record_error",
20042 ToolInvocationTransitionKind::ReconcileResult => "reconcile_result",
20043 ToolInvocationTransitionKind::ReconcileError => "reconcile_error",
20044 }
20045}
20046
20047const fn model_invocation_status_text(status: ModelInvocationStatus) -> &'static str {
20048 match status {
20049 ModelInvocationStatus::Prepared => "prepared",
20050 ModelInvocationStatus::Executing => "executing",
20051 ModelInvocationStatus::Committed => "committed",
20052 ModelInvocationStatus::Failed => "failed",
20053 }
20054}
20055
20056const fn model_invocation_transition_kind_text(
20057 kind: ModelInvocationTransitionKind,
20058) -> &'static str {
20059 match kind {
20060 ModelInvocationTransitionKind::StartAttempt => "start_attempt",
20061 ModelInvocationTransitionKind::RecordResponse => "record_response",
20062 ModelInvocationTransitionKind::RecordError => "record_error",
20063 }
20064}
20065
20066const fn pending_node_result_control_kind_text(kind: NodeControlKind) -> &'static str {
20067 match kind {
20068 NodeControlKind::Continue => "continue",
20069 NodeControlKind::Route => "route",
20070 NodeControlKind::Wait => "wait",
20071 NodeControlKind::Terminal => "terminal",
20072 }
20073}
20074
20075fn map_pending_node_result_commit_error(error: &PendingNodeResultError) -> StoreError {
20076 match error {
20077 PendingNodeResultError::JournalNotAfterBinding
20078 | PendingNodeResultError::BindingClockRegression => {
20079 StoreError::InvalidPendingNodeResultBinding
20080 }
20081 _ => StoreError::InvalidPendingNodeResult,
20082 }
20083}
20084
20085fn map_event_commit_error(error: &JournalEventError) -> StoreError {
20086 match error {
20087 JournalEventError::SequenceOverflow => StoreError::JournalSequenceExhausted,
20088 _ => StoreError::encoding("journal event"),
20089 }
20090}
20091
20092fn has_database_error_code(error: &sqlx_core::Error, expected: &str) -> bool {
20093 matches!(
20094 error,
20095 sqlx_core::Error::Database(database)
20096 if database.code().is_some_and(|code| code.as_ref() == expected)
20097 )
20098}
20099
20100fn has_database_constraint(error: &sqlx_core::Error, expected: &str) -> bool {
20101 matches!(
20102 error,
20103 sqlx_core::Error::Database(database)
20104 if database.constraint().is_some_and(|constraint| constraint == expected)
20105 )
20106}
20107
20108fn is_invalid_pending_binding_constraint(error: &sqlx_core::Error) -> bool {
20109 matches!(
20110 error,
20111 sqlx_core::Error::Database(database)
20112 if database.constraint().is_some_and(|constraint| matches!(
20113 constraint,
20114 "pending_node_result_tool_bindings_activation_fk"
20115 | "pending_node_result_tool_bindings_revision_fk"
20116 | "pending_node_result_tool_bindings_status_valid"
20117 | "pending_node_result_tool_bindings_causal"
20118 | "pending_node_result_tool_bindings_once"
20119 | "pending_node_result_tool_bindings_pkey"
20120 | "pending_node_result_model_bindings_activation_fk"
20121 | "pending_node_result_model_bindings_revision_fk"
20122 | "pending_node_result_model_bindings_status_valid"
20123 | "pending_node_result_model_bindings_causal"
20124 | "pending_node_result_model_bindings_once"
20125 | "pending_node_result_model_bindings_pkey"
20126 ))
20127 )
20128}
20129
20130#[cfg(test)]
20131mod tests {
20132 use super::*;
20133 use crate::{ConfigurationError, PostgresTransportSecurity};
20134
20135 #[tokio::test]
20136 async fn complete_on_drop_finishes_an_in_flight_future_after_cancellation() {
20137 let (started_sender, started_receiver) = tokio::sync::oneshot::channel();
20138 let (release_sender, release_receiver) = tokio::sync::oneshot::channel();
20139 let (completed_sender, completed_receiver) = tokio::sync::oneshot::channel();
20140
20141 let task = tokio::spawn(CompleteOnDrop::new(async move {
20142 started_sender.send(()).expect("start observer is live");
20143 release_receiver.await.expect("release signal is live");
20144 completed_sender
20145 .send(())
20146 .expect("completion observer is live");
20147 }));
20148
20149 started_receiver.await.expect("future started");
20150 task.abort();
20151 assert!(task.await.expect_err("task was cancelled").is_cancelled());
20152 release_sender.send(()).expect("cleanup task is live");
20153 tokio::time::timeout(Duration::from_secs(1), completed_receiver)
20154 .await
20155 .expect("cleanup completed before the deadline")
20156 .expect("completion signal was delivered");
20157 }
20158
20159 #[test]
20160 fn store_debug_and_configuration_do_not_expose_urls() {
20161 let options = PostgresStoreOptions::default();
20162 assert_eq!(
20163 options.transport_security(),
20164 PostgresTransportSecurity::VerifyFull
20165 );
20166 assert_eq!(
20167 options.clone().with_pool_size(2, 1).validate(),
20168 Err(ConfigurationError::PoolMinimumExceedsMaximum)
20169 );
20170 assert_eq!(
20171 options
20172 .clone()
20173 .with_transaction_timeouts(Duration::from_secs(2), Duration::from_secs(1))
20174 .validate(),
20175 Err(ConfigurationError::LockTimeoutNotBelowStatementTimeout)
20176 );
20177 assert_eq!(
20178 options
20179 .with_lease_timing(Duration::from_secs(60), Duration::from_secs(30))
20180 .validate(),
20181 Err(ConfigurationError::LeaseDurationExceedsMaximumHorizon)
20182 );
20183 }
20184
20185 #[test]
20186 fn database_timing_configuration_is_exactly_representable() {
20187 let rounded = PostgresStoreOptions::default()
20188 .with_transaction_timeouts(Duration::from_nanos(1), Duration::from_nanos(1_000_001));
20189 assert_eq!(rounded.lock_timeout_setting(), "1ms");
20190 assert_eq!(rounded.statement_timeout_setting(), "2ms");
20191 assert!(rounded.validate().is_ok());
20192
20193 assert_eq!(
20194 PostgresStoreOptions::default()
20195 .with_transaction_timeouts(Duration::from_nanos(1), Duration::from_nanos(2))
20196 .validate(),
20197 Err(ConfigurationError::LockTimeoutNotBelowStatementTimeout)
20198 );
20199 assert_eq!(
20200 PostgresStoreOptions::default()
20201 .with_lease_timing(Duration::from_nanos(1_500), Duration::from_secs(1))
20202 .validate(),
20203 Err(ConfigurationError::LeaseTimingNotMicrosecondAligned {
20204 name: "lease duration"
20205 })
20206 );
20207 assert_eq!(
20208 PostgresStoreOptions::default()
20209 .with_transaction_timeouts(
20210 Duration::from_millis(i32::MAX as u64 + 1),
20211 Duration::from_millis(i32::MAX as u64 + 2),
20212 )
20213 .validate(),
20214 Err(ConfigurationError::PostgresTimeoutTooLarge {
20215 name: "lock timeout"
20216 })
20217 );
20218 assert_eq!(
20219 PostgresStoreOptions::default()
20220 .with_outbox_attempt_lease(Duration::ZERO)
20221 .validate(),
20222 Err(ConfigurationError::ZeroDuration {
20223 name: "outbox attempt lease duration"
20224 })
20225 );
20226 assert_eq!(
20227 PostgresStoreOptions::default()
20228 .with_outbox_attempt_lease(Duration::from_nanos(1_500))
20229 .validate(),
20230 Err(ConfigurationError::LeaseTimingNotMicrosecondAligned {
20231 name: "outbox attempt lease duration"
20232 })
20233 );
20234 assert!(
20235 PostgresStoreOptions::default()
20236 .with_outbox_attempt_lease(Duration::from_secs(5 * 60))
20237 .validate()
20238 .is_ok()
20239 );
20240 assert_eq!(
20241 PostgresStoreOptions::default()
20242 .with_outbox_attempt_lease(Duration::from_micros(300_000_001))
20243 .validate(),
20244 Err(ConfigurationError::OutboxAttemptLeaseTooLong)
20245 );
20246 }
20247
20248 #[test]
20249 fn page_size_is_strictly_bounded() {
20250 assert!(JournalPageSize::new(1).is_ok());
20251 assert!(JournalPageSize::new(JournalPageSize::MAX).is_ok());
20252 assert!(JournalPageSize::new(0).is_err());
20253 assert!(JournalPageSize::new(JournalPageSize::MAX + 1).is_err());
20254
20255 assert!(CheckpointLineagePageSize::new(1).is_ok());
20256 assert!(CheckpointLineagePageSize::new(CheckpointLineagePageSize::MAX).is_ok());
20257 assert!(CheckpointLineagePageSize::new(0).is_err());
20258 assert!(CheckpointLineagePageSize::new(CheckpointLineagePageSize::MAX + 1).is_err());
20259
20260 assert!(ToolInvocationHistoryPageSize::new(1).is_ok());
20261 assert!(ToolInvocationHistoryPageSize::new(ToolInvocationHistoryPageSize::MAX).is_ok());
20262 assert!(ToolInvocationHistoryPageSize::new(0).is_err());
20263 assert!(
20264 ToolInvocationHistoryPageSize::new(ToolInvocationHistoryPageSize::MAX + 1).is_err()
20265 );
20266
20267 assert!(ModelInvocationHistoryPageSize::new(1).is_ok());
20268 assert!(ModelInvocationHistoryPageSize::new(0).is_err());
20269 assert!(ModelInvocationHistoryPageSize::new(2).is_err());
20270
20271 assert!(PendingNodeResultPageSize::new(1).is_ok());
20272 assert!(PendingNodeResultPageSize::new(PendingNodeResultPageSize::MAX).is_ok());
20273 assert!(PendingNodeResultPageSize::new(0).is_err());
20274 assert!(PendingNodeResultPageSize::new(PendingNodeResultPageSize::MAX + 1).is_err());
20275
20276 assert!(RunnableRunPageSize::new(1).is_ok());
20277 assert!(RunnableRunPageSize::new(RunnableRunPageSize::MAX).is_ok());
20278 assert!(RunnableRunPageSize::new(0).is_err());
20279 assert!(RunnableRunPageSize::new(RunnableRunPageSize::MAX + 1).is_err());
20280
20281 assert!(OutboxAttemptHistoryPageSize::new(1).is_ok());
20282 assert!(OutboxAttemptHistoryPageSize::new(OutboxAttemptHistoryPageSize::MAX).is_ok());
20283 assert!(OutboxAttemptHistoryPageSize::new(0).is_err());
20284 assert!(OutboxAttemptHistoryPageSize::new(OutboxAttemptHistoryPageSize::MAX + 1).is_err());
20285 }
20286
20287 #[test]
20288 fn retry_classification_is_conservative() {
20289 assert!(StoreError::database("test", sqlx_core::Error::PoolTimedOut).is_retryable());
20290 assert!(!StoreError::RunNotFound.is_retryable());
20291 }
20292
20293 #[test]
20294 fn projection_intent_digests_are_versioned_and_frozen() {
20295 let unchanged = projection_digest(&RunProjection::unchanged()).unwrap();
20296 assert_eq!(
20297 unchanged.to_string(),
20298 "sha256:c1bee6a7d79cfc7b48f7dc9bbf625ccd3f1bf87bfdb2d624fe40517ad19534d8"
20299 );
20300
20301 let transition = RunProjection::transition(
20302 RunRevision::ZERO,
20303 RunTransition::Start {
20304 started_at: "2030-01-01T00:00:01.000000Z".parse().unwrap(),
20305 },
20306 );
20307 assert_eq!(
20308 projection_digest(&transition).unwrap().to_string(),
20309 "sha256:9f2dfecd7af4cee03f6cc1b9f95ac3e3f191303def4b037ea1c6ac29df0e225b"
20310 );
20311
20312 let barrier_intent =
20313 "sha256:3b454071dfa5f18b7dd7ac7084236afdf88e5f73fc3aa4382d32f82ca974112b"
20314 .parse()
20315 .unwrap();
20316 assert_eq!(
20317 barrier_projection_digest(&RunProjection::unchanged(), barrier_intent)
20318 .unwrap()
20319 .to_string(),
20320 "sha256:dafe1bdef91699c3db6ce24b1272af5bb6da7fbd7055d25bc0cd2223bb29b195"
20321 );
20322 }
20323}