Skip to main content

runledger_postgres/
migrations.rs

1use std::collections::HashMap;
2use std::fmt;
3
4use sqlx::migrate::{AppliedMigration, Migrate, MigrateError, Migrator};
5
6use crate::DbPool;
7
8/// Raw SQLx migrator for inspecting the migrations bundled with this crate
9/// version.
10///
11/// Iterating this value to inspect bundled versions and checksums is supported.
12/// Calling [`Migrator::run`] or [`Migrator::undo`] on it with a shared
13/// application pool is not. SQLx rejects applied versions absent from the exact
14/// bundle, and PostgreSQL migration locks are session-scoped; SQLx can return
15/// from a validation error before unlocking and put the still-locked session
16/// back into the pool.
17///
18/// Use [`migrate_after_idempotency_cutover`] to apply Runledger migrations, or
19/// [`ensure_schema_compatible_after_idempotency_cutover`] when DDL is managed
20/// externally. If a compatibility diagnostic intentionally executes a raw
21/// migrator that may mismatch history, give it a disposable connection or
22/// single-use pool and close that connection or pool on every error path.
23///
24/// During an additive compatibility window, an exact older binary that cannot
25/// use the filtered API must be patched before startup or explicitly accept the
26/// data-loss boundary of reverting newer migrations. Reverting the 0.8
27/// migrations erases workflow-recovery lineage/idempotency, active claims,
28/// execution-resource keys and claims, retry audit fields, and workflow-step
29/// continuation opt-ins. Reverting the post-v0.6 successful-replay migration
30/// also erases relational replay lineage and replay-request idempotency state
31/// while retaining the underlying replay-created queue rows.
32pub static MIGRATOR: Migrator = sqlx::migrate!("./migrations");
33
34type PgPoolConnection = sqlx::pool::PoolConnection<sqlx::Postgres>;
35type RunledgerMigrationMap = HashMap<i64, &'static sqlx::migrate::Migration>;
36
37const WORKFLOW_STEP_JOB_LINK_CONTRACT_MIGRATION_VERSION: i64 = 202608240002;
38const AFTER_ROW_INSERT_OR_UPDATE_TRIGGER_TYPE: i16 = 21;
39
40// Trigger names remain exact because the contract migration drops these named
41// objects before removing job_queue.workflow_step_id. The validator below
42// otherwise checks the safety properties needed during the mixed-version
43// window, not byte-for-byte DDL identity: firing on additional UPDATE columns
44// is safe, while changing events, timing, functions, or deferral is not.
45// Migration checksums establish the function source applied initially. This
46// read-only guard validates live catalog wiring and reciprocal data; it does
47// not try to detect privileged post-migration function-body replacement by
48// embedding a second, brittle copy of each PL/pgSQL body.
49#[derive(Clone, Copy)]
50struct WorkflowJobLinkTriggerSpec {
51    table_name: &'static str,
52    trigger_name: &'static str,
53    function_name: &'static str,
54    update_column_name: &'static str,
55    constraint_mode: WorkflowJobLinkTriggerConstraintMode,
56}
57
58#[derive(Clone, Copy)]
59enum WorkflowJobLinkTriggerConstraintMode {
60    Deferred,
61    Ordinary,
62}
63
64const WORKFLOW_JOB_LINK_EXPAND_TRIGGER_SPECS: [WorkflowJobLinkTriggerSpec; 3] = [
65    WorkflowJobLinkTriggerSpec {
66        table_name: "job_queue",
67        trigger_name: "trg_job_queue_workflow_step_linkage_symmetry",
68        function_name: "enforce_workflow_job_linkage_symmetry",
69        update_column_name: "workflow_step_id",
70        constraint_mode: WorkflowJobLinkTriggerConstraintMode::Deferred,
71    },
72    WorkflowJobLinkTriggerSpec {
73        table_name: "workflow_steps",
74        trigger_name: "trg_workflow_steps_job_linkage_symmetry",
75        function_name: "enforce_workflow_job_linkage_symmetry",
76        update_column_name: "job_id",
77        constraint_mode: WorkflowJobLinkTriggerConstraintMode::Deferred,
78    },
79    WorkflowJobLinkTriggerSpec {
80        table_name: "workflow_steps",
81        trigger_name: "trg_workflow_steps_job_linkage_compatibility",
82        function_name: "project_workflow_step_job_linkage_compatibility",
83        update_column_name: "job_id",
84        constraint_mode: WorkflowJobLinkTriggerConstraintMode::Ordinary,
85    },
86];
87
88/// One reason an expand-window workflow/job-link trigger is unsafe.
89#[derive(Clone, Copy, Debug, Eq, PartialEq)]
90#[non_exhaustive]
91pub enum WorkflowJobLinkTriggerProblem {
92    Missing,
93    WrongFunction,
94    NotEnabledForOriginWrites,
95    InternallyGenerated,
96    WrongFiringEvents,
97    UpdateColumnNotCovered,
98    UnexpectedTriggerArguments,
99    UnexpectedWhenCondition,
100    WrongConstraintMode,
101}
102
103impl fmt::Display for WorkflowJobLinkTriggerProblem {
104    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
105        f.write_str(match self {
106            Self::Missing => "missing from the expected public table",
107            Self::WrongFunction => {
108                "does not call the expected public zero-argument trigger function"
109            }
110            Self::NotEnabledForOriginWrites => "does not fire for origin/local writes",
111            Self::InternallyGenerated => "is internally generated instead of user-defined",
112            Self::WrongFiringEvents => "is not an AFTER ROW INSERT OR UPDATE trigger",
113            Self::UpdateColumnNotCovered => "does not fire when the relationship column is updated",
114            Self::UnexpectedTriggerArguments => "passes unexpected trigger arguments",
115            Self::UnexpectedWhenCondition => "has an unexpected WHEN condition",
116            Self::WrongConstraintMode => "has the wrong constraint or deferral mode",
117        })
118    }
119}
120
121/// Validation details for one unsafe expand-window workflow/job-link trigger.
122#[derive(Clone, Debug, Eq, PartialEq)]
123#[non_exhaustive]
124pub struct WorkflowJobLinkTriggerDiagnostic {
125    table_name: &'static str,
126    trigger_name: &'static str,
127    problems: Vec<WorkflowJobLinkTriggerProblem>,
128}
129
130impl WorkflowJobLinkTriggerDiagnostic {
131    #[must_use]
132    pub const fn table_name(&self) -> &str {
133        self.table_name
134    }
135
136    #[must_use]
137    pub const fn trigger_name(&self) -> &str {
138        self.trigger_name
139    }
140
141    #[must_use]
142    pub fn problems(&self) -> &[WorkflowJobLinkTriggerProblem] {
143        &self.problems
144    }
145}
146
147impl fmt::Display for WorkflowJobLinkTriggerDiagnostic {
148    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
149        write!(f, "public.{}.{}: ", self.table_name, self.trigger_name)?;
150        for (index, problem) in self.problems.iter().enumerate() {
151            if index != 0 {
152                f.write_str(", ")?;
153            }
154            write!(f, "{problem}")?;
155        }
156        Ok(())
157    }
158}
159
160#[derive(Debug)]
161#[non_exhaustive]
162pub enum SchemaCompatibilityError {
163    Query(sqlx::Error),
164    MissingMigrationHistory {
165        required_first_migration_version: i64,
166    },
167    LegacyIdempotencySnapshotsMissing {
168        job_count: i64,
169        workflow_count: i64,
170    },
171    WorkflowJobLinkExpandInvalid {
172        compatibility_trigger_count: i64,
173        inconsistent_link_count: i64,
174    },
175    WorkflowJobLinkExpandTriggersInvalid {
176        trigger_diagnostics: Vec<WorkflowJobLinkTriggerDiagnostic>,
177        inconsistent_link_count: i64,
178    },
179    Incompatible(MigrateError),
180    MigrationUnlock(MigrateError),
181}
182
183impl fmt::Display for SchemaCompatibilityError {
184    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
185        match self {
186            Self::Query(error) => write!(
187                f,
188                "Runledger schema compatibility check could not query PostgreSQL state: {error}"
189            ),
190            Self::MissingMigrationHistory {
191                required_first_migration_version,
192            } => write!(
193                f,
194                "Runledger schema compatibility check requires the _sqlx_migrations table; apply or record Runledger migrations first (expected migration history starting at version {required_first_migration_version})"
195            ),
196            Self::LegacyIdempotencySnapshotsMissing {
197                job_count,
198                workflow_count,
199            } => write!(
200                f,
201                "Runledger idempotency cutover requires enqueue_request snapshots for all keyed rows; found {job_count} legacy job rows and {workflow_count} legacy workflow rows"
202            ),
203            Self::WorkflowJobLinkExpandInvalid {
204                compatibility_trigger_count,
205                inconsistent_link_count,
206            } => write!(
207                f,
208                "Runledger workflow-step/job expand schema requires all three expand-window triggers and empty reciprocal anti-joins before the contract migration; found {compatibility_trigger_count} valid triggers and {inconsistent_link_count} inconsistent relationships"
209            ),
210            Self::WorkflowJobLinkExpandTriggersInvalid {
211                trigger_diagnostics,
212                inconsistent_link_count,
213            } => {
214                f.write_str(
215                    "Runledger workflow-step/job expand schema has invalid expand-window triggers: ",
216                )?;
217                for (index, diagnostic) in trigger_diagnostics.iter().enumerate() {
218                    if index != 0 {
219                        f.write_str("; ")?;
220                    }
221                    write!(f, "{diagnostic}")?;
222                }
223                write!(
224                    f,
225                    "; found {inconsistent_link_count} inconsistent relationships"
226                )
227            }
228            Self::Incompatible(error) => write!(f, "{error}"),
229            Self::MigrationUnlock(error) => {
230                write!(
231                    f,
232                    "Runledger schema migration lock could not be released: {error}"
233                )
234            }
235        }
236    }
237}
238
239impl std::error::Error for SchemaCompatibilityError {
240    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
241        match self {
242            Self::Query(error) => Some(error),
243            Self::MissingMigrationHistory { .. } => None,
244            Self::LegacyIdempotencySnapshotsMissing { .. } => None,
245            Self::WorkflowJobLinkExpandInvalid { .. }
246            | Self::WorkflowJobLinkExpandTriggersInvalid { .. } => None,
247            Self::Incompatible(error) | Self::MigrationUnlock(error) => Some(error),
248        }
249    }
250}
251
252impl From<MigrateError> for SchemaCompatibilityError {
253    fn from(error: MigrateError) -> Self {
254        Self::Incompatible(error)
255    }
256}
257
258impl From<sqlx::Error> for SchemaCompatibilityError {
259    fn from(error: sqlx::Error) -> Self {
260        Self::Query(error)
261    }
262}
263
264/// Apply the bundled Runledger schema migrations to a PostgreSQL pool, then
265/// enforce the idempotency snapshot cutover.
266///
267/// This is intentionally named as a hard-cutover API. Downstream applications
268/// upgrading from older Runledger versions must update their startup code and
269/// verify no keyed legacy rows remain without `enqueue_request` snapshots.
270/// This function applies every pending bundled migration immediately, including
271/// the workflow-step/job-link contract migration that removes
272/// `job_queue.workflow_step_id` and crosses the 0.10 rollback boundary. For a
273/// mixed-version rollout, apply the expand migration externally, deploy and
274/// drain all 0.10 writers and leases, then apply the contract migration; use
275/// [`ensure_schema_compatible_after_idempotency_cutover`] for startup checks
276/// during that staged rollout.
277///
278/// Unlike raw [`MIGRATOR`] execution, this filters shared SQLx history through
279/// Runledger's migration compatibility fence so declared additive migrations
280/// can coexist with older compatible startup code.
281pub async fn migrate_after_idempotency_cutover(
282    pool: &DbPool,
283) -> Result<(), SchemaCompatibilityError> {
284    let mut conn = pool.acquire().await?;
285
286    if MIGRATOR.locking {
287        // PostgreSQL advisory migration locks are session-scoped; never return
288        // a possibly locked session to the pool if this future is cancelled.
289        conn.close_on_drop();
290        (*conn)
291            .lock()
292            .await
293            .map_err(SchemaCompatibilityError::Incompatible)?;
294    }
295
296    let result = run_migrations_with_filtered_history(&mut conn).await;
297    let unlock_result = if MIGRATOR.locking {
298        (*conn).unlock().await
299    } else {
300        Ok(())
301    };
302
303    match (result, unlock_result) {
304        (Err(migration_error), Err(unlock_error)) => {
305            tracing::error!(
306                error = %unlock_error,
307                "failed to unlock migration lock after migration failure"
308            );
309            Err(SchemaCompatibilityError::Incompatible(migration_error))
310        }
311        (Err(error), Ok(())) => Err(SchemaCompatibilityError::Incompatible(error)),
312        (Ok(()), Err(error)) => Err(SchemaCompatibilityError::MigrationUnlock(error)),
313        (Ok(()), Ok(())) => {
314            // The DDL migration lock is no longer needed here: the NOT VALID
315            // cutover constraints already block new violating rows, and
316            // validation is idempotent if another startup validates first.
317            reject_legacy_idempotency_rows(&mut conn).await?;
318            validate_idempotency_cutover_constraints(&mut conn).await
319        }
320    }
321}
322
323/// Apply the bundled Runledger schema migrations to a PostgreSQL pool.
324///
325/// Deprecated compatibility alias for [`migrate_after_idempotency_cutover`].
326/// The current migration set enforces the enqueue request snapshot cutover, so
327/// this function has the same strict behavior as the new explicit API.
328#[deprecated(
329    since = "0.1.2",
330    note = "use migrate_after_idempotency_cutover to make the enqueue request snapshot cutover explicit"
331)]
332pub async fn migrate(pool: &DbPool) -> Result<(), SchemaCompatibilityError> {
333    migrate_after_idempotency_cutover(pool).await
334}
335
336/// Validate that the target database's SQLx migration history matches the
337/// bundled Runledger migrations.
338///
339/// Unlike [`migrate_after_idempotency_cutover`], this does not apply pending
340/// migrations. It is intended
341/// for deployments that manage DDL outside the application process but still
342/// want a startup guardrail. This check is read-only, but it relies on the
343/// `_sqlx_migrations` history table being present and up to date. When present,
344/// it also uses Runledger's own `runledger_migration_history` compatibility
345/// fence to detect newer releases whose schema is not declared backward
346/// compatible. Additive migrations may deliberately rely only on SQLx history
347/// so older guards can coexist during expand-first rollout.
348/// The workflow-step/job contract migration may also remain pending while its
349/// expand migration's compatibility projection is present and consistent.
350/// This differs from invoking raw [`MIGRATOR`] execution, which rejects any
351/// applied migration version absent from that exact binary's bundle.
352///
353/// This read-only path does not validate `NOT VALID` cutover constraints after
354/// legacy rows are remediated. Deployments that apply DDL externally can run
355/// PostgreSQL `VALIDATE CONSTRAINT` for the idempotency cutover constraints
356/// after this check passes, or use [`migrate_after_idempotency_cutover`] to let
357/// Runledger do that promotion.
358pub async fn ensure_schema_compatible_after_idempotency_cutover(
359    pool: &DbPool,
360) -> Result<(), SchemaCompatibilityError> {
361    let mut conn = pool.acquire().await?;
362
363    if !has_migrations_table(&mut conn).await? {
364        return Err(SchemaCompatibilityError::MissingMigrationHistory {
365            required_first_migration_version: first_up_migration_version(),
366        });
367    }
368
369    let expected_migrations = expected_runledger_migrations();
370    let history = list_migration_history(&mut conn).await?;
371
372    if let Some(version) = first_conflicting_runledger_version(&history, &expected_migrations) {
373        return Err(SchemaCompatibilityError::Incompatible(
374            MigrateError::VersionMismatch(version),
375        ));
376    }
377
378    if let Some(version) = first_dirty_runledger_version(&history, &expected_migrations) {
379        return Err(SchemaCompatibilityError::Incompatible(MigrateError::Dirty(
380            version,
381        )));
382    }
383
384    if has_runledger_migration_history_table(&mut conn).await? {
385        let recorded_versions = list_recorded_runledger_migrations(&mut conn).await?;
386        if let Some(version) =
387            first_missing_runledger_version(&recorded_versions, &expected_migrations)
388        {
389            return Err(SchemaCompatibilityError::Incompatible(
390                MigrateError::VersionMissing(version),
391            ));
392        }
393    }
394
395    let applied = applied_runledger_migrations(&history, &expected_migrations);
396    let applied_by_version: HashMap<_, _> = applied
397        .iter()
398        .map(|applied_migration| (applied_migration.version, applied_migration))
399        .collect();
400    let latest_applied_version = applied.iter().map(|migration| migration.version).max();
401
402    for migration in MIGRATOR
403        .iter()
404        .filter(|migration| migration.migration_type.is_up_migration())
405    {
406        match applied_by_version.get(&migration.version) {
407            Some(applied_migration) => {
408                validate_checksum(migration.version, applied_migration, migration)
409                    .map_err(SchemaCompatibilityError::from)?
410            }
411            None => {
412                if migration.version == WORKFLOW_STEP_JOB_LINK_CONTRACT_MIGRATION_VERSION {
413                    continue;
414                }
415                return Err(SchemaCompatibilityError::Incompatible(
416                    MigrateError::VersionTooNew(
417                        migration.version,
418                        latest_applied_version.unwrap_or_default(),
419                    ),
420                ));
421            }
422        }
423    }
424
425    validate_workflow_job_link_expand_schema(&mut conn).await?;
426    reject_legacy_idempotency_rows(&mut conn).await
427}
428
429/// Validate that the target database's SQLx migration history matches the
430/// bundled Runledger migrations.
431///
432/// Deprecated compatibility alias for
433/// [`ensure_schema_compatible_after_idempotency_cutover`]. The current schema
434/// compatibility check rejects keyed legacy rows without enqueue request
435/// snapshots, matching the stricter cutover API.
436#[deprecated(
437    since = "0.1.2",
438    note = "use ensure_schema_compatible_after_idempotency_cutover to make the enqueue request snapshot cutover explicit"
439)]
440pub async fn ensure_schema_compatible(pool: &DbPool) -> Result<(), SchemaCompatibilityError> {
441    ensure_schema_compatible_after_idempotency_cutover(pool).await
442}
443
444async fn has_migrations_table(conn: &mut PgPoolConnection) -> Result<bool, sqlx::Error> {
445    sqlx::query_scalar::<_, bool>("SELECT to_regclass('_sqlx_migrations') IS NOT NULL")
446        .fetch_one(&mut **conn)
447        .await
448}
449
450async fn has_runledger_migration_history_table(
451    conn: &mut PgPoolConnection,
452) -> Result<bool, sqlx::Error> {
453    sqlx::query_scalar::<_, bool>("SELECT to_regclass('runledger_migration_history') IS NOT NULL")
454        .fetch_one(&mut **conn)
455        .await
456}
457
458async fn list_migration_history(
459    conn: &mut PgPoolConnection,
460) -> Result<Vec<MigrationHistoryRow>, sqlx::Error> {
461    sqlx::query_as::<_, MigrationHistoryRow>(
462        "SELECT version, checksum, success
463         FROM _sqlx_migrations
464         ORDER BY version",
465    )
466    .fetch_all(&mut **conn)
467    .await
468}
469
470async fn list_recorded_runledger_migrations(
471    conn: &mut PgPoolConnection,
472) -> Result<Vec<i64>, sqlx::Error> {
473    sqlx::query_scalar::<_, i64>(
474        "SELECT version
475         FROM runledger_migration_history
476         ORDER BY version",
477    )
478    .fetch_all(&mut **conn)
479    .await
480}
481
482async fn reject_legacy_idempotency_rows(
483    conn: &mut PgPoolConnection,
484) -> Result<(), SchemaCompatibilityError> {
485    if idempotency_cutover_constraints_valid(conn).await? {
486        return Ok(());
487    }
488
489    let row = sqlx::query!(
490        r#"SELECT
491            (
492                SELECT COUNT(*)::bigint
493                FROM job_queue
494                WHERE idempotency_key IS NOT NULL
495                  AND enqueue_request IS NULL
496            ) AS "job_count!",
497            (
498                SELECT COUNT(*)::bigint
499                FROM workflow_runs
500                WHERE idempotency_key IS NOT NULL
501                  AND enqueue_request IS NULL
502            ) AS "workflow_count!""#,
503    )
504    .fetch_one(&mut **conn)
505    .await?;
506
507    if row.job_count == 0 && row.workflow_count == 0 {
508        return Ok(());
509    }
510
511    Err(
512        SchemaCompatibilityError::LegacyIdempotencySnapshotsMissing {
513            job_count: row.job_count,
514            workflow_count: row.workflow_count,
515        },
516    )
517}
518
519async fn validate_workflow_job_link_expand_schema(
520    conn: &mut PgPoolConnection,
521) -> Result<(), SchemaCompatibilityError> {
522    let deprecated_column_exists = sqlx::query_scalar::<_, bool>(
523        "SELECT EXISTS (
524            SELECT 1
525            FROM information_schema.columns
526            WHERE table_schema = 'public'
527              AND table_name = 'job_queue'
528              AND column_name = 'workflow_step_id'
529         )",
530    )
531    .fetch_one(&mut **conn)
532    .await?;
533    if !deprecated_column_exists {
534        return Ok(());
535    }
536
537    let trigger_catalog = workflow_job_link_trigger_catalog(conn).await?;
538    let trigger_diagnostics = workflow_job_link_trigger_diagnostics(&trigger_catalog);
539    let inconsistent_link_count = sqlx::query_scalar::<_, i64>(
540        "SELECT count(*)
541         FROM (
542            SELECT jq.id
543            FROM job_queue jq
544            WHERE jq.workflow_step_id IS NOT NULL
545              AND NOT EXISTS (
546                  SELECT 1
547                  FROM workflow_steps ws
548                  WHERE ws.id = jq.workflow_step_id
549                    AND ws.job_id = jq.id
550              )
551
552            UNION ALL
553
554            SELECT ws.id
555            FROM workflow_steps ws
556            WHERE ws.job_id IS NOT NULL
557              AND NOT EXISTS (
558                  SELECT 1
559                  FROM job_queue jq
560                  WHERE jq.id = ws.job_id
561                    AND jq.workflow_step_id = ws.id
562              )
563         ) inconsistencies",
564    )
565    .fetch_one(&mut **conn)
566    .await?;
567
568    if !trigger_diagnostics.is_empty() {
569        return Err(
570            SchemaCompatibilityError::WorkflowJobLinkExpandTriggersInvalid {
571                trigger_diagnostics,
572                inconsistent_link_count,
573            },
574        );
575    }
576
577    if inconsistent_link_count == 0 {
578        return Ok(());
579    }
580
581    Err(SchemaCompatibilityError::WorkflowJobLinkExpandInvalid {
582        compatibility_trigger_count: i64::try_from(WORKFLOW_JOB_LINK_EXPAND_TRIGGER_SPECS.len())
583            .expect("workflow job-link trigger count fits i64"),
584        inconsistent_link_count,
585    })
586}
587
588#[derive(Clone, Debug, sqlx::FromRow)]
589struct WorkflowJobLinkTriggerCatalogRow {
590    table_name: String,
591    trigger_name: String,
592    function_schema: String,
593    function_name: String,
594    function_argument_count: i16,
595    returns_trigger: bool,
596    enabled_mode: String,
597    is_internal: bool,
598    trigger_type: i16,
599    update_column_names: Vec<String>,
600    trigger_argument_count: i16,
601    has_when_condition: bool,
602    is_constraint: bool,
603    is_deferrable: bool,
604    is_initially_deferred: bool,
605}
606
607async fn workflow_job_link_trigger_catalog(
608    conn: &mut PgPoolConnection,
609) -> Result<Vec<WorkflowJobLinkTriggerCatalogRow>, sqlx::Error> {
610    sqlx::query_as(
611        "SELECT
612            relation.relname::text AS table_name,
613            trigger_row.tgname::text AS trigger_name,
614            function_namespace.nspname::text AS function_schema,
615            function_row.proname::text AS function_name,
616            function_row.pronargs AS function_argument_count,
617            function_row.prorettype = 'pg_catalog.trigger'::regtype AS returns_trigger,
618            trigger_row.tgenabled::text AS enabled_mode,
619            trigger_row.tgisinternal AS is_internal,
620            trigger_row.tgtype AS trigger_type,
621            ARRAY(
622                SELECT attribute.attname::text
623                FROM unnest(trigger_row.tgattr::smallint[]) WITH ORDINALITY
624                    AS trigger_column(attnum, ordinal)
625                JOIN pg_attribute AS attribute
626                  ON attribute.attrelid = relation.oid
627                 AND attribute.attnum = trigger_column.attnum
628                 AND NOT attribute.attisdropped
629                ORDER BY trigger_column.ordinal
630            ) AS update_column_names,
631            trigger_row.tgnargs AS trigger_argument_count,
632            trigger_row.tgqual IS NOT NULL AS has_when_condition,
633            trigger_row.tgconstraint <> 0 AS is_constraint,
634            trigger_row.tgdeferrable AS is_deferrable,
635            trigger_row.tginitdeferred AS is_initially_deferred
636         FROM pg_namespace AS table_namespace
637         JOIN pg_class AS relation
638           ON relation.relnamespace = table_namespace.oid
639          AND relation.relkind IN ('r', 'p')
640         JOIN pg_trigger AS trigger_row
641           ON trigger_row.tgrelid = relation.oid
642         JOIN pg_proc AS function_row
643           ON function_row.oid = trigger_row.tgfoid
644         JOIN pg_namespace AS function_namespace
645           ON function_namespace.oid = function_row.pronamespace
646         WHERE table_namespace.nspname = 'public'
647           AND (
648                (relation.relname, trigger_row.tgname) = (
649                    'job_queue',
650                    'trg_job_queue_workflow_step_linkage_symmetry'
651                )
652                OR (relation.relname, trigger_row.tgname) = (
653                    'workflow_steps',
654                    'trg_workflow_steps_job_linkage_symmetry'
655                )
656                OR (relation.relname, trigger_row.tgname) = (
657                    'workflow_steps',
658                    'trg_workflow_steps_job_linkage_compatibility'
659                )
660           )",
661    )
662    .fetch_all(&mut **conn)
663    .await
664}
665
666fn workflow_job_link_trigger_diagnostics(
667    catalog: &[WorkflowJobLinkTriggerCatalogRow],
668) -> Vec<WorkflowJobLinkTriggerDiagnostic> {
669    WORKFLOW_JOB_LINK_EXPAND_TRIGGER_SPECS
670        .iter()
671        .filter_map(|spec| {
672            let Some(trigger) = catalog.iter().find(|trigger| {
673                trigger.table_name == spec.table_name && trigger.trigger_name == spec.trigger_name
674            }) else {
675                return Some(WorkflowJobLinkTriggerDiagnostic {
676                    table_name: spec.table_name,
677                    trigger_name: spec.trigger_name,
678                    problems: vec![WorkflowJobLinkTriggerProblem::Missing],
679                });
680            };
681
682            let problems = workflow_job_link_trigger_problems(spec, trigger);
683            (!problems.is_empty()).then_some(WorkflowJobLinkTriggerDiagnostic {
684                table_name: spec.table_name,
685                trigger_name: spec.trigger_name,
686                problems,
687            })
688        })
689        .collect()
690}
691
692fn workflow_job_link_trigger_problems(
693    spec: &WorkflowJobLinkTriggerSpec,
694    trigger: &WorkflowJobLinkTriggerCatalogRow,
695) -> Vec<WorkflowJobLinkTriggerProblem> {
696    let mut problems = Vec::new();
697
698    if trigger.function_schema != "public"
699        || trigger.function_name != spec.function_name
700        || trigger.function_argument_count != 0
701        || !trigger.returns_trigger
702    {
703        problems.push(WorkflowJobLinkTriggerProblem::WrongFunction);
704    }
705    if !matches!(trigger.enabled_mode.as_str(), "O" | "A") {
706        problems.push(WorkflowJobLinkTriggerProblem::NotEnabledForOriginWrites);
707    }
708    if trigger.is_internal {
709        problems.push(WorkflowJobLinkTriggerProblem::InternallyGenerated);
710    }
711    if trigger.trigger_type != AFTER_ROW_INSERT_OR_UPDATE_TRIGGER_TYPE {
712        problems.push(WorkflowJobLinkTriggerProblem::WrongFiringEvents);
713    }
714    if !trigger.update_column_names.is_empty()
715        && !trigger
716            .update_column_names
717            .iter()
718            .any(|column_name| column_name == spec.update_column_name)
719    {
720        problems.push(WorkflowJobLinkTriggerProblem::UpdateColumnNotCovered);
721    }
722    if trigger.trigger_argument_count != 0 {
723        problems.push(WorkflowJobLinkTriggerProblem::UnexpectedTriggerArguments);
724    }
725    if trigger.has_when_condition {
726        problems.push(WorkflowJobLinkTriggerProblem::UnexpectedWhenCondition);
727    }
728
729    let constraint_mode_is_valid = match spec.constraint_mode {
730        WorkflowJobLinkTriggerConstraintMode::Deferred => {
731            trigger.is_constraint && trigger.is_deferrable && trigger.is_initially_deferred
732        }
733        WorkflowJobLinkTriggerConstraintMode::Ordinary => {
734            !trigger.is_constraint && !trigger.is_deferrable && !trigger.is_initially_deferred
735        }
736    };
737    if !constraint_mode_is_valid {
738        problems.push(WorkflowJobLinkTriggerProblem::WrongConstraintMode);
739    }
740
741    problems
742}
743
744async fn validate_idempotency_cutover_constraints(
745    conn: &mut PgPoolConnection,
746) -> Result<(), SchemaCompatibilityError> {
747    if idempotency_cutover_constraints_valid(conn).await? {
748        return Ok(());
749    }
750
751    // PostgreSQL validates each table constraint independently. If one
752    // validation succeeds and the other fails, the next startup skips the valid
753    // constraint and retries the remaining one.
754    sqlx::query(
755        "ALTER TABLE job_queue
756         VALIDATE CONSTRAINT ck_job_queue_idempotency_enqueue_request",
757    )
758    .execute(&mut **conn)
759    .await
760    .map_err(|error| {
761        tracing::warn!(
762            error = %error,
763            "failed to validate job_queue idempotency cutover constraint"
764        );
765        SchemaCompatibilityError::Query(error)
766    })?;
767
768    sqlx::query(
769        "ALTER TABLE workflow_runs
770         VALIDATE CONSTRAINT ck_workflow_runs_idempotency_enqueue_request",
771    )
772    .execute(&mut **conn)
773    .await
774    .map_err(|error| {
775        tracing::warn!(
776            error = %error,
777            "failed to validate workflow_runs idempotency cutover constraint"
778        );
779        SchemaCompatibilityError::Query(error)
780    })?;
781
782    Ok(())
783}
784
785async fn idempotency_cutover_constraints_valid(
786    conn: &mut PgPoolConnection,
787) -> Result<bool, sqlx::Error> {
788    // A validated cutover constraint is the durable proof that legacy keyed rows
789    // without enqueue_request snapshots cannot exist for that table. If future
790    // migrations replace these constraints, they must preserve that invariant
791    // before this short-circuit remains valid.
792    sqlx::query_scalar::<_, bool>(
793        "SELECT COUNT(*) FILTER (WHERE c.convalidated) = 2
794         FROM pg_constraint c
795         JOIN pg_class t ON t.oid = c.conrelid
796         WHERE (t.relname, c.conname) IN (
797             ('job_queue', 'ck_job_queue_idempotency_enqueue_request'),
798             ('workflow_runs', 'ck_workflow_runs_idempotency_enqueue_request')
799         )",
800    )
801    .fetch_one(&mut **conn)
802    .await
803}
804
805fn first_up_migration_version() -> i64 {
806    MIGRATOR
807        .iter()
808        .find(|migration| migration.migration_type.is_up_migration())
809        .map(|migration| migration.version)
810        .unwrap_or_default()
811}
812
813fn expected_runledger_migrations() -> RunledgerMigrationMap {
814    MIGRATOR
815        .iter()
816        .filter(|migration| migration.migration_type.is_up_migration())
817        .map(|migration| (migration.version, migration))
818        .collect()
819}
820
821fn first_conflicting_runledger_version(
822    history: &[MigrationHistoryRow],
823    expected_migrations: &RunledgerMigrationMap,
824) -> Option<i64> {
825    history.iter().find_map(|row| {
826        expected_migrations
827            .get(&row.version)
828            .filter(|migration| row.checksum.as_slice() != migration.checksum.as_ref())
829            .map(|_| row.version)
830    })
831}
832
833fn first_dirty_runledger_version(
834    history: &[MigrationHistoryRow],
835    expected_migrations: &RunledgerMigrationMap,
836) -> Option<i64> {
837    history.iter().filter(|row| !row.success).find_map(|row| {
838        expected_migrations
839            .get(&row.version)
840            .filter(|migration| row.checksum.as_slice() == migration.checksum.as_ref())
841            .map(|_| row.version)
842    })
843}
844
845fn first_missing_runledger_version(
846    recorded_versions: &[i64],
847    expected_migrations: &RunledgerMigrationMap,
848) -> Option<i64> {
849    recorded_versions
850        .iter()
851        .copied()
852        .find(|version| !expected_migrations.contains_key(version))
853}
854
855fn applied_runledger_migrations(
856    history: &[MigrationHistoryRow],
857    expected_migrations: &RunledgerMigrationMap,
858) -> Vec<AppliedMigration> {
859    history
860        .iter()
861        .filter(|row| row.success)
862        .filter(|row| {
863            expected_migrations
864                .get(&row.version)
865                .is_some_and(|migration| row.checksum.as_slice() == migration.checksum.as_ref())
866        })
867        .map(|row| AppliedMigration {
868            version: row.version,
869            checksum: row.checksum.clone().into(),
870        })
871        .collect()
872}
873
874async fn run_migrations_with_filtered_history(
875    conn: &mut PgPoolConnection,
876) -> Result<(), MigrateError> {
877    (**conn).ensure_migrations_table().await?;
878
879    let expected_migrations = expected_runledger_migrations();
880    let history = list_migration_history(conn).await?;
881
882    if let Some(version) = first_conflicting_runledger_version(&history, &expected_migrations) {
883        return Err(MigrateError::VersionMismatch(version));
884    }
885
886    if let Some(version) = first_dirty_runledger_version(&history, &expected_migrations) {
887        return Err(MigrateError::Dirty(version));
888    }
889
890    if has_runledger_migration_history_table(conn).await? {
891        let recorded_versions = list_recorded_runledger_migrations(conn).await?;
892        if let Some(version) =
893            first_missing_runledger_version(&recorded_versions, &expected_migrations)
894        {
895            return Err(MigrateError::VersionMissing(version));
896        }
897    }
898
899    let applied = applied_runledger_migrations(&history, &expected_migrations);
900    let applied_by_version: HashMap<_, _> = applied
901        .into_iter()
902        .map(|migration| (migration.version, migration))
903        .collect();
904
905    for migration in MIGRATOR
906        .iter()
907        .filter(|migration| migration.migration_type.is_up_migration())
908    {
909        match applied_by_version.get(&migration.version) {
910            Some(applied_migration) => {
911                validate_checksum(migration.version, applied_migration, migration)?
912            }
913            None => {
914                (**conn).apply(migration).await?;
915            }
916        }
917    }
918
919    Ok(())
920}
921
922#[derive(sqlx::FromRow)]
923struct MigrationHistoryRow {
924    version: i64,
925    checksum: Vec<u8>,
926    success: bool,
927}
928
929fn validate_checksum(
930    version: i64,
931    applied_migration: &AppliedMigration,
932    expected_migration: &sqlx::migrate::Migration,
933) -> Result<(), MigrateError> {
934    if applied_migration.checksum != expected_migration.checksum {
935        return Err(MigrateError::VersionMismatch(version));
936    }
937
938    Ok(())
939}
940
941#[cfg(test)]
942mod workflow_job_link_trigger_validation_tests {
943    use super::*;
944
945    #[test]
946    fn valid_catalog_rows_have_no_diagnostics() {
947        let catalog = WORKFLOW_JOB_LINK_EXPAND_TRIGGER_SPECS
948            .iter()
949            .map(valid_catalog_row)
950            .collect::<Vec<_>>();
951
952        assert!(workflow_job_link_trigger_diagnostics(&catalog).is_empty());
953    }
954
955    #[test]
956    fn missing_triggers_are_reported_individually() {
957        let diagnostics = workflow_job_link_trigger_diagnostics(&[]);
958
959        assert_eq!(
960            diagnostics.len(),
961            WORKFLOW_JOB_LINK_EXPAND_TRIGGER_SPECS.len()
962        );
963        for (diagnostic, spec) in diagnostics
964            .iter()
965            .zip(WORKFLOW_JOB_LINK_EXPAND_TRIGGER_SPECS)
966        {
967            assert_eq!(diagnostic.table_name(), spec.table_name);
968            assert_eq!(diagnostic.trigger_name(), spec.trigger_name);
969            assert_eq!(
970                diagnostic.problems(),
971                &[WorkflowJobLinkTriggerProblem::Missing]
972            );
973        }
974    }
975
976    #[test]
977    fn every_unsafe_catalog_property_has_a_typed_problem() {
978        let spec = &WORKFLOW_JOB_LINK_EXPAND_TRIGGER_SPECS[0];
979        let valid = valid_catalog_row(spec);
980
981        let mut trigger = valid.clone();
982        trigger.function_schema = "shadow".to_owned();
983        assert_only_problem(spec, &trigger, WorkflowJobLinkTriggerProblem::WrongFunction);
984
985        let mut trigger = valid.clone();
986        trigger.enabled_mode = "R".to_owned();
987        assert_only_problem(
988            spec,
989            &trigger,
990            WorkflowJobLinkTriggerProblem::NotEnabledForOriginWrites,
991        );
992
993        let mut trigger = valid.clone();
994        trigger.is_internal = true;
995        assert_only_problem(
996            spec,
997            &trigger,
998            WorkflowJobLinkTriggerProblem::InternallyGenerated,
999        );
1000
1001        let mut trigger = valid.clone();
1002        trigger.trigger_type = 20;
1003        assert_only_problem(
1004            spec,
1005            &trigger,
1006            WorkflowJobLinkTriggerProblem::WrongFiringEvents,
1007        );
1008
1009        let mut trigger = valid.clone();
1010        trigger.update_column_names = vec!["stage".to_owned()];
1011        assert_only_problem(
1012            spec,
1013            &trigger,
1014            WorkflowJobLinkTriggerProblem::UpdateColumnNotCovered,
1015        );
1016
1017        let mut trigger = valid.clone();
1018        trigger.trigger_argument_count = 1;
1019        assert_only_problem(
1020            spec,
1021            &trigger,
1022            WorkflowJobLinkTriggerProblem::UnexpectedTriggerArguments,
1023        );
1024
1025        let mut trigger = valid.clone();
1026        trigger.has_when_condition = true;
1027        assert_only_problem(
1028            spec,
1029            &trigger,
1030            WorkflowJobLinkTriggerProblem::UnexpectedWhenCondition,
1031        );
1032
1033        let mut trigger = valid;
1034        trigger.is_deferrable = false;
1035        trigger.is_initially_deferred = false;
1036        assert_only_problem(
1037            spec,
1038            &trigger,
1039            WorkflowJobLinkTriggerProblem::WrongConstraintMode,
1040        );
1041    }
1042
1043    #[test]
1044    fn safe_update_column_supersets_and_always_enabled_mode_are_accepted() {
1045        let spec = &WORKFLOW_JOB_LINK_EXPAND_TRIGGER_SPECS[2];
1046
1047        let mut all_updates = valid_catalog_row(spec);
1048        all_updates.update_column_names.clear();
1049        assert!(workflow_job_link_trigger_problems(spec, &all_updates).is_empty());
1050
1051        let mut additional_columns = valid_catalog_row(spec);
1052        additional_columns
1053            .update_column_names
1054            .push("stage".to_owned());
1055        additional_columns.enabled_mode = "A".to_owned();
1056        assert!(workflow_job_link_trigger_problems(spec, &additional_columns).is_empty());
1057    }
1058
1059    fn valid_catalog_row(spec: &WorkflowJobLinkTriggerSpec) -> WorkflowJobLinkTriggerCatalogRow {
1060        let (is_constraint, is_deferrable, is_initially_deferred) = match spec.constraint_mode {
1061            WorkflowJobLinkTriggerConstraintMode::Deferred => (true, true, true),
1062            WorkflowJobLinkTriggerConstraintMode::Ordinary => (false, false, false),
1063        };
1064
1065        WorkflowJobLinkTriggerCatalogRow {
1066            table_name: spec.table_name.to_owned(),
1067            trigger_name: spec.trigger_name.to_owned(),
1068            function_schema: "public".to_owned(),
1069            function_name: spec.function_name.to_owned(),
1070            function_argument_count: 0,
1071            returns_trigger: true,
1072            enabled_mode: "O".to_owned(),
1073            is_internal: false,
1074            trigger_type: AFTER_ROW_INSERT_OR_UPDATE_TRIGGER_TYPE,
1075            update_column_names: vec![spec.update_column_name.to_owned()],
1076            trigger_argument_count: 0,
1077            has_when_condition: false,
1078            is_constraint,
1079            is_deferrable,
1080            is_initially_deferred,
1081        }
1082    }
1083
1084    fn assert_only_problem(
1085        spec: &WorkflowJobLinkTriggerSpec,
1086        trigger: &WorkflowJobLinkTriggerCatalogRow,
1087        expected: WorkflowJobLinkTriggerProblem,
1088    ) {
1089        assert_eq!(
1090            workflow_job_link_trigger_problems(spec, trigger),
1091            vec![expected]
1092        );
1093    }
1094}