Skip to main content

minco_sqlx_sqlite/
jobs.rs

1//! Durable `SQLite` job storage: job rows, publication generations,
2//! fenced execution leases and overlap locks.
3//!
4//! The job row owns execution state and the publication row owns pending
5//! transport delivery; the queue message is never authoritative. `SQLite`
6//! has one writer at a time, so claims run inside `BEGIN IMMEDIATE`
7//! transactions; every mutation re-checks the claim's opaque lease
8//! identity, and retry state plus the next publication generation commit
9//! in one transaction.
10
11use chrono::{DateTime, Utc};
12use minco_plugin_jobs::{
13    EnqueueOutcome, IngestOutcome, JobAttempt, JobClaim, JobError, JobPublication, JobRecord,
14    JobStatus, PublicationStatus, semantic_fingerprint, validate_worker_claim,
15};
16use sqlx::{Row, SqlitePool, Transaction, sqlite::SqliteRow};
17use uuid::Uuid;
18
19/// SQLite-backed job store implementing every job port.
20#[derive(Debug, Clone)]
21pub struct SqliteJobStore {
22    pool: SqlitePool,
23}
24
25impl SqliteJobStore {
26    #[must_use]
27    pub const fn new(pool: SqlitePool) -> Self {
28        Self { pool }
29    }
30
31    /// Insert the job and its first publication generation inside the
32    /// caller's transaction so the business mutation and the durable
33    /// dispatch commit atomically.
34    pub async fn enqueue_in(
35        &self,
36        transaction: &mut Transaction<'_, sqlx::Sqlite>,
37        record: JobRecord,
38    ) -> Result<EnqueueOutcome, JobError> {
39        Self::enqueue_record_in(transaction, record, "pending").await
40    }
41
42    /// Insert the job with its publication recorded as already delivered
43    /// (Scheduler ingestion) inside the caller's transaction.
44    pub async fn ingest_in(
45        &self,
46        transaction: &mut Transaction<'_, sqlx::Sqlite>,
47        record: JobRecord,
48    ) -> Result<IngestOutcome, JobError> {
49        match Self::enqueue_record_in(transaction, record, "published").await? {
50            EnqueueOutcome::Inserted(job_id) => Ok(IngestOutcome::Ingested(job_id)),
51            EnqueueOutcome::Duplicate(existing) => Ok(IngestOutcome::Duplicate(existing)),
52        }
53    }
54
55    async fn enqueue_record_in(
56        transaction: &mut Transaction<'_, sqlx::Sqlite>,
57        record: JobRecord,
58        publication_status: &str,
59    ) -> Result<EnqueueOutcome, JobError> {
60        if record.status != JobStatus::Pending {
61            return Err(JobError::InvalidJob("jobs must be enqueued pending".into()));
62        }
63        record.envelope.validate()?;
64        let envelope_json = serde_json::to_string(&record.envelope).map_err(|error| {
65            JobError::Infrastructure(format!("job envelope encode failed: {error}"))
66        })?;
67        let attempts = serde_json::to_string(&record.attempts)
68            .map_err(|error| JobError::Infrastructure(format!("attempt encode failed: {error}")))?;
69        let fingerprint = semantic_fingerprint(&record.envelope);
70        let result = sqlx::query(
71            "INSERT INTO minco_jobs (job_id, worker_profile, envelope, fingerprint, status, \
72             revision, available_at, attempt_count, lease_id, lease_expires_at, attempts, \
73             dedupe_key, failure_code, completed_at) \
74             VALUES ($1, $2, $3, $4, 'pending', $5, $6, $7, NULL, NULL, $8, $9, NULL, NULL)",
75        )
76        .bind(record.envelope.job_id.to_string())
77        .bind(&record.envelope.worker_profile)
78        .bind(&envelope_json)
79        .bind(&fingerprint)
80        .bind(i64::try_from(record.revision).unwrap_or(1))
81        .bind(record.envelope.available_at.to_rfc3339())
82        .bind(i64::from(record.attempt_count))
83        .bind(&attempts)
84        .bind(record.envelope.dedupe_key.as_deref())
85        .execute(&mut **transaction)
86        .await;
87        if let Err(error) = result {
88            let message = format!("{error}");
89            if message.contains("minco_jobs.job_id") {
90                // An identical re-ingestion (same job identity and
91                // fingerprint) is an idempotent duplicate; anything else is
92                // an error.
93                let existing: Option<(String,)> =
94                    sqlx::query_as("SELECT fingerprint FROM minco_jobs WHERE job_id = $1")
95                        .bind(record.envelope.job_id.to_string())
96                        .fetch_optional(&mut **transaction)
97                        .await
98                        .map_err(|_| {
99                            JobError::Infrastructure("job identity probe failed".into())
100                        })?;
101                return match existing {
102                    Some((existing_fingerprint,)) if existing_fingerprint == fingerprint => {
103                        Ok(EnqueueOutcome::Duplicate(record.envelope.job_id))
104                    }
105                    _ => Err(JobError::DuplicateJobIdentity(record.envelope.job_id)),
106                };
107            }
108            if message.contains("minco_jobs.dedupe_key") {
109                return dedupe_outcome(&record, &fingerprint, transaction).await;
110            }
111            return Err(infrastructure(&error));
112        }
113        sqlx::query(
114            "INSERT INTO minco_job_publications (publication_id, job_id, generation, \
115             worker_profile, status, attempt_count, available_at, claimed_by, \
116             claim_expires_at, lease_id, last_error) \
117             VALUES ($1, $2, 1, $3, $4, 0, $5, NULL, NULL, NULL, NULL)",
118        )
119        .bind(Uuid::now_v7().to_string())
120        .bind(record.envelope.job_id.to_string())
121        .bind(&record.envelope.worker_profile)
122        .bind(publication_status)
123        .bind(record.envelope.available_at.to_rfc3339())
124        .execute(&mut **transaction)
125        .await
126        .map_err(|_| JobError::Infrastructure("job publication insert failed".into()))?;
127        Ok(EnqueueOutcome::Inserted(record.envelope.job_id))
128    }
129}
130
131async fn dedupe_outcome(
132    record: &JobRecord,
133    fingerprint: &str,
134    transaction: &mut Transaction<'_, sqlx::Sqlite>,
135) -> Result<EnqueueOutcome, JobError> {
136    let existing: Option<(String, String)> =
137        sqlx::query_as("SELECT envelope, fingerprint FROM minco_jobs WHERE dedupe_key = $1")
138            .bind(record.envelope.dedupe_key.as_deref())
139            .fetch_optional(&mut **transaction)
140            .await
141            .map_err(|_| JobError::Infrastructure("job dedupe lookup failed".into()))?;
142    let Some((existing_envelope, existing_fingerprint)) = existing else {
143        return Err(JobError::Infrastructure(
144            "job dedupe conflict vanished during insert".into(),
145        ));
146    };
147    let existing: serde_json::Value = serde_json::from_str(&existing_envelope)
148        .map_err(|_| JobError::Infrastructure("existing job envelope decode failed".into()))?;
149    let existing_id = existing
150        .get("job_id")
151        .and_then(serde_json::Value::as_str)
152        .and_then(|value| value.parse::<Uuid>().ok());
153    match (existing_fingerprint == fingerprint, existing_id) {
154        (true, Some(existing_job_id)) => Ok(EnqueueOutcome::Duplicate(existing_job_id)),
155        (false, Some(existing_job_id)) => {
156            Err(JobError::DuplicateSubmissionConflict { existing_job_id })
157        }
158        _ => Err(JobError::Infrastructure(
159            "job dedupe conflict could not be resolved".into(),
160        )),
161    }
162}
163
164fn infrastructure(error: &sqlx::Error) -> JobError {
165    JobError::Infrastructure(format!("sqlite job store failed: {error}"))
166}
167
168fn parse_status(status: &str) -> Result<JobStatus, JobError> {
169    match status {
170        "pending" => Ok(JobStatus::Pending),
171        "running" => Ok(JobStatus::Running),
172        "succeeded" => Ok(JobStatus::Succeeded),
173        "failed_permanently" => Ok(JobStatus::FailedPermanently),
174        "cancelled" => Ok(JobStatus::Cancelled),
175        other => Err(JobError::Infrastructure(format!(
176            "unknown job status {other}"
177        ))),
178    }
179}
180
181fn parse_timestamp(value: &str) -> Result<DateTime<Utc>, JobError> {
182    DateTime::parse_from_rfc3339(value)
183        .map(|parsed| parsed.with_timezone(&Utc))
184        .map_err(|_| JobError::Infrastructure("timestamp decode failed".into()))
185}
186
187fn optional_timestamp(value: Option<&str>) -> Result<Option<DateTime<Utc>>, JobError> {
188    value.map(parse_timestamp).transpose()
189}
190
191fn optional_uuid(value: Option<&str>) -> Option<Uuid> {
192    value.and_then(|text| text.parse::<Uuid>().ok())
193}
194
195fn decode_job_row(row: &SqliteRow) -> Result<JobRecord, JobError> {
196    let envelope_text: String = row
197        .try_get("envelope")
198        .map_err(|_| JobError::Infrastructure("job envelope column unreadable".into()))?;
199    let mut envelope: minco_plugin_jobs::JobEnvelope = serde_json::from_str(&envelope_text)
200        .map_err(|error| {
201            JobError::Infrastructure(format!("job envelope decode failed: {error}"))
202        })?;
203    let status: String = row
204        .try_get("status")
205        .map_err(|_| JobError::Infrastructure("job status column unreadable".into()))?;
206    let attempt_count: i64 = row
207        .try_get("attempt_count")
208        .map_err(|_| JobError::Infrastructure("job attempt column unreadable".into()))?;
209    envelope.available_at = parse_timestamp(
210        row.try_get("available_at")
211            .map_err(|_| JobError::Infrastructure("job availability unreadable".into()))?,
212    )?;
213    envelope.attempt = u32::try_from(attempt_count)
214        .unwrap_or(envelope.attempt)
215        .max(1);
216    let attempts_text: String = row
217        .try_get("attempts")
218        .map_err(|_| JobError::Infrastructure("job attempts unreadable".into()))?;
219    let attempts: Vec<JobAttempt> = serde_json::from_str(&attempts_text)
220        .map_err(|error| JobError::Infrastructure(format!("attempt decode failed: {error}")))?;
221    Ok(JobRecord {
222        envelope,
223        status: parse_status(&status)?,
224        revision: u64::try_from(
225            row.try_get::<i64, _>("revision")
226                .map_err(|_| JobError::Infrastructure("job revision unreadable".into()))?,
227        )
228        .unwrap_or(1),
229        lease_id: optional_uuid(
230            row.try_get::<Option<String>, _>("lease_id")
231                .map_err(|_| JobError::Infrastructure("job lease identity unreadable".into()))?
232                .as_deref(),
233        ),
234        lease_expires_at: optional_timestamp(
235            row.try_get::<Option<String>, _>("lease_expires_at")
236                .map_err(|_| JobError::Infrastructure("job lease expiry unreadable".into()))?
237                .as_deref(),
238        )?,
239        attempt_count: u32::try_from(attempt_count).unwrap_or(0),
240        attempts,
241        failure_code: row
242            .try_get("failure_code")
243            .map_err(|_| JobError::Infrastructure("job failure code unreadable".into()))?,
244        completed_at: optional_timestamp(
245            row.try_get::<Option<String>, _>("completed_at")
246                .map_err(|_| JobError::Infrastructure("job completion unreadable".into()))?
247                .as_deref(),
248        )?,
249    })
250}
251
252fn attempt_entry(
253    attempt: u32,
254    worker_execution_id: &str,
255    now: DateTime<Utc>,
256    outcome: minco_plugin_jobs::JobAttemptOutcome,
257) -> Result<String, JobError> {
258    serde_json::to_string(&JobAttempt {
259        attempt,
260        at: now,
261        worker_execution_id: worker_execution_id.to_owned(),
262        outcome,
263    })
264    .map_err(|error| JobError::Infrastructure(format!("attempt encode failed: {error}")))
265}
266
267fn decode_publication_row(row: &SqliteRow) -> Result<JobPublication, JobError> {
268    let status: String = row
269        .try_get("status")
270        .map_err(|_| JobError::Infrastructure("publication status unreadable".into()))?;
271    let status = match status.as_str() {
272        "pending" => PublicationStatus::Pending,
273        "claimed" => PublicationStatus::Claimed,
274        "published" => PublicationStatus::Published,
275        "failed" => PublicationStatus::Failed,
276        other => {
277            return Err(JobError::Infrastructure(format!(
278                "unknown publication status {other}"
279            )));
280        }
281    };
282    let publication_id: String = row
283        .try_get("publication_id")
284        .map_err(|_| JobError::Infrastructure("publication identity unreadable".into()))?;
285    let job_id: String = row
286        .try_get("job_id")
287        .map_err(|_| JobError::Infrastructure("publication job unreadable".into()))?;
288    Ok(JobPublication {
289        publication_id: Uuid::parse_str(&publication_id)
290            .map_err(|_| JobError::Infrastructure("publication identity decode failed".into()))?,
291        job_id: Uuid::parse_str(&job_id)
292            .map_err(|_| JobError::Infrastructure("publication job decode failed".into()))?,
293        generation: u32::try_from(
294            row.try_get::<i64, _>("generation").map_err(|_| {
295                JobError::Infrastructure("publication generation unreadable".into())
296            })?,
297        )
298        .unwrap_or(1),
299        worker_profile: row
300            .try_get("worker_profile")
301            .map_err(|_| JobError::Infrastructure("publication profile unreadable".into()))?,
302        status,
303        attempt_count: u32::try_from(
304            row.try_get::<i64, _>("attempt_count")
305                .map_err(|_| JobError::Infrastructure("publication attempts unreadable".into()))?,
306        )
307        .unwrap_or(0),
308        available_at: parse_timestamp(row.try_get("available_at").map_err(|_| {
309            JobError::Infrastructure("publication availability unreadable".into())
310        })?)?,
311        claimed_by: row
312            .try_get("claimed_by")
313            .map_err(|_| JobError::Infrastructure("publication claimant unreadable".into()))?,
314        claim_expires_at: optional_timestamp(
315            row.try_get::<Option<String>, _>("claim_expires_at")
316                .map_err(|_| {
317                    JobError::Infrastructure("publication claim expiry unreadable".into())
318                })?
319                .as_deref(),
320        )?,
321        lease_id: optional_uuid(
322            row.try_get::<Option<String>, _>("lease_id")
323                .map_err(|_| JobError::Infrastructure("publication lease unreadable".into()))?
324                .as_deref(),
325        ),
326        last_error: row
327            .try_get("last_error")
328            .map_err(|_| JobError::Infrastructure("publication error unreadable".into()))?,
329    })
330}
331
332async fn verify_job_update(
333    pool: &SqlitePool,
334    job_id: Uuid,
335    result: &sqlx::sqlite::SqliteQueryResult,
336) -> Result<(), JobError> {
337    if result.rows_affected() == 1 {
338        return Ok(());
339    }
340    let exists: bool =
341        sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM minco_jobs WHERE job_id = $1)")
342            .bind(job_id.to_string())
343            .fetch_one(pool)
344            .await
345            .map_err(|_| JobError::Infrastructure("job existence probe failed".into()))?;
346    if exists {
347        Err(JobError::LeaseFencedOut { job_id })
348    } else {
349        Err(JobError::MissingJob(job_id))
350    }
351}
352
353async fn verify_revision_guard(
354    pool: &SqlitePool,
355    job_id: Uuid,
356    expected_revision: u64,
357    result: &sqlx::sqlite::SqliteQueryResult,
358) -> Result<(), JobError> {
359    if result.rows_affected() == 1 {
360        return Ok(());
361    }
362    let current: Option<i64> =
363        sqlx::query_scalar("SELECT revision FROM minco_jobs WHERE job_id = $1")
364            .bind(job_id.to_string())
365            .fetch_optional(pool)
366            .await
367            .map_err(|error| infrastructure(&error))?;
368    match current {
369        Some(revision) if u64::try_from(revision).unwrap_or(0) != expected_revision => {
370            Err(JobError::RevisionConflict {
371                job_id,
372                expected_revision,
373            })
374        }
375        Some(_) => Err(JobError::InvalidTransition { job_id }),
376        None => Err(JobError::MissingJob(job_id)),
377    }
378}
379
380async fn verify_publication_update(
381    pool: &SqlitePool,
382    publication_id: Uuid,
383    result: &sqlx::sqlite::SqliteQueryResult,
384) -> Result<(), JobError> {
385    if result.rows_affected() == 1 {
386        return Ok(());
387    }
388    let exists: bool = sqlx::query_scalar(
389        "SELECT EXISTS(SELECT 1 FROM minco_job_publications WHERE publication_id = $1)",
390    )
391    .bind(publication_id.to_string())
392    .fetch_one(pool)
393    .await
394    .map_err(|_| JobError::Infrastructure("publication probe failed".into()))?;
395    if exists {
396        Err(JobError::PublicationFencedOut { publication_id })
397    } else {
398        Err(JobError::MissingJob(publication_id))
399    }
400}
401
402#[async_trait::async_trait]
403impl minco_plugin_jobs::JobStore for SqliteJobStore {
404    async fn enqueue_with_intent(&self, record: JobRecord) -> Result<EnqueueOutcome, JobError> {
405        let mut transaction = self
406            .pool
407            .begin_with("BEGIN IMMEDIATE")
408            .await
409            .map_err(|error| infrastructure(&error))?;
410        let outcome = self.enqueue_in(&mut transaction, record).await?;
411        transaction
412            .commit()
413            .await
414            .map_err(|error| infrastructure(&error))?;
415        Ok(outcome)
416    }
417
418    async fn ingest_existing_delivery(&self, record: JobRecord) -> Result<IngestOutcome, JobError> {
419        let mut transaction = self
420            .pool
421            .begin_with("BEGIN IMMEDIATE")
422            .await
423            .map_err(|error| infrastructure(&error))?;
424        let outcome = self.ingest_in(&mut transaction, record).await?;
425        transaction
426            .commit()
427            .await
428            .map_err(|error| infrastructure(&error))?;
429        Ok(outcome)
430    }
431
432    async fn claim_execution(
433        &self,
434        job_id: Uuid,
435        worker_execution_id: &str,
436        lease_expires_at: DateTime<Utc>,
437        now: DateTime<Utc>,
438    ) -> Result<Option<JobClaim>, JobError> {
439        validate_worker_claim(worker_execution_id, 1, lease_expires_at, now)?;
440        let mut transaction = self
441            .pool
442            .begin_with("BEGIN IMMEDIATE")
443            .await
444            .map_err(|error| infrastructure(&error))?;
445        let claimable: Option<i64> = sqlx::query_scalar(
446            "SELECT CASE WHEN (status = 'pending' AND available_at <= $2) \
447             OR (status = 'running' AND lease_expires_at IS NOT NULL AND lease_expires_at <= $2) \
448             THEN 1 ELSE 0 END FROM minco_jobs WHERE job_id = $1",
449        )
450        .bind(job_id.to_string())
451        .bind(now.to_rfc3339())
452        .fetch_optional(&mut *transaction)
453        .await
454        .map_err(|error| infrastructure(&error))?;
455        if claimable != Some(1) {
456            transaction
457                .commit()
458                .await
459                .map_err(|error| infrastructure(&error))?;
460            return Ok(None);
461        }
462        let lease_id = Uuid::now_v7();
463        sqlx::query(
464            "UPDATE minco_jobs SET status = 'running', lease_id = $2, lease_expires_at = $3, \
465             attempt_count = attempt_count + 1, revision = revision + 1 WHERE job_id = $1",
466        )
467        .bind(job_id.to_string())
468        .bind(lease_id.to_string())
469        .bind(lease_expires_at.to_rfc3339())
470        .execute(&mut *transaction)
471        .await
472        .map_err(|error| infrastructure(&error))?;
473        let row = sqlx::query(
474            "SELECT job_id, worker_profile, envelope, status, revision, available_at, \
475             attempt_count, lease_id, lease_expires_at, attempts, dedupe_key, failure_code, \
476             completed_at FROM minco_jobs WHERE job_id = $1",
477        )
478        .bind(job_id.to_string())
479        .fetch_one(&mut *transaction)
480        .await
481        .map_err(|error| infrastructure(&error))?;
482        transaction
483            .commit()
484            .await
485            .map_err(|error| infrastructure(&error))?;
486        let record = decode_job_row(&row)?;
487        Ok(Some(JobClaim {
488            fence: record.revision,
489            record,
490            lease_id,
491        }))
492    }
493
494    async fn complete(&self, claim: &JobClaim, now: DateTime<Utc>) -> Result<(), JobError> {
495        let job_id = claim.record.envelope.job_id;
496        let entry = attempt_entry(
497            claim.record.attempt_count,
498            &format!("lease-{}", claim.lease_id),
499            now,
500            minco_plugin_jobs::JobAttemptOutcome::Succeeded,
501        )?;
502        let result = sqlx::query(
503            "UPDATE minco_jobs SET status = 'succeeded', lease_id = NULL, lease_expires_at = \
504             NULL, failure_code = NULL, completed_at = $3, revision = revision + 1, \
505             attempts = json_insert(CASE WHEN json_array_length(attempts) >= 25 \
506             THEN json_remove(attempts, '$[0]') ELSE attempts END, '$[#]', json($4)) \
507             WHERE job_id = $1 AND status = 'running' AND lease_id = $2",
508        )
509        .bind(job_id.to_string())
510        .bind(claim.lease_id.to_string())
511        .bind(now.to_rfc3339())
512        .bind(entry)
513        .execute(&self.pool)
514        .await
515        .map_err(|error| infrastructure(&error))?;
516        verify_job_update(&self.pool, job_id, &result).await
517    }
518
519    async fn schedule_retry_and_publish(
520        &self,
521        claim: &JobClaim,
522        failure_code: &str,
523        next_available_at: DateTime<Utc>,
524        now: DateTime<Utc>,
525    ) -> Result<Uuid, JobError> {
526        let job_id = claim.record.envelope.job_id;
527        let entry = attempt_entry(
528            claim.record.attempt_count,
529            &format!("lease-{}", claim.lease_id),
530            now,
531            minco_plugin_jobs::JobAttemptOutcome::Retried {
532                code: failure_code.to_owned(),
533            },
534        )?;
535        let mut transaction = self
536            .pool
537            .begin_with("BEGIN IMMEDIATE")
538            .await
539            .map_err(|error| infrastructure(&error))?;
540        let result = sqlx::query(
541            "UPDATE minco_jobs SET status = 'pending', lease_id = NULL, lease_expires_at = \
542             NULL, available_at = $4, failure_code = $3, revision = revision + 1, \
543             attempts = json_insert(CASE WHEN json_array_length(attempts) >= 25 \
544             THEN json_remove(attempts, '$[0]') ELSE attempts END, '$[#]', json($5)) \
545             WHERE job_id = $1 AND status = 'running' AND lease_id = $2",
546        )
547        .bind(job_id.to_string())
548        .bind(claim.lease_id.to_string())
549        .bind(failure_code)
550        .bind(next_available_at.to_rfc3339())
551        .bind(entry)
552        .execute(&mut *transaction)
553        .await
554        .map_err(|error| infrastructure(&error))?;
555        if result.rows_affected() != 1 {
556            transaction
557                .rollback()
558                .await
559                .map_err(|error| infrastructure(&error))?;
560            verify_job_update(&self.pool, job_id, &result).await?;
561            return Err(JobError::LeaseFencedOut { job_id });
562        }
563        let publication_id = Uuid::now_v7();
564        sqlx::query(
565            "INSERT INTO minco_job_publications (publication_id, job_id, generation, \
566             worker_profile, status, attempt_count, available_at, claimed_by, \
567             claim_expires_at, lease_id, last_error) \
568             SELECT $1, $2, COALESCE(MAX(generation), 0) + 1, $3, 'pending', 0, $4, NULL, \
569             NULL, NULL, NULL FROM minco_job_publications WHERE job_id = $2",
570        )
571        .bind(publication_id.to_string())
572        .bind(job_id.to_string())
573        .bind(&claim.record.envelope.worker_profile)
574        .bind(next_available_at.to_rfc3339())
575        .execute(&mut *transaction)
576        .await
577        .map_err(|error| infrastructure(&error))?;
578        transaction
579            .commit()
580            .await
581            .map_err(|error| infrastructure(&error))?;
582        Ok(publication_id)
583    }
584
585    async fn fail_permanently(
586        &self,
587        claim: &JobClaim,
588        failure_code: &str,
589        now: DateTime<Utc>,
590    ) -> Result<(), JobError> {
591        let job_id = claim.record.envelope.job_id;
592        let entry = attempt_entry(
593            claim.record.attempt_count,
594            &format!("lease-{}", claim.lease_id),
595            now,
596            minco_plugin_jobs::JobAttemptOutcome::FailedPermanently {
597                code: failure_code.to_owned(),
598            },
599        )?;
600        let result = sqlx::query(
601            "UPDATE minco_jobs SET status = 'failed_permanently', lease_id = NULL, \
602             lease_expires_at = NULL, failure_code = $3, completed_at = $4, revision = revision + 1, \
603             attempts = json_insert(CASE WHEN json_array_length(attempts) >= 25 \
604             THEN json_remove(attempts, '$[0]') ELSE attempts END, '$[#]', json($5)) \
605             WHERE job_id = $1 AND status = 'running' AND lease_id = $2",
606        )
607        .bind(job_id.to_string())
608        .bind(claim.lease_id.to_string())
609        .bind(failure_code)
610        .bind(now.to_rfc3339())
611        .bind(entry)
612        .execute(&self.pool)
613        .await
614        .map_err(|error| infrastructure(&error))?;
615        verify_job_update(&self.pool, job_id, &result).await
616    }
617
618    async fn cancel(
619        &self,
620        job_id: Uuid,
621        expected_revision: u64,
622        now: DateTime<Utc>,
623    ) -> Result<(), JobError> {
624        let result = sqlx::query(
625            "UPDATE minco_jobs SET status = 'cancelled', completed_at = $3, revision = revision + 1 \
626             WHERE job_id = $1 AND revision = $2 AND status = 'pending'",
627        )
628        .bind(job_id.to_string())
629        .bind(i64::try_from(expected_revision).unwrap_or(-1))
630        .bind(now.to_rfc3339())
631        .execute(&self.pool)
632        .await
633        .map_err(|error| infrastructure(&error))?;
634        verify_revision_guard(&self.pool, job_id, expected_revision, &result).await
635    }
636
637    async fn retry_failed(
638        &self,
639        job_id: Uuid,
640        expected_revision: u64,
641        now: DateTime<Utc>,
642    ) -> Result<DateTime<Utc>, JobError> {
643        let mut transaction = self
644            .pool
645            .begin_with("BEGIN IMMEDIATE")
646            .await
647            .map_err(|error| infrastructure(&error))?;
648        let result = sqlx::query(
649            "UPDATE minco_jobs SET status = 'pending', failure_code = NULL, completed_at = NULL, \
650             available_at = $3, attempt_count = 0, lease_id = NULL, lease_expires_at = NULL, \
651             revision = revision + 1 WHERE job_id = $1 AND revision = $2 AND status = \
652             'failed_permanently'",
653        )
654        .bind(job_id.to_string())
655        .bind(i64::try_from(expected_revision).unwrap_or(-1))
656        .bind(now.to_rfc3339())
657        .execute(&mut *transaction)
658        .await
659        .map_err(|error| infrastructure(&error))?;
660        if result.rows_affected() != 1 {
661            transaction
662                .rollback()
663                .await
664                .map_err(|error| infrastructure(&error))?;
665            verify_revision_guard(&self.pool, job_id, expected_revision, &result).await?;
666            return Err(JobError::RevisionConflict {
667                job_id,
668                expected_revision,
669            });
670        }
671        let profile: String =
672            sqlx::query_scalar("SELECT worker_profile FROM minco_jobs WHERE job_id = $1")
673                .bind(job_id.to_string())
674                .fetch_one(&mut *transaction)
675                .await
676                .map_err(|error| infrastructure(&error))?;
677        let publication_id = Uuid::now_v7();
678        sqlx::query(
679            "INSERT INTO minco_job_publications (publication_id, job_id, generation, \
680             worker_profile, status, attempt_count, available_at, claimed_by, \
681             claim_expires_at, lease_id, last_error) \
682             SELECT $1, $2, COALESCE(MAX(generation), 0) + 1, $3, 'pending', 0, $4, NULL, \
683             NULL, NULL, NULL FROM minco_job_publications WHERE job_id = $2",
684        )
685        .bind(publication_id.to_string())
686        .bind(job_id.to_string())
687        .bind(&profile)
688        .bind(now.to_rfc3339())
689        .execute(&mut *transaction)
690        .await
691        .map_err(|error| infrastructure(&error))?;
692        transaction
693            .commit()
694            .await
695            .map_err(|error| infrastructure(&error))?;
696        Ok(now)
697    }
698
699    async fn recover_expired_leases(&self, now: DateTime<Utc>) -> Result<usize, JobError> {
700        let mut transaction = self
701            .pool
702            .begin_with("BEGIN IMMEDIATE")
703            .await
704            .map_err(|error| infrastructure(&error))?;
705        let recovered: Vec<(String, String)> = sqlx::query_as(
706            "UPDATE minco_jobs SET status = 'pending', lease_id = NULL, lease_expires_at = \
707             NULL, available_at = $1, revision = revision + 1 \
708             WHERE status = 'running' AND lease_expires_at IS NOT NULL AND lease_expires_at <= $1 \
709             RETURNING job_id, worker_profile",
710        )
711        .bind(now.to_rfc3339())
712        .fetch_all(&mut *transaction)
713        .await
714        .map_err(|error| infrastructure(&error))?;
715        let count = recovered.len();
716        for (job_id, profile) in &recovered {
717            let publication_id = Uuid::now_v7();
718            sqlx::query(
719                "INSERT INTO minco_job_publications (publication_id, job_id, generation, \
720                 worker_profile, status, attempt_count, available_at, claimed_by, \
721                 claim_expires_at, lease_id, last_error) \
722                 SELECT $1, $2, COALESCE(MAX(generation), 0) + 1, $3, 'pending', 0, $4, NULL, \
723                 NULL, NULL, NULL FROM minco_job_publications WHERE job_id = $2",
724            )
725            .bind(publication_id.to_string())
726            .bind(job_id)
727            .bind(profile)
728            .bind(now.to_rfc3339())
729            .execute(&mut *transaction)
730            .await
731            .map_err(|error| infrastructure(&error))?;
732        }
733        transaction
734            .commit()
735            .await
736            .map_err(|error| infrastructure(&error))?;
737        Ok(count)
738    }
739
740    async fn get(&self, job_id: Uuid) -> Result<Option<JobRecord>, JobError> {
741        let row = sqlx::query(
742            "SELECT job_id, worker_profile, envelope, status, revision, available_at, \
743             attempt_count, lease_id, lease_expires_at, attempts, dedupe_key, failure_code, \
744             completed_at FROM minco_jobs WHERE job_id = $1",
745        )
746        .bind(job_id.to_string())
747        .fetch_optional(&self.pool)
748        .await
749        .map_err(|error| infrastructure(&error))?;
750        row.map(|row| decode_job_row(&row)).transpose()
751    }
752
753    async fn list_failed(&self, limit: usize) -> Result<Vec<JobRecord>, JobError> {
754        let rows = sqlx::query(
755            "SELECT job_id, worker_profile, envelope, status, revision, available_at, \
756             attempt_count, lease_id, lease_expires_at, attempts, dedupe_key, failure_code, \
757             completed_at FROM minco_jobs WHERE status = 'failed_permanently' \
758             ORDER BY completed_at, job_id LIMIT $1",
759        )
760        .bind(i64::try_from(limit.min(100)).unwrap_or(100))
761        .fetch_all(&self.pool)
762        .await
763        .map_err(|error| infrastructure(&error))?;
764        rows.iter().map(decode_job_row).collect()
765    }
766}
767
768#[async_trait::async_trait]
769impl minco_plugin_jobs::JobPublicationStore for SqliteJobStore {
770    async fn claim_due(
771        &self,
772        worker_execution_id: &str,
773        limit: usize,
774        claim_expires_at: DateTime<Utc>,
775        now: DateTime<Utc>,
776    ) -> Result<Vec<JobPublication>, JobError> {
777        validate_worker_claim(worker_execution_id, limit, claim_expires_at, now)?;
778        let mut transaction = self
779            .pool
780            .begin_with("BEGIN IMMEDIATE")
781            .await
782            .map_err(|error| infrastructure(&error))?;
783        let ids: Vec<(String,)> = sqlx::query_as(
784            "SELECT publication_id FROM minco_job_publications \
785             WHERE (status IN ('pending', 'failed') AND available_at <= $1) \
786             OR (status = 'claimed' AND claim_expires_at IS NOT NULL AND claim_expires_at <= $1) \
787             ORDER BY available_at, publication_id LIMIT $2",
788        )
789        .bind(now.to_rfc3339())
790        .bind(i64::try_from(limit.min(100)).unwrap_or(100))
791        .fetch_all(&mut *transaction)
792        .await
793        .map_err(|error| infrastructure(&error))?;
794        let claimed: Vec<JobPublication> = if ids.is_empty() {
795            Vec::new()
796        } else {
797            let mut builder = sqlx::QueryBuilder::new(
798                "UPDATE minco_job_publications SET status = 'claimed', claimed_by = ",
799            );
800            builder.push_bind(worker_execution_id.to_owned());
801            builder.push(", claim_expires_at = ");
802            builder.push_bind(claim_expires_at.to_rfc3339());
803            builder.push(", lease_id = ");
804            builder.push_bind(Uuid::now_v7().to_string());
805            builder.push(", attempt_count = attempt_count + 1 WHERE publication_id IN (");
806            let mut separated = builder.separated(", ");
807            for (publication_id,) in &ids {
808                separated.push_bind(publication_id.clone());
809            }
810            builder.push(
811                ") RETURNING publication_id, job_id, generation, worker_profile, status, \
812                 attempt_count, available_at, claimed_by, claim_expires_at, lease_id, \
813                 last_error",
814            );
815            let rows = builder
816                .build()
817                .fetch_all(&mut *transaction)
818                .await
819                .map_err(|error| infrastructure(&error))?;
820            rows.iter()
821                .map(decode_publication_row)
822                .collect::<Result<Vec<_>, _>>()?
823        };
824        transaction
825            .commit()
826            .await
827            .map_err(|error| infrastructure(&error))?;
828        Ok(claimed)
829    }
830
831    async fn mark_published(&self, publication_id: Uuid, lease_id: Uuid) -> Result<(), JobError> {
832        let result = sqlx::query(
833            "UPDATE minco_job_publications SET status = 'published', claimed_by = NULL, \
834             claim_expires_at = NULL, lease_id = NULL, last_error = NULL \
835             WHERE publication_id = $1 AND status = 'claimed' AND lease_id = $2",
836        )
837        .bind(publication_id.to_string())
838        .bind(lease_id.to_string())
839        .execute(&self.pool)
840        .await
841        .map_err(|error| infrastructure(&error))?;
842        verify_publication_update(&self.pool, publication_id, &result).await
843    }
844
845    async fn mark_failed(
846        &self,
847        publication_id: Uuid,
848        lease_id: Uuid,
849        error: &str,
850        retry_at: DateTime<Utc>,
851    ) -> Result<(), JobError> {
852        let result = sqlx::query(
853            "UPDATE minco_job_publications SET status = 'failed', claimed_by = NULL, \
854             claim_expires_at = NULL, lease_id = NULL, available_at = $3, last_error = $4 \
855             WHERE publication_id = $1 AND status = 'claimed' AND lease_id = $2",
856        )
857        .bind(publication_id.to_string())
858        .bind(lease_id.to_string())
859        .bind(retry_at.to_rfc3339())
860        .bind(error)
861        .execute(&self.pool)
862        .await
863        .map_err(|error| infrastructure(&error))?;
864        verify_publication_update(&self.pool, publication_id, &result).await
865    }
866
867    async fn recover_expired_claims(&self, now: DateTime<Utc>) -> Result<usize, JobError> {
868        let result = sqlx::query(
869            "UPDATE minco_job_publications SET status = 'pending', claimed_by = NULL, \
870             claim_expires_at = NULL, lease_id = NULL \
871             WHERE status = 'claimed' AND claim_expires_at IS NOT NULL AND claim_expires_at <= $1",
872        )
873        .bind(now.to_rfc3339())
874        .execute(&self.pool)
875        .await
876        .map_err(|error| infrastructure(&error))?;
877        Ok(usize::try_from(result.rows_affected()).unwrap_or(usize::MAX))
878    }
879}
880
881#[async_trait::async_trait]
882impl minco_plugin_jobs::OverlapLockStore for SqliteJobStore {
883    async fn acquire(
884        &self,
885        overlap_key: &str,
886        lease_id: Uuid,
887        expires_at: DateTime<Utc>,
888        now: DateTime<Utc>,
889    ) -> Result<bool, JobError> {
890        let mut transaction = self
891            .pool
892            .begin_with("BEGIN IMMEDIATE")
893            .await
894            .map_err(|error| infrastructure(&error))?;
895        let expired: bool = sqlx::query_scalar(
896            "SELECT EXISTS(SELECT 1 FROM minco_job_locks WHERE overlap_key = $1 AND expires_at <= $2)",
897        )
898        .bind(overlap_key)
899        .bind(now.to_rfc3339())
900        .fetch_one(&mut *transaction)
901        .await
902        .map_err(|error| infrastructure(&error))?;
903        let acquired = if expired {
904            sqlx::query(
905                "UPDATE minco_job_locks SET owner = $2, expires_at = $3 \
906                 WHERE overlap_key = $1 AND expires_at <= $4",
907            )
908            .bind(overlap_key)
909            .bind(lease_id.to_string())
910            .bind(expires_at.to_rfc3339())
911            .bind(now.to_rfc3339())
912            .execute(&mut *transaction)
913            .await
914            .map_err(|error| infrastructure(&error))?
915            .rows_affected()
916                == 1
917        } else {
918            sqlx::query("INSERT OR IGNORE INTO minco_job_locks (overlap_key, owner, expires_at) VALUES ($1, $2, $3)")
919                .bind(overlap_key)
920                .bind(lease_id.to_string())
921                .bind(expires_at.to_rfc3339())
922                .execute(&mut *transaction)
923                .await
924                .map_err(|error| infrastructure(&error))?
925                .rows_affected()
926                == 1
927        };
928        transaction
929            .commit()
930            .await
931            .map_err(|error| infrastructure(&error))?;
932        Ok(acquired)
933    }
934
935    async fn refresh(
936        &self,
937        overlap_key: &str,
938        lease_id: Uuid,
939        expires_at: DateTime<Utc>,
940        now: DateTime<Utc>,
941    ) -> Result<bool, JobError> {
942        let result = sqlx::query(
943            "UPDATE minco_job_locks SET expires_at = $3 \
944             WHERE overlap_key = $1 AND owner = $2 AND expires_at > $4",
945        )
946        .bind(overlap_key)
947        .bind(lease_id.to_string())
948        .bind(expires_at.to_rfc3339())
949        .bind(now.to_rfc3339())
950        .execute(&self.pool)
951        .await
952        .map_err(|error| infrastructure(&error))?;
953        Ok(result.rows_affected() == 1)
954    }
955
956    async fn release(&self, overlap_key: &str, lease_id: Uuid) -> Result<(), JobError> {
957        sqlx::query("DELETE FROM minco_job_locks WHERE overlap_key = $1 AND owner = $2")
958            .bind(overlap_key)
959            .bind(lease_id.to_string())
960            .execute(&self.pool)
961            .await
962            .map_err(|error| infrastructure(&error))?;
963        Ok(())
964    }
965
966    async fn recover_expired(&self, now: DateTime<Utc>) -> Result<usize, JobError> {
967        let result = sqlx::query("DELETE FROM minco_job_locks WHERE expires_at <= $1")
968            .bind(now.to_rfc3339())
969            .execute(&self.pool)
970            .await
971            .map_err(|error| infrastructure(&error))?;
972        Ok(usize::try_from(result.rows_affected()).unwrap_or(usize::MAX))
973    }
974}
975
976#[cfg(test)]
977mod tests {
978    use super::*;
979    use minco_plugin_jobs::{
980        EnqueueOutcome, JobEnvelope, JobOptions, JobPublicationStore as _, JobStore as _,
981        OverlapLockStore as _, RetryPolicy,
982    };
983
984    async fn pool() -> SqlitePool {
985        let directory = tempfile::tempdir().expect("temp dir");
986        let path = directory.path().join("jobs.sqlite");
987        std::mem::forget(directory);
988        crate::plugin_adapters::migrate_plugin_storage(
989            &SqlitePool::connect(&format!("sqlite://{}?mode=rwc", path.display()))
990                .await
991                .expect("sqlite pool"),
992        )
993        .await
994        .expect("migrations");
995        SqlitePool::connect(&format!("sqlite://{}", path.display()))
996            .await
997            .expect("reconnect")
998    }
999
1000    fn record(dedupe_key: Option<&str>) -> JobRecord {
1001        let mut options = JobOptions::default().with_retry(RetryPolicy::fixed(5, 1));
1002        if let Some(key) = dedupe_key {
1003            options = options.with_dedupe_key(key);
1004        }
1005        let envelope = JobEnvelope::for_parts(
1006            "orders.send-confirmation",
1007            1,
1008            serde_json::json!({ "order_id": "o-1" }),
1009            "orders-notifications",
1010            Uuid::now_v7(),
1011        )
1012        .expect("valid envelope")
1013        .with(options);
1014        minco_plugin_jobs::pending_record(envelope)
1015    }
1016
1017    #[tokio::test]
1018    async fn enqueue_in_rolls_back_with_the_callers_transaction() {
1019        let pool = pool().await;
1020        let store = SqliteJobStore::new(pool.clone());
1021        let mut transaction = pool.begin_with("BEGIN IMMEDIATE").await.expect("begin");
1022        let record = record(None);
1023        store
1024            .enqueue_in(&mut transaction, record.clone())
1025            .await
1026            .expect("enqueue in tx");
1027        transaction.rollback().await.expect("rollback");
1028        assert!(
1029            store
1030                .get(record.envelope.job_id)
1031                .await
1032                .expect("get")
1033                .is_none()
1034        );
1035        let publications = store
1036            .claim_due(
1037                "probe",
1038                10,
1039                Utc::now() + chrono::TimeDelta::minutes(1),
1040                Utc::now(),
1041            )
1042            .await
1043            .expect("probe");
1044        assert!(publications.is_empty(), "rollback leaves no intent");
1045    }
1046
1047    #[tokio::test]
1048    async fn enqueue_in_commits_exactly_one_recoverable_generation() {
1049        let pool = pool().await;
1050        let store = SqliteJobStore::new(pool.clone());
1051        let mut transaction = pool.begin_with("BEGIN IMMEDIATE").await.expect("begin");
1052        let record = record(None);
1053        match store
1054            .enqueue_in(&mut transaction, record.clone())
1055            .await
1056            .expect("enqueue")
1057        {
1058            EnqueueOutcome::Inserted(job_id) => assert_eq!(job_id, record.envelope.job_id),
1059            EnqueueOutcome::Duplicate(existing) => {
1060                panic!("expected insertion, got duplicate {existing}")
1061            }
1062        }
1063        transaction.commit().await.expect("commit");
1064        let publications = store
1065            .claim_due(
1066                "probe",
1067                10,
1068                Utc::now() + chrono::TimeDelta::minutes(1),
1069                Utc::now(),
1070            )
1071            .await
1072            .expect("claim");
1073        assert_eq!(publications.len(), 1);
1074        assert_eq!(publications[0].generation, 1);
1075        assert!(publications[0].lease_id.is_some());
1076    }
1077
1078    #[tokio::test]
1079    async fn stale_claims_cannot_mutate_newer_claims_even_with_one_worker_name() {
1080        let pool = pool().await;
1081        let store = SqliteJobStore::new(pool);
1082        let record = record(None);
1083        store
1084            .enqueue_with_intent(record.clone())
1085            .await
1086            .expect("enqueue");
1087        let start = Utc::now();
1088        let stale = store
1089            .claim_execution(
1090                record.envelope.job_id,
1091                "same-worker-name",
1092                start + chrono::TimeDelta::minutes(1),
1093                start,
1094            )
1095            .await
1096            .expect("first claim")
1097            .expect("claimed");
1098        let newer = store
1099            .claim_execution(
1100                record.envelope.job_id,
1101                "same-worker-name",
1102                start + chrono::TimeDelta::minutes(30),
1103                start + chrono::TimeDelta::minutes(2),
1104            )
1105            .await
1106            .expect("reclaim")
1107            .expect("reclaimed");
1108        assert_ne!(stale.lease_id, newer.lease_id);
1109        let error = store.complete(&stale, start).await.unwrap_err();
1110        assert!(matches!(error, JobError::LeaseFencedOut { .. }));
1111        let error = store
1112            .schedule_retry_and_publish(
1113                &stale,
1114                "stale",
1115                start + chrono::TimeDelta::minutes(3),
1116                start,
1117            )
1118            .await
1119            .unwrap_err();
1120        assert!(matches!(error, JobError::LeaseFencedOut { .. }));
1121        let error = store
1122            .fail_permanently(&stale, "stale", start)
1123            .await
1124            .unwrap_err();
1125        assert!(matches!(error, JobError::LeaseFencedOut { .. }));
1126        store.complete(&newer, start).await.expect("newer owns it");
1127    }
1128
1129    #[tokio::test]
1130    async fn retry_state_and_next_generation_commit_together() {
1131        let pool = pool().await;
1132        let store = SqliteJobStore::new(pool);
1133        let record = record(None);
1134        store
1135            .enqueue_with_intent(record.clone())
1136            .await
1137            .expect("enqueue");
1138        let now = Utc::now();
1139        let delivered = store
1140            .claim_due("dispatcher-1", 10, now + chrono::TimeDelta::minutes(1), now)
1141            .await
1142            .expect("deliver generation 1");
1143        assert_eq!(delivered.len(), 1);
1144        store
1145            .mark_published(
1146                delivered[0].publication_id,
1147                delivered[0].lease_id.expect("lease"),
1148            )
1149            .await
1150            .expect("published");
1151        let claim = store
1152            .claim_execution(
1153                record.envelope.job_id,
1154                "worker-exec-1",
1155                now + chrono::TimeDelta::minutes(5),
1156                now,
1157            )
1158            .await
1159            .expect("claim")
1160            .expect("claimed");
1161        let retry_at = now + chrono::TimeDelta::seconds(60);
1162        let publication_id = store
1163            .schedule_retry_and_publish(&claim, "notification-unavailable", retry_at, now)
1164            .await
1165            .expect("atomic retry");
1166        assert_ne!(publication_id, Uuid::nil());
1167        let generations: Vec<(i64, String)> = sqlx::query_as(
1168            "SELECT generation, status FROM minco_job_publications WHERE job_id = $1 ORDER BY \
1169             generation",
1170        )
1171        .bind(record.envelope.job_id.to_string())
1172        .fetch_all(&store.pool)
1173        .await
1174        .expect("generations");
1175        assert_eq!(
1176            generations.len(),
1177            2,
1178            "generation 2 committed with the retry"
1179        );
1180        assert_eq!(generations[1].1, "pending");
1181        let early = store
1182            .claim_due("d", 10, now + chrono::TimeDelta::minutes(1), now)
1183            .await
1184            .expect("early");
1185        assert!(early.is_empty(), "generation 2 is not due yet");
1186        let due = store
1187            .claim_due("d", 10, retry_at + chrono::TimeDelta::minutes(1), retry_at)
1188            .await
1189            .expect("due");
1190        assert_eq!(due.len(), 1);
1191        assert_eq!(due[0].generation, 2);
1192    }
1193
1194    #[tokio::test]
1195    async fn execution_claims_admit_exactly_one_owner() {
1196        let pool = pool().await;
1197        let store = SqliteJobStore::new(pool);
1198        let record = record(None);
1199        store
1200            .enqueue_with_intent(record.clone())
1201            .await
1202            .expect("enqueue");
1203        let now = Utc::now();
1204        assert!(
1205            store
1206                .claim_execution(
1207                    record.envelope.job_id,
1208                    "worker-a",
1209                    now + chrono::TimeDelta::minutes(10),
1210                    now
1211                )
1212                .await
1213                .expect("first claim")
1214                .is_some()
1215        );
1216        assert!(
1217            store
1218                .claim_execution(
1219                    record.envelope.job_id,
1220                    "worker-b",
1221                    now + chrono::TimeDelta::minutes(10),
1222                    now
1223                )
1224                .await
1225                .expect("second claim")
1226                .is_none(),
1227            "a live lease blocks other owners"
1228        );
1229    }
1230
1231    #[tokio::test]
1232    async fn duplicate_dedupe_uses_the_semantic_fingerprint() {
1233        let pool = pool().await;
1234        let store = SqliteJobStore::new(pool);
1235        store
1236            .enqueue_with_intent(record(Some("orders.confirm:o-1")))
1237            .await
1238            .expect("first");
1239        match store
1240            .enqueue_with_intent(record(Some("orders.confirm:o-1")))
1241            .await
1242            .expect("second")
1243        {
1244            EnqueueOutcome::Duplicate(_) => {}
1245            EnqueueOutcome::Inserted(inserted) => {
1246                panic!("identical resubmission must be idempotent, got {inserted}")
1247            }
1248        }
1249        let conflicting = record(None);
1250        let mut envelope = conflicting.envelope.clone();
1251        envelope.payload = serde_json::json!({ "order_id": "o-2" });
1252        envelope.dedupe_key = Some("orders.confirm:o-1".into());
1253        let error = store
1254            .enqueue_with_intent(minco_plugin_jobs::pending_record(envelope))
1255            .await
1256            .expect_err("conflict");
1257        assert!(matches!(
1258            error,
1259            JobError::DuplicateSubmissionConflict { .. }
1260        ));
1261    }
1262
1263    #[tokio::test]
1264    async fn ingestion_creates_no_pending_publication() {
1265        let pool = pool().await;
1266        let store = SqliteJobStore::new(pool);
1267        let mut occurrence = record(None);
1268        occurrence.envelope.dedupe_key = Some("orders-nightly:2026-08-22T13:00:00Z".into());
1269        match store
1270            .ingest_existing_delivery(occurrence.clone())
1271            .await
1272            .expect("ingest")
1273        {
1274            minco_plugin_jobs::IngestOutcome::Ingested(job_id) => {
1275                assert_eq!(job_id, occurrence.envelope.job_id);
1276            }
1277            minco_plugin_jobs::IngestOutcome::Duplicate(existing) => {
1278                panic!("first occurrence ingests, got duplicate {existing}")
1279            }
1280        }
1281        let statuses: Vec<(String,)> =
1282            sqlx::query_as("SELECT status FROM minco_job_publications WHERE job_id = $1")
1283                .bind(occurrence.envelope.job_id.to_string())
1284                .fetch_all(&store.pool)
1285                .await
1286                .expect("statuses");
1287        assert_eq!(statuses.len(), 1);
1288        assert_eq!(statuses[0].0, "published", "no pending generation appears");
1289        match store
1290            .ingest_existing_delivery(occurrence)
1291            .await
1292            .expect("re")
1293        {
1294            minco_plugin_jobs::IngestOutcome::Duplicate(_) => {}
1295            minco_plugin_jobs::IngestOutcome::Ingested(job_id) => {
1296                panic!("re-ingestion is idempotent, got {job_id}")
1297            }
1298        }
1299    }
1300
1301    #[tokio::test]
1302    async fn stale_overlap_owner_cannot_release_a_newer_lock() {
1303        let pool = pool().await;
1304        let store = SqliteJobStore::new(pool);
1305        let now = Utc::now();
1306        let stale_lease = Uuid::now_v7();
1307        assert!(
1308            store
1309                .acquire(
1310                    "orders.confirm:o-1",
1311                    stale_lease,
1312                    now + chrono::TimeDelta::minutes(1),
1313                    now
1314                )
1315                .await
1316                .expect("acquire")
1317        );
1318        let newer_lease = Uuid::now_v7();
1319        assert!(
1320            store
1321                .acquire(
1322                    "orders.confirm:o-1",
1323                    newer_lease,
1324                    now + chrono::TimeDelta::minutes(30),
1325                    now + chrono::TimeDelta::minutes(2)
1326                )
1327                .await
1328                .expect("reclaim"),
1329            "the expired lock is reclaimable"
1330        );
1331        store
1332            .release("orders.confirm:o-1", stale_lease)
1333            .await
1334            .expect("stale release is a no-op");
1335        let held: Option<(String,)> =
1336            sqlx::query_as("SELECT owner FROM minco_job_locks WHERE overlap_key = $1")
1337                .bind("orders.confirm:o-1")
1338                .fetch_optional(&store.pool)
1339                .await
1340                .expect("held");
1341        assert_eq!(
1342            held.map(|(owner,)| owner),
1343            Some(newer_lease.to_string()),
1344            "the stale owner cannot release the newer lock"
1345        );
1346    }
1347
1348    #[tokio::test]
1349    async fn stale_publication_claimant_cannot_mark_delivery() {
1350        let pool = pool().await;
1351        let store = SqliteJobStore::new(pool);
1352        store
1353            .enqueue_with_intent(record(None))
1354            .await
1355            .expect("enqueue");
1356        let now = Utc::now();
1357        let stale = store
1358            .claim_due("dispatcher-a", 10, now + chrono::TimeDelta::minutes(1), now)
1359            .await
1360            .expect("stale claim");
1361        assert_eq!(stale.len(), 1);
1362        let stale_lease = stale[0].lease_id.expect("lease");
1363        let newer = store
1364            .claim_due(
1365                "dispatcher-b",
1366                10,
1367                now + chrono::TimeDelta::minutes(30),
1368                now + chrono::TimeDelta::minutes(2),
1369            )
1370            .await
1371            .expect("reclaim");
1372        assert_eq!(newer.len(), 1);
1373        let error = store
1374            .mark_published(stale[0].publication_id, stale_lease)
1375            .await
1376            .expect_err("fenced");
1377        assert!(matches!(error, JobError::PublicationFencedOut { .. }));
1378        store
1379            .mark_published(
1380                newer[0].publication_id,
1381                newer[0].lease_id.expect("newer lease"),
1382            )
1383            .await
1384            .expect("newer marks delivery");
1385    }
1386
1387    #[tokio::test]
1388    async fn attempt_history_stays_bounded() {
1389        let pool = pool().await;
1390        let store = SqliteJobStore::new(pool);
1391        let record = record(None);
1392        store
1393            .enqueue_with_intent(record.clone())
1394            .await
1395            .expect("enqueue");
1396        let mut now = Utc::now();
1397        for _ in 0..30 {
1398            let claim = store
1399                .claim_execution(
1400                    record.envelope.job_id,
1401                    "worker-exec",
1402                    now + chrono::TimeDelta::minutes(5),
1403                    now,
1404                )
1405                .await
1406                .expect("claim")
1407                .expect("claimed");
1408            store
1409                .schedule_retry_and_publish(
1410                    &claim,
1411                    "always-busy",
1412                    now + chrono::TimeDelta::seconds(1),
1413                    now,
1414                )
1415                .await
1416                .expect("retry");
1417            now += chrono::TimeDelta::seconds(2);
1418        }
1419        let final_record = store
1420            .get(record.envelope.job_id)
1421            .await
1422            .expect("get")
1423            .expect("present");
1424        assert_eq!(final_record.attempts.len(), 25, "history is bounded");
1425        assert_eq!(final_record.attempt_count, 30);
1426    }
1427
1428    #[tokio::test]
1429    async fn operator_transitions_are_revision_guarded() {
1430        let pool = pool().await;
1431        let store = SqliteJobStore::new(pool);
1432        let record = record(None);
1433        store
1434            .enqueue_with_intent(record.clone())
1435            .await
1436            .expect("enqueue");
1437        let job_id = record.envelope.job_id;
1438        let current = store.get(job_id).await.unwrap().unwrap();
1439        let error = store
1440            .cancel(job_id, current.revision + 1, Utc::now())
1441            .await
1442            .unwrap_err();
1443        assert!(matches!(error, JobError::RevisionConflict { .. }));
1444        store
1445            .cancel(job_id, current.revision, Utc::now())
1446            .await
1447            .unwrap();
1448    }
1449}