1use chrono::{DateTime, Utc};
12use minco_plugin_jobs::{
13 EnqueueOutcome, IngestOutcome, JobAttempt, JobClaim, JobError, JobPublication, JobRecord,
14 JobStatus, PublicationStatus, semantic_fingerprint, validate_worker_claim,
15};
16use sqlx::{Row, SqlitePool, Transaction, sqlite::SqliteRow};
17use uuid::Uuid;
18
19#[derive(Debug, Clone)]
21pub struct SqliteJobStore {
22 pool: SqlitePool,
23}
24
25impl SqliteJobStore {
26 #[must_use]
27 pub const fn new(pool: SqlitePool) -> Self {
28 Self { pool }
29 }
30
31 pub async fn enqueue_in(
35 &self,
36 transaction: &mut Transaction<'_, sqlx::Sqlite>,
37 record: JobRecord,
38 ) -> Result<EnqueueOutcome, JobError> {
39 Self::enqueue_record_in(transaction, record, "pending").await
40 }
41
42 pub async fn ingest_in(
45 &self,
46 transaction: &mut Transaction<'_, sqlx::Sqlite>,
47 record: JobRecord,
48 ) -> Result<IngestOutcome, JobError> {
49 match Self::enqueue_record_in(transaction, record, "published").await? {
50 EnqueueOutcome::Inserted(job_id) => Ok(IngestOutcome::Ingested(job_id)),
51 EnqueueOutcome::Duplicate(existing) => Ok(IngestOutcome::Duplicate(existing)),
52 }
53 }
54
55 async fn enqueue_record_in(
56 transaction: &mut Transaction<'_, sqlx::Sqlite>,
57 record: JobRecord,
58 publication_status: &str,
59 ) -> Result<EnqueueOutcome, JobError> {
60 if record.status != JobStatus::Pending {
61 return Err(JobError::InvalidJob("jobs must be enqueued pending".into()));
62 }
63 record.envelope.validate()?;
64 let envelope_json = serde_json::to_string(&record.envelope).map_err(|error| {
65 JobError::Infrastructure(format!("job envelope encode failed: {error}"))
66 })?;
67 let attempts = serde_json::to_string(&record.attempts)
68 .map_err(|error| JobError::Infrastructure(format!("attempt encode failed: {error}")))?;
69 let fingerprint = semantic_fingerprint(&record.envelope);
70 let result = sqlx::query(
71 "INSERT INTO minco_jobs (job_id, worker_profile, envelope, fingerprint, status, \
72 revision, available_at, attempt_count, lease_id, lease_expires_at, attempts, \
73 dedupe_key, failure_code, completed_at) \
74 VALUES ($1, $2, $3, $4, 'pending', $5, $6, $7, NULL, NULL, $8, $9, NULL, NULL)",
75 )
76 .bind(record.envelope.job_id.to_string())
77 .bind(&record.envelope.worker_profile)
78 .bind(&envelope_json)
79 .bind(&fingerprint)
80 .bind(i64::try_from(record.revision).unwrap_or(1))
81 .bind(record.envelope.available_at.to_rfc3339())
82 .bind(i64::from(record.attempt_count))
83 .bind(&attempts)
84 .bind(record.envelope.dedupe_key.as_deref())
85 .execute(&mut **transaction)
86 .await;
87 if let Err(error) = result {
88 let message = format!("{error}");
89 if message.contains("minco_jobs.job_id") {
90 let existing: Option<(String,)> =
94 sqlx::query_as("SELECT fingerprint FROM minco_jobs WHERE job_id = $1")
95 .bind(record.envelope.job_id.to_string())
96 .fetch_optional(&mut **transaction)
97 .await
98 .map_err(|_| {
99 JobError::Infrastructure("job identity probe failed".into())
100 })?;
101 return match existing {
102 Some((existing_fingerprint,)) if existing_fingerprint == fingerprint => {
103 Ok(EnqueueOutcome::Duplicate(record.envelope.job_id))
104 }
105 _ => Err(JobError::DuplicateJobIdentity(record.envelope.job_id)),
106 };
107 }
108 if message.contains("minco_jobs.dedupe_key") {
109 return dedupe_outcome(&record, &fingerprint, transaction).await;
110 }
111 return Err(infrastructure(&error));
112 }
113 sqlx::query(
114 "INSERT INTO minco_job_publications (publication_id, job_id, generation, \
115 worker_profile, status, attempt_count, available_at, claimed_by, \
116 claim_expires_at, lease_id, last_error) \
117 VALUES ($1, $2, 1, $3, $4, 0, $5, NULL, NULL, NULL, NULL)",
118 )
119 .bind(Uuid::now_v7().to_string())
120 .bind(record.envelope.job_id.to_string())
121 .bind(&record.envelope.worker_profile)
122 .bind(publication_status)
123 .bind(record.envelope.available_at.to_rfc3339())
124 .execute(&mut **transaction)
125 .await
126 .map_err(|_| JobError::Infrastructure("job publication insert failed".into()))?;
127 Ok(EnqueueOutcome::Inserted(record.envelope.job_id))
128 }
129}
130
131async fn dedupe_outcome(
132 record: &JobRecord,
133 fingerprint: &str,
134 transaction: &mut Transaction<'_, sqlx::Sqlite>,
135) -> Result<EnqueueOutcome, JobError> {
136 let existing: Option<(String, String)> =
137 sqlx::query_as("SELECT envelope, fingerprint FROM minco_jobs WHERE dedupe_key = $1")
138 .bind(record.envelope.dedupe_key.as_deref())
139 .fetch_optional(&mut **transaction)
140 .await
141 .map_err(|_| JobError::Infrastructure("job dedupe lookup failed".into()))?;
142 let Some((existing_envelope, existing_fingerprint)) = existing else {
143 return Err(JobError::Infrastructure(
144 "job dedupe conflict vanished during insert".into(),
145 ));
146 };
147 let existing: serde_json::Value = serde_json::from_str(&existing_envelope)
148 .map_err(|_| JobError::Infrastructure("existing job envelope decode failed".into()))?;
149 let existing_id = existing
150 .get("job_id")
151 .and_then(serde_json::Value::as_str)
152 .and_then(|value| value.parse::<Uuid>().ok());
153 match (existing_fingerprint == fingerprint, existing_id) {
154 (true, Some(existing_job_id)) => Ok(EnqueueOutcome::Duplicate(existing_job_id)),
155 (false, Some(existing_job_id)) => {
156 Err(JobError::DuplicateSubmissionConflict { existing_job_id })
157 }
158 _ => Err(JobError::Infrastructure(
159 "job dedupe conflict could not be resolved".into(),
160 )),
161 }
162}
163
164fn infrastructure(error: &sqlx::Error) -> JobError {
165 JobError::Infrastructure(format!("sqlite job store failed: {error}"))
166}
167
168fn parse_status(status: &str) -> Result<JobStatus, JobError> {
169 match status {
170 "pending" => Ok(JobStatus::Pending),
171 "running" => Ok(JobStatus::Running),
172 "succeeded" => Ok(JobStatus::Succeeded),
173 "failed_permanently" => Ok(JobStatus::FailedPermanently),
174 "cancelled" => Ok(JobStatus::Cancelled),
175 other => Err(JobError::Infrastructure(format!(
176 "unknown job status {other}"
177 ))),
178 }
179}
180
181fn parse_timestamp(value: &str) -> Result<DateTime<Utc>, JobError> {
182 DateTime::parse_from_rfc3339(value)
183 .map(|parsed| parsed.with_timezone(&Utc))
184 .map_err(|_| JobError::Infrastructure("timestamp decode failed".into()))
185}
186
187fn optional_timestamp(value: Option<&str>) -> Result<Option<DateTime<Utc>>, JobError> {
188 value.map(parse_timestamp).transpose()
189}
190
191fn optional_uuid(value: Option<&str>) -> Option<Uuid> {
192 value.and_then(|text| text.parse::<Uuid>().ok())
193}
194
195fn decode_job_row(row: &SqliteRow) -> Result<JobRecord, JobError> {
196 let envelope_text: String = row
197 .try_get("envelope")
198 .map_err(|_| JobError::Infrastructure("job envelope column unreadable".into()))?;
199 let mut envelope: minco_plugin_jobs::JobEnvelope = serde_json::from_str(&envelope_text)
200 .map_err(|error| {
201 JobError::Infrastructure(format!("job envelope decode failed: {error}"))
202 })?;
203 let status: String = row
204 .try_get("status")
205 .map_err(|_| JobError::Infrastructure("job status column unreadable".into()))?;
206 let attempt_count: i64 = row
207 .try_get("attempt_count")
208 .map_err(|_| JobError::Infrastructure("job attempt column unreadable".into()))?;
209 envelope.available_at = parse_timestamp(
210 row.try_get("available_at")
211 .map_err(|_| JobError::Infrastructure("job availability unreadable".into()))?,
212 )?;
213 envelope.attempt = u32::try_from(attempt_count)
214 .unwrap_or(envelope.attempt)
215 .max(1);
216 let attempts_text: String = row
217 .try_get("attempts")
218 .map_err(|_| JobError::Infrastructure("job attempts unreadable".into()))?;
219 let attempts: Vec<JobAttempt> = serde_json::from_str(&attempts_text)
220 .map_err(|error| JobError::Infrastructure(format!("attempt decode failed: {error}")))?;
221 Ok(JobRecord {
222 envelope,
223 status: parse_status(&status)?,
224 revision: u64::try_from(
225 row.try_get::<i64, _>("revision")
226 .map_err(|_| JobError::Infrastructure("job revision unreadable".into()))?,
227 )
228 .unwrap_or(1),
229 lease_id: optional_uuid(
230 row.try_get::<Option<String>, _>("lease_id")
231 .map_err(|_| JobError::Infrastructure("job lease identity unreadable".into()))?
232 .as_deref(),
233 ),
234 lease_expires_at: optional_timestamp(
235 row.try_get::<Option<String>, _>("lease_expires_at")
236 .map_err(|_| JobError::Infrastructure("job lease expiry unreadable".into()))?
237 .as_deref(),
238 )?,
239 attempt_count: u32::try_from(attempt_count).unwrap_or(0),
240 attempts,
241 failure_code: row
242 .try_get("failure_code")
243 .map_err(|_| JobError::Infrastructure("job failure code unreadable".into()))?,
244 completed_at: optional_timestamp(
245 row.try_get::<Option<String>, _>("completed_at")
246 .map_err(|_| JobError::Infrastructure("job completion unreadable".into()))?
247 .as_deref(),
248 )?,
249 })
250}
251
252fn attempt_entry(
253 attempt: u32,
254 worker_execution_id: &str,
255 now: DateTime<Utc>,
256 outcome: minco_plugin_jobs::JobAttemptOutcome,
257) -> Result<String, JobError> {
258 serde_json::to_string(&JobAttempt {
259 attempt,
260 at: now,
261 worker_execution_id: worker_execution_id.to_owned(),
262 outcome,
263 })
264 .map_err(|error| JobError::Infrastructure(format!("attempt encode failed: {error}")))
265}
266
267fn decode_publication_row(row: &SqliteRow) -> Result<JobPublication, JobError> {
268 let status: String = row
269 .try_get("status")
270 .map_err(|_| JobError::Infrastructure("publication status unreadable".into()))?;
271 let status = match status.as_str() {
272 "pending" => PublicationStatus::Pending,
273 "claimed" => PublicationStatus::Claimed,
274 "published" => PublicationStatus::Published,
275 "failed" => PublicationStatus::Failed,
276 other => {
277 return Err(JobError::Infrastructure(format!(
278 "unknown publication status {other}"
279 )));
280 }
281 };
282 let publication_id: String = row
283 .try_get("publication_id")
284 .map_err(|_| JobError::Infrastructure("publication identity unreadable".into()))?;
285 let job_id: String = row
286 .try_get("job_id")
287 .map_err(|_| JobError::Infrastructure("publication job unreadable".into()))?;
288 Ok(JobPublication {
289 publication_id: Uuid::parse_str(&publication_id)
290 .map_err(|_| JobError::Infrastructure("publication identity decode failed".into()))?,
291 job_id: Uuid::parse_str(&job_id)
292 .map_err(|_| JobError::Infrastructure("publication job decode failed".into()))?,
293 generation: u32::try_from(
294 row.try_get::<i64, _>("generation").map_err(|_| {
295 JobError::Infrastructure("publication generation unreadable".into())
296 })?,
297 )
298 .unwrap_or(1),
299 worker_profile: row
300 .try_get("worker_profile")
301 .map_err(|_| JobError::Infrastructure("publication profile unreadable".into()))?,
302 status,
303 attempt_count: u32::try_from(
304 row.try_get::<i64, _>("attempt_count")
305 .map_err(|_| JobError::Infrastructure("publication attempts unreadable".into()))?,
306 )
307 .unwrap_or(0),
308 available_at: parse_timestamp(row.try_get("available_at").map_err(|_| {
309 JobError::Infrastructure("publication availability unreadable".into())
310 })?)?,
311 claimed_by: row
312 .try_get("claimed_by")
313 .map_err(|_| JobError::Infrastructure("publication claimant unreadable".into()))?,
314 claim_expires_at: optional_timestamp(
315 row.try_get::<Option<String>, _>("claim_expires_at")
316 .map_err(|_| {
317 JobError::Infrastructure("publication claim expiry unreadable".into())
318 })?
319 .as_deref(),
320 )?,
321 lease_id: optional_uuid(
322 row.try_get::<Option<String>, _>("lease_id")
323 .map_err(|_| JobError::Infrastructure("publication lease unreadable".into()))?
324 .as_deref(),
325 ),
326 last_error: row
327 .try_get("last_error")
328 .map_err(|_| JobError::Infrastructure("publication error unreadable".into()))?,
329 })
330}
331
332async fn verify_job_update(
333 pool: &SqlitePool,
334 job_id: Uuid,
335 result: &sqlx::sqlite::SqliteQueryResult,
336) -> Result<(), JobError> {
337 if result.rows_affected() == 1 {
338 return Ok(());
339 }
340 let exists: bool =
341 sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM minco_jobs WHERE job_id = $1)")
342 .bind(job_id.to_string())
343 .fetch_one(pool)
344 .await
345 .map_err(|_| JobError::Infrastructure("job existence probe failed".into()))?;
346 if exists {
347 Err(JobError::LeaseFencedOut { job_id })
348 } else {
349 Err(JobError::MissingJob(job_id))
350 }
351}
352
353async fn verify_revision_guard(
354 pool: &SqlitePool,
355 job_id: Uuid,
356 expected_revision: u64,
357 result: &sqlx::sqlite::SqliteQueryResult,
358) -> Result<(), JobError> {
359 if result.rows_affected() == 1 {
360 return Ok(());
361 }
362 let current: Option<i64> =
363 sqlx::query_scalar("SELECT revision FROM minco_jobs WHERE job_id = $1")
364 .bind(job_id.to_string())
365 .fetch_optional(pool)
366 .await
367 .map_err(|error| infrastructure(&error))?;
368 match current {
369 Some(revision) if u64::try_from(revision).unwrap_or(0) != expected_revision => {
370 Err(JobError::RevisionConflict {
371 job_id,
372 expected_revision,
373 })
374 }
375 Some(_) => Err(JobError::InvalidTransition { job_id }),
376 None => Err(JobError::MissingJob(job_id)),
377 }
378}
379
380async fn verify_publication_update(
381 pool: &SqlitePool,
382 publication_id: Uuid,
383 result: &sqlx::sqlite::SqliteQueryResult,
384) -> Result<(), JobError> {
385 if result.rows_affected() == 1 {
386 return Ok(());
387 }
388 let exists: bool = sqlx::query_scalar(
389 "SELECT EXISTS(SELECT 1 FROM minco_job_publications WHERE publication_id = $1)",
390 )
391 .bind(publication_id.to_string())
392 .fetch_one(pool)
393 .await
394 .map_err(|_| JobError::Infrastructure("publication probe failed".into()))?;
395 if exists {
396 Err(JobError::PublicationFencedOut { publication_id })
397 } else {
398 Err(JobError::MissingJob(publication_id))
399 }
400}
401
402#[async_trait::async_trait]
403impl minco_plugin_jobs::JobStore for SqliteJobStore {
404 async fn enqueue_with_intent(&self, record: JobRecord) -> Result<EnqueueOutcome, JobError> {
405 let mut transaction = self
406 .pool
407 .begin_with("BEGIN IMMEDIATE")
408 .await
409 .map_err(|error| infrastructure(&error))?;
410 let outcome = self.enqueue_in(&mut transaction, record).await?;
411 transaction
412 .commit()
413 .await
414 .map_err(|error| infrastructure(&error))?;
415 Ok(outcome)
416 }
417
418 async fn ingest_existing_delivery(&self, record: JobRecord) -> Result<IngestOutcome, JobError> {
419 let mut transaction = self
420 .pool
421 .begin_with("BEGIN IMMEDIATE")
422 .await
423 .map_err(|error| infrastructure(&error))?;
424 let outcome = self.ingest_in(&mut transaction, record).await?;
425 transaction
426 .commit()
427 .await
428 .map_err(|error| infrastructure(&error))?;
429 Ok(outcome)
430 }
431
432 async fn claim_execution(
433 &self,
434 job_id: Uuid,
435 worker_execution_id: &str,
436 lease_expires_at: DateTime<Utc>,
437 now: DateTime<Utc>,
438 ) -> Result<Option<JobClaim>, JobError> {
439 validate_worker_claim(worker_execution_id, 1, lease_expires_at, now)?;
440 let mut transaction = self
441 .pool
442 .begin_with("BEGIN IMMEDIATE")
443 .await
444 .map_err(|error| infrastructure(&error))?;
445 let claimable: Option<i64> = sqlx::query_scalar(
446 "SELECT CASE WHEN (status = 'pending' AND available_at <= $2) \
447 OR (status = 'running' AND lease_expires_at IS NOT NULL AND lease_expires_at <= $2) \
448 THEN 1 ELSE 0 END FROM minco_jobs WHERE job_id = $1",
449 )
450 .bind(job_id.to_string())
451 .bind(now.to_rfc3339())
452 .fetch_optional(&mut *transaction)
453 .await
454 .map_err(|error| infrastructure(&error))?;
455 if claimable != Some(1) {
456 transaction
457 .commit()
458 .await
459 .map_err(|error| infrastructure(&error))?;
460 return Ok(None);
461 }
462 let lease_id = Uuid::now_v7();
463 sqlx::query(
464 "UPDATE minco_jobs SET status = 'running', lease_id = $2, lease_expires_at = $3, \
465 attempt_count = attempt_count + 1, revision = revision + 1 WHERE job_id = $1",
466 )
467 .bind(job_id.to_string())
468 .bind(lease_id.to_string())
469 .bind(lease_expires_at.to_rfc3339())
470 .execute(&mut *transaction)
471 .await
472 .map_err(|error| infrastructure(&error))?;
473 let row = sqlx::query(
474 "SELECT job_id, worker_profile, envelope, status, revision, available_at, \
475 attempt_count, lease_id, lease_expires_at, attempts, dedupe_key, failure_code, \
476 completed_at FROM minco_jobs WHERE job_id = $1",
477 )
478 .bind(job_id.to_string())
479 .fetch_one(&mut *transaction)
480 .await
481 .map_err(|error| infrastructure(&error))?;
482 transaction
483 .commit()
484 .await
485 .map_err(|error| infrastructure(&error))?;
486 let record = decode_job_row(&row)?;
487 Ok(Some(JobClaim {
488 fence: record.revision,
489 record,
490 lease_id,
491 }))
492 }
493
494 async fn complete(&self, claim: &JobClaim, now: DateTime<Utc>) -> Result<(), JobError> {
495 let job_id = claim.record.envelope.job_id;
496 let entry = attempt_entry(
497 claim.record.attempt_count,
498 &format!("lease-{}", claim.lease_id),
499 now,
500 minco_plugin_jobs::JobAttemptOutcome::Succeeded,
501 )?;
502 let result = sqlx::query(
503 "UPDATE minco_jobs SET status = 'succeeded', lease_id = NULL, lease_expires_at = \
504 NULL, failure_code = NULL, completed_at = $3, revision = revision + 1, \
505 attempts = json_insert(CASE WHEN json_array_length(attempts) >= 25 \
506 THEN json_remove(attempts, '$[0]') ELSE attempts END, '$[#]', json($4)) \
507 WHERE job_id = $1 AND status = 'running' AND lease_id = $2",
508 )
509 .bind(job_id.to_string())
510 .bind(claim.lease_id.to_string())
511 .bind(now.to_rfc3339())
512 .bind(entry)
513 .execute(&self.pool)
514 .await
515 .map_err(|error| infrastructure(&error))?;
516 verify_job_update(&self.pool, job_id, &result).await
517 }
518
519 async fn schedule_retry_and_publish(
520 &self,
521 claim: &JobClaim,
522 failure_code: &str,
523 next_available_at: DateTime<Utc>,
524 now: DateTime<Utc>,
525 ) -> Result<Uuid, JobError> {
526 let job_id = claim.record.envelope.job_id;
527 let entry = attempt_entry(
528 claim.record.attempt_count,
529 &format!("lease-{}", claim.lease_id),
530 now,
531 minco_plugin_jobs::JobAttemptOutcome::Retried {
532 code: failure_code.to_owned(),
533 },
534 )?;
535 let mut transaction = self
536 .pool
537 .begin_with("BEGIN IMMEDIATE")
538 .await
539 .map_err(|error| infrastructure(&error))?;
540 let result = sqlx::query(
541 "UPDATE minco_jobs SET status = 'pending', lease_id = NULL, lease_expires_at = \
542 NULL, available_at = $4, failure_code = $3, revision = revision + 1, \
543 attempts = json_insert(CASE WHEN json_array_length(attempts) >= 25 \
544 THEN json_remove(attempts, '$[0]') ELSE attempts END, '$[#]', json($5)) \
545 WHERE job_id = $1 AND status = 'running' AND lease_id = $2",
546 )
547 .bind(job_id.to_string())
548 .bind(claim.lease_id.to_string())
549 .bind(failure_code)
550 .bind(next_available_at.to_rfc3339())
551 .bind(entry)
552 .execute(&mut *transaction)
553 .await
554 .map_err(|error| infrastructure(&error))?;
555 if result.rows_affected() != 1 {
556 transaction
557 .rollback()
558 .await
559 .map_err(|error| infrastructure(&error))?;
560 verify_job_update(&self.pool, job_id, &result).await?;
561 return Err(JobError::LeaseFencedOut { job_id });
562 }
563 let publication_id = Uuid::now_v7();
564 sqlx::query(
565 "INSERT INTO minco_job_publications (publication_id, job_id, generation, \
566 worker_profile, status, attempt_count, available_at, claimed_by, \
567 claim_expires_at, lease_id, last_error) \
568 SELECT $1, $2, COALESCE(MAX(generation), 0) + 1, $3, 'pending', 0, $4, NULL, \
569 NULL, NULL, NULL FROM minco_job_publications WHERE job_id = $2",
570 )
571 .bind(publication_id.to_string())
572 .bind(job_id.to_string())
573 .bind(&claim.record.envelope.worker_profile)
574 .bind(next_available_at.to_rfc3339())
575 .execute(&mut *transaction)
576 .await
577 .map_err(|error| infrastructure(&error))?;
578 transaction
579 .commit()
580 .await
581 .map_err(|error| infrastructure(&error))?;
582 Ok(publication_id)
583 }
584
585 async fn fail_permanently(
586 &self,
587 claim: &JobClaim,
588 failure_code: &str,
589 now: DateTime<Utc>,
590 ) -> Result<(), JobError> {
591 let job_id = claim.record.envelope.job_id;
592 let entry = attempt_entry(
593 claim.record.attempt_count,
594 &format!("lease-{}", claim.lease_id),
595 now,
596 minco_plugin_jobs::JobAttemptOutcome::FailedPermanently {
597 code: failure_code.to_owned(),
598 },
599 )?;
600 let result = sqlx::query(
601 "UPDATE minco_jobs SET status = 'failed_permanently', lease_id = NULL, \
602 lease_expires_at = NULL, failure_code = $3, completed_at = $4, revision = revision + 1, \
603 attempts = json_insert(CASE WHEN json_array_length(attempts) >= 25 \
604 THEN json_remove(attempts, '$[0]') ELSE attempts END, '$[#]', json($5)) \
605 WHERE job_id = $1 AND status = 'running' AND lease_id = $2",
606 )
607 .bind(job_id.to_string())
608 .bind(claim.lease_id.to_string())
609 .bind(failure_code)
610 .bind(now.to_rfc3339())
611 .bind(entry)
612 .execute(&self.pool)
613 .await
614 .map_err(|error| infrastructure(&error))?;
615 verify_job_update(&self.pool, job_id, &result).await
616 }
617
618 async fn cancel(
619 &self,
620 job_id: Uuid,
621 expected_revision: u64,
622 now: DateTime<Utc>,
623 ) -> Result<(), JobError> {
624 let result = sqlx::query(
625 "UPDATE minco_jobs SET status = 'cancelled', completed_at = $3, revision = revision + 1 \
626 WHERE job_id = $1 AND revision = $2 AND status = 'pending'",
627 )
628 .bind(job_id.to_string())
629 .bind(i64::try_from(expected_revision).unwrap_or(-1))
630 .bind(now.to_rfc3339())
631 .execute(&self.pool)
632 .await
633 .map_err(|error| infrastructure(&error))?;
634 verify_revision_guard(&self.pool, job_id, expected_revision, &result).await
635 }
636
637 async fn retry_failed(
638 &self,
639 job_id: Uuid,
640 expected_revision: u64,
641 now: DateTime<Utc>,
642 ) -> Result<DateTime<Utc>, JobError> {
643 let mut transaction = self
644 .pool
645 .begin_with("BEGIN IMMEDIATE")
646 .await
647 .map_err(|error| infrastructure(&error))?;
648 let result = sqlx::query(
649 "UPDATE minco_jobs SET status = 'pending', failure_code = NULL, completed_at = NULL, \
650 available_at = $3, attempt_count = 0, lease_id = NULL, lease_expires_at = NULL, \
651 revision = revision + 1 WHERE job_id = $1 AND revision = $2 AND status = \
652 'failed_permanently'",
653 )
654 .bind(job_id.to_string())
655 .bind(i64::try_from(expected_revision).unwrap_or(-1))
656 .bind(now.to_rfc3339())
657 .execute(&mut *transaction)
658 .await
659 .map_err(|error| infrastructure(&error))?;
660 if result.rows_affected() != 1 {
661 transaction
662 .rollback()
663 .await
664 .map_err(|error| infrastructure(&error))?;
665 verify_revision_guard(&self.pool, job_id, expected_revision, &result).await?;
666 return Err(JobError::RevisionConflict {
667 job_id,
668 expected_revision,
669 });
670 }
671 let profile: String =
672 sqlx::query_scalar("SELECT worker_profile FROM minco_jobs WHERE job_id = $1")
673 .bind(job_id.to_string())
674 .fetch_one(&mut *transaction)
675 .await
676 .map_err(|error| infrastructure(&error))?;
677 let publication_id = Uuid::now_v7();
678 sqlx::query(
679 "INSERT INTO minco_job_publications (publication_id, job_id, generation, \
680 worker_profile, status, attempt_count, available_at, claimed_by, \
681 claim_expires_at, lease_id, last_error) \
682 SELECT $1, $2, COALESCE(MAX(generation), 0) + 1, $3, 'pending', 0, $4, NULL, \
683 NULL, NULL, NULL FROM minco_job_publications WHERE job_id = $2",
684 )
685 .bind(publication_id.to_string())
686 .bind(job_id.to_string())
687 .bind(&profile)
688 .bind(now.to_rfc3339())
689 .execute(&mut *transaction)
690 .await
691 .map_err(|error| infrastructure(&error))?;
692 transaction
693 .commit()
694 .await
695 .map_err(|error| infrastructure(&error))?;
696 Ok(now)
697 }
698
699 async fn recover_expired_leases(&self, now: DateTime<Utc>) -> Result<usize, JobError> {
700 let mut transaction = self
701 .pool
702 .begin_with("BEGIN IMMEDIATE")
703 .await
704 .map_err(|error| infrastructure(&error))?;
705 let recovered: Vec<(String, String)> = sqlx::query_as(
706 "UPDATE minco_jobs SET status = 'pending', lease_id = NULL, lease_expires_at = \
707 NULL, available_at = $1, revision = revision + 1 \
708 WHERE status = 'running' AND lease_expires_at IS NOT NULL AND lease_expires_at <= $1 \
709 RETURNING job_id, worker_profile",
710 )
711 .bind(now.to_rfc3339())
712 .fetch_all(&mut *transaction)
713 .await
714 .map_err(|error| infrastructure(&error))?;
715 let count = recovered.len();
716 for (job_id, profile) in &recovered {
717 let publication_id = Uuid::now_v7();
718 sqlx::query(
719 "INSERT INTO minco_job_publications (publication_id, job_id, generation, \
720 worker_profile, status, attempt_count, available_at, claimed_by, \
721 claim_expires_at, lease_id, last_error) \
722 SELECT $1, $2, COALESCE(MAX(generation), 0) + 1, $3, 'pending', 0, $4, NULL, \
723 NULL, NULL, NULL FROM minco_job_publications WHERE job_id = $2",
724 )
725 .bind(publication_id.to_string())
726 .bind(job_id)
727 .bind(profile)
728 .bind(now.to_rfc3339())
729 .execute(&mut *transaction)
730 .await
731 .map_err(|error| infrastructure(&error))?;
732 }
733 transaction
734 .commit()
735 .await
736 .map_err(|error| infrastructure(&error))?;
737 Ok(count)
738 }
739
740 async fn get(&self, job_id: Uuid) -> Result<Option<JobRecord>, JobError> {
741 let row = sqlx::query(
742 "SELECT job_id, worker_profile, envelope, status, revision, available_at, \
743 attempt_count, lease_id, lease_expires_at, attempts, dedupe_key, failure_code, \
744 completed_at FROM minco_jobs WHERE job_id = $1",
745 )
746 .bind(job_id.to_string())
747 .fetch_optional(&self.pool)
748 .await
749 .map_err(|error| infrastructure(&error))?;
750 row.map(|row| decode_job_row(&row)).transpose()
751 }
752
753 async fn list_failed(&self, limit: usize) -> Result<Vec<JobRecord>, JobError> {
754 let rows = sqlx::query(
755 "SELECT job_id, worker_profile, envelope, status, revision, available_at, \
756 attempt_count, lease_id, lease_expires_at, attempts, dedupe_key, failure_code, \
757 completed_at FROM minco_jobs WHERE status = 'failed_permanently' \
758 ORDER BY completed_at, job_id LIMIT $1",
759 )
760 .bind(i64::try_from(limit.min(100)).unwrap_or(100))
761 .fetch_all(&self.pool)
762 .await
763 .map_err(|error| infrastructure(&error))?;
764 rows.iter().map(decode_job_row).collect()
765 }
766}
767
768#[async_trait::async_trait]
769impl minco_plugin_jobs::JobPublicationStore for SqliteJobStore {
770 async fn claim_due(
771 &self,
772 worker_execution_id: &str,
773 limit: usize,
774 claim_expires_at: DateTime<Utc>,
775 now: DateTime<Utc>,
776 ) -> Result<Vec<JobPublication>, JobError> {
777 validate_worker_claim(worker_execution_id, limit, claim_expires_at, now)?;
778 let mut transaction = self
779 .pool
780 .begin_with("BEGIN IMMEDIATE")
781 .await
782 .map_err(|error| infrastructure(&error))?;
783 let ids: Vec<(String,)> = sqlx::query_as(
784 "SELECT publication_id FROM minco_job_publications \
785 WHERE (status IN ('pending', 'failed') AND available_at <= $1) \
786 OR (status = 'claimed' AND claim_expires_at IS NOT NULL AND claim_expires_at <= $1) \
787 ORDER BY available_at, publication_id LIMIT $2",
788 )
789 .bind(now.to_rfc3339())
790 .bind(i64::try_from(limit.min(100)).unwrap_or(100))
791 .fetch_all(&mut *transaction)
792 .await
793 .map_err(|error| infrastructure(&error))?;
794 let claimed: Vec<JobPublication> = if ids.is_empty() {
795 Vec::new()
796 } else {
797 let mut builder = sqlx::QueryBuilder::new(
798 "UPDATE minco_job_publications SET status = 'claimed', claimed_by = ",
799 );
800 builder.push_bind(worker_execution_id.to_owned());
801 builder.push(", claim_expires_at = ");
802 builder.push_bind(claim_expires_at.to_rfc3339());
803 builder.push(", lease_id = ");
804 builder.push_bind(Uuid::now_v7().to_string());
805 builder.push(", attempt_count = attempt_count + 1 WHERE publication_id IN (");
806 let mut separated = builder.separated(", ");
807 for (publication_id,) in &ids {
808 separated.push_bind(publication_id.clone());
809 }
810 builder.push(
811 ") RETURNING publication_id, job_id, generation, worker_profile, status, \
812 attempt_count, available_at, claimed_by, claim_expires_at, lease_id, \
813 last_error",
814 );
815 let rows = builder
816 .build()
817 .fetch_all(&mut *transaction)
818 .await
819 .map_err(|error| infrastructure(&error))?;
820 rows.iter()
821 .map(decode_publication_row)
822 .collect::<Result<Vec<_>, _>>()?
823 };
824 transaction
825 .commit()
826 .await
827 .map_err(|error| infrastructure(&error))?;
828 Ok(claimed)
829 }
830
831 async fn mark_published(&self, publication_id: Uuid, lease_id: Uuid) -> Result<(), JobError> {
832 let result = sqlx::query(
833 "UPDATE minco_job_publications SET status = 'published', claimed_by = NULL, \
834 claim_expires_at = NULL, lease_id = NULL, last_error = NULL \
835 WHERE publication_id = $1 AND status = 'claimed' AND lease_id = $2",
836 )
837 .bind(publication_id.to_string())
838 .bind(lease_id.to_string())
839 .execute(&self.pool)
840 .await
841 .map_err(|error| infrastructure(&error))?;
842 verify_publication_update(&self.pool, publication_id, &result).await
843 }
844
845 async fn mark_failed(
846 &self,
847 publication_id: Uuid,
848 lease_id: Uuid,
849 error: &str,
850 retry_at: DateTime<Utc>,
851 ) -> Result<(), JobError> {
852 let result = sqlx::query(
853 "UPDATE minco_job_publications SET status = 'failed', claimed_by = NULL, \
854 claim_expires_at = NULL, lease_id = NULL, available_at = $3, last_error = $4 \
855 WHERE publication_id = $1 AND status = 'claimed' AND lease_id = $2",
856 )
857 .bind(publication_id.to_string())
858 .bind(lease_id.to_string())
859 .bind(retry_at.to_rfc3339())
860 .bind(error)
861 .execute(&self.pool)
862 .await
863 .map_err(|error| infrastructure(&error))?;
864 verify_publication_update(&self.pool, publication_id, &result).await
865 }
866
867 async fn recover_expired_claims(&self, now: DateTime<Utc>) -> Result<usize, JobError> {
868 let result = sqlx::query(
869 "UPDATE minco_job_publications SET status = 'pending', claimed_by = NULL, \
870 claim_expires_at = NULL, lease_id = NULL \
871 WHERE status = 'claimed' AND claim_expires_at IS NOT NULL AND claim_expires_at <= $1",
872 )
873 .bind(now.to_rfc3339())
874 .execute(&self.pool)
875 .await
876 .map_err(|error| infrastructure(&error))?;
877 Ok(usize::try_from(result.rows_affected()).unwrap_or(usize::MAX))
878 }
879}
880
881#[async_trait::async_trait]
882impl minco_plugin_jobs::OverlapLockStore for SqliteJobStore {
883 async fn acquire(
884 &self,
885 overlap_key: &str,
886 lease_id: Uuid,
887 expires_at: DateTime<Utc>,
888 now: DateTime<Utc>,
889 ) -> Result<bool, JobError> {
890 let mut transaction = self
891 .pool
892 .begin_with("BEGIN IMMEDIATE")
893 .await
894 .map_err(|error| infrastructure(&error))?;
895 let expired: bool = sqlx::query_scalar(
896 "SELECT EXISTS(SELECT 1 FROM minco_job_locks WHERE overlap_key = $1 AND expires_at <= $2)",
897 )
898 .bind(overlap_key)
899 .bind(now.to_rfc3339())
900 .fetch_one(&mut *transaction)
901 .await
902 .map_err(|error| infrastructure(&error))?;
903 let acquired = if expired {
904 sqlx::query(
905 "UPDATE minco_job_locks SET owner = $2, expires_at = $3 \
906 WHERE overlap_key = $1 AND expires_at <= $4",
907 )
908 .bind(overlap_key)
909 .bind(lease_id.to_string())
910 .bind(expires_at.to_rfc3339())
911 .bind(now.to_rfc3339())
912 .execute(&mut *transaction)
913 .await
914 .map_err(|error| infrastructure(&error))?
915 .rows_affected()
916 == 1
917 } else {
918 sqlx::query("INSERT OR IGNORE INTO minco_job_locks (overlap_key, owner, expires_at) VALUES ($1, $2, $3)")
919 .bind(overlap_key)
920 .bind(lease_id.to_string())
921 .bind(expires_at.to_rfc3339())
922 .execute(&mut *transaction)
923 .await
924 .map_err(|error| infrastructure(&error))?
925 .rows_affected()
926 == 1
927 };
928 transaction
929 .commit()
930 .await
931 .map_err(|error| infrastructure(&error))?;
932 Ok(acquired)
933 }
934
935 async fn refresh(
936 &self,
937 overlap_key: &str,
938 lease_id: Uuid,
939 expires_at: DateTime<Utc>,
940 now: DateTime<Utc>,
941 ) -> Result<bool, JobError> {
942 let result = sqlx::query(
943 "UPDATE minco_job_locks SET expires_at = $3 \
944 WHERE overlap_key = $1 AND owner = $2 AND expires_at > $4",
945 )
946 .bind(overlap_key)
947 .bind(lease_id.to_string())
948 .bind(expires_at.to_rfc3339())
949 .bind(now.to_rfc3339())
950 .execute(&self.pool)
951 .await
952 .map_err(|error| infrastructure(&error))?;
953 Ok(result.rows_affected() == 1)
954 }
955
956 async fn release(&self, overlap_key: &str, lease_id: Uuid) -> Result<(), JobError> {
957 sqlx::query("DELETE FROM minco_job_locks WHERE overlap_key = $1 AND owner = $2")
958 .bind(overlap_key)
959 .bind(lease_id.to_string())
960 .execute(&self.pool)
961 .await
962 .map_err(|error| infrastructure(&error))?;
963 Ok(())
964 }
965
966 async fn recover_expired(&self, now: DateTime<Utc>) -> Result<usize, JobError> {
967 let result = sqlx::query("DELETE FROM minco_job_locks WHERE expires_at <= $1")
968 .bind(now.to_rfc3339())
969 .execute(&self.pool)
970 .await
971 .map_err(|error| infrastructure(&error))?;
972 Ok(usize::try_from(result.rows_affected()).unwrap_or(usize::MAX))
973 }
974}
975
976#[cfg(test)]
977mod tests {
978 use super::*;
979 use minco_plugin_jobs::{
980 EnqueueOutcome, JobEnvelope, JobOptions, JobPublicationStore as _, JobStore as _,
981 OverlapLockStore as _, RetryPolicy,
982 };
983
984 async fn pool() -> SqlitePool {
985 let directory = tempfile::tempdir().expect("temp dir");
986 let path = directory.path().join("jobs.sqlite");
987 std::mem::forget(directory);
988 crate::plugin_adapters::migrate_plugin_storage(
989 &SqlitePool::connect(&format!("sqlite://{}?mode=rwc", path.display()))
990 .await
991 .expect("sqlite pool"),
992 )
993 .await
994 .expect("migrations");
995 SqlitePool::connect(&format!("sqlite://{}", path.display()))
996 .await
997 .expect("reconnect")
998 }
999
1000 fn record(dedupe_key: Option<&str>) -> JobRecord {
1001 let mut options = JobOptions::default().with_retry(RetryPolicy::fixed(5, 1));
1002 if let Some(key) = dedupe_key {
1003 options = options.with_dedupe_key(key);
1004 }
1005 let envelope = JobEnvelope::for_parts(
1006 "orders.send-confirmation",
1007 1,
1008 serde_json::json!({ "order_id": "o-1" }),
1009 "orders-notifications",
1010 Uuid::now_v7(),
1011 )
1012 .expect("valid envelope")
1013 .with(options);
1014 minco_plugin_jobs::pending_record(envelope)
1015 }
1016
1017 #[tokio::test]
1018 async fn enqueue_in_rolls_back_with_the_callers_transaction() {
1019 let pool = pool().await;
1020 let store = SqliteJobStore::new(pool.clone());
1021 let mut transaction = pool.begin_with("BEGIN IMMEDIATE").await.expect("begin");
1022 let record = record(None);
1023 store
1024 .enqueue_in(&mut transaction, record.clone())
1025 .await
1026 .expect("enqueue in tx");
1027 transaction.rollback().await.expect("rollback");
1028 assert!(
1029 store
1030 .get(record.envelope.job_id)
1031 .await
1032 .expect("get")
1033 .is_none()
1034 );
1035 let publications = store
1036 .claim_due(
1037 "probe",
1038 10,
1039 Utc::now() + chrono::TimeDelta::minutes(1),
1040 Utc::now(),
1041 )
1042 .await
1043 .expect("probe");
1044 assert!(publications.is_empty(), "rollback leaves no intent");
1045 }
1046
1047 #[tokio::test]
1048 async fn enqueue_in_commits_exactly_one_recoverable_generation() {
1049 let pool = pool().await;
1050 let store = SqliteJobStore::new(pool.clone());
1051 let mut transaction = pool.begin_with("BEGIN IMMEDIATE").await.expect("begin");
1052 let record = record(None);
1053 match store
1054 .enqueue_in(&mut transaction, record.clone())
1055 .await
1056 .expect("enqueue")
1057 {
1058 EnqueueOutcome::Inserted(job_id) => assert_eq!(job_id, record.envelope.job_id),
1059 EnqueueOutcome::Duplicate(existing) => {
1060 panic!("expected insertion, got duplicate {existing}")
1061 }
1062 }
1063 transaction.commit().await.expect("commit");
1064 let publications = store
1065 .claim_due(
1066 "probe",
1067 10,
1068 Utc::now() + chrono::TimeDelta::minutes(1),
1069 Utc::now(),
1070 )
1071 .await
1072 .expect("claim");
1073 assert_eq!(publications.len(), 1);
1074 assert_eq!(publications[0].generation, 1);
1075 assert!(publications[0].lease_id.is_some());
1076 }
1077
1078 #[tokio::test]
1079 async fn stale_claims_cannot_mutate_newer_claims_even_with_one_worker_name() {
1080 let pool = pool().await;
1081 let store = SqliteJobStore::new(pool);
1082 let record = record(None);
1083 store
1084 .enqueue_with_intent(record.clone())
1085 .await
1086 .expect("enqueue");
1087 let start = Utc::now();
1088 let stale = store
1089 .claim_execution(
1090 record.envelope.job_id,
1091 "same-worker-name",
1092 start + chrono::TimeDelta::minutes(1),
1093 start,
1094 )
1095 .await
1096 .expect("first claim")
1097 .expect("claimed");
1098 let newer = store
1099 .claim_execution(
1100 record.envelope.job_id,
1101 "same-worker-name",
1102 start + chrono::TimeDelta::minutes(30),
1103 start + chrono::TimeDelta::minutes(2),
1104 )
1105 .await
1106 .expect("reclaim")
1107 .expect("reclaimed");
1108 assert_ne!(stale.lease_id, newer.lease_id);
1109 let error = store.complete(&stale, start).await.unwrap_err();
1110 assert!(matches!(error, JobError::LeaseFencedOut { .. }));
1111 let error = store
1112 .schedule_retry_and_publish(
1113 &stale,
1114 "stale",
1115 start + chrono::TimeDelta::minutes(3),
1116 start,
1117 )
1118 .await
1119 .unwrap_err();
1120 assert!(matches!(error, JobError::LeaseFencedOut { .. }));
1121 let error = store
1122 .fail_permanently(&stale, "stale", start)
1123 .await
1124 .unwrap_err();
1125 assert!(matches!(error, JobError::LeaseFencedOut { .. }));
1126 store.complete(&newer, start).await.expect("newer owns it");
1127 }
1128
1129 #[tokio::test]
1130 async fn retry_state_and_next_generation_commit_together() {
1131 let pool = pool().await;
1132 let store = SqliteJobStore::new(pool);
1133 let record = record(None);
1134 store
1135 .enqueue_with_intent(record.clone())
1136 .await
1137 .expect("enqueue");
1138 let now = Utc::now();
1139 let delivered = store
1140 .claim_due("dispatcher-1", 10, now + chrono::TimeDelta::minutes(1), now)
1141 .await
1142 .expect("deliver generation 1");
1143 assert_eq!(delivered.len(), 1);
1144 store
1145 .mark_published(
1146 delivered[0].publication_id,
1147 delivered[0].lease_id.expect("lease"),
1148 )
1149 .await
1150 .expect("published");
1151 let claim = store
1152 .claim_execution(
1153 record.envelope.job_id,
1154 "worker-exec-1",
1155 now + chrono::TimeDelta::minutes(5),
1156 now,
1157 )
1158 .await
1159 .expect("claim")
1160 .expect("claimed");
1161 let retry_at = now + chrono::TimeDelta::seconds(60);
1162 let publication_id = store
1163 .schedule_retry_and_publish(&claim, "notification-unavailable", retry_at, now)
1164 .await
1165 .expect("atomic retry");
1166 assert_ne!(publication_id, Uuid::nil());
1167 let generations: Vec<(i64, String)> = sqlx::query_as(
1168 "SELECT generation, status FROM minco_job_publications WHERE job_id = $1 ORDER BY \
1169 generation",
1170 )
1171 .bind(record.envelope.job_id.to_string())
1172 .fetch_all(&store.pool)
1173 .await
1174 .expect("generations");
1175 assert_eq!(
1176 generations.len(),
1177 2,
1178 "generation 2 committed with the retry"
1179 );
1180 assert_eq!(generations[1].1, "pending");
1181 let early = store
1182 .claim_due("d", 10, now + chrono::TimeDelta::minutes(1), now)
1183 .await
1184 .expect("early");
1185 assert!(early.is_empty(), "generation 2 is not due yet");
1186 let due = store
1187 .claim_due("d", 10, retry_at + chrono::TimeDelta::minutes(1), retry_at)
1188 .await
1189 .expect("due");
1190 assert_eq!(due.len(), 1);
1191 assert_eq!(due[0].generation, 2);
1192 }
1193
1194 #[tokio::test]
1195 async fn execution_claims_admit_exactly_one_owner() {
1196 let pool = pool().await;
1197 let store = SqliteJobStore::new(pool);
1198 let record = record(None);
1199 store
1200 .enqueue_with_intent(record.clone())
1201 .await
1202 .expect("enqueue");
1203 let now = Utc::now();
1204 assert!(
1205 store
1206 .claim_execution(
1207 record.envelope.job_id,
1208 "worker-a",
1209 now + chrono::TimeDelta::minutes(10),
1210 now
1211 )
1212 .await
1213 .expect("first claim")
1214 .is_some()
1215 );
1216 assert!(
1217 store
1218 .claim_execution(
1219 record.envelope.job_id,
1220 "worker-b",
1221 now + chrono::TimeDelta::minutes(10),
1222 now
1223 )
1224 .await
1225 .expect("second claim")
1226 .is_none(),
1227 "a live lease blocks other owners"
1228 );
1229 }
1230
1231 #[tokio::test]
1232 async fn duplicate_dedupe_uses_the_semantic_fingerprint() {
1233 let pool = pool().await;
1234 let store = SqliteJobStore::new(pool);
1235 store
1236 .enqueue_with_intent(record(Some("orders.confirm:o-1")))
1237 .await
1238 .expect("first");
1239 match store
1240 .enqueue_with_intent(record(Some("orders.confirm:o-1")))
1241 .await
1242 .expect("second")
1243 {
1244 EnqueueOutcome::Duplicate(_) => {}
1245 EnqueueOutcome::Inserted(inserted) => {
1246 panic!("identical resubmission must be idempotent, got {inserted}")
1247 }
1248 }
1249 let conflicting = record(None);
1250 let mut envelope = conflicting.envelope.clone();
1251 envelope.payload = serde_json::json!({ "order_id": "o-2" });
1252 envelope.dedupe_key = Some("orders.confirm:o-1".into());
1253 let error = store
1254 .enqueue_with_intent(minco_plugin_jobs::pending_record(envelope))
1255 .await
1256 .expect_err("conflict");
1257 assert!(matches!(
1258 error,
1259 JobError::DuplicateSubmissionConflict { .. }
1260 ));
1261 }
1262
1263 #[tokio::test]
1264 async fn ingestion_creates_no_pending_publication() {
1265 let pool = pool().await;
1266 let store = SqliteJobStore::new(pool);
1267 let mut occurrence = record(None);
1268 occurrence.envelope.dedupe_key = Some("orders-nightly:2026-08-22T13:00:00Z".into());
1269 match store
1270 .ingest_existing_delivery(occurrence.clone())
1271 .await
1272 .expect("ingest")
1273 {
1274 minco_plugin_jobs::IngestOutcome::Ingested(job_id) => {
1275 assert_eq!(job_id, occurrence.envelope.job_id);
1276 }
1277 minco_plugin_jobs::IngestOutcome::Duplicate(existing) => {
1278 panic!("first occurrence ingests, got duplicate {existing}")
1279 }
1280 }
1281 let statuses: Vec<(String,)> =
1282 sqlx::query_as("SELECT status FROM minco_job_publications WHERE job_id = $1")
1283 .bind(occurrence.envelope.job_id.to_string())
1284 .fetch_all(&store.pool)
1285 .await
1286 .expect("statuses");
1287 assert_eq!(statuses.len(), 1);
1288 assert_eq!(statuses[0].0, "published", "no pending generation appears");
1289 match store
1290 .ingest_existing_delivery(occurrence)
1291 .await
1292 .expect("re")
1293 {
1294 minco_plugin_jobs::IngestOutcome::Duplicate(_) => {}
1295 minco_plugin_jobs::IngestOutcome::Ingested(job_id) => {
1296 panic!("re-ingestion is idempotent, got {job_id}")
1297 }
1298 }
1299 }
1300
1301 #[tokio::test]
1302 async fn stale_overlap_owner_cannot_release_a_newer_lock() {
1303 let pool = pool().await;
1304 let store = SqliteJobStore::new(pool);
1305 let now = Utc::now();
1306 let stale_lease = Uuid::now_v7();
1307 assert!(
1308 store
1309 .acquire(
1310 "orders.confirm:o-1",
1311 stale_lease,
1312 now + chrono::TimeDelta::minutes(1),
1313 now
1314 )
1315 .await
1316 .expect("acquire")
1317 );
1318 let newer_lease = Uuid::now_v7();
1319 assert!(
1320 store
1321 .acquire(
1322 "orders.confirm:o-1",
1323 newer_lease,
1324 now + chrono::TimeDelta::minutes(30),
1325 now + chrono::TimeDelta::minutes(2)
1326 )
1327 .await
1328 .expect("reclaim"),
1329 "the expired lock is reclaimable"
1330 );
1331 store
1332 .release("orders.confirm:o-1", stale_lease)
1333 .await
1334 .expect("stale release is a no-op");
1335 let held: Option<(String,)> =
1336 sqlx::query_as("SELECT owner FROM minco_job_locks WHERE overlap_key = $1")
1337 .bind("orders.confirm:o-1")
1338 .fetch_optional(&store.pool)
1339 .await
1340 .expect("held");
1341 assert_eq!(
1342 held.map(|(owner,)| owner),
1343 Some(newer_lease.to_string()),
1344 "the stale owner cannot release the newer lock"
1345 );
1346 }
1347
1348 #[tokio::test]
1349 async fn stale_publication_claimant_cannot_mark_delivery() {
1350 let pool = pool().await;
1351 let store = SqliteJobStore::new(pool);
1352 store
1353 .enqueue_with_intent(record(None))
1354 .await
1355 .expect("enqueue");
1356 let now = Utc::now();
1357 let stale = store
1358 .claim_due("dispatcher-a", 10, now + chrono::TimeDelta::minutes(1), now)
1359 .await
1360 .expect("stale claim");
1361 assert_eq!(stale.len(), 1);
1362 let stale_lease = stale[0].lease_id.expect("lease");
1363 let newer = store
1364 .claim_due(
1365 "dispatcher-b",
1366 10,
1367 now + chrono::TimeDelta::minutes(30),
1368 now + chrono::TimeDelta::minutes(2),
1369 )
1370 .await
1371 .expect("reclaim");
1372 assert_eq!(newer.len(), 1);
1373 let error = store
1374 .mark_published(stale[0].publication_id, stale_lease)
1375 .await
1376 .expect_err("fenced");
1377 assert!(matches!(error, JobError::PublicationFencedOut { .. }));
1378 store
1379 .mark_published(
1380 newer[0].publication_id,
1381 newer[0].lease_id.expect("newer lease"),
1382 )
1383 .await
1384 .expect("newer marks delivery");
1385 }
1386
1387 #[tokio::test]
1388 async fn attempt_history_stays_bounded() {
1389 let pool = pool().await;
1390 let store = SqliteJobStore::new(pool);
1391 let record = record(None);
1392 store
1393 .enqueue_with_intent(record.clone())
1394 .await
1395 .expect("enqueue");
1396 let mut now = Utc::now();
1397 for _ in 0..30 {
1398 let claim = store
1399 .claim_execution(
1400 record.envelope.job_id,
1401 "worker-exec",
1402 now + chrono::TimeDelta::minutes(5),
1403 now,
1404 )
1405 .await
1406 .expect("claim")
1407 .expect("claimed");
1408 store
1409 .schedule_retry_and_publish(
1410 &claim,
1411 "always-busy",
1412 now + chrono::TimeDelta::seconds(1),
1413 now,
1414 )
1415 .await
1416 .expect("retry");
1417 now += chrono::TimeDelta::seconds(2);
1418 }
1419 let final_record = store
1420 .get(record.envelope.job_id)
1421 .await
1422 .expect("get")
1423 .expect("present");
1424 assert_eq!(final_record.attempts.len(), 25, "history is bounded");
1425 assert_eq!(final_record.attempt_count, 30);
1426 }
1427
1428 #[tokio::test]
1429 async fn operator_transitions_are_revision_guarded() {
1430 let pool = pool().await;
1431 let store = SqliteJobStore::new(pool);
1432 let record = record(None);
1433 store
1434 .enqueue_with_intent(record.clone())
1435 .await
1436 .expect("enqueue");
1437 let job_id = record.envelope.job_id;
1438 let current = store.get(job_id).await.unwrap().unwrap();
1439 let error = store
1440 .cancel(job_id, current.revision + 1, Utc::now())
1441 .await
1442 .unwrap_err();
1443 assert!(matches!(error, JobError::RevisionConflict { .. }));
1444 store
1445 .cancel(job_id, current.revision, Utc::now())
1446 .await
1447 .unwrap();
1448 }
1449}