use crate::job::{BoxFuture, FailedJob, QueuedJob, ReservedJob};
use crate::queue::{Queue, record_pushed};
use crate::time::unix_now;
use rustlavel_core::{Error, Json, Result};
use rustlavel_db::schema::{Schema, Table};
use rustlavel_db::dialect::quote_qualified;
use rustlavel_db::{Database, Value};
use std::time::Duration;
pub const JOBS_TABLE: &str = "jobs";
pub const FAILED_JOBS_TABLE: &str = "failed_jobs";
pub async fn create_tables(schema: &Schema<'_>, jobs: &str, failed: &str) -> Result<()> {
schema.create(jobs, define_jobs_table).await?;
schema.create(failed, define_failed_jobs_table).await?;
Ok(())
}
pub fn define_jobs_table(t: &mut Table) {
t.id();
t.string("queue");
t.string("name");
t.json("payload");
t.integer("attempts").default_int(0);
t.integer("max_tries").default_int(crate::job::DEFAULT_TRIES as i64);
t.integer("retry_after").default_int(60);
t.big_integer("available_at");
t.big_integer("reserved_at").nullable();
t.big_integer("created_at");
t.index(&["queue", "reserved_at", "available_at"]);
}
pub fn define_failed_jobs_table(t: &mut Table) {
t.id();
t.string("queue");
t.string("name");
t.json("payload");
t.integer("attempts");
t.text("error");
t.big_integer("failed_at");
}
rustlavel_db::migration!(
CreateQueueTables,
"2026_08_29_000001_create_queue_tables",
up: |schema| { crate::database::create_tables(schema, JOBS_TABLE, FAILED_JOBS_TABLE).await },
down: |schema| {
schema.drop(FAILED_JOBS_TABLE).await?;
schema.drop(JOBS_TABLE).await
},
);
pub const JOB_PROGRESS_TABLE: &str = "job_progress";
pub fn define_job_progress_table(t: &mut Table) {
t.id();
t.string("key").unique();
t.integer("percent");
t.string("stage");
t.big_integer("updated_at");
t.boolean("finished").default_bool(false);
t.text("failed").nullable();
}
rustlavel_db::migration!(
CreateJobProgressTable,
"2026_09_17_000001_create_job_progress_table",
up: |schema| { schema.create(JOB_PROGRESS_TABLE, crate::database::define_job_progress_table).await },
down: |schema| { schema.drop(JOB_PROGRESS_TABLE).await },
);
#[derive(Clone)]
pub struct DatabaseProgress {
db: Database,
table: String,
}
impl DatabaseProgress {
pub fn new(db: Database) -> DatabaseProgress {
DatabaseProgress { db, table: JOB_PROGRESS_TABLE.to_string() }
}
pub fn with_table(mut self, table: impl Into<String>) -> DatabaseProgress {
self.table = table.into();
self
}
}
impl crate::progress::ProgressStore for DatabaseProgress {
fn set<'a>(&'a self, key: &'a str, progress: crate::progress::Progress) -> crate::progress::ProgressFuture<'a, ()> {
Box::pin(async move {
let values: Vec<(&str, Value)> = vec![
("percent", Value::from(progress.percent as i64)),
("stage", Value::from(progress.stage.as_str())),
("updated_at", Value::from(progress.updated_at as i64)),
("finished", Value::from(progress.finished)),
("failed", progress.failed.as_deref().map_or(Value::Null, Value::from)),
];
let updated = self.db.table(&self.table).filter("key", key).update(&self.db, &values).await?;
if updated > 0 {
return Ok(());
}
let mut insert = values.clone();
insert.push(("key", Value::from(key)));
if self.db.table(&self.table).insert_without_id(&self.db, &insert).await.is_err() {
self.db.table(&self.table).filter("key", key).update(&self.db, &values).await?;
}
Ok(())
})
}
fn get<'a>(&'a self, key: &'a str) -> crate::progress::ProgressFuture<'a, Option<crate::progress::Progress>> {
Box::pin(async move {
let row = self.db.table(&self.table).filter("key", key).first(&self.db).await?;
Ok(row.map(|row| crate::progress::Progress {
percent: row.get::<i64>("percent").unwrap_or(0).clamp(0, 100) as u8,
stage: row.get::<String>("stage").unwrap_or_default(),
updated_at: row.get::<i64>("updated_at").unwrap_or(0).max(0) as u64,
finished: row.get::<bool>("finished").unwrap_or(false),
failed: row.get::<String>("failed").ok().filter(|f| !f.is_empty()),
}))
})
}
fn purge<'a>(&'a self, before: u64) -> crate::progress::ProgressFuture<'a, usize> {
Box::pin(async move {
let removed = self
.db
.table(&self.table)
.filter_op("updated_at", "<", before as i64)
.delete(&self.db)
.await?;
Ok(removed as usize)
})
}
}
#[derive(Clone)]
pub struct DatabaseQueue {
db: Database,
jobs: String,
failed: String,
jobs_sql: String,
failed_sql: String,
}
fn reserve_sql(jobs_sql: &str, dialect: &dyn rustlavel_db::Dialect) -> String {
let (hint, lock) = dialect.skip_locked();
format!(
"select id, name, payload, attempts, max_tries, retry_after \
from {jobs_sql}{hint} \
where queue = {} and reserved_at is null and available_at <= {} \
order by id{lock}{}",
dialect.placeholder(1),
dialect.placeholder(2),
dialect.limit_offset(Some(1), None, true)
)
}
impl DatabaseQueue {
pub fn new(db: Database) -> Self {
DatabaseQueue::with_tables(db, JOBS_TABLE, FAILED_JOBS_TABLE)
.expect("the built-in table names are valid identifiers")
}
pub fn with_tables(db: Database, jobs: &str, failed: &str) -> Result<Self> {
Ok(DatabaseQueue {
jobs_sql: quote_qualified(db.dialect(), jobs)?,
failed_sql: quote_qualified(db.dialect(), failed)?,
jobs: jobs.to_string(),
failed: failed.to_string(),
db,
})
}
pub fn database(&self) -> &Database {
&self.db
}
pub async fn migrate(&self) -> Result<()> {
let schema = Schema::new(&self.db);
create_tables(&schema, &self.jobs, &self.failed).await?;
schema.create_if_missing(JOB_PROGRESS_TABLE, define_job_progress_table).await
}
pub async fn drop_tables(&self) -> Result<()> {
let schema = Schema::new(&self.db);
schema.drop(&self.failed).await?;
schema.drop(&self.jobs).await
}
async fn reserve(&self, queue: &str) -> Result<Option<ReservedJob>> {
let now = unix_now();
let mut tx = self.db.begin().await?;
let dialect = self.db.dialect();
let sql = reserve_sql(&self.jobs_sql, dialect);
let row = tx.select_one(&sql, &[Value::from(queue), Value::from(now)]).await?;
let Some(row) = row else {
tx.rollback().await?;
return Ok(None);
};
let id = row.get::<i64>("id")?;
let attempts = row.get::<i64>("attempts")? as u32 + 1;
let retry_after = Duration::from_secs(row.get::<i64>("retry_after")?.max(0) as u64);
tx.execute(
&format!(
"update {} set attempts = {}, reserved_at = {} where id = {}",
self.jobs_sql,
dialect.placeholder(1),
dialect.placeholder(2),
dialect.placeholder(3)
),
&[Value::from(attempts), Value::from(now), Value::from(id)],
)
.await?;
tx.commit().await?;
let stored: Json = row.get::<Json>("payload")?;
let job = QueuedJob {
name: row.get::<String>("name")?,
payload: Json::Null,
queue: queue.to_string(),
max_tries: row.get::<i64>("max_tries")?.max(1) as u32,
retry_after,
delay: Duration::ZERO,
chain: None,
then: Vec::new(),
}
.with_stored_payload(stored);
Ok(Some(ReservedJob {
id: id.to_string(),
job,
attempts,
}))
}
pub async fn reclaim_expired(&self, queue: &str) -> Result<u64> {
let now = unix_now();
self.db
.execute(
&format!(
"update {} set reserved_at = null, available_at = {} \
where queue = {} and reserved_at is not null and reserved_at + retry_after < {}",
self.jobs_sql,
self.db.dialect().placeholder(1),
self.db.dialect().placeholder(2),
self.db.dialect().placeholder(3)
),
&[Value::from(now), Value::from(queue), Value::from(now)],
)
.await
}
fn row_to_failed(row: &rustlavel_db::Row) -> Result<FailedJob> {
Ok(FailedJob {
id: row.get::<i64>("id")?.to_string(),
name: row.get::<String>("name")?,
queue: row.get::<String>("queue")?,
payload: row.get::<Json>("payload")?,
attempts: row.get::<i64>("attempts")?.max(0) as u32,
error: row.get::<String>("error")?,
failed_at: row.get::<i64>("failed_at")?,
})
}
fn job_id(job: &ReservedJob) -> Result<i64> {
job.id.parse::<i64>().map_err(|_| {
Error::msg(format!(
"`{}` is not an id from the database queue. Was this job reserved by a \
different driver?",
job.id
))
})
}
}
impl Queue for DatabaseQueue {
fn driver(&self) -> &'static str {
"database"
}
fn push(&self, job: QueuedJob) -> BoxFuture<'_, Result<String>> {
Box::pin(async move {
let now = unix_now();
let id = self
.db
.table(&self.jobs)
.insert(
&self.db,
&[
("queue", Value::from(job.queue.as_str())),
("name", Value::from(job.name.as_str())),
("payload", Value::from(job.stored_payload())),
("attempts", Value::from(0)),
("max_tries", Value::from(job.max_tries)),
("retry_after", Value::from(job.retry_after.as_secs() as i64)),
("available_at", Value::from(now + job.delay.as_secs() as i64)),
("created_at", Value::from(now)),
],
)
.await?
.to_string();
record_pushed(self.driver(), &job, &id);
Ok(id)
})
}
fn pop<'a>(&'a self, queue: &'a str) -> BoxFuture<'a, Result<Option<ReservedJob>>> {
Box::pin(async move {
if let Some(job) = self.reserve(queue).await? {
return Ok(Some(job));
}
if self.reclaim_expired(queue).await? > 0 {
return self.reserve(queue).await;
}
Ok(None)
})
}
fn size<'a>(&'a self, queue: &'a str) -> BoxFuture<'a, Result<u64>> {
Box::pin(async move {
let count = self.db.table(&self.jobs).filter("queue", queue).count(&self.db).await?;
Ok(count.max(0) as u64)
})
}
fn delete<'a>(&'a self, job: &'a ReservedJob) -> BoxFuture<'a, Result<()>> {
Box::pin(async move {
self.db
.table(&self.jobs)
.filter("id", DatabaseQueue::job_id(job)?)
.delete(&self.db)
.await?;
Ok(())
})
}
fn release<'a>(&'a self, job: &'a ReservedJob, delay: Duration) -> BoxFuture<'a, Result<()>> {
Box::pin(async move {
self.db
.table(&self.jobs)
.filter("id", DatabaseQueue::job_id(job)?)
.update(
&self.db,
&[
("reserved_at", Value::Null),
("available_at", Value::from(unix_now() + delay.as_secs() as i64)),
],
)
.await?;
Ok(())
})
}
fn fail<'a>(&'a self, job: &'a ReservedJob, error: &'a str) -> BoxFuture<'a, Result<()>> {
Box::pin(async move {
let id = DatabaseQueue::job_id(job)?;
let mut tx = self.db.begin().await?;
tx.execute(
&format!(
"insert into {} (queue, name, payload, attempts, error, failed_at) \
values ({}, {}, {}, {}, {}, {})",
self.failed_sql,
self.db.dialect().placeholder(1),
self.db.dialect().placeholder(2),
self.db.dialect().placeholder(3),
self.db.dialect().placeholder(4),
self.db.dialect().placeholder(5),
self.db.dialect().placeholder(6)
),
&[
Value::from(job.job.queue.as_str()),
Value::from(job.job.name.as_str()),
Value::from(job.job.stored_payload()),
Value::from(job.attempts),
Value::from(error),
Value::from(unix_now()),
],
)
.await?;
tx.execute(
&format!(
"delete from {} where id = {}",
self.jobs_sql,
self.db.dialect().placeholder(1)
),
&[Value::from(id)],
)
.await?;
tx.commit().await
})
}
fn failed_jobs(&self) -> BoxFuture<'_, Result<Vec<FailedJob>>> {
Box::pin(async move {
let rows = self
.db
.select(&format!("select * from {} order by id", self.failed_sql), &[])
.await?;
rows.iter().map(DatabaseQueue::row_to_failed).collect()
})
}
fn clear<'a>(&'a self, queue: &'a str) -> BoxFuture<'a, Result<u64>> {
Box::pin(async move {
self.db.table(&self.jobs).filter("queue", queue).delete(&self.db).await
})
}
}
#[cfg(test)]
mod portability {
#[test]
fn no_statement_writes_a_placeholder_literally() {
let source = include_str!("database.rs");
let offenders: Vec<(usize, &str)> = source
.lines()
.enumerate()
.map(|(number, line)| (number + 1, line))
.take_while(|(_, line)| !line.contains("mod portability"))
.filter(|(_, line)| !line.trim_start().starts_with("///"))
.filter(|(_, line)| {
["$1", "$2", "$3", "$4", "$5", "$6"].iter().any(|p| line.contains(p))
})
.collect();
assert!(
offenders.is_empty(),
"these lines write a PostgreSQL placeholder literally; use \
`dialect.placeholder(n)`: {offenders:?}"
);
}
#[test]
fn the_claiming_statement_is_built_for_each_database() {
use rustlavel_db::dialect::{MySql, Postgres, Sqlite, SqlServer};
use rustlavel_db::dialect::quote_qualified;
let sql = |dialect: &dyn rustlavel_db::Dialect| {
super::reserve_sql("e_qualified(dialect, "jobs").unwrap(), dialect)
};
assert_eq!(
sql(&Postgres),
"select id, name, payload, attempts, max_tries, retry_after \
from \"jobs\" \
where queue = $1 and reserved_at is null and available_at <= $2 \
order by id for update skip locked limit 1"
);
assert_eq!(
sql(&MySql),
"select id, name, payload, attempts, max_tries, retry_after \
from `jobs` \
where queue = ? and reserved_at is null and available_at <= ? \
order by id for update skip locked limit 1"
);
assert_eq!(
sql(&SqlServer),
"select id, name, payload, attempts, max_tries, retry_after \
from [jobs] with (updlock, readpast, rowlock) \
where queue = @P1 and reserved_at is null and available_at <= @P2 \
order by id offset 0 rows fetch next 1 rows only"
);
assert_eq!(
sql(&Sqlite),
"select id, name, payload, attempts, max_tries, retry_after \
from \"jobs\" \
where queue = ? and reserved_at is null and available_at <= ? \
order by id limit 1"
);
}
#[test]
fn the_row_lock_is_asked_of_the_dialect_not_written_here() {
let source = include_str!("database.rs");
let written_literally = source
.lines()
.take_while(|line| !line.contains("mod portability"))
.any(|line| {
!line.trim_start().starts_with("///") && line.contains("for update skip locked")
});
assert!(
!written_literally,
"`for update skip locked` is written into a statement here; ask \
`dialect.skip_locked()` instead — SQL Server needs a table hint"
);
}
}
#[cfg(test)]
mod tests {
use super::*;
use rustlavel_db::Migration;
#[test]
fn table_names_are_validated_once_at_construction() {
let db = Database::lazy(rustlavel_db::DatabaseConfig::default())
.expect("the default configuration names a driver that exists");
assert!(DatabaseQueue::with_tables(db.clone(), "app_jobs", "app_failed").is_ok());
let error = match DatabaseQueue::with_tables(db, "jobs; drop table users", "failed") {
Err(error) => error.to_string(),
Ok(_) => panic!("an injected table name should have been rejected"),
};
assert!(error.contains("not a valid SQL identifier"), "{error}");
}
#[test]
fn the_jobs_table_has_the_columns_the_driver_reads() {
let jobs = rustlavel_db::schema::create_statements(
&rustlavel_db::dialect::Postgres,
JOBS_TABLE,
define_jobs_table,
)
.unwrap();
for column in [
r#""id" bigserial primary key"#,
r#""queue" varchar(255) not null"#,
r#""name" varchar(255) not null"#,
r#""payload" jsonb not null"#,
r#""attempts" integer not null default 0"#,
r#""max_tries" integer not null default 3"#,
r#""retry_after" integer not null default 60"#,
r#""available_at" bigint not null"#,
r#""created_at" bigint not null"#,
] {
assert!(jobs[0].contains(column), "expected {column} in:\n{}", jobs[0]);
}
assert!(jobs[0].contains(r#""reserved_at" bigint,"#), "{}", jobs[0]);
assert!(!jobs[0].contains(r#""reserved_at" bigint not null"#), "{}", jobs[0]);
assert!(
jobs[1].contains(r#"on "jobs" ("queue", "reserved_at", "available_at")"#),
"the reservation query needs a covering index:\n{}",
jobs[1]
);
}
#[test]
fn the_failed_jobs_table_keeps_everything_needed_to_diagnose_a_job() {
let failed =
rustlavel_db::schema::create_statements(
&rustlavel_db::dialect::Postgres,
FAILED_JOBS_TABLE,
define_failed_jobs_table,
)
.unwrap();
for column in [
r#""queue" varchar(255) not null"#,
r#""name" varchar(255) not null"#,
r#""payload" jsonb not null"#,
r#""attempts" integer not null"#,
r#""error" text not null"#,
r#""failed_at" bigint not null"#,
] {
assert!(failed[0].contains(column), "expected {column} in:\n{}", failed[0]);
}
}
#[test]
fn the_migration_is_registered_under_a_sortable_name() {
assert_eq!(CreateQueueTables.name(), "2026_08_29_000001_create_queue_tables");
}
#[test]
fn an_id_from_another_driver_is_rejected_rather_than_silently_ignored() {
let job = ReservedJob {
id: "memory-7".to_string(),
job: QueuedJob::new("x", Json::Null),
attempts: 1,
};
let error = DatabaseQueue::job_id(&job).unwrap_err().to_string();
assert!(error.contains("not an id from the database queue"), "{error}");
}
}