Skip to main content

boson_backend_scylla/
lib.rs

1//! `ScyllaDB` [`QueueBackend`] for Boson.
2//!
3//! ## Local builds
4//!
5//! Prefer a single Cargo job on resource-constrained hosts:
6//!
7//! ```bash
8//! export CARGO_BUILD_JOBS=1
9//! export CARGO_TARGET_DIR=target-boson-scylla
10//! ```
11//!
12//! Multi-node / multi-worker Scylla E2E must run against CI or cloud contact points —
13//! not a local multi-node Docker cluster.
14
15mod bootstrap;
16mod config;
17mod enqueue_rate;
18mod error_map;
19mod schema;
20
21pub use bootstrap::{
22    install_default_scylla_backend, isolated_keyspace, scylla_test_contact_points,
23};
24
25use std::collections::HashSet;
26use std::fmt;
27use std::hash::{Hash, Hasher};
28use std::num::NonZeroUsize;
29use std::sync::atomic::{AtomicU64, Ordering};
30use std::sync::Arc;
31
32use async_trait::async_trait;
33use chrono::{DateTime, Duration, Utc};
34use futures::future::join_all;
35use scylla::client::session::Session;
36use scylla::client::session_builder::SessionBuilder;
37use scylla::client::PoolSize;
38use scylla::DeserializeRow;
39use uuid::Uuid;
40
41use boson_core::{
42    BosonError, IdempotencyMode, Job, JobEnqueueDisposition, JobStatus, QueueBackend, Result, Run,
43    RunStatus, TaskConfig, TaskRunStats,
44};
45
46use config::{expiry_bucket, shard_for_job};
47use enqueue_rate::EnqueueRateLimiter;
48use error_map::{map_connect_err_source, map_err_source};
49
50pub use config::ScyllaQueueConfig;
51
52/// Scylla-backed queue backend.
53pub struct ScyllaQueueBackend {
54    session: Arc<Session>,
55    keyspace: String,
56    ready_shard_count: u32,
57    shard_concurrency: u32,
58    parallel_writes: bool,
59    enqueue_rate: EnqueueRateLimiter,
60    list_cursor: AtomicU64,
61    // prepared statements
62    insert_job: scylla::statement::prepared::PreparedStatement,
63    select_job: scylla::statement::prepared::PreparedStatement,
64    update_job_claim: scylla::statement::prepared::PreparedStatement,
65    update_job_status: scylla::statement::prepared::PreparedStatement,
66    update_job_revert: scylla::statement::prepared::PreparedStatement,
67    insert_ready: scylla::statement::prepared::PreparedStatement,
68    select_ready: scylla::statement::prepared::PreparedStatement,
69    delete_ready: scylla::statement::prepared::PreparedStatement,
70    insert_idempotency: scylla::statement::prepared::PreparedStatement,
71    upsert_idempotency: scylla::statement::prepared::PreparedStatement,
72    select_idempotency: scylla::statement::prepared::PreparedStatement,
73    insert_lease: scylla::statement::prepared::PreparedStatement,
74    steal_lease: scylla::statement::prepared::PreparedStatement,
75    select_lease: scylla::statement::prepared::PreparedStatement,
76    update_lease_ttl: scylla::statement::prepared::PreparedStatement,
77    delete_lease: scylla::statement::prepared::PreparedStatement,
78    insert_lease_by_id: scylla::statement::prepared::PreparedStatement,
79    select_lease_by_id: scylla::statement::prepared::PreparedStatement,
80    delete_lease_by_id: scylla::statement::prepared::PreparedStatement,
81    insert_lease_expiry: scylla::statement::prepared::PreparedStatement,
82    delete_lease_expiry: scylla::statement::prepared::PreparedStatement,
83    select_lease_expiry: scylla::statement::prepared::PreparedStatement,
84    insert_status_idx: scylla::statement::prepared::PreparedStatement,
85    delete_status_idx: scylla::statement::prepared::PreparedStatement,
86    select_status_idx: scylla::statement::prepared::PreparedStatement,
87    insert_run: scylla::statement::prepared::PreparedStatement,
88    select_run: scylla::statement::prepared::PreparedStatement,
89    update_run_finish: scylla::statement::prepared::PreparedStatement,
90    insert_run_by_job: scylla::statement::prepared::PreparedStatement,
91    select_run_by_job: scylla::statement::prepared::PreparedStatement,
92    insert_task_config: scylla::statement::prepared::PreparedStatement,
93    select_task_config: scylla::statement::prepared::PreparedStatement,
94}
95
96#[derive(DeserializeRow)]
97struct JobRow {
98    job_id: String,
99    task_name: String,
100    actor_json: String,
101    params_json: String,
102    priority: i32,
103    pool: String,
104    status: String,
105    idempotency_key: Option<String>,
106    created_at: i64,
107    signature_hash: i64,
108    attempt: i32,
109}
110
111#[derive(DeserializeRow)]
112struct ReadyRow {
113    priority: i32,
114    created_at: i64,
115    job_id: String,
116}
117
118#[derive(DeserializeRow)]
119struct IdRow {
120    job_id: String,
121}
122
123#[derive(DeserializeRow)]
124struct LeaseRow {
125    #[allow(dead_code)]
126    lease_id: String,
127    #[allow(dead_code)]
128    worker_id: String,
129    expires_at: i64,
130}
131
132#[derive(DeserializeRow)]
133struct LeaseExpiryRow {
134    expires_at: i64,
135    job_id: String,
136    lease_id: String,
137}
138
139#[derive(DeserializeRow)]
140struct StatusIdxRow {
141    #[allow(dead_code)]
142    created_at: i64,
143    job_id: String,
144}
145
146#[derive(DeserializeRow)]
147struct RunRow {
148    run_id: String,
149    job_id: String,
150    task_name: String,
151    attempt: i32,
152    status: String,
153    started_at: i64,
154    finished_at: Option<i64>,
155    duration_ms: Option<i64>,
156    error_message: Option<String>,
157}
158
159#[derive(DeserializeRow)]
160struct RunIdRow {
161    run_id: String,
162}
163
164#[derive(DeserializeRow)]
165struct TaskConfigRow {
166    task_name: String,
167    priority: i32,
168    pool: String,
169    retry_policy_json: String,
170    rate_limit_policy_json: String,
171    idempotency_mode: Option<String>,
172    updated_at: i64,
173}
174
175const fn to_millis(dt: DateTime<Utc>) -> i64 {
176    dt.timestamp_millis()
177}
178
179fn from_millis(ms: i64) -> DateTime<Utc> {
180    DateTime::from_timestamp_millis(ms).unwrap_or_else(Utc::now)
181}
182
183impl ScyllaQueueBackend {
184    /// Connect to a Scylla cluster and bootstrap schema.
185    ///
186    /// # Errors
187    ///
188    /// Returns an error when the session cannot connect or schema bootstrap fails.
189    pub async fn connect(config: ScyllaQueueConfig) -> Result<Self> {
190        let mut builder = SessionBuilder::new();
191        for cp in &config.contact_points {
192            builder = builder.known_node(cp.as_str());
193        }
194        if let Some(dc) = &config.datacenter {
195            builder = builder.prefer_datacenter(dc.clone());
196        }
197        if let (Some(user), Some(pass)) = (&config.username, &config.password) {
198            builder = builder.user(user.clone(), pass.clone());
199        }
200        if let Some(n) = config
201            .pool_per_shard
202            .and_then(|n| NonZeroUsize::new(n.max(1) as usize))
203        {
204            builder = builder.pool_size(PoolSize::PerShard(n));
205        }
206        let contact_label = config.contact_points.join(",");
207        let session = Box::pin(builder.build())
208            .await
209            .map_err(|e| map_connect_err_source(&contact_label, e))?;
210        Self::from_session(Arc::new(session), &config).await
211    }
212
213    /// Wrap an existing session (schema bootstrap runs).
214    ///
215    /// # Errors
216    ///
217    /// Returns an error when schema bootstrap or prepare fails.
218    #[allow(clippy::cognitive_complexity, clippy::too_many_lines)]
219    pub async fn from_session(session: Arc<Session>, config: &ScyllaQueueConfig) -> Result<Self> {
220        schema::ensure_schema(&session, &config.keyspace, config.replication_factor).await?;
221        let keyspace = config.keyspace.clone();
222        let prefix = format!("{keyspace}.");
223        let q = |sql: &str| sql.replace("boson.", &prefix);
224
225        let prepare = |sql: String| {
226            let session = Arc::clone(&session);
227            async move { session.prepare(sql).await.map_err(map_err_source) }
228        };
229
230        Ok(Self {
231            session: Arc::clone(&session),
232            keyspace,
233            ready_shard_count: config.ready_shard_count.max(1),
234            shard_concurrency: config.shard_concurrency.max(1),
235            parallel_writes: config.parallel_writes,
236            enqueue_rate: EnqueueRateLimiter::new(),
237            list_cursor: AtomicU64::new(0),
238            insert_job: prepare(q(
239                "INSERT INTO boson.boson_job (job_id, task_name, actor_json, params_json, priority, pool, status, idempotency_key, created_at, signature_hash, attempt) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
240            ))
241            .await?,
242            select_job: prepare(q(
243                "SELECT job_id, task_name, actor_json, params_json, priority, pool, status, idempotency_key, created_at, signature_hash, attempt FROM boson.boson_job WHERE job_id = ?",
244            ))
245            .await?,
246            update_job_claim: prepare(q(
247                "UPDATE boson.boson_job SET status = ? WHERE job_id = ? IF status = ?",
248            ))
249            .await?,
250            update_job_status: prepare(q(
251                "UPDATE boson.boson_job SET status = ?, attempt = ? WHERE job_id = ?",
252            ))
253            .await?,
254            update_job_revert: prepare(q(
255                "UPDATE boson.boson_job SET status = ? WHERE job_id = ? IF status = ?",
256            ))
257            .await?,
258            insert_ready: prepare(q(
259                "INSERT INTO boson.boson_ready (pool, shard, priority, created_at, job_id) VALUES (?, ?, ?, ?, ?)",
260            ))
261            .await?,
262            select_ready: prepare(q(
263                "SELECT priority, created_at, job_id FROM boson.boson_ready WHERE pool = ? AND shard = ? LIMIT ?",
264            ))
265            .await?,
266            delete_ready: prepare(q(
267                "DELETE FROM boson.boson_ready WHERE pool = ? AND shard = ? AND priority = ? AND created_at = ? AND job_id = ?",
268            ))
269            .await?,
270            insert_idempotency: prepare(q(
271                "INSERT INTO boson.boson_idempotency (idempotency_key, job_id) VALUES (?, ?) IF NOT EXISTS",
272            ))
273            .await?,
274            upsert_idempotency: prepare(q(
275                "INSERT INTO boson.boson_idempotency (idempotency_key, job_id) VALUES (?, ?)",
276            ))
277            .await?,
278            select_idempotency: prepare(q(
279                "SELECT job_id FROM boson.boson_idempotency WHERE idempotency_key = ?",
280            ))
281            .await?,
282            insert_lease: prepare(q(
283                "INSERT INTO boson.boson_lease (job_id, lease_id, worker_id, expires_at) VALUES (?, ?, ?, ?) IF NOT EXISTS",
284            ))
285            .await?,
286            steal_lease: prepare(q(
287                "UPDATE boson.boson_lease SET lease_id = ?, worker_id = ?, expires_at = ? WHERE job_id = ? IF expires_at < ?",
288            ))
289            .await?,
290            select_lease: prepare(q(
291                "SELECT lease_id, worker_id, expires_at FROM boson.boson_lease WHERE job_id = ?",
292            ))
293            .await?,
294            update_lease_ttl: prepare(q(
295                "UPDATE boson.boson_lease SET expires_at = ? WHERE job_id = ?",
296            ))
297            .await?,
298            delete_lease: prepare(q("DELETE FROM boson.boson_lease WHERE job_id = ?")).await?,
299            insert_lease_by_id: prepare(q(
300                "INSERT INTO boson.boson_lease_by_id (lease_id, job_id) VALUES (?, ?)",
301            ))
302            .await?,
303            select_lease_by_id: prepare(q(
304                "SELECT job_id FROM boson.boson_lease_by_id WHERE lease_id = ?",
305            ))
306            .await?,
307            delete_lease_by_id: prepare(q(
308                "DELETE FROM boson.boson_lease_by_id WHERE lease_id = ?",
309            ))
310            .await?,
311            insert_lease_expiry: prepare(q(
312                "INSERT INTO boson.boson_lease_by_expiry (bucket, expires_at, job_id, lease_id) VALUES (?, ?, ?, ?)",
313            ))
314            .await?,
315            delete_lease_expiry: prepare(q(
316                "DELETE FROM boson.boson_lease_by_expiry WHERE bucket = ? AND expires_at = ? AND job_id = ?",
317            ))
318            .await?,
319            select_lease_expiry: prepare(q(
320                "SELECT expires_at, job_id, lease_id FROM boson.boson_lease_by_expiry WHERE bucket = ?",
321            ))
322            .await?,
323            insert_status_idx: prepare(q(
324                "INSERT INTO boson.boson_job_by_status (status, created_at, job_id) VALUES (?, ?, ?)",
325            ))
326            .await?,
327            delete_status_idx: prepare(q(
328                "DELETE FROM boson.boson_job_by_status WHERE status = ? AND created_at = ? AND job_id = ?",
329            ))
330            .await?,
331            select_status_idx: prepare(q(
332                "SELECT created_at, job_id FROM boson.boson_job_by_status WHERE status = ? LIMIT ?",
333            ))
334            .await?,
335            insert_run: prepare(q(
336                "INSERT INTO boson.boson_run (run_id, job_id, task_name, attempt, status, started_at, finished_at, duration_ms, error_message) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
337            ))
338            .await?,
339            select_run: prepare(q(
340                "SELECT run_id, job_id, task_name, attempt, status, started_at, finished_at, duration_ms, error_message FROM boson.boson_run WHERE run_id = ?",
341            ))
342            .await?,
343            update_run_finish: prepare(q(
344                "UPDATE boson.boson_run SET status = ?, finished_at = ?, duration_ms = ?, error_message = ? WHERE run_id = ?",
345            ))
346            .await?,
347            insert_run_by_job: prepare(q(
348                "INSERT INTO boson.boson_run_by_job (job_id, started_at, run_id) VALUES (?, ?, ?)",
349            ))
350            .await?,
351            select_run_by_job: prepare(q(
352                "SELECT run_id FROM boson.boson_run_by_job WHERE job_id = ? LIMIT ?",
353            ))
354            .await?,
355            insert_task_config: prepare(q(
356                "INSERT INTO boson.boson_task_config (task_name, priority, pool, retry_policy_json, rate_limit_policy_json, idempotency_mode, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)",
357            ))
358            .await?,
359            select_task_config: prepare(q(
360                "SELECT task_name, priority, pool, retry_policy_json, rate_limit_policy_json, idempotency_mode, updated_at FROM boson.boson_task_config WHERE task_name = ?",
361            ))
362            .await?,
363        })
364    }
365
366    /// Underlying driver session.
367    #[must_use]
368    pub const fn session(&self) -> &Arc<Session> {
369        &self.session
370    }
371
372    /// Configured keyspace name.
373    #[must_use]
374    pub fn keyspace(&self) -> &str {
375        &self.keyspace
376    }
377
378    /// Ready-queue shard count.
379    #[must_use]
380    pub const fn ready_shard_count(&self) -> u32 {
381        self.ready_shard_count
382    }
383
384    async fn exec(
385        &self,
386        stmt: &scylla::statement::prepared::PreparedStatement,
387        values: impl scylla::serialize::row::SerializeRow,
388    ) -> Result<scylla::response::query_result::QueryResult> {
389        self.session
390            .execute_unpaged(stmt, values)
391            .await
392            .map_err(map_err_source)
393    }
394
395    fn job_from_row(row: JobRow) -> Result<Job> {
396        Ok(Job {
397            job_id: row.job_id,
398            task_name: row.task_name,
399            actor_json: serde_json::from_str(&row.actor_json)?,
400            params_json: serde_json::from_str(&row.params_json)?,
401            priority: row.priority,
402            pool: row.pool,
403            status: parse_job_status(&row.status)?,
404            idempotency_key: row.idempotency_key,
405            created_at: from_millis(row.created_at),
406            signature_hash: u64::try_from(row.signature_hash).unwrap_or(0),
407            attempt: row.attempt,
408        })
409    }
410
411    async fn load_job(&self, job_id: &str) -> Result<Option<Job>> {
412        let rows = self.exec(&self.select_job, (job_id,)).await?;
413        maybe_first_row::<JobRow>(rows)
414            .map(Self::job_from_row)
415            .transpose()
416    }
417
418    async fn insert_job_body(&self, job: &Job) -> Result<()> {
419        let actor = serde_json::to_string(&job.actor_json)?;
420        let params = serde_json::to_string(&job.params_json)?;
421        self.exec(
422            &self.insert_job,
423            (
424                job.job_id.as_str(),
425                job.task_name.as_str(),
426                actor.as_str(),
427                params.as_str(),
428                job.priority,
429                job.pool.as_str(),
430                job.status.to_string(),
431                job.idempotency_key.as_deref(),
432                to_millis(job.created_at),
433                i64::try_from(job.signature_hash).unwrap_or(i64::MAX),
434                job.attempt,
435            ),
436        )
437        .await?;
438        Ok(())
439    }
440
441    async fn insert_status_for_job(&self, job: &Job) -> Result<()> {
442        self.exec(
443            &self.insert_status_idx,
444            (
445                job.status.to_string(),
446                to_millis(job.created_at),
447                job.job_id.as_str(),
448            ),
449        )
450        .await?;
451        Ok(())
452    }
453
454    async fn insert_job_row(&self, job: &Job) -> Result<()> {
455        if self.parallel_writes {
456            let (job_res, status_res) =
457                tokio::join!(self.insert_job_body(job), self.insert_status_for_job(job));
458            job_res?;
459            status_res?;
460        } else {
461            self.insert_job_body(job).await?;
462            self.insert_status_for_job(job).await?;
463        }
464        Ok(())
465    }
466
467    async fn insert_ready_row(&self, job: &Job) -> Result<()> {
468        let shard = shard_for_job(&job.job_id, self.ready_shard_count);
469        self.exec(
470            &self.insert_ready,
471            (
472                job.pool.as_str(),
473                shard,
474                job.priority,
475                to_millis(job.created_at),
476                job.job_id.as_str(),
477            ),
478        )
479        .await?;
480        Ok(())
481    }
482
483    async fn delete_ready_row(&self, job: &Job) -> Result<()> {
484        let shard = shard_for_job(&job.job_id, self.ready_shard_count);
485        self.exec(
486            &self.delete_ready,
487            (
488                job.pool.as_str(),
489                shard,
490                job.priority,
491                to_millis(job.created_at),
492                job.job_id.as_str(),
493            ),
494        )
495        .await?;
496        Ok(())
497    }
498
499    async fn move_status_idx(
500        &self,
501        job_id: &str,
502        created_at: DateTime<Utc>,
503        from: JobStatus,
504        to: JobStatus,
505    ) -> Result<()> {
506        let ms = to_millis(created_at);
507        let from_s = from.to_string();
508        let to_s = to.to_string();
509        if self.parallel_writes {
510            let (del_res, ins_res) = tokio::join!(
511                self.exec(&self.delete_status_idx, (from_s.as_str(), ms, job_id)),
512                self.exec(&self.insert_status_idx, (to_s.as_str(), ms, job_id)),
513            );
514            del_res?;
515            ins_res?;
516        } else {
517            self.exec(&self.delete_status_idx, (from_s.as_str(), ms, job_id))
518                .await?;
519            self.exec(&self.insert_status_idx, (to_s.as_str(), ms, job_id))
520                .await?;
521        }
522        Ok(())
523    }
524}
525
526#[async_trait]
527impl QueueBackend for ScyllaQueueBackend {
528    async fn upsert_job(&self, job: &Job) -> Result<()> {
529        if let Some(existing) = self.load_job(&job.job_id).await? {
530            if existing.status != job.status {
531                self.move_status_idx(
532                    &job.job_id,
533                    existing.created_at,
534                    existing.status,
535                    job.status,
536                )
537                .await?;
538                if existing.status == JobStatus::Queued {
539                    let _ = self.delete_ready_row(&existing).await;
540                }
541            } else if existing.status == JobStatus::Queued {
542                // Same status but fields may change (e.g. attempt on retry); refresh ready PK.
543                let _ = self.delete_ready_row(&existing).await;
544            }
545        }
546        self.insert_job_row(job).await?;
547        if job.status == JobStatus::Queued {
548            self.insert_ready_row(job).await?;
549        }
550        Ok(())
551    }
552
553    async fn enqueue_with_policies(
554        &self,
555        job: Job,
556        task_config: &TaskConfig,
557    ) -> Result<(String, JobEnqueueDisposition)> {
558        let idempotency = task_config.resolved_idempotency_mode(IdempotencyMode::Lwt);
559        let mut job = job;
560        if idempotency == IdempotencyMode::Lwt {
561            if let Some(ref key) = job.idempotency_key {
562                if !key.is_empty() {
563                    let result = self
564                        .exec(
565                            &self.insert_idempotency,
566                            (key.as_str(), job.job_id.as_str()),
567                        )
568                        .await?;
569                    if !lwt_applied(result) {
570                        let existing = self.exec(&self.select_idempotency, (key.as_str(),)).await?;
571                        let Some(row) = maybe_first_row::<IdRow>(existing) else {
572                            return Err(BosonError::backend(
573                                "idempotency insert not applied but row missing",
574                            ));
575                        };
576                        // Reuse only while the prior job is still active (mem/SQL parity).
577                        if let Some(prior) = self.load_job(&row.job_id).await? {
578                            if matches!(prior.status, JobStatus::Queued | JobStatus::Running) {
579                                return Ok((row.job_id, JobEnqueueDisposition::ReusedIdempotent));
580                            }
581                        }
582                        // Terminal or missing: point the key at the new job id.
583                        self.exec(
584                            &self.upsert_idempotency,
585                            (key.as_str(), job.job_id.as_str()),
586                        )
587                        .await?;
588                    }
589                }
590            }
591        } else {
592            job.idempotency_key = None;
593        }
594
595        let policy = &task_config.rate_limit_policy;
596        if policy.max_in_flight > 0 {
597            let count = self.count_active_jobs_for_task(&job.task_name).await?;
598            if count >= policy.max_in_flight {
599                return Err(BosonError::RateLimited(job.task_name.clone()));
600            }
601        }
602        if policy.max_enqueue_per_second > 0
603            && !self
604                .enqueue_rate
605                .try_record(&job.task_name, policy.max_enqueue_per_second)
606        {
607            return Err(BosonError::RateLimited(job.task_name.clone()));
608        }
609
610        let job_id = job.job_id.clone();
611        if self.parallel_writes {
612            let (job_res, status_res, ready_res) = tokio::join!(
613                self.insert_job_body(&job),
614                self.insert_status_for_job(&job),
615                self.insert_ready_row(&job),
616            );
617            job_res?;
618            status_res?;
619            ready_res?;
620        } else {
621            self.insert_job_row(&job).await?;
622            self.insert_ready_row(&job).await?;
623        }
624        Ok((job_id, JobEnqueueDisposition::InsertedNew))
625    }
626
627    async fn get_job(&self, job_id: &str) -> Result<Option<Job>> {
628        self.load_job(job_id).await
629    }
630
631    async fn list_jobs(
632        &self,
633        status_filter: Option<JobStatus>,
634        offset: usize,
635        limit: usize,
636    ) -> Result<Vec<Job>> {
637        let statuses: Vec<JobStatus> = status_filter.map_or_else(
638            || {
639                vec![
640                    JobStatus::Queued,
641                    JobStatus::Running,
642                    JobStatus::Success,
643                    JobStatus::Failed,
644                    JobStatus::Canceled,
645                ]
646            },
647            |status| vec![status],
648        );
649        let mut jobs = Vec::new();
650        let fetch = offset.saturating_add(limit).max(1);
651        for status in statuses {
652            let rows = self
653                .exec(
654                    &self.select_status_idx,
655                    (status.to_string(), i32::try_from(fetch).unwrap_or(i32::MAX)),
656                )
657                .await?;
658            for idx in collect_rows::<StatusIdxRow>(rows) {
659                if let Some(job) = self.load_job(&idx.job_id).await? {
660                    jobs.push(job);
661                }
662            }
663        }
664        jobs.sort_by_key(|j| j.created_at);
665        Ok(jobs.into_iter().skip(offset).take(limit).collect())
666    }
667
668    async fn cancel_job_if_active(&self, job_id: &str) -> Result<()> {
669        let Some(job) = self.load_job(job_id).await? else {
670            return Err(BosonError::JobNotFound(job_id.to_string()));
671        };
672        if !matches!(job.status, JobStatus::Queued | JobStatus::Running) {
673            return Ok(());
674        }
675        let from = job.status;
676        self.exec(
677            &self.update_job_status,
678            (JobStatus::Canceled.to_string(), job.attempt, job_id),
679        )
680        .await?;
681        self.move_status_idx(job_id, job.created_at, from, JobStatus::Canceled)
682            .await?;
683        if from == JobStatus::Queued {
684            self.delete_ready_row(&job).await?;
685        }
686        Ok(())
687    }
688
689    async fn try_claim_job(&self, job_id: &str) -> Result<Option<Job>> {
690        let result = self
691            .exec(
692                &self.update_job_claim,
693                (
694                    JobStatus::Running.to_string(),
695                    job_id,
696                    JobStatus::Queued.to_string(),
697                ),
698            )
699            .await?;
700        if !lwt_applied(result) {
701            return Ok(None);
702        }
703        let Some(job) = self.load_job(job_id).await? else {
704            return Ok(None);
705        };
706        if self.parallel_writes {
707            let (status_res, ready_res) = tokio::join!(
708                self.move_status_idx(
709                    job_id,
710                    job.created_at,
711                    JobStatus::Queued,
712                    JobStatus::Running
713                ),
714                self.delete_ready_row(&job),
715            );
716            status_res?;
717            let _ = ready_res;
718        } else {
719            self.move_status_idx(
720                job_id,
721                job.created_at,
722                JobStatus::Queued,
723                JobStatus::Running,
724            )
725            .await?;
726            let _ = self.delete_ready_row(&job).await;
727        }
728        Ok(Some(job))
729    }
730
731    async fn revert_job_to_queued(&self, job_id: &str) -> Result<()> {
732        let Some(job) = self.load_job(job_id).await? else {
733            return Ok(());
734        };
735        if job.status != JobStatus::Running {
736            return Ok(());
737        }
738        let result = self
739            .exec(
740                &self.update_job_revert,
741                (
742                    JobStatus::Queued.to_string(),
743                    job_id,
744                    JobStatus::Running.to_string(),
745                ),
746            )
747            .await?;
748        if lwt_applied(result) {
749            self.move_status_idx(
750                job_id,
751                job.created_at,
752                JobStatus::Running,
753                JobStatus::Queued,
754            )
755            .await?;
756            let mut queued = job;
757            queued.status = JobStatus::Queued;
758            self.insert_ready_row(&queued).await?;
759        }
760        Ok(())
761    }
762
763    async fn distinct_pools_queued(&self) -> Result<Vec<String>> {
764        // Sample ready shards for pool discovery (admin path).
765        let mut pools = HashSet::new();
766        let sample = self.ready_shard_count.min(16);
767        for shard in 0..i32::try_from(sample).unwrap_or(1) {
768            // We do not have pool list without scanning; use a known default plus any from status index.
769            let _ = shard;
770        }
771        let rows = self
772            .exec(
773                &self.select_status_idx,
774                (JobStatus::Queued.to_string(), 10_000_i32),
775            )
776            .await?;
777        for idx in collect_rows::<StatusIdxRow>(rows) {
778            if let Some(job) = self.load_job(&idx.job_id).await? {
779                pools.insert(job.pool);
780            }
781        }
782        let mut out: Vec<String> = pools.into_iter().collect();
783        out.sort();
784        Ok(out)
785    }
786
787    async fn list_queued_for_pool_sorted(&self, pool: &str, limit: usize) -> Result<Vec<Job>> {
788        let limit = limit.max(1);
789        let shard_count = self.ready_shard_count;
790        // Scan every shard so priority order and claim candidates are correct. Sampling a
791        // subset of 256 shards routinely missed ready jobs (empty list / stuck Queued).
792        let start = self.list_cursor.fetch_add(1, Ordering::Relaxed);
793        let per_shard = i32::try_from(limit).unwrap_or(i32::MAX);
794        let concurrency = self.shard_concurrency.max(1) as usize;
795        let shards: Vec<i32> = (0..shard_count)
796            .map(|i| i32::try_from((start + u64::from(i)) % u64::from(shard_count)).unwrap_or(0))
797            .collect();
798
799        let mut candidates: Vec<(i32, i64, String)> = Vec::new();
800        for chunk in shards.chunks(concurrency) {
801            let futs: Vec<_> = chunk
802                .iter()
803                .map(|&shard| self.exec(&self.select_ready, (pool, shard, per_shard)))
804                .collect();
805            for result in join_all(futs).await {
806                let rows = result?;
807                for row in collect_rows::<ReadyRow>(rows) {
808                    candidates.push((row.priority, row.created_at, row.job_id));
809                }
810            }
811        }
812        candidates.sort_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.cmp(&b.1)));
813        candidates.dedup_by(|a, b| a.2 == b.2);
814        let job_ids: Vec<String> = candidates
815            .into_iter()
816            .take(limit)
817            .map(|(_, _, id)| id)
818            .collect();
819        let load_futs: Vec<_> = job_ids.iter().map(|id| self.load_job(id)).collect();
820        let mut jobs = Vec::new();
821        for result in join_all(load_futs).await {
822            if let Some(job) = result? {
823                if job.status == JobStatus::Queued && job.pool == pool {
824                    jobs.push(job);
825                }
826            }
827        }
828        Ok(jobs)
829    }
830
831    async fn count_jobs(&self, status_filter: Option<JobStatus>) -> Result<u64> {
832        let jobs = self.list_jobs(status_filter, 0, 100_000).await?;
833        Ok(u64::try_from(jobs.len()).unwrap_or(u64::MAX))
834    }
835
836    async fn count_jobs_for_task(&self, task_name: &str, status: Option<JobStatus>) -> Result<u64> {
837        let jobs = self.list_jobs(status, 0, 100_000).await?;
838        let count = jobs.iter().filter(|j| j.task_name == task_name).count();
839        Ok(u64::try_from(count).unwrap_or(u64::MAX))
840    }
841
842    async fn count_active_jobs_for_task(&self, task_name: &str) -> Result<u32> {
843        let queued = self
844            .count_jobs_for_task(task_name, Some(JobStatus::Queued))
845            .await?;
846        let running = self
847            .count_jobs_for_task(task_name, Some(JobStatus::Running))
848            .await?;
849        Ok(u32::try_from(queued.saturating_add(running)).unwrap_or(u32::MAX))
850    }
851
852    async fn find_nonterminal_by_idempotency_key(&self, key: &str) -> Result<Option<String>> {
853        let rows = self.exec(&self.select_idempotency, (key,)).await?;
854        Ok(maybe_first_row::<IdRow>(rows).map(|r| r.job_id))
855    }
856
857    async fn upsert_run(&self, run: &Run) -> Result<()> {
858        self.exec(
859            &self.insert_run,
860            (
861                run.run_id.as_str(),
862                run.job_id.as_str(),
863                run.task_name.as_str(),
864                run.attempt,
865                run.status.to_string(),
866                to_millis(run.started_at),
867                run.finished_at.map(to_millis),
868                run.duration_ms,
869                run.error_message.as_deref(),
870            ),
871        )
872        .await?;
873        self.exec(
874            &self.insert_run_by_job,
875            (
876                run.job_id.as_str(),
877                to_millis(run.started_at),
878                run.run_id.as_str(),
879            ),
880        )
881        .await?;
882        Ok(())
883    }
884
885    async fn get_run(&self, run_id: &str) -> Result<Option<Run>> {
886        let rows = self.exec(&self.select_run, (run_id,)).await?;
887        Ok(maybe_first_row::<RunRow>(rows)
888            .map(run_from_row)
889            .transpose()?)
890    }
891
892    async fn list_runs(
893        &self,
894        job_id_filter: Option<&str>,
895        offset: usize,
896        limit: usize,
897    ) -> Result<Vec<Run>> {
898        let Some(job_id) = job_id_filter else {
899            return Ok(vec![]);
900        };
901        let fetch = offset.saturating_add(limit).max(1);
902        let rows = self
903            .exec(
904                &self.select_run_by_job,
905                (job_id, i32::try_from(fetch).unwrap_or(i32::MAX)),
906            )
907            .await?;
908        let mut runs = Vec::new();
909        for id in collect_rows::<RunIdRow>(rows) {
910            if let Some(run) = self.get_run(&id.run_id).await? {
911                runs.push(run);
912            }
913        }
914        runs.sort_by_key(|r| r.started_at);
915        Ok(runs.into_iter().skip(offset).take(limit).collect())
916    }
917
918    async fn finish_run(
919        &self,
920        run_id: &str,
921        status: RunStatus,
922        duration_ms: Option<i64>,
923        error_message: Option<String>,
924    ) -> Result<()> {
925        let finished_at = to_millis(Utc::now());
926        self.exec(
927            &self.update_run_finish,
928            (
929                status.to_string(),
930                finished_at,
931                duration_ms,
932                error_message.as_deref(),
933                run_id,
934            ),
935        )
936        .await?;
937        Ok(())
938    }
939
940    async fn count_runs(&self, job_id_filter: Option<&str>) -> Result<u64> {
941        let runs = self.list_runs(job_id_filter, 0, 100_000).await?;
942        Ok(u64::try_from(runs.len()).unwrap_or(u64::MAX))
943    }
944
945    async fn count_runs_since(&self, _since: DateTime<Utc>) -> Result<u64> {
946        Ok(0)
947    }
948
949    async fn task_run_stats(&self, _task_name: &str) -> Result<TaskRunStats> {
950        Ok(TaskRunStats {
951            runs_total: 0,
952            success_count: 0,
953        })
954    }
955
956    async fn get_task_config(&self, task_name: &str) -> Result<Option<TaskConfig>> {
957        let rows = self.exec(&self.select_task_config, (task_name,)).await?;
958        let Some(row) = maybe_first_row::<TaskConfigRow>(rows) else {
959            return Ok(None);
960        };
961        Ok(Some(TaskConfig {
962            task_name: row.task_name,
963            priority: row.priority,
964            pool: row.pool,
965            retry_policy: serde_json::from_str(&row.retry_policy_json)?,
966            rate_limit_policy: serde_json::from_str(&row.rate_limit_policy_json)?,
967            idempotency_mode: row
968                .idempotency_mode
969                .as_deref()
970                .and_then(IdempotencyMode::parse),
971            updated_at: from_millis(row.updated_at),
972        }))
973    }
974
975    async fn upsert_task_config(&self, config: &TaskConfig) -> Result<()> {
976        let mode = config.idempotency_mode.map(|m| match m {
977            IdempotencyMode::Lwt => "lwt",
978            IdempotencyMode::None => "none",
979        });
980        self.exec(
981            &self.insert_task_config,
982            (
983                config.task_name.as_str(),
984                config.priority,
985                config.pool.as_str(),
986                serde_json::to_string(&config.retry_policy)?,
987                serde_json::to_string(&config.rate_limit_policy)?,
988                mode,
989                to_millis(config.updated_at),
990            ),
991        )
992        .await?;
993        Ok(())
994    }
995
996    async fn try_claim_run_lease(
997        &self,
998        job_id: &str,
999        worker_id: &str,
1000        ttl_secs: i64,
1001    ) -> Result<Option<String>> {
1002        let now_ms = to_millis(Utc::now());
1003        let expires_at = to_millis(Utc::now() + Duration::seconds(ttl_secs));
1004        let lease_id = Uuid::new_v4().to_string();
1005        let insert = self
1006            .exec(
1007                &self.insert_lease,
1008                (job_id, lease_id.as_str(), worker_id, expires_at),
1009            )
1010            .await?;
1011        if lwt_applied(insert) {
1012            self.record_lease(job_id, &lease_id, expires_at).await?;
1013            return Ok(Some(lease_id));
1014        }
1015        let steal = self
1016            .exec(
1017                &self.steal_lease,
1018                (lease_id.as_str(), worker_id, expires_at, job_id, now_ms),
1019            )
1020            .await?;
1021        if lwt_applied(steal) {
1022            self.record_lease(job_id, &lease_id, expires_at).await?;
1023            return Ok(Some(lease_id));
1024        }
1025        Ok(None)
1026    }
1027
1028    async fn extend_lease(&self, lease_id: &str, ttl_secs: i64) -> Result<()> {
1029        let Some(job_id) = self.job_id_for_lease(lease_id).await? else {
1030            return Ok(());
1031        };
1032        let expires_at = to_millis(Utc::now() + Duration::seconds(ttl_secs));
1033        self.exec(&self.update_lease_ttl, (expires_at, job_id.as_str()))
1034            .await?;
1035        self.insert_expiry_index(&job_id, lease_id, expires_at)
1036            .await?;
1037        Ok(())
1038    }
1039
1040    async fn release_lease(&self, lease_id: &str) -> Result<()> {
1041        let Some(job_id) = self.job_id_for_lease(lease_id).await? else {
1042            return Ok(());
1043        };
1044        if let Some(lease) = self.load_lease(&job_id).await? {
1045            let bucket = expiry_bucket(lease.expires_at / 1000);
1046            let _ = self
1047                .exec(
1048                    &self.delete_lease_expiry,
1049                    (bucket, lease.expires_at, job_id.as_str()),
1050                )
1051                .await;
1052        }
1053        self.exec(&self.delete_lease, (job_id.as_str(),)).await?;
1054        self.exec(&self.delete_lease_by_id, (lease_id,)).await?;
1055        Ok(())
1056    }
1057
1058    async fn expired_lease_job_pairs(&self) -> Result<Vec<(String, String)>> {
1059        let now_ms = to_millis(Utc::now());
1060        let bucket = expiry_bucket(now_ms / 1000);
1061        let mut pairs = Vec::new();
1062        for b in [bucket - 1, bucket, bucket + 1] {
1063            if b < 0 {
1064                continue;
1065            }
1066            let rows = self.exec(&self.select_lease_expiry, (b,)).await?;
1067            for row in collect_rows::<LeaseExpiryRow>(rows) {
1068                if row.expires_at <= now_ms {
1069                    pairs.push((row.lease_id, row.job_id));
1070                }
1071            }
1072        }
1073        Ok(pairs)
1074    }
1075}
1076
1077impl ScyllaQueueBackend {
1078    async fn insert_expiry_index(
1079        &self,
1080        job_id: &str,
1081        lease_id: &str,
1082        expires_at_ms: i64,
1083    ) -> Result<()> {
1084        let bucket = expiry_bucket(expires_at_ms / 1000);
1085        self.exec(
1086            &self.insert_lease_expiry,
1087            (bucket, expires_at_ms, job_id, lease_id),
1088        )
1089        .await?;
1090        Ok(())
1091    }
1092
1093    async fn record_lease(&self, job_id: &str, lease_id: &str, expires_at_ms: i64) -> Result<()> {
1094        self.exec(&self.insert_lease_by_id, (lease_id, job_id))
1095            .await?;
1096        self.insert_expiry_index(job_id, lease_id, expires_at_ms)
1097            .await?;
1098        Ok(())
1099    }
1100
1101    async fn job_id_for_lease(&self, lease_id: &str) -> Result<Option<String>> {
1102        let rows = self.exec(&self.select_lease_by_id, (lease_id,)).await?;
1103        Ok(maybe_first_row::<IdRow>(rows).map(|r| r.job_id))
1104    }
1105
1106    async fn load_lease(&self, job_id: &str) -> Result<Option<LeaseRow>> {
1107        let rows = self.exec(&self.select_lease, (job_id,)).await?;
1108        Ok(maybe_first_row::<LeaseRow>(rows))
1109    }
1110}
1111
1112impl fmt::Debug for ScyllaQueueBackend {
1113    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1114        f.debug_struct("ScyllaQueueBackend")
1115            .field("keyspace", &self.keyspace)
1116            .field("ready_shard_count", &self.ready_shard_count)
1117            .finish_non_exhaustive()
1118    }
1119}
1120
1121fn parse_job_status(s: &str) -> Result<JobStatus> {
1122    match s {
1123        "queued" => Ok(JobStatus::Queued),
1124        "running" => Ok(JobStatus::Running),
1125        "success" => Ok(JobStatus::Success),
1126        "failed" => Ok(JobStatus::Failed),
1127        "canceled" => Ok(JobStatus::Canceled),
1128        other => Err(BosonError::backend(format!("unknown job status: {other}"))),
1129    }
1130}
1131
1132fn parse_run_status(s: &str) -> Result<RunStatus> {
1133    match s {
1134        "running" => Ok(RunStatus::Running),
1135        "success" => Ok(RunStatus::Success),
1136        "failed" => Ok(RunStatus::Failed),
1137        "canceled" => Ok(RunStatus::Canceled),
1138        "timeout" => Ok(RunStatus::Timeout),
1139        other => Err(BosonError::backend(format!("unknown run status: {other}"))),
1140    }
1141}
1142
1143fn run_from_row(row: RunRow) -> Result<Run> {
1144    Ok(Run {
1145        run_id: row.run_id,
1146        job_id: row.job_id,
1147        task_name: row.task_name,
1148        attempt: row.attempt,
1149        status: parse_run_status(&row.status)?,
1150        started_at: from_millis(row.started_at),
1151        finished_at: row.finished_at.map(from_millis),
1152        duration_ms: row.duration_ms,
1153        error_message: row.error_message,
1154    })
1155}
1156
1157fn lwt_applied(result: scylla::response::query_result::QueryResult) -> bool {
1158    use scylla::deserialize::row::ColumnIterator;
1159    use scylla::deserialize::value::DeserializeValue;
1160
1161    // Scylla LWT responses always include `[applied]` plus primary-key / IF columns.
1162    // Derive-based row types reject unknown columns, so read the first column only.
1163    let Ok(rows) = result.into_rows_result() else {
1164        return false;
1165    };
1166    let Ok(Some(mut cols)) = rows.maybe_first_row::<ColumnIterator<'_, '_>>() else {
1167        return false;
1168    };
1169    let Some(Ok(first)) = cols.next() else {
1170        return false;
1171    };
1172    bool::deserialize(first.spec.typ(), first.slice).unwrap_or(false)
1173}
1174
1175fn maybe_first_row<R>(result: scylla::response::query_result::QueryResult) -> Option<R>
1176where
1177    R: for<'frame> scylla::deserialize::row::DeserializeRow<'frame, 'frame>,
1178{
1179    result
1180        .into_rows_result()
1181        .ok()
1182        .and_then(|rows| rows.maybe_first_row::<R>().ok().flatten())
1183}
1184
1185fn collect_rows<R>(result: scylla::response::query_result::QueryResult) -> Vec<R>
1186where
1187    R: for<'frame> scylla::deserialize::row::DeserializeRow<'frame, 'frame>,
1188{
1189    result
1190        .into_rows_result()
1191        .ok()
1192        .and_then(|rows| {
1193            rows.rows::<R>()
1194                .ok()
1195                .map(|iter| iter.filter_map(std::result::Result::ok).collect())
1196        })
1197        .unwrap_or_default()
1198}
1199
1200// Silence unused import warnings for Hash/Hasher if any.
1201#[allow(dead_code)]
1202fn _hash_str(s: &str) -> u64 {
1203    let mut h = std::collections::hash_map::DefaultHasher::new();
1204    s.hash(&mut h);
1205    h.finish()
1206}