1mod bootstrap;
16mod config;
17mod enqueue_rate;
18mod error_map;
19mod schema;
20
21pub use bootstrap::{
22 install_default_scylla_backend, isolated_keyspace, scylla_test_contact_points,
23};
24
25use std::collections::HashSet;
26use std::fmt;
27use std::hash::{Hash, Hasher};
28use std::num::NonZeroUsize;
29use std::sync::atomic::{AtomicU64, Ordering};
30use std::sync::Arc;
31
32use async_trait::async_trait;
33use chrono::{DateTime, Duration, Utc};
34use futures::future::join_all;
35use scylla::client::session::Session;
36use scylla::client::session_builder::SessionBuilder;
37use scylla::client::PoolSize;
38use scylla::DeserializeRow;
39use uuid::Uuid;
40
41use boson_core::{
42 BosonError, IdempotencyMode, Job, JobEnqueueDisposition, JobStatus, QueueBackend, Result, Run,
43 RunStatus, TaskConfig, TaskRunStats,
44};
45
46use config::{expiry_bucket, shard_for_job};
47use enqueue_rate::EnqueueRateLimiter;
48use error_map::{map_connect_err_source, map_err_source};
49
50pub use config::ScyllaQueueConfig;
51
52pub struct ScyllaQueueBackend {
54 session: Arc<Session>,
55 keyspace: String,
56 ready_shard_count: u32,
57 shard_concurrency: u32,
58 parallel_writes: bool,
59 enqueue_rate: EnqueueRateLimiter,
60 list_cursor: AtomicU64,
61 insert_job: scylla::statement::prepared::PreparedStatement,
63 select_job: scylla::statement::prepared::PreparedStatement,
64 update_job_claim: scylla::statement::prepared::PreparedStatement,
65 update_job_status: scylla::statement::prepared::PreparedStatement,
66 update_job_revert: scylla::statement::prepared::PreparedStatement,
67 insert_ready: scylla::statement::prepared::PreparedStatement,
68 select_ready: scylla::statement::prepared::PreparedStatement,
69 delete_ready: scylla::statement::prepared::PreparedStatement,
70 insert_idempotency: scylla::statement::prepared::PreparedStatement,
71 upsert_idempotency: scylla::statement::prepared::PreparedStatement,
72 select_idempotency: scylla::statement::prepared::PreparedStatement,
73 insert_lease: scylla::statement::prepared::PreparedStatement,
74 steal_lease: scylla::statement::prepared::PreparedStatement,
75 select_lease: scylla::statement::prepared::PreparedStatement,
76 update_lease_ttl: scylla::statement::prepared::PreparedStatement,
77 delete_lease: scylla::statement::prepared::PreparedStatement,
78 insert_lease_by_id: scylla::statement::prepared::PreparedStatement,
79 select_lease_by_id: scylla::statement::prepared::PreparedStatement,
80 delete_lease_by_id: scylla::statement::prepared::PreparedStatement,
81 insert_lease_expiry: scylla::statement::prepared::PreparedStatement,
82 delete_lease_expiry: scylla::statement::prepared::PreparedStatement,
83 select_lease_expiry: scylla::statement::prepared::PreparedStatement,
84 insert_status_idx: scylla::statement::prepared::PreparedStatement,
85 delete_status_idx: scylla::statement::prepared::PreparedStatement,
86 select_status_idx: scylla::statement::prepared::PreparedStatement,
87 insert_run: scylla::statement::prepared::PreparedStatement,
88 select_run: scylla::statement::prepared::PreparedStatement,
89 update_run_finish: scylla::statement::prepared::PreparedStatement,
90 insert_run_by_job: scylla::statement::prepared::PreparedStatement,
91 select_run_by_job: scylla::statement::prepared::PreparedStatement,
92 insert_task_config: scylla::statement::prepared::PreparedStatement,
93 select_task_config: scylla::statement::prepared::PreparedStatement,
94}
95
96#[derive(DeserializeRow)]
97struct JobRow {
98 job_id: String,
99 task_name: String,
100 actor_json: String,
101 params_json: String,
102 priority: i32,
103 pool: String,
104 status: String,
105 idempotency_key: Option<String>,
106 created_at: i64,
107 signature_hash: i64,
108 attempt: i32,
109}
110
111#[derive(DeserializeRow)]
112struct ReadyRow {
113 priority: i32,
114 created_at: i64,
115 job_id: String,
116}
117
118#[derive(DeserializeRow)]
119struct IdRow {
120 job_id: String,
121}
122
123#[derive(DeserializeRow)]
124struct LeaseRow {
125 #[allow(dead_code)]
126 lease_id: String,
127 #[allow(dead_code)]
128 worker_id: String,
129 expires_at: i64,
130}
131
132#[derive(DeserializeRow)]
133struct LeaseExpiryRow {
134 expires_at: i64,
135 job_id: String,
136 lease_id: String,
137}
138
139#[derive(DeserializeRow)]
140struct StatusIdxRow {
141 #[allow(dead_code)]
142 created_at: i64,
143 job_id: String,
144}
145
146#[derive(DeserializeRow)]
147struct RunRow {
148 run_id: String,
149 job_id: String,
150 task_name: String,
151 attempt: i32,
152 status: String,
153 started_at: i64,
154 finished_at: Option<i64>,
155 duration_ms: Option<i64>,
156 error_message: Option<String>,
157}
158
159#[derive(DeserializeRow)]
160struct RunIdRow {
161 run_id: String,
162}
163
164#[derive(DeserializeRow)]
165struct TaskConfigRow {
166 task_name: String,
167 priority: i32,
168 pool: String,
169 retry_policy_json: String,
170 rate_limit_policy_json: String,
171 idempotency_mode: Option<String>,
172 updated_at: i64,
173}
174
175const fn to_millis(dt: DateTime<Utc>) -> i64 {
176 dt.timestamp_millis()
177}
178
179fn from_millis(ms: i64) -> DateTime<Utc> {
180 DateTime::from_timestamp_millis(ms).unwrap_or_else(Utc::now)
181}
182
183impl ScyllaQueueBackend {
184 pub async fn connect(config: ScyllaQueueConfig) -> Result<Self> {
190 let mut builder = SessionBuilder::new();
191 for cp in &config.contact_points {
192 builder = builder.known_node(cp.as_str());
193 }
194 if let Some(dc) = &config.datacenter {
195 builder = builder.prefer_datacenter(dc.clone());
196 }
197 if let (Some(user), Some(pass)) = (&config.username, &config.password) {
198 builder = builder.user(user.clone(), pass.clone());
199 }
200 if let Some(n) = config
201 .pool_per_shard
202 .and_then(|n| NonZeroUsize::new(n.max(1) as usize))
203 {
204 builder = builder.pool_size(PoolSize::PerShard(n));
205 }
206 let contact_label = config.contact_points.join(",");
207 let session = Box::pin(builder.build())
208 .await
209 .map_err(|e| map_connect_err_source(&contact_label, e))?;
210 Self::from_session(Arc::new(session), &config).await
211 }
212
213 #[allow(clippy::cognitive_complexity, clippy::too_many_lines)]
219 pub async fn from_session(session: Arc<Session>, config: &ScyllaQueueConfig) -> Result<Self> {
220 schema::ensure_schema(&session, &config.keyspace, config.replication_factor).await?;
221 let keyspace = config.keyspace.clone();
222 let prefix = format!("{keyspace}.");
223 let q = |sql: &str| sql.replace("boson.", &prefix);
224
225 let prepare = |sql: String| {
226 let session = Arc::clone(&session);
227 async move { session.prepare(sql).await.map_err(map_err_source) }
228 };
229
230 Ok(Self {
231 session: Arc::clone(&session),
232 keyspace,
233 ready_shard_count: config.ready_shard_count.max(1),
234 shard_concurrency: config.shard_concurrency.max(1),
235 parallel_writes: config.parallel_writes,
236 enqueue_rate: EnqueueRateLimiter::new(),
237 list_cursor: AtomicU64::new(0),
238 insert_job: prepare(q(
239 "INSERT INTO boson.boson_job (job_id, task_name, actor_json, params_json, priority, pool, status, idempotency_key, created_at, signature_hash, attempt) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
240 ))
241 .await?,
242 select_job: prepare(q(
243 "SELECT job_id, task_name, actor_json, params_json, priority, pool, status, idempotency_key, created_at, signature_hash, attempt FROM boson.boson_job WHERE job_id = ?",
244 ))
245 .await?,
246 update_job_claim: prepare(q(
247 "UPDATE boson.boson_job SET status = ? WHERE job_id = ? IF status = ?",
248 ))
249 .await?,
250 update_job_status: prepare(q(
251 "UPDATE boson.boson_job SET status = ?, attempt = ? WHERE job_id = ?",
252 ))
253 .await?,
254 update_job_revert: prepare(q(
255 "UPDATE boson.boson_job SET status = ? WHERE job_id = ? IF status = ?",
256 ))
257 .await?,
258 insert_ready: prepare(q(
259 "INSERT INTO boson.boson_ready (pool, shard, priority, created_at, job_id) VALUES (?, ?, ?, ?, ?)",
260 ))
261 .await?,
262 select_ready: prepare(q(
263 "SELECT priority, created_at, job_id FROM boson.boson_ready WHERE pool = ? AND shard = ? LIMIT ?",
264 ))
265 .await?,
266 delete_ready: prepare(q(
267 "DELETE FROM boson.boson_ready WHERE pool = ? AND shard = ? AND priority = ? AND created_at = ? AND job_id = ?",
268 ))
269 .await?,
270 insert_idempotency: prepare(q(
271 "INSERT INTO boson.boson_idempotency (idempotency_key, job_id) VALUES (?, ?) IF NOT EXISTS",
272 ))
273 .await?,
274 upsert_idempotency: prepare(q(
275 "INSERT INTO boson.boson_idempotency (idempotency_key, job_id) VALUES (?, ?)",
276 ))
277 .await?,
278 select_idempotency: prepare(q(
279 "SELECT job_id FROM boson.boson_idempotency WHERE idempotency_key = ?",
280 ))
281 .await?,
282 insert_lease: prepare(q(
283 "INSERT INTO boson.boson_lease (job_id, lease_id, worker_id, expires_at) VALUES (?, ?, ?, ?) IF NOT EXISTS",
284 ))
285 .await?,
286 steal_lease: prepare(q(
287 "UPDATE boson.boson_lease SET lease_id = ?, worker_id = ?, expires_at = ? WHERE job_id = ? IF expires_at < ?",
288 ))
289 .await?,
290 select_lease: prepare(q(
291 "SELECT lease_id, worker_id, expires_at FROM boson.boson_lease WHERE job_id = ?",
292 ))
293 .await?,
294 update_lease_ttl: prepare(q(
295 "UPDATE boson.boson_lease SET expires_at = ? WHERE job_id = ?",
296 ))
297 .await?,
298 delete_lease: prepare(q("DELETE FROM boson.boson_lease WHERE job_id = ?")).await?,
299 insert_lease_by_id: prepare(q(
300 "INSERT INTO boson.boson_lease_by_id (lease_id, job_id) VALUES (?, ?)",
301 ))
302 .await?,
303 select_lease_by_id: prepare(q(
304 "SELECT job_id FROM boson.boson_lease_by_id WHERE lease_id = ?",
305 ))
306 .await?,
307 delete_lease_by_id: prepare(q(
308 "DELETE FROM boson.boson_lease_by_id WHERE lease_id = ?",
309 ))
310 .await?,
311 insert_lease_expiry: prepare(q(
312 "INSERT INTO boson.boson_lease_by_expiry (bucket, expires_at, job_id, lease_id) VALUES (?, ?, ?, ?)",
313 ))
314 .await?,
315 delete_lease_expiry: prepare(q(
316 "DELETE FROM boson.boson_lease_by_expiry WHERE bucket = ? AND expires_at = ? AND job_id = ?",
317 ))
318 .await?,
319 select_lease_expiry: prepare(q(
320 "SELECT expires_at, job_id, lease_id FROM boson.boson_lease_by_expiry WHERE bucket = ?",
321 ))
322 .await?,
323 insert_status_idx: prepare(q(
324 "INSERT INTO boson.boson_job_by_status (status, created_at, job_id) VALUES (?, ?, ?)",
325 ))
326 .await?,
327 delete_status_idx: prepare(q(
328 "DELETE FROM boson.boson_job_by_status WHERE status = ? AND created_at = ? AND job_id = ?",
329 ))
330 .await?,
331 select_status_idx: prepare(q(
332 "SELECT created_at, job_id FROM boson.boson_job_by_status WHERE status = ? LIMIT ?",
333 ))
334 .await?,
335 insert_run: prepare(q(
336 "INSERT INTO boson.boson_run (run_id, job_id, task_name, attempt, status, started_at, finished_at, duration_ms, error_message) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
337 ))
338 .await?,
339 select_run: prepare(q(
340 "SELECT run_id, job_id, task_name, attempt, status, started_at, finished_at, duration_ms, error_message FROM boson.boson_run WHERE run_id = ?",
341 ))
342 .await?,
343 update_run_finish: prepare(q(
344 "UPDATE boson.boson_run SET status = ?, finished_at = ?, duration_ms = ?, error_message = ? WHERE run_id = ?",
345 ))
346 .await?,
347 insert_run_by_job: prepare(q(
348 "INSERT INTO boson.boson_run_by_job (job_id, started_at, run_id) VALUES (?, ?, ?)",
349 ))
350 .await?,
351 select_run_by_job: prepare(q(
352 "SELECT run_id FROM boson.boson_run_by_job WHERE job_id = ? LIMIT ?",
353 ))
354 .await?,
355 insert_task_config: prepare(q(
356 "INSERT INTO boson.boson_task_config (task_name, priority, pool, retry_policy_json, rate_limit_policy_json, idempotency_mode, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)",
357 ))
358 .await?,
359 select_task_config: prepare(q(
360 "SELECT task_name, priority, pool, retry_policy_json, rate_limit_policy_json, idempotency_mode, updated_at FROM boson.boson_task_config WHERE task_name = ?",
361 ))
362 .await?,
363 })
364 }
365
366 #[must_use]
368 pub const fn session(&self) -> &Arc<Session> {
369 &self.session
370 }
371
372 #[must_use]
374 pub fn keyspace(&self) -> &str {
375 &self.keyspace
376 }
377
378 #[must_use]
380 pub const fn ready_shard_count(&self) -> u32 {
381 self.ready_shard_count
382 }
383
384 async fn exec(
385 &self,
386 stmt: &scylla::statement::prepared::PreparedStatement,
387 values: impl scylla::serialize::row::SerializeRow,
388 ) -> Result<scylla::response::query_result::QueryResult> {
389 self.session
390 .execute_unpaged(stmt, values)
391 .await
392 .map_err(map_err_source)
393 }
394
395 fn job_from_row(row: JobRow) -> Result<Job> {
396 Ok(Job {
397 job_id: row.job_id,
398 task_name: row.task_name,
399 actor_json: serde_json::from_str(&row.actor_json)?,
400 params_json: serde_json::from_str(&row.params_json)?,
401 priority: row.priority,
402 pool: row.pool,
403 status: parse_job_status(&row.status)?,
404 idempotency_key: row.idempotency_key,
405 created_at: from_millis(row.created_at),
406 signature_hash: u64::try_from(row.signature_hash).unwrap_or(0),
407 attempt: row.attempt,
408 })
409 }
410
411 async fn load_job(&self, job_id: &str) -> Result<Option<Job>> {
412 let rows = self.exec(&self.select_job, (job_id,)).await?;
413 maybe_first_row::<JobRow>(rows)
414 .map(Self::job_from_row)
415 .transpose()
416 }
417
418 async fn insert_job_body(&self, job: &Job) -> Result<()> {
419 let actor = serde_json::to_string(&job.actor_json)?;
420 let params = serde_json::to_string(&job.params_json)?;
421 self.exec(
422 &self.insert_job,
423 (
424 job.job_id.as_str(),
425 job.task_name.as_str(),
426 actor.as_str(),
427 params.as_str(),
428 job.priority,
429 job.pool.as_str(),
430 job.status.to_string(),
431 job.idempotency_key.as_deref(),
432 to_millis(job.created_at),
433 i64::try_from(job.signature_hash).unwrap_or(i64::MAX),
434 job.attempt,
435 ),
436 )
437 .await?;
438 Ok(())
439 }
440
441 async fn insert_status_for_job(&self, job: &Job) -> Result<()> {
442 self.exec(
443 &self.insert_status_idx,
444 (
445 job.status.to_string(),
446 to_millis(job.created_at),
447 job.job_id.as_str(),
448 ),
449 )
450 .await?;
451 Ok(())
452 }
453
454 async fn insert_job_row(&self, job: &Job) -> Result<()> {
455 if self.parallel_writes {
456 let (job_res, status_res) =
457 tokio::join!(self.insert_job_body(job), self.insert_status_for_job(job));
458 job_res?;
459 status_res?;
460 } else {
461 self.insert_job_body(job).await?;
462 self.insert_status_for_job(job).await?;
463 }
464 Ok(())
465 }
466
467 async fn insert_ready_row(&self, job: &Job) -> Result<()> {
468 let shard = shard_for_job(&job.job_id, self.ready_shard_count);
469 self.exec(
470 &self.insert_ready,
471 (
472 job.pool.as_str(),
473 shard,
474 job.priority,
475 to_millis(job.created_at),
476 job.job_id.as_str(),
477 ),
478 )
479 .await?;
480 Ok(())
481 }
482
483 async fn delete_ready_row(&self, job: &Job) -> Result<()> {
484 let shard = shard_for_job(&job.job_id, self.ready_shard_count);
485 self.exec(
486 &self.delete_ready,
487 (
488 job.pool.as_str(),
489 shard,
490 job.priority,
491 to_millis(job.created_at),
492 job.job_id.as_str(),
493 ),
494 )
495 .await?;
496 Ok(())
497 }
498
499 async fn move_status_idx(
500 &self,
501 job_id: &str,
502 created_at: DateTime<Utc>,
503 from: JobStatus,
504 to: JobStatus,
505 ) -> Result<()> {
506 let ms = to_millis(created_at);
507 let from_s = from.to_string();
508 let to_s = to.to_string();
509 if self.parallel_writes {
510 let (del_res, ins_res) = tokio::join!(
511 self.exec(&self.delete_status_idx, (from_s.as_str(), ms, job_id)),
512 self.exec(&self.insert_status_idx, (to_s.as_str(), ms, job_id)),
513 );
514 del_res?;
515 ins_res?;
516 } else {
517 self.exec(&self.delete_status_idx, (from_s.as_str(), ms, job_id))
518 .await?;
519 self.exec(&self.insert_status_idx, (to_s.as_str(), ms, job_id))
520 .await?;
521 }
522 Ok(())
523 }
524}
525
526#[async_trait]
527impl QueueBackend for ScyllaQueueBackend {
528 async fn upsert_job(&self, job: &Job) -> Result<()> {
529 if let Some(existing) = self.load_job(&job.job_id).await? {
530 if existing.status != job.status {
531 self.move_status_idx(
532 &job.job_id,
533 existing.created_at,
534 existing.status,
535 job.status,
536 )
537 .await?;
538 if existing.status == JobStatus::Queued {
539 let _ = self.delete_ready_row(&existing).await;
540 }
541 } else if existing.status == JobStatus::Queued {
542 let _ = self.delete_ready_row(&existing).await;
544 }
545 }
546 self.insert_job_row(job).await?;
547 if job.status == JobStatus::Queued {
548 self.insert_ready_row(job).await?;
549 }
550 Ok(())
551 }
552
553 async fn enqueue_with_policies(
554 &self,
555 job: Job,
556 task_config: &TaskConfig,
557 ) -> Result<(String, JobEnqueueDisposition)> {
558 let idempotency = task_config.resolved_idempotency_mode(IdempotencyMode::Lwt);
559 let mut job = job;
560 if idempotency == IdempotencyMode::Lwt {
561 if let Some(ref key) = job.idempotency_key {
562 if !key.is_empty() {
563 let result = self
564 .exec(
565 &self.insert_idempotency,
566 (key.as_str(), job.job_id.as_str()),
567 )
568 .await?;
569 if !lwt_applied(result) {
570 let existing = self.exec(&self.select_idempotency, (key.as_str(),)).await?;
571 let Some(row) = maybe_first_row::<IdRow>(existing) else {
572 return Err(BosonError::backend(
573 "idempotency insert not applied but row missing",
574 ));
575 };
576 if let Some(prior) = self.load_job(&row.job_id).await? {
578 if matches!(prior.status, JobStatus::Queued | JobStatus::Running) {
579 return Ok((row.job_id, JobEnqueueDisposition::ReusedIdempotent));
580 }
581 }
582 self.exec(
584 &self.upsert_idempotency,
585 (key.as_str(), job.job_id.as_str()),
586 )
587 .await?;
588 }
589 }
590 }
591 } else {
592 job.idempotency_key = None;
593 }
594
595 let policy = &task_config.rate_limit_policy;
596 if policy.max_in_flight > 0 {
597 let count = self.count_active_jobs_for_task(&job.task_name).await?;
598 if count >= policy.max_in_flight {
599 return Err(BosonError::RateLimited(job.task_name.clone()));
600 }
601 }
602 if policy.max_enqueue_per_second > 0
603 && !self
604 .enqueue_rate
605 .try_record(&job.task_name, policy.max_enqueue_per_second)
606 {
607 return Err(BosonError::RateLimited(job.task_name.clone()));
608 }
609
610 let job_id = job.job_id.clone();
611 if self.parallel_writes {
612 let (job_res, status_res, ready_res) = tokio::join!(
613 self.insert_job_body(&job),
614 self.insert_status_for_job(&job),
615 self.insert_ready_row(&job),
616 );
617 job_res?;
618 status_res?;
619 ready_res?;
620 } else {
621 self.insert_job_row(&job).await?;
622 self.insert_ready_row(&job).await?;
623 }
624 Ok((job_id, JobEnqueueDisposition::InsertedNew))
625 }
626
627 async fn get_job(&self, job_id: &str) -> Result<Option<Job>> {
628 self.load_job(job_id).await
629 }
630
631 async fn list_jobs(
632 &self,
633 status_filter: Option<JobStatus>,
634 offset: usize,
635 limit: usize,
636 ) -> Result<Vec<Job>> {
637 let statuses: Vec<JobStatus> = status_filter.map_or_else(
638 || {
639 vec![
640 JobStatus::Queued,
641 JobStatus::Running,
642 JobStatus::Success,
643 JobStatus::Failed,
644 JobStatus::Canceled,
645 ]
646 },
647 |status| vec![status],
648 );
649 let mut jobs = Vec::new();
650 let fetch = offset.saturating_add(limit).max(1);
651 for status in statuses {
652 let rows = self
653 .exec(
654 &self.select_status_idx,
655 (status.to_string(), i32::try_from(fetch).unwrap_or(i32::MAX)),
656 )
657 .await?;
658 for idx in collect_rows::<StatusIdxRow>(rows) {
659 if let Some(job) = self.load_job(&idx.job_id).await? {
660 jobs.push(job);
661 }
662 }
663 }
664 jobs.sort_by_key(|j| j.created_at);
665 Ok(jobs.into_iter().skip(offset).take(limit).collect())
666 }
667
668 async fn cancel_job_if_active(&self, job_id: &str) -> Result<()> {
669 let Some(job) = self.load_job(job_id).await? else {
670 return Err(BosonError::JobNotFound(job_id.to_string()));
671 };
672 if !matches!(job.status, JobStatus::Queued | JobStatus::Running) {
673 return Ok(());
674 }
675 let from = job.status;
676 self.exec(
677 &self.update_job_status,
678 (JobStatus::Canceled.to_string(), job.attempt, job_id),
679 )
680 .await?;
681 self.move_status_idx(job_id, job.created_at, from, JobStatus::Canceled)
682 .await?;
683 if from == JobStatus::Queued {
684 self.delete_ready_row(&job).await?;
685 }
686 Ok(())
687 }
688
689 async fn try_claim_job(&self, job_id: &str) -> Result<Option<Job>> {
690 let result = self
691 .exec(
692 &self.update_job_claim,
693 (
694 JobStatus::Running.to_string(),
695 job_id,
696 JobStatus::Queued.to_string(),
697 ),
698 )
699 .await?;
700 if !lwt_applied(result) {
701 return Ok(None);
702 }
703 let Some(job) = self.load_job(job_id).await? else {
704 return Ok(None);
705 };
706 if self.parallel_writes {
707 let (status_res, ready_res) = tokio::join!(
708 self.move_status_idx(
709 job_id,
710 job.created_at,
711 JobStatus::Queued,
712 JobStatus::Running
713 ),
714 self.delete_ready_row(&job),
715 );
716 status_res?;
717 let _ = ready_res;
718 } else {
719 self.move_status_idx(
720 job_id,
721 job.created_at,
722 JobStatus::Queued,
723 JobStatus::Running,
724 )
725 .await?;
726 let _ = self.delete_ready_row(&job).await;
727 }
728 Ok(Some(job))
729 }
730
731 async fn revert_job_to_queued(&self, job_id: &str) -> Result<()> {
732 let Some(job) = self.load_job(job_id).await? else {
733 return Ok(());
734 };
735 if job.status != JobStatus::Running {
736 return Ok(());
737 }
738 let result = self
739 .exec(
740 &self.update_job_revert,
741 (
742 JobStatus::Queued.to_string(),
743 job_id,
744 JobStatus::Running.to_string(),
745 ),
746 )
747 .await?;
748 if lwt_applied(result) {
749 self.move_status_idx(
750 job_id,
751 job.created_at,
752 JobStatus::Running,
753 JobStatus::Queued,
754 )
755 .await?;
756 let mut queued = job;
757 queued.status = JobStatus::Queued;
758 self.insert_ready_row(&queued).await?;
759 }
760 Ok(())
761 }
762
763 async fn distinct_pools_queued(&self) -> Result<Vec<String>> {
764 let mut pools = HashSet::new();
766 let sample = self.ready_shard_count.min(16);
767 for shard in 0..i32::try_from(sample).unwrap_or(1) {
768 let _ = shard;
770 }
771 let rows = self
772 .exec(
773 &self.select_status_idx,
774 (JobStatus::Queued.to_string(), 10_000_i32),
775 )
776 .await?;
777 for idx in collect_rows::<StatusIdxRow>(rows) {
778 if let Some(job) = self.load_job(&idx.job_id).await? {
779 pools.insert(job.pool);
780 }
781 }
782 let mut out: Vec<String> = pools.into_iter().collect();
783 out.sort();
784 Ok(out)
785 }
786
787 async fn list_queued_for_pool_sorted(&self, pool: &str, limit: usize) -> Result<Vec<Job>> {
788 let limit = limit.max(1);
789 let shard_count = self.ready_shard_count;
790 let start = self.list_cursor.fetch_add(1, Ordering::Relaxed);
793 let per_shard = i32::try_from(limit).unwrap_or(i32::MAX);
794 let concurrency = self.shard_concurrency.max(1) as usize;
795 let shards: Vec<i32> = (0..shard_count)
796 .map(|i| i32::try_from((start + u64::from(i)) % u64::from(shard_count)).unwrap_or(0))
797 .collect();
798
799 let mut candidates: Vec<(i32, i64, String)> = Vec::new();
800 for chunk in shards.chunks(concurrency) {
801 let futs: Vec<_> = chunk
802 .iter()
803 .map(|&shard| self.exec(&self.select_ready, (pool, shard, per_shard)))
804 .collect();
805 for result in join_all(futs).await {
806 let rows = result?;
807 for row in collect_rows::<ReadyRow>(rows) {
808 candidates.push((row.priority, row.created_at, row.job_id));
809 }
810 }
811 }
812 candidates.sort_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.cmp(&b.1)));
813 candidates.dedup_by(|a, b| a.2 == b.2);
814 let job_ids: Vec<String> = candidates
815 .into_iter()
816 .take(limit)
817 .map(|(_, _, id)| id)
818 .collect();
819 let load_futs: Vec<_> = job_ids.iter().map(|id| self.load_job(id)).collect();
820 let mut jobs = Vec::new();
821 for result in join_all(load_futs).await {
822 if let Some(job) = result? {
823 if job.status == JobStatus::Queued && job.pool == pool {
824 jobs.push(job);
825 }
826 }
827 }
828 Ok(jobs)
829 }
830
831 async fn count_jobs(&self, status_filter: Option<JobStatus>) -> Result<u64> {
832 let jobs = self.list_jobs(status_filter, 0, 100_000).await?;
833 Ok(u64::try_from(jobs.len()).unwrap_or(u64::MAX))
834 }
835
836 async fn count_jobs_for_task(&self, task_name: &str, status: Option<JobStatus>) -> Result<u64> {
837 let jobs = self.list_jobs(status, 0, 100_000).await?;
838 let count = jobs.iter().filter(|j| j.task_name == task_name).count();
839 Ok(u64::try_from(count).unwrap_or(u64::MAX))
840 }
841
842 async fn count_active_jobs_for_task(&self, task_name: &str) -> Result<u32> {
843 let queued = self
844 .count_jobs_for_task(task_name, Some(JobStatus::Queued))
845 .await?;
846 let running = self
847 .count_jobs_for_task(task_name, Some(JobStatus::Running))
848 .await?;
849 Ok(u32::try_from(queued.saturating_add(running)).unwrap_or(u32::MAX))
850 }
851
852 async fn find_nonterminal_by_idempotency_key(&self, key: &str) -> Result<Option<String>> {
853 let rows = self.exec(&self.select_idempotency, (key,)).await?;
854 Ok(maybe_first_row::<IdRow>(rows).map(|r| r.job_id))
855 }
856
857 async fn upsert_run(&self, run: &Run) -> Result<()> {
858 self.exec(
859 &self.insert_run,
860 (
861 run.run_id.as_str(),
862 run.job_id.as_str(),
863 run.task_name.as_str(),
864 run.attempt,
865 run.status.to_string(),
866 to_millis(run.started_at),
867 run.finished_at.map(to_millis),
868 run.duration_ms,
869 run.error_message.as_deref(),
870 ),
871 )
872 .await?;
873 self.exec(
874 &self.insert_run_by_job,
875 (
876 run.job_id.as_str(),
877 to_millis(run.started_at),
878 run.run_id.as_str(),
879 ),
880 )
881 .await?;
882 Ok(())
883 }
884
885 async fn get_run(&self, run_id: &str) -> Result<Option<Run>> {
886 let rows = self.exec(&self.select_run, (run_id,)).await?;
887 Ok(maybe_first_row::<RunRow>(rows)
888 .map(run_from_row)
889 .transpose()?)
890 }
891
892 async fn list_runs(
893 &self,
894 job_id_filter: Option<&str>,
895 offset: usize,
896 limit: usize,
897 ) -> Result<Vec<Run>> {
898 let Some(job_id) = job_id_filter else {
899 return Ok(vec![]);
900 };
901 let fetch = offset.saturating_add(limit).max(1);
902 let rows = self
903 .exec(
904 &self.select_run_by_job,
905 (job_id, i32::try_from(fetch).unwrap_or(i32::MAX)),
906 )
907 .await?;
908 let mut runs = Vec::new();
909 for id in collect_rows::<RunIdRow>(rows) {
910 if let Some(run) = self.get_run(&id.run_id).await? {
911 runs.push(run);
912 }
913 }
914 runs.sort_by_key(|r| r.started_at);
915 Ok(runs.into_iter().skip(offset).take(limit).collect())
916 }
917
918 async fn finish_run(
919 &self,
920 run_id: &str,
921 status: RunStatus,
922 duration_ms: Option<i64>,
923 error_message: Option<String>,
924 ) -> Result<()> {
925 let finished_at = to_millis(Utc::now());
926 self.exec(
927 &self.update_run_finish,
928 (
929 status.to_string(),
930 finished_at,
931 duration_ms,
932 error_message.as_deref(),
933 run_id,
934 ),
935 )
936 .await?;
937 Ok(())
938 }
939
940 async fn count_runs(&self, job_id_filter: Option<&str>) -> Result<u64> {
941 let runs = self.list_runs(job_id_filter, 0, 100_000).await?;
942 Ok(u64::try_from(runs.len()).unwrap_or(u64::MAX))
943 }
944
945 async fn count_runs_since(&self, _since: DateTime<Utc>) -> Result<u64> {
946 Ok(0)
947 }
948
949 async fn task_run_stats(&self, _task_name: &str) -> Result<TaskRunStats> {
950 Ok(TaskRunStats {
951 runs_total: 0,
952 success_count: 0,
953 })
954 }
955
956 async fn get_task_config(&self, task_name: &str) -> Result<Option<TaskConfig>> {
957 let rows = self.exec(&self.select_task_config, (task_name,)).await?;
958 let Some(row) = maybe_first_row::<TaskConfigRow>(rows) else {
959 return Ok(None);
960 };
961 Ok(Some(TaskConfig {
962 task_name: row.task_name,
963 priority: row.priority,
964 pool: row.pool,
965 retry_policy: serde_json::from_str(&row.retry_policy_json)?,
966 rate_limit_policy: serde_json::from_str(&row.rate_limit_policy_json)?,
967 idempotency_mode: row
968 .idempotency_mode
969 .as_deref()
970 .and_then(IdempotencyMode::parse),
971 updated_at: from_millis(row.updated_at),
972 }))
973 }
974
975 async fn upsert_task_config(&self, config: &TaskConfig) -> Result<()> {
976 let mode = config.idempotency_mode.map(|m| match m {
977 IdempotencyMode::Lwt => "lwt",
978 IdempotencyMode::None => "none",
979 });
980 self.exec(
981 &self.insert_task_config,
982 (
983 config.task_name.as_str(),
984 config.priority,
985 config.pool.as_str(),
986 serde_json::to_string(&config.retry_policy)?,
987 serde_json::to_string(&config.rate_limit_policy)?,
988 mode,
989 to_millis(config.updated_at),
990 ),
991 )
992 .await?;
993 Ok(())
994 }
995
996 async fn try_claim_run_lease(
997 &self,
998 job_id: &str,
999 worker_id: &str,
1000 ttl_secs: i64,
1001 ) -> Result<Option<String>> {
1002 let now_ms = to_millis(Utc::now());
1003 let expires_at = to_millis(Utc::now() + Duration::seconds(ttl_secs));
1004 let lease_id = Uuid::new_v4().to_string();
1005 let insert = self
1006 .exec(
1007 &self.insert_lease,
1008 (job_id, lease_id.as_str(), worker_id, expires_at),
1009 )
1010 .await?;
1011 if lwt_applied(insert) {
1012 self.record_lease(job_id, &lease_id, expires_at).await?;
1013 return Ok(Some(lease_id));
1014 }
1015 let steal = self
1016 .exec(
1017 &self.steal_lease,
1018 (lease_id.as_str(), worker_id, expires_at, job_id, now_ms),
1019 )
1020 .await?;
1021 if lwt_applied(steal) {
1022 self.record_lease(job_id, &lease_id, expires_at).await?;
1023 return Ok(Some(lease_id));
1024 }
1025 Ok(None)
1026 }
1027
1028 async fn extend_lease(&self, lease_id: &str, ttl_secs: i64) -> Result<()> {
1029 let Some(job_id) = self.job_id_for_lease(lease_id).await? else {
1030 return Ok(());
1031 };
1032 let expires_at = to_millis(Utc::now() + Duration::seconds(ttl_secs));
1033 self.exec(&self.update_lease_ttl, (expires_at, job_id.as_str()))
1034 .await?;
1035 self.insert_expiry_index(&job_id, lease_id, expires_at)
1036 .await?;
1037 Ok(())
1038 }
1039
1040 async fn release_lease(&self, lease_id: &str) -> Result<()> {
1041 let Some(job_id) = self.job_id_for_lease(lease_id).await? else {
1042 return Ok(());
1043 };
1044 if let Some(lease) = self.load_lease(&job_id).await? {
1045 let bucket = expiry_bucket(lease.expires_at / 1000);
1046 let _ = self
1047 .exec(
1048 &self.delete_lease_expiry,
1049 (bucket, lease.expires_at, job_id.as_str()),
1050 )
1051 .await;
1052 }
1053 self.exec(&self.delete_lease, (job_id.as_str(),)).await?;
1054 self.exec(&self.delete_lease_by_id, (lease_id,)).await?;
1055 Ok(())
1056 }
1057
1058 async fn expired_lease_job_pairs(&self) -> Result<Vec<(String, String)>> {
1059 let now_ms = to_millis(Utc::now());
1060 let bucket = expiry_bucket(now_ms / 1000);
1061 let mut pairs = Vec::new();
1062 for b in [bucket - 1, bucket, bucket + 1] {
1063 if b < 0 {
1064 continue;
1065 }
1066 let rows = self.exec(&self.select_lease_expiry, (b,)).await?;
1067 for row in collect_rows::<LeaseExpiryRow>(rows) {
1068 if row.expires_at <= now_ms {
1069 pairs.push((row.lease_id, row.job_id));
1070 }
1071 }
1072 }
1073 Ok(pairs)
1074 }
1075}
1076
1077impl ScyllaQueueBackend {
1078 async fn insert_expiry_index(
1079 &self,
1080 job_id: &str,
1081 lease_id: &str,
1082 expires_at_ms: i64,
1083 ) -> Result<()> {
1084 let bucket = expiry_bucket(expires_at_ms / 1000);
1085 self.exec(
1086 &self.insert_lease_expiry,
1087 (bucket, expires_at_ms, job_id, lease_id),
1088 )
1089 .await?;
1090 Ok(())
1091 }
1092
1093 async fn record_lease(&self, job_id: &str, lease_id: &str, expires_at_ms: i64) -> Result<()> {
1094 self.exec(&self.insert_lease_by_id, (lease_id, job_id))
1095 .await?;
1096 self.insert_expiry_index(job_id, lease_id, expires_at_ms)
1097 .await?;
1098 Ok(())
1099 }
1100
1101 async fn job_id_for_lease(&self, lease_id: &str) -> Result<Option<String>> {
1102 let rows = self.exec(&self.select_lease_by_id, (lease_id,)).await?;
1103 Ok(maybe_first_row::<IdRow>(rows).map(|r| r.job_id))
1104 }
1105
1106 async fn load_lease(&self, job_id: &str) -> Result<Option<LeaseRow>> {
1107 let rows = self.exec(&self.select_lease, (job_id,)).await?;
1108 Ok(maybe_first_row::<LeaseRow>(rows))
1109 }
1110}
1111
1112impl fmt::Debug for ScyllaQueueBackend {
1113 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1114 f.debug_struct("ScyllaQueueBackend")
1115 .field("keyspace", &self.keyspace)
1116 .field("ready_shard_count", &self.ready_shard_count)
1117 .finish_non_exhaustive()
1118 }
1119}
1120
1121fn parse_job_status(s: &str) -> Result<JobStatus> {
1122 match s {
1123 "queued" => Ok(JobStatus::Queued),
1124 "running" => Ok(JobStatus::Running),
1125 "success" => Ok(JobStatus::Success),
1126 "failed" => Ok(JobStatus::Failed),
1127 "canceled" => Ok(JobStatus::Canceled),
1128 other => Err(BosonError::backend(format!("unknown job status: {other}"))),
1129 }
1130}
1131
1132fn parse_run_status(s: &str) -> Result<RunStatus> {
1133 match s {
1134 "running" => Ok(RunStatus::Running),
1135 "success" => Ok(RunStatus::Success),
1136 "failed" => Ok(RunStatus::Failed),
1137 "canceled" => Ok(RunStatus::Canceled),
1138 "timeout" => Ok(RunStatus::Timeout),
1139 other => Err(BosonError::backend(format!("unknown run status: {other}"))),
1140 }
1141}
1142
1143fn run_from_row(row: RunRow) -> Result<Run> {
1144 Ok(Run {
1145 run_id: row.run_id,
1146 job_id: row.job_id,
1147 task_name: row.task_name,
1148 attempt: row.attempt,
1149 status: parse_run_status(&row.status)?,
1150 started_at: from_millis(row.started_at),
1151 finished_at: row.finished_at.map(from_millis),
1152 duration_ms: row.duration_ms,
1153 error_message: row.error_message,
1154 })
1155}
1156
1157fn lwt_applied(result: scylla::response::query_result::QueryResult) -> bool {
1158 use scylla::deserialize::row::ColumnIterator;
1159 use scylla::deserialize::value::DeserializeValue;
1160
1161 let Ok(rows) = result.into_rows_result() else {
1164 return false;
1165 };
1166 let Ok(Some(mut cols)) = rows.maybe_first_row::<ColumnIterator<'_, '_>>() else {
1167 return false;
1168 };
1169 let Some(Ok(first)) = cols.next() else {
1170 return false;
1171 };
1172 bool::deserialize(first.spec.typ(), first.slice).unwrap_or(false)
1173}
1174
1175fn maybe_first_row<R>(result: scylla::response::query_result::QueryResult) -> Option<R>
1176where
1177 R: for<'frame> scylla::deserialize::row::DeserializeRow<'frame, 'frame>,
1178{
1179 result
1180 .into_rows_result()
1181 .ok()
1182 .and_then(|rows| rows.maybe_first_row::<R>().ok().flatten())
1183}
1184
1185fn collect_rows<R>(result: scylla::response::query_result::QueryResult) -> Vec<R>
1186where
1187 R: for<'frame> scylla::deserialize::row::DeserializeRow<'frame, 'frame>,
1188{
1189 result
1190 .into_rows_result()
1191 .ok()
1192 .and_then(|rows| {
1193 rows.rows::<R>()
1194 .ok()
1195 .map(|iter| iter.filter_map(std::result::Result::ok).collect())
1196 })
1197 .unwrap_or_default()
1198}
1199
1200#[allow(dead_code)]
1202fn _hash_str(s: &str) -> u64 {
1203 let mut h = std::collections::hash_map::DefaultHasher::new();
1204 s.hash(&mut h);
1205 h.finish()
1206}