use std::collections::HashMap;
use std::ops::Bound;
use serde::{Deserialize, Serialize};
use crate::commit::tx::{ReadAssertion, ReadSet, TransactionEnvelope, WriteClass, WriteIntent};
use crate::commit::validation::Mutation;
use crate::error::AedbError;
use crate::permission::CallerContext;
use crate::query::plan::ConsistencyMode;
use crate::storage::keyspace::KvEntry;
use crate::{AedbInstance, FencedCommit};
pub const QUEUE_KEY_PREFIX: &[u8] = b"\x00aedb:queue:";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum TaskState {
Ready,
InFlight,
Dead,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct EnqueueOptions {
pub payload_json: String,
pub delay_micros: u64,
pub priority: u64,
pub max_attempts: u32,
pub backoff_base_micros: u64,
pub backoff_max_micros: u64,
pub idempotency_key: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct EnqueueSpec {
pub queue: String,
pub options: EnqueueOptions,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct EnqueueOutcome {
pub task_id: String,
pub seq: u64,
pub deduplicated: bool,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ClaimedTask {
pub task_id: String,
pub queue: String,
pub payload_json: String,
pub attempts: u32,
pub fencing_token: u64,
pub lease_deadline_micros: u64,
pub priority: u64,
pub enqueued_micros: u64,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TaskRecord {
pub task_id: String,
pub queue: String,
pub state: TaskState,
pub not_before_micros: u64,
pub priority: u64,
pub attempts: u32,
pub max_attempts: u32,
pub owner_id: Option<String>,
pub fencing_token: u64,
pub lease_deadline_micros: u64,
pub payload_json: String,
pub progress_json: String,
pub result_json: String,
pub last_error: Option<String>,
pub created_micros: u64,
pub updated_micros: u64,
}
const MAX_CAS_RETRIES: usize = 16;
const DEFAULT_MAX_ATTEMPTS: u32 = 5;
const DEFAULT_BACKOFF_BASE_MICROS: u64 = 1_000_000;
const DEFAULT_BACKOFF_MAX_MICROS: u64 = 60_000_000;
const KIND_PRIMARY: u8 = b'p'; const KIND_SCHEDULED: u8 = b's'; const KIND_READY: u8 = b'r'; const KIND_INFLIGHT: u8 = b'f'; const KIND_IDEM: u8 = b'k'; const KIND_COUNTER: u8 = b'n';
const STATE_READY: u8 = 0;
const STATE_INFLIGHT: u8 = 1;
const STATE_DEAD: u8 = 2;
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
struct StoredTask {
id: String,
queue: String,
state: u8,
not_before_micros: u64,
priority: u64,
seq: u64,
#[serde(default)]
in_ready: bool,
attempts: u32,
max_attempts: u32,
backoff_base_micros: u64,
backoff_max_micros: u64,
#[serde(default)]
owner_id: Option<String>,
#[serde(default)]
fencing_token: u64,
#[serde(default)]
lease_deadline_micros: u64,
#[serde(default)]
payload_json: String,
#[serde(default)]
progress_json: String,
#[serde(default)]
result_json: String,
#[serde(default)]
last_error: Option<String>,
created_micros: u64,
updated_micros: u64,
}
fn be(v: u64) -> [u8; 8] {
v.to_be_bytes()
}
fn q_base(queue: &str) -> Vec<u8> {
let qbytes = queue.as_bytes();
let mut k = Vec::with_capacity(QUEUE_KEY_PREFIX.len() + 2 + qbytes.len() + 24);
k.extend_from_slice(QUEUE_KEY_PREFIX);
k.extend_from_slice(&(qbytes.len() as u16).to_be_bytes());
k.extend_from_slice(qbytes);
k
}
fn primary_key(queue: &str, id: &str) -> Vec<u8> {
let mut k = q_base(queue);
k.push(KIND_PRIMARY);
k.extend_from_slice(id.as_bytes());
k
}
fn primary_prefix(queue: &str) -> Vec<u8> {
let mut k = q_base(queue);
k.push(KIND_PRIMARY);
k
}
fn counter_key(queue: &str) -> Vec<u8> {
let mut k = q_base(queue);
k.push(KIND_COUNTER);
k
}
fn idem_key(queue: &str, key: &str) -> Vec<u8> {
let mut k = q_base(queue);
k.push(KIND_IDEM);
k.extend_from_slice(key.as_bytes());
k
}
fn scheduled_key(queue: &str, not_before: u64, seq: u64, id: &str) -> Vec<u8> {
let mut k = q_base(queue);
k.push(KIND_SCHEDULED);
k.extend_from_slice(&be(not_before));
k.extend_from_slice(&be(seq));
k.extend_from_slice(id.as_bytes());
k
}
fn ready_key(queue: &str, priority: u64, seq: u64, id: &str) -> Vec<u8> {
let mut k = q_base(queue);
k.push(KIND_READY);
k.extend_from_slice(&be(u64::MAX - priority)); k.extend_from_slice(&be(seq));
k.extend_from_slice(id.as_bytes());
k
}
fn ready_prefix(queue: &str) -> Vec<u8> {
let mut k = q_base(queue);
k.push(KIND_READY);
k
}
fn ready_state_index_key(queue: &str, task: &StoredTask) -> Vec<u8> {
if task.in_ready {
ready_key(queue, task.priority, task.seq, &task.id)
} else {
scheduled_key(queue, task.not_before_micros, task.seq, &task.id)
}
}
fn inflight_key(queue: &str, deadline: u64, seq: u64, id: &str) -> Vec<u8> {
let mut k = q_base(queue);
k.push(KIND_INFLIGHT);
k.extend_from_slice(&be(deadline));
k.extend_from_slice(&be(seq));
k.extend_from_slice(id.as_bytes());
k
}
fn due_bounds(queue: &str, kind: u8, now: u64) -> (Bound<Vec<u8>>, Bound<Vec<u8>>) {
let mut lo = q_base(queue);
lo.push(kind);
let mut hi = lo.clone();
hi.extend_from_slice(&be(now.saturating_add(1)));
(Bound::Included(lo), Bound::Excluded(hi))
}
fn is_retryable_conflict(err: &AedbError) -> bool {
matches!(
err,
AedbError::Conflict(_) | AedbError::AssertionFailed { .. }
)
}
fn key_assertion(
project_id: &str,
scope_id: &str,
key: &[u8],
entry: Option<&KvEntry>,
) -> ReadAssertion {
match entry {
Some(entry) => ReadAssertion::KeyVersion {
project_id: project_id.to_string(),
scope_id: scope_id.to_string(),
key: key.to_vec(),
expected_seq: entry.version,
},
None => ReadAssertion::KeyExists {
project_id: project_id.to_string(),
scope_id: scope_id.to_string(),
key: key.to_vec(),
expected: false,
},
}
}
fn set_mut(project_id: &str, scope_id: &str, key: Vec<u8>, value: Vec<u8>) -> Mutation {
Mutation::KvSet {
project_id: project_id.to_string(),
scope_id: scope_id.to_string(),
key,
value,
}
}
fn del_mut(project_id: &str, scope_id: &str, key: Vec<u8>) -> Mutation {
Mutation::KvDel {
project_id: project_id.to_string(),
scope_id: scope_id.to_string(),
key,
}
}
fn encode_task(task: &StoredTask) -> Result<Vec<u8>, AedbError> {
serde_json::to_vec(task).map_err(|e| AedbError::Encode(format!("task encode failed: {e}")))
}
fn decode_task(entry: &KvEntry) -> Result<StoredTask, AedbError> {
serde_json::from_slice(&entry.value)
.map_err(|_| AedbError::Decode("task record is corrupt".into()))
}
fn next_backoff(task: &StoredTask, now: u64) -> u64 {
let base = if task.backoff_base_micros == 0 {
DEFAULT_BACKOFF_BASE_MICROS
} else {
task.backoff_base_micros
};
let ceil = if task.backoff_max_micros == 0 {
DEFAULT_BACKOFF_MAX_MICROS
} else {
task.backoff_max_micros
};
let shift = task.attempts.saturating_sub(1).min(32);
let delay = base.saturating_mul(1u64 << shift).min(ceil);
now.saturating_add(delay)
}
fn ensure_valid_json(s: &str, what: &str) -> Result<(), AedbError> {
if s.is_empty() {
return Ok(());
}
serde_json::from_str::<serde_json::Value>(s)
.map(|_| ())
.map_err(|_| AedbError::Validation(format!("{what} must be empty or valid JSON")))
}
struct CounterCursor {
entry_version: Option<u64>,
next_seq: u64,
}
struct EnqueuePlan {
mutations: Vec<Mutation>,
assertions: Vec<ReadAssertion>,
counters: HashMap<String, CounterCursor>,
}
impl EnqueuePlan {
fn new() -> Self {
Self {
mutations: Vec::new(),
assertions: Vec::new(),
counters: HashMap::new(),
}
}
}
impl AedbInstance {
pub async fn queue_enqueue(
&self,
project_id: &str,
scope_id: &str,
queue: &str,
options: EnqueueOptions,
extra_mutations: Vec<Mutation>,
) -> Result<EnqueueOutcome, AedbError> {
self.queue_enqueue_inner(None, project_id, scope_id, queue, options, extra_mutations)
.await
}
pub async fn queue_enqueue_as(
&self,
caller: CallerContext,
project_id: &str,
scope_id: &str,
queue: &str,
options: EnqueueOptions,
extra_mutations: Vec<Mutation>,
) -> Result<EnqueueOutcome, AedbError> {
self.queue_enqueue_inner(
Some(caller),
project_id,
scope_id,
queue,
options,
extra_mutations,
)
.await
}
async fn queue_enqueue_inner(
&self,
caller: Option<CallerContext>,
project_id: &str,
scope_id: &str,
queue: &str,
options: EnqueueOptions,
extra_mutations: Vec<Mutation>,
) -> Result<EnqueueOutcome, AedbError> {
if queue.is_empty() {
return Err(AedbError::Validation("queue name cannot be empty".into()));
}
ensure_valid_json(&options.payload_json, "payload_json")?;
let spec = EnqueueSpec {
queue: queue.to_string(),
options,
};
for _ in 0..MAX_CAS_RETRIES {
let lease = self.acquire_snapshot(ConsistencyMode::AtLatest).await?;
let base_seq = lease.view.seq;
let view = &lease.view.keyspace;
let now = crate::system_now_micros();
if let Some(idem) = &spec.options.idempotency_key
&& let Some(entry) =
view.try_kv_get(project_id, scope_id, &idem_key(queue, idem))?
{
let task_id = String::from_utf8_lossy(&entry.value).into_owned();
return Ok(EnqueueOutcome {
task_id,
seq: 0,
deduplicated: true,
});
}
let mut plan = EnqueuePlan::new();
let outcome = self.plan_enqueue(project_id, scope_id, view, &spec, now, &mut plan)?;
plan.mutations.extend(extra_mutations.clone());
self.finalize_counters(project_id, scope_id, view, &mut plan);
match self.commit_plan(caller.clone(), plan, base_seq).await {
Ok(_) => return Ok(outcome),
Err(err) if is_retryable_conflict(&err) => continue,
Err(err) => return Err(err),
}
}
Err(AedbError::Conflict(
"queue_enqueue exhausted retries".into(),
))
}
pub async fn queue_claim(
&self,
project_id: &str,
scope_id: &str,
queue: &str,
owner_id: &str,
lease_micros: u64,
max: usize,
) -> Result<Vec<ClaimedTask>, AedbError> {
self.queue_claim_inner(
None,
project_id,
scope_id,
queue,
owner_id,
lease_micros,
max,
)
.await
}
#[allow(clippy::too_many_arguments)]
pub async fn queue_claim_as(
&self,
caller: CallerContext,
project_id: &str,
scope_id: &str,
queue: &str,
owner_id: &str,
lease_micros: u64,
max: usize,
) -> Result<Vec<ClaimedTask>, AedbError> {
self.queue_claim_inner(
Some(caller),
project_id,
scope_id,
queue,
owner_id,
lease_micros,
max,
)
.await
}
#[allow(clippy::too_many_arguments)]
async fn queue_claim_inner(
&self,
caller: Option<CallerContext>,
project_id: &str,
scope_id: &str,
queue: &str,
owner_id: &str,
lease_micros: u64,
max: usize,
) -> Result<Vec<ClaimedTask>, AedbError> {
if queue.is_empty() || owner_id.is_empty() {
return Err(AedbError::Validation(
"queue and owner_id cannot be empty".into(),
));
}
if max == 0 {
return Ok(Vec::new());
}
self.promote_due(caller.clone(), project_id, scope_id, queue, max.max(8))
.await?;
self.reclaim_expired(caller.clone(), project_id, scope_id, queue, max.max(8))
.await?;
let mut claimed = Vec::with_capacity(max);
while claimed.len() < max {
match self
.try_claim_one(
caller.clone(),
project_id,
scope_id,
queue,
owner_id,
lease_micros,
)
.await?
{
Some(task) => claimed.push(task),
None => break,
}
}
Ok(claimed)
}
async fn try_claim_one(
&self,
caller: Option<CallerContext>,
project_id: &str,
scope_id: &str,
queue: &str,
owner_id: &str,
lease_micros: u64,
) -> Result<Option<ClaimedTask>, AedbError> {
for _ in 0..MAX_CAS_RETRIES {
let lease = self.acquire_snapshot(ConsistencyMode::AtLatest).await?;
let base_seq = lease.view.seq;
let view = &lease.view.keyspace;
let now = crate::system_now_micros();
let ready = view.try_kv_scan_prefix(project_id, scope_id, &ready_prefix(queue), 1)?;
let Some((ready_k, ready_entry)) = ready.into_iter().next() else {
return Ok(None);
};
let id = String::from_utf8_lossy(&ready_entry.value).into_owned();
let pk = primary_key(queue, &id);
let Some(primary) = view.try_kv_get(project_id, scope_id, &pk)? else {
let env = self.envelope(
caller.clone(),
vec![del_mut(project_id, scope_id, ready_k)],
Vec::new(),
base_seq,
);
let _ = self.commit_envelope(env).await;
continue;
};
let mut task = decode_task(&primary)?;
task.attempts = task.attempts.saturating_add(1);
task.state = STATE_INFLIGHT;
task.owner_id = Some(owner_id.to_string());
task.fencing_token = task.fencing_token.saturating_add(1);
task.lease_deadline_micros = now.saturating_add(lease_micros);
task.updated_micros = now;
let inflight_k = inflight_key(queue, task.lease_deadline_micros, task.seq, &task.id);
let mutations = vec![
set_mut(project_id, scope_id, pk.clone(), encode_task(&task)?),
del_mut(project_id, scope_id, ready_k),
set_mut(project_id, scope_id, inflight_k, id.as_bytes().to_vec()),
];
let assertions = vec![key_assertion(project_id, scope_id, &pk, Some(&primary))];
let env = self.envelope(caller.clone(), mutations, assertions, base_seq);
match self.commit_envelope(env).await {
Ok(_) => {
return Ok(Some(ClaimedTask {
task_id: task.id,
queue: task.queue,
payload_json: task.payload_json,
attempts: task.attempts,
fencing_token: task.fencing_token,
lease_deadline_micros: task.lease_deadline_micros,
priority: task.priority,
enqueued_micros: task.created_micros,
}));
}
Err(err) if is_retryable_conflict(&err) => continue,
Err(err) => return Err(err),
}
}
Ok(None)
}
async fn promote_due(
&self,
caller: Option<CallerContext>,
project_id: &str,
scope_id: &str,
queue: &str,
cap: usize,
) -> Result<(), AedbError> {
for _ in 0..cap {
let lease = self.acquire_snapshot(ConsistencyMode::AtLatest).await?;
let base_seq = lease.view.seq;
let view = &lease.view.keyspace;
let now = crate::system_now_micros();
let (lo, hi) = due_bounds(queue, KIND_SCHEDULED, now);
let due = view.try_kv_scan_range(project_id, scope_id, lo, hi, 1)?;
let Some((sched_k, sched_entry)) = due.into_iter().next() else {
return Ok(());
};
let id = String::from_utf8_lossy(&sched_entry.value).into_owned();
let pk = primary_key(queue, &id);
let Some(primary) = view.try_kv_get(project_id, scope_id, &pk)? else {
let env = self.envelope(
caller.clone(),
vec![del_mut(project_id, scope_id, sched_k)],
Vec::new(),
base_seq,
);
let _ = self.commit_envelope(env).await;
continue;
};
let mut task = decode_task(&primary)?;
if task.state != STATE_READY || task.in_ready {
let _ = self
.commit_envelope(self.envelope(
caller.clone(),
vec![del_mut(project_id, scope_id, sched_k)],
Vec::new(),
base_seq,
))
.await;
continue;
}
task.in_ready = true;
let mutations = vec![
del_mut(project_id, scope_id, sched_k),
set_mut(
project_id,
scope_id,
ready_key(queue, task.priority, task.seq, &task.id),
id.as_bytes().to_vec(),
),
set_mut(project_id, scope_id, pk.clone(), encode_task(&task)?),
];
let assertions = vec![key_assertion(project_id, scope_id, &pk, Some(&primary))];
let env = self.envelope(caller.clone(), mutations, assertions, base_seq);
let _ = self.commit_envelope(env).await;
}
Ok(())
}
async fn reclaim_expired(
&self,
caller: Option<CallerContext>,
project_id: &str,
scope_id: &str,
queue: &str,
cap: usize,
) -> Result<(), AedbError> {
for _ in 0..cap {
let lease = self.acquire_snapshot(ConsistencyMode::AtLatest).await?;
let base_seq = lease.view.seq;
let view = &lease.view.keyspace;
let now = crate::system_now_micros();
let (lo, hi) = due_bounds(queue, KIND_INFLIGHT, now);
let expired = view.try_kv_scan_range(project_id, scope_id, lo, hi, 1)?;
let Some((inflight_k, inflight_entry)) = expired.into_iter().next() else {
return Ok(());
};
let id = String::from_utf8_lossy(&inflight_entry.value).into_owned();
let pk = primary_key(queue, &id);
let Some(primary) = view.try_kv_get(project_id, scope_id, &pk)? else {
let env = self.envelope(
caller.clone(),
vec![del_mut(project_id, scope_id, inflight_k)],
Vec::new(),
base_seq,
);
let _ = self.commit_envelope(env).await;
continue;
};
let mut task = decode_task(&primary)?;
if task.state != STATE_INFLIGHT || task.lease_deadline_micros > now {
let _ = self
.commit_envelope(self.envelope(
caller.clone(),
vec![del_mut(project_id, scope_id, inflight_k.clone())],
vec![ReadAssertion::KeyExists {
project_id: project_id.to_string(),
scope_id: scope_id.to_string(),
key: inflight_k,
expected: true,
}],
base_seq,
))
.await;
continue;
}
let max_attempts = effective_max_attempts(&task);
let mut mutations = vec![del_mut(project_id, scope_id, inflight_k)];
task.owner_id = None;
task.updated_micros = now;
if task.attempts >= max_attempts {
task.state = STATE_DEAD;
task.last_error = Some("lease expired; attempts exhausted".into());
} else {
task.state = STATE_READY;
task.in_ready = true;
task.not_before_micros = now;
task.last_error = Some("lease expired; requeued".into());
mutations.push(set_mut(
project_id,
scope_id,
ready_key(queue, task.priority, task.seq, &task.id),
task.id.as_bytes().to_vec(),
));
}
mutations.push(set_mut(
project_id,
scope_id,
pk.clone(),
encode_task(&task)?,
));
let assertions = vec![key_assertion(project_id, scope_id, &pk, Some(&primary))];
let env = self.envelope(caller.clone(), mutations, assertions, base_seq);
let _ = self.commit_envelope(env).await;
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
pub async fn queue_complete(
&self,
project_id: &str,
scope_id: &str,
queue: &str,
task_id: &str,
fencing_token: u64,
result_json: String,
follow_ups: Vec<EnqueueSpec>,
extra_mutations: Vec<Mutation>,
) -> Result<FencedCommit, AedbError> {
self.queue_complete_inner(
None,
project_id,
scope_id,
queue,
task_id,
fencing_token,
result_json,
follow_ups,
extra_mutations,
)
.await
}
#[allow(clippy::too_many_arguments)]
pub async fn queue_complete_as(
&self,
caller: CallerContext,
project_id: &str,
scope_id: &str,
queue: &str,
task_id: &str,
fencing_token: u64,
result_json: String,
follow_ups: Vec<EnqueueSpec>,
extra_mutations: Vec<Mutation>,
) -> Result<FencedCommit, AedbError> {
self.queue_complete_inner(
Some(caller),
project_id,
scope_id,
queue,
task_id,
fencing_token,
result_json,
follow_ups,
extra_mutations,
)
.await
}
#[allow(clippy::too_many_arguments)]
async fn queue_complete_inner(
&self,
caller: Option<CallerContext>,
project_id: &str,
scope_id: &str,
queue: &str,
task_id: &str,
fencing_token: u64,
result_json: String,
follow_ups: Vec<EnqueueSpec>,
extra_mutations: Vec<Mutation>,
) -> Result<FencedCommit, AedbError> {
ensure_valid_json(&result_json, "result_json")?;
for spec in &follow_ups {
ensure_valid_json(&spec.options.payload_json, "follow-up payload_json")?;
}
let pk = primary_key(queue, task_id);
for _ in 0..MAX_CAS_RETRIES {
let lease = self.acquire_snapshot(ConsistencyMode::AtLatest).await?;
let base_seq = lease.view.seq;
let view = &lease.view.keyspace;
let Some(primary) = view.try_kv_get(project_id, scope_id, &pk)? else {
return Ok(FencedCommit::LeaseLost);
};
let task = decode_task(&primary)?;
if task.state != STATE_INFLIGHT || task.fencing_token != fencing_token {
return Ok(FencedCommit::LeaseLost);
}
let now = crate::system_now_micros();
let mut plan = EnqueuePlan::new();
plan.mutations
.push(del_mut(project_id, scope_id, pk.clone()));
plan.mutations.push(del_mut(
project_id,
scope_id,
inflight_key(queue, task.lease_deadline_micros, task.seq, &task.id),
));
plan.assertions
.push(key_assertion(project_id, scope_id, &pk, Some(&primary)));
plan.mutations.extend(extra_mutations.clone());
for spec in &follow_ups {
self.plan_enqueue(project_id, scope_id, view, spec, now, &mut plan)?;
}
self.finalize_counters(project_id, scope_id, view, &mut plan);
match self.commit_plan(caller.clone(), plan, base_seq).await {
Ok(commit) => return Ok(FencedCommit::Applied(commit)),
Err(err) if is_retryable_conflict(&err) => continue,
Err(err) => return Err(err),
}
}
Err(AedbError::Conflict(
"queue_complete exhausted retries".into(),
))
}
#[allow(clippy::too_many_arguments)]
pub async fn queue_fail(
&self,
project_id: &str,
scope_id: &str,
queue: &str,
task_id: &str,
fencing_token: u64,
error: String,
) -> Result<FencedCommit, AedbError> {
self.queue_fail_inner(
None,
project_id,
scope_id,
queue,
task_id,
fencing_token,
error,
)
.await
}
#[allow(clippy::too_many_arguments)]
pub async fn queue_fail_as(
&self,
caller: CallerContext,
project_id: &str,
scope_id: &str,
queue: &str,
task_id: &str,
fencing_token: u64,
error: String,
) -> Result<FencedCommit, AedbError> {
self.queue_fail_inner(
Some(caller),
project_id,
scope_id,
queue,
task_id,
fencing_token,
error,
)
.await
}
#[allow(clippy::too_many_arguments)]
async fn queue_fail_inner(
&self,
caller: Option<CallerContext>,
project_id: &str,
scope_id: &str,
queue: &str,
task_id: &str,
fencing_token: u64,
error: String,
) -> Result<FencedCommit, AedbError> {
let pk = primary_key(queue, task_id);
for _ in 0..MAX_CAS_RETRIES {
let lease = self.acquire_snapshot(ConsistencyMode::AtLatest).await?;
let base_seq = lease.view.seq;
let view = &lease.view.keyspace;
let Some(primary) = view.try_kv_get(project_id, scope_id, &pk)? else {
return Ok(FencedCommit::LeaseLost);
};
let mut task = decode_task(&primary)?;
if task.state != STATE_INFLIGHT || task.fencing_token != fencing_token {
return Ok(FencedCommit::LeaseLost);
}
let now = crate::system_now_micros();
let max_attempts = effective_max_attempts(&task);
let mut mutations = vec![del_mut(
project_id,
scope_id,
inflight_key(queue, task.lease_deadline_micros, task.seq, &task.id),
)];
task.owner_id = None;
task.last_error = Some(error.clone());
task.updated_micros = now;
if task.attempts >= max_attempts {
task.state = STATE_DEAD;
} else {
task.state = STATE_READY;
task.in_ready = false;
task.not_before_micros = next_backoff(&task, now);
mutations.push(set_mut(
project_id,
scope_id,
scheduled_key(queue, task.not_before_micros, task.seq, &task.id),
task.id.as_bytes().to_vec(),
));
}
mutations.push(set_mut(
project_id,
scope_id,
pk.clone(),
encode_task(&task)?,
));
let assertions = vec![key_assertion(project_id, scope_id, &pk, Some(&primary))];
let env = self.envelope(caller.clone(), mutations, assertions, base_seq);
match self.commit_envelope(env).await {
Ok(commit) => return Ok(FencedCommit::Applied(commit)),
Err(err) if is_retryable_conflict(&err) => continue,
Err(err) => return Err(err),
}
}
Err(AedbError::Conflict("queue_fail exhausted retries".into()))
}
#[allow(clippy::too_many_arguments)]
pub async fn queue_heartbeat(
&self,
project_id: &str,
scope_id: &str,
queue: &str,
task_id: &str,
fencing_token: u64,
extend_micros: u64,
progress_json: Option<String>,
) -> Result<FencedCommit, AedbError> {
self.queue_heartbeat_inner(
None,
project_id,
scope_id,
queue,
task_id,
fencing_token,
extend_micros,
progress_json,
)
.await
}
#[allow(clippy::too_many_arguments)]
pub async fn queue_heartbeat_as(
&self,
caller: CallerContext,
project_id: &str,
scope_id: &str,
queue: &str,
task_id: &str,
fencing_token: u64,
extend_micros: u64,
progress_json: Option<String>,
) -> Result<FencedCommit, AedbError> {
self.queue_heartbeat_inner(
Some(caller),
project_id,
scope_id,
queue,
task_id,
fencing_token,
extend_micros,
progress_json,
)
.await
}
#[allow(clippy::too_many_arguments)]
async fn queue_heartbeat_inner(
&self,
caller: Option<CallerContext>,
project_id: &str,
scope_id: &str,
queue: &str,
task_id: &str,
fencing_token: u64,
extend_micros: u64,
progress_json: Option<String>,
) -> Result<FencedCommit, AedbError> {
if let Some(p) = &progress_json {
ensure_valid_json(p, "progress_json")?;
}
let pk = primary_key(queue, task_id);
for _ in 0..MAX_CAS_RETRIES {
let lease = self.acquire_snapshot(ConsistencyMode::AtLatest).await?;
let base_seq = lease.view.seq;
let view = &lease.view.keyspace;
let Some(primary) = view.try_kv_get(project_id, scope_id, &pk)? else {
return Ok(FencedCommit::LeaseLost);
};
let mut task = decode_task(&primary)?;
if task.state != STATE_INFLIGHT || task.fencing_token != fencing_token {
return Ok(FencedCommit::LeaseLost);
}
let now = crate::system_now_micros();
let old_inflight = inflight_key(queue, task.lease_deadline_micros, task.seq, &task.id);
task.lease_deadline_micros = now.saturating_add(extend_micros);
if let Some(p) = &progress_json {
task.progress_json = p.clone();
}
task.updated_micros = now;
let new_inflight = inflight_key(queue, task.lease_deadline_micros, task.seq, &task.id);
let mut mutations = Vec::with_capacity(3);
if new_inflight != old_inflight {
mutations.push(del_mut(project_id, scope_id, old_inflight));
mutations.push(set_mut(
project_id,
scope_id,
new_inflight,
task.id.as_bytes().to_vec(),
));
}
mutations.push(set_mut(
project_id,
scope_id,
pk.clone(),
encode_task(&task)?,
));
let assertions = vec![key_assertion(project_id, scope_id, &pk, Some(&primary))];
let env = self.envelope(caller.clone(), mutations, assertions, base_seq);
match self.commit_envelope(env).await {
Ok(commit) => return Ok(FencedCommit::Applied(commit)),
Err(err) if is_retryable_conflict(&err) => continue,
Err(err) => return Err(err),
}
}
Err(AedbError::Conflict(
"queue_heartbeat exhausted retries".into(),
))
}
pub async fn queue_cancel(
&self,
project_id: &str,
scope_id: &str,
queue: &str,
task_id: &str,
) -> Result<bool, AedbError> {
self.queue_cancel_inner(None, project_id, scope_id, queue, task_id)
.await
}
pub async fn queue_cancel_as(
&self,
caller: CallerContext,
project_id: &str,
scope_id: &str,
queue: &str,
task_id: &str,
) -> Result<bool, AedbError> {
self.queue_cancel_inner(Some(caller), project_id, scope_id, queue, task_id)
.await
}
async fn queue_cancel_inner(
&self,
caller: Option<CallerContext>,
project_id: &str,
scope_id: &str,
queue: &str,
task_id: &str,
) -> Result<bool, AedbError> {
let pk = primary_key(queue, task_id);
for _ in 0..MAX_CAS_RETRIES {
let lease = self.acquire_snapshot(ConsistencyMode::AtLatest).await?;
let base_seq = lease.view.seq;
let view = &lease.view.keyspace;
let Some(primary) = view.try_kv_get(project_id, scope_id, &pk)? else {
return Ok(false);
};
let task = decode_task(&primary)?;
let index_k = match task.state {
STATE_READY => Some(ready_state_index_key(queue, &task)),
STATE_INFLIGHT => Some(inflight_key(
queue,
task.lease_deadline_micros,
task.seq,
&task.id,
)),
_ => None,
};
let mut mutations = vec![del_mut(project_id, scope_id, pk.clone())];
if let Some(k) = index_k {
mutations.push(del_mut(project_id, scope_id, k));
}
let assertions = vec![key_assertion(project_id, scope_id, &pk, Some(&primary))];
let env = self.envelope(caller.clone(), mutations, assertions, base_seq);
match self.commit_envelope(env).await {
Ok(_) => return Ok(true),
Err(err) if is_retryable_conflict(&err) => continue,
Err(err) => return Err(err),
}
}
Err(AedbError::Conflict("queue_cancel exhausted retries".into()))
}
pub async fn queue_get(
&self,
project_id: &str,
scope_id: &str,
queue: &str,
task_id: &str,
consistency: ConsistencyMode,
) -> Result<Option<TaskRecord>, AedbError> {
if self.require_authenticated_calls {
return Err(AedbError::PermissionDenied(
"authenticated caller required in secure mode".into(),
));
}
let lease = self.acquire_snapshot(consistency).await?;
let Some(entry) =
lease
.view
.keyspace
.try_kv_get(project_id, scope_id, &primary_key(queue, task_id))?
else {
return Ok(None);
};
Ok(Some(task_to_record(decode_task(&entry)?)))
}
pub async fn queue_list(
&self,
project_id: &str,
scope_id: &str,
queue: &str,
state: Option<TaskState>,
limit: usize,
consistency: ConsistencyMode,
) -> Result<Vec<TaskRecord>, AedbError> {
if self.require_authenticated_calls {
return Err(AedbError::PermissionDenied(
"authenticated caller required in secure mode".into(),
));
}
let lease = self.acquire_snapshot(consistency).await?;
let view = &lease.view.keyspace;
let prefix = primary_prefix(queue);
let scan_limit = if state.is_some() {
view.try_kv_scan_prefix(project_id, scope_id, &prefix, self._config.max_scan_rows)?
} else {
view.try_kv_scan_prefix(project_id, scope_id, &prefix, limit)?
};
let want = state.map(state_to_u8);
let mut out = Vec::new();
for (_, entry) in scan_limit {
let task = decode_task(&entry)?;
if want.is_none_or(|w| w == task.state) {
out.push(task_to_record(task));
if out.len() >= limit {
break;
}
}
}
Ok(out)
}
fn plan_enqueue(
&self,
project_id: &str,
scope_id: &str,
view: &crate::storage::keyspace::KeyspaceSnapshot,
spec: &EnqueueSpec,
now: u64,
plan: &mut EnqueuePlan,
) -> Result<EnqueueOutcome, AedbError> {
let queue = &spec.queue;
if !plan.counters.contains_key(queue) {
let entry = view.try_kv_get(project_id, scope_id, &counter_key(queue))?;
let next_seq = entry
.as_ref()
.and_then(|e| e.value.get(..8))
.and_then(|b| <[u8; 8]>::try_from(b).ok())
.map(u64::from_be_bytes)
.unwrap_or(0);
plan.counters.insert(
queue.clone(),
CounterCursor {
entry_version: entry.as_ref().map(|e| e.version),
next_seq,
},
);
}
let cursor = plan.counters.get_mut(queue).expect("counter present");
let seq = cursor.next_seq;
cursor.next_seq = cursor.next_seq.saturating_add(1);
let id = format!("{seq:020}");
let not_before = now.saturating_add(spec.options.delay_micros);
let in_ready = not_before <= now; let task = StoredTask {
id: id.clone(),
queue: queue.clone(),
state: STATE_READY,
not_before_micros: not_before,
priority: spec.options.priority,
seq,
in_ready,
attempts: 0,
max_attempts: spec.options.max_attempts,
backoff_base_micros: spec.options.backoff_base_micros,
backoff_max_micros: spec.options.backoff_max_micros,
owner_id: None,
fencing_token: 0,
lease_deadline_micros: 0,
payload_json: spec.options.payload_json.clone(),
progress_json: String::new(),
result_json: String::new(),
last_error: None,
created_micros: now,
updated_micros: now,
};
plan.mutations.push(set_mut(
project_id,
scope_id,
primary_key(queue, &id),
encode_task(&task)?,
));
plan.mutations.push(set_mut(
project_id,
scope_id,
ready_state_index_key(queue, &task),
id.as_bytes().to_vec(),
));
if let Some(idem) = &spec.options.idempotency_key {
let ik = idem_key(queue, idem);
plan.assertions.push(ReadAssertion::KeyExists {
project_id: project_id.to_string(),
scope_id: scope_id.to_string(),
key: ik.clone(),
expected: false,
});
plan.mutations
.push(set_mut(project_id, scope_id, ik, id.as_bytes().to_vec()));
}
Ok(EnqueueOutcome {
task_id: id,
seq,
deduplicated: false,
})
}
fn finalize_counters(
&self,
project_id: &str,
scope_id: &str,
_view: &crate::storage::keyspace::KeyspaceSnapshot,
plan: &mut EnqueuePlan,
) {
for (queue, cursor) in &plan.counters {
let ck = counter_key(queue);
plan.assertions.push(match cursor.entry_version {
Some(version) => ReadAssertion::KeyVersion {
project_id: project_id.to_string(),
scope_id: scope_id.to_string(),
key: ck.clone(),
expected_seq: version,
},
None => ReadAssertion::KeyExists {
project_id: project_id.to_string(),
scope_id: scope_id.to_string(),
key: ck.clone(),
expected: false,
},
});
plan.mutations.push(set_mut(
project_id,
scope_id,
ck,
cursor.next_seq.to_be_bytes().to_vec(),
));
}
}
fn envelope(
&self,
caller: Option<CallerContext>,
mutations: Vec<Mutation>,
assertions: Vec<ReadAssertion>,
base_seq: u64,
) -> TransactionEnvelope {
TransactionEnvelope {
caller,
idempotency_key: None,
write_class: WriteClass::Standard,
assertions,
read_set: ReadSet::default(),
write_intent: WriteIntent { mutations },
base_seq,
}
}
async fn commit_plan(
&self,
caller: Option<CallerContext>,
plan: EnqueuePlan,
base_seq: u64,
) -> Result<crate::CommitResult, AedbError> {
let env = self.envelope(caller, plan.mutations, plan.assertions, base_seq);
self.commit_envelope(env).await
}
}
fn effective_max_attempts(task: &StoredTask) -> u32 {
if task.max_attempts == 0 {
DEFAULT_MAX_ATTEMPTS
} else {
task.max_attempts
}
}
fn state_to_u8(state: TaskState) -> u8 {
match state {
TaskState::Ready => STATE_READY,
TaskState::InFlight => STATE_INFLIGHT,
TaskState::Dead => STATE_DEAD,
}
}
fn u8_to_state(state: u8) -> TaskState {
match state {
STATE_INFLIGHT => TaskState::InFlight,
STATE_DEAD => TaskState::Dead,
_ => TaskState::Ready,
}
}
fn task_to_record(task: StoredTask) -> TaskRecord {
TaskRecord {
task_id: task.id,
queue: task.queue,
state: u8_to_state(task.state),
not_before_micros: task.not_before_micros,
priority: task.priority,
attempts: task.attempts,
max_attempts: effective_max_attempts_val(task.max_attempts),
owner_id: task.owner_id,
fencing_token: task.fencing_token,
lease_deadline_micros: task.lease_deadline_micros,
payload_json: task.payload_json,
progress_json: task.progress_json,
result_json: task.result_json,
last_error: task.last_error,
created_micros: task.created_micros,
updated_micros: task.updated_micros,
}
}
fn effective_max_attempts_val(max_attempts: u32) -> u32 {
if max_attempts == 0 {
DEFAULT_MAX_ATTEMPTS
} else {
max_attempts
}
}
#[cfg(test)]
mod tests;