use async_trait::async_trait;
use aws_sdk_dynamodb::Client;
use aws_sdk_dynamodb::types::AttributeValue;
use std::collections::HashMap;
use std::sync::LazyLock;
use turul_a2a_types::state_machine::TERMINAL_STATES;
use turul_a2a_types::{Artifact, Message, Task, TaskState, TaskStatus};
use super::atomic::A2aAtomicStore;
use super::error::A2aStorageError;
use super::event_store::A2aEventStore;
use super::filter::{PushConfigListPage, TaskFilter, TaskListPage};
use super::traits::{A2aPushNotificationStorage, A2aTaskStorage};
use crate::push::{
A2aPushDeliveryStore, AbandonedReason, ClaimStatus, DeliveryClaim, DeliveryErrorClass,
DeliveryOutcome, FailedDelivery, GaveUpReason,
};
use crate::streaming::StreamEvent;
#[derive(Debug, Clone)]
pub struct DynamoDbConfig {
pub tasks_table: String,
pub push_configs_table: String,
pub events_table: String,
pub push_deliveries_table: String,
pub push_pending_dispatches_table: String,
pub task_ttl_seconds: u64,
pub event_ttl_seconds: u64,
pub push_delivery_ttl_seconds: u64,
}
static TERMINAL_STATE_CONDITION: LazyLock<String> = LazyLock::new(|| {
let placeholders = (0..TERMINAL_STATES.len())
.map(|i| format!(":terminal{i}"))
.collect::<Vec<_>>()
.join(", ");
format!("NOT statusState IN ({placeholders})")
});
fn terminal_condition_values<const N: usize>(
extra: [(&str, AttributeValue); N],
) -> HashMap<String, AttributeValue> {
TERMINAL_STATES
.iter()
.enumerate()
.map(|(i, s)| {
(
format!(":terminal{i}"),
AttributeValue::S(s.wire_name().to_string()),
)
})
.chain(extra.into_iter().map(|(k, v)| (k.to_string(), v)))
.collect()
}
const DEFAULT_TASK_TTL: u64 = 24 * 3600;
const DEFAULT_EVENT_TTL: u64 = 24 * 3600;
const DEFAULT_PUSH_DELIVERY_TTL: u64 = 7 * 24 * 3600;
impl Default for DynamoDbConfig {
fn default() -> Self {
Self {
tasks_table: "a2a_tasks".into(),
push_configs_table: "a2a_push_configs".into(),
events_table: "a2a_task_events".into(),
push_deliveries_table: "a2a_push_deliveries".into(),
push_pending_dispatches_table: "a2a_push_pending_dispatches".into(),
task_ttl_seconds: DEFAULT_TASK_TTL,
event_ttl_seconds: DEFAULT_EVENT_TTL,
push_delivery_ttl_seconds: DEFAULT_PUSH_DELIVERY_TTL,
}
}
}
struct EventSequenceAllocation {
observed: u64,
sequences: Vec<u64>,
counter_write: Option<aws_sdk_dynamodb::types::TransactWriteItem>,
}
impl EventSequenceAllocation {
fn sequences_after(observed: u64, count: usize) -> Vec<u64> {
(1..=count as u64).map(|i| observed + i).collect()
}
fn latest(&self) -> u64 {
self.sequences.last().copied().unwrap_or(self.observed)
}
fn attach(
&mut self,
items: &mut Vec<aws_sdk_dynamodb::types::TransactWriteItem>,
event_items: Vec<aws_sdk_dynamodb::types::TransactWriteItem>,
) -> Option<SequenceItems> {
let events_start = items.len();
items.extend(event_items);
let events = events_start..items.len();
self.counter_write.take().map(|write| {
items.push(write);
SequenceItems {
events,
counter: items.len() - 1,
}
})
}
}
struct SequenceItems {
events: std::ops::Range<usize>,
counter: usize,
}
fn lost_sequence_race(
reasons: &[aws_sdk_dynamodb::types::CancellationReason],
sequence_items: Option<&SequenceItems>,
) -> bool {
let Some(SequenceItems { events, counter }) = sequence_items else {
return false;
};
let code = |i: usize| reasons.get(i).and_then(|r| r.code());
let final_condition_failed = (0..reasons.len()).any(|i| {
i != *counter && !events.contains(&i) && code(i) == Some("ConditionalCheckFailed")
});
let raced = code(*counter) == Some("ConditionalCheckFailed")
|| reasons
.iter()
.any(|r| r.code() == Some("TransactionConflict"));
raced && !final_condition_failed
}
fn cancellation_reasons(
err: &aws_sdk_dynamodb::error::SdkError<
aws_sdk_dynamodb::operation::transact_write_items::TransactWriteItemsError,
>,
) -> &[aws_sdk_dynamodb::types::CancellationReason] {
match err.as_service_error() {
Some(
aws_sdk_dynamodb::operation::transact_write_items::TransactWriteItemsError::TransactionCanceledException(e),
) => e.cancellation_reasons(),
_ => &[],
}
}
#[derive(Default)]
struct SequenceCommitRetry {
attempt: u32,
}
impl SequenceCommitRetry {
const MAX_ATTEMPTS: u32 = 32;
async fn backoff(&mut self, pk: &str) -> Result<(), A2aStorageError> {
use rand::RngExt;
self.attempt += 1;
if self.attempt >= Self::MAX_ATTEMPTS {
return Err(A2aStorageError::ConcurrentModification(format!(
"event sequence allocation for {pk} lost {} consecutive races",
self.attempt
)));
}
let base_ms = 5u64 << self.attempt.min(5);
let jitter_ms = rand::rng().random_range(0..=base_ms);
tokio::time::sleep(std::time::Duration::from_millis(base_ms + jitter_ms)).await;
Ok(())
}
}
#[derive(Clone)]
pub struct DynamoDbA2aStorage {
client: Client,
config: DynamoDbConfig,
push_dispatch_enabled: bool,
}
impl DynamoDbA2aStorage {
pub async fn new(config: DynamoDbConfig) -> Result<Self, A2aStorageError> {
let aws_config = aws_config::load_defaults(aws_config::BehaviorVersion::latest()).await;
let client = Client::new(&aws_config);
Ok(Self {
client,
config,
push_dispatch_enabled: false,
})
}
pub fn from_client(client: Client, config: DynamoDbConfig) -> Self {
Self {
client,
config,
push_dispatch_enabled: false,
}
}
pub fn with_push_dispatch_enabled(mut self, enabled: bool) -> Self {
self.push_dispatch_enabled = enabled;
self
}
fn task_key(tenant: &str, task_id: &str) -> String {
format!("{tenant}#{task_id}")
}
async fn task_has_push_config(
&self,
tenant: &str,
task_id: &str,
) -> Result<bool, A2aStorageError> {
let mut start_key = None;
loop {
let page = self
.client
.scan()
.table_name(&self.config.push_configs_table)
.filter_expression("tenant = :t AND taskId = :tid")
.expression_attribute_values(":t", AttributeValue::S(tenant.to_string()))
.expression_attribute_values(":tid", AttributeValue::S(task_id.to_string()))
.projection_expression("pk")
.set_exclusive_start_key(start_key)
.send()
.await
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?;
if page.count() > 0 {
return Ok(true);
}
match page.last_evaluated_key {
Some(k) => start_key = Some(k),
None => return Ok(false),
}
}
}
fn enc_key_component(s: &str) -> String {
s.replace('%', "%25")
.replace('|', "%7C")
.replace('#', "%23")
}
fn artifact_pk(tenant: &str, task_id: &str) -> String {
format!(
"art|{}|{}",
Self::enc_key_component(tenant),
Self::enc_key_component(task_id)
)
}
fn lease_pk(tenant: &str, task_id: &str) -> String {
format!(
"lease|{}|{}",
Self::enc_key_component(tenant),
Self::enc_key_component(task_id)
)
}
async fn stamp_child_ttl(&self, tenant: &str, task_id: &str, ttl: Option<&AttributeValue>) {
let Some(ttl) = ttl else { return };
if let Err(e) = self.try_stamp_child_ttl(tenant, task_id, ttl).await {
tracing::warn!(
target: "turul_a2a::retention_child_ttl_error",
tenant,
task_id,
error = %e,
"failed to stamp ttl on a terminal task's child items"
);
}
}
async fn try_stamp_child_ttl(
&self,
tenant: &str,
task_id: &str,
ttl: &AttributeValue,
) -> Result<(), A2aStorageError> {
let db_err = |e: &dyn std::fmt::Debug| A2aStorageError::DatabaseError(format!("{e:?}"));
let mut targets: Vec<(&str, HashMap<String, AttributeValue>)> = Vec::new();
let art_pk = Self::artifact_pk(tenant, task_id);
let mut start_key = None;
loop {
let out = self
.client
.query()
.table_name(&self.config.events_table)
.key_condition_expression("pk = :pk")
.expression_attribute_values(":pk", AttributeValue::S(art_pk.clone()))
.projection_expression("pk, sk")
.set_exclusive_start_key(start_key)
.send()
.await
.map_err(|e| db_err(&e))?;
targets.extend(
out.items
.unwrap_or_default()
.into_iter()
.map(|key| (self.config.events_table.as_str(), key)),
);
match out.last_evaluated_key {
Some(k) if !k.is_empty() => start_key = Some(k),
_ => break,
}
}
let mut start_key = None;
loop {
let out = self
.client
.scan()
.table_name(&self.config.push_configs_table)
.filter_expression("tenant = :t AND taskId = :tid")
.expression_attribute_values(":t", AttributeValue::S(tenant.to_string()))
.expression_attribute_values(":tid", AttributeValue::S(task_id.to_string()))
.projection_expression("pk")
.set_exclusive_start_key(start_key)
.send()
.await
.map_err(|e| db_err(&e))?;
targets.extend(
out.items
.unwrap_or_default()
.into_iter()
.map(|key| (self.config.push_configs_table.as_str(), key)),
);
match out.last_evaluated_key {
Some(k) => start_key = Some(k),
None => break,
}
}
let mut first_error = None;
for chunk in targets.chunks(32) {
let results = futures::future::join_all(chunk.iter().map(|(table, key)| {
self.client
.update_item()
.table_name(*table)
.set_key(Some(key.clone()))
.condition_expression("attribute_exists(pk)")
.update_expression("SET #ttl = :ttl")
.expression_attribute_names("#ttl", "ttl")
.expression_attribute_values(":ttl", ttl.clone())
.send()
}))
.await;
for result in results {
if let Err(e) = result
&& !e
.as_service_error()
.is_some_and(|s| s.is_conditional_check_failed_exception())
&& first_error.is_none()
{
first_error = Some(db_err(&e));
}
}
}
first_error.map_or(Ok(()), Err)
}
fn split_task(
task: &Task,
) -> Result<(String, Vec<turul_a2a_proto::Artifact>), A2aStorageError> {
let mut proto = task.as_proto().clone();
let artifacts = std::mem::take(&mut proto.artifacts);
let stripped = Task::try_from(proto).map_err(A2aStorageError::TypeError)?;
Ok((Self::task_to_json(&stripped)?, artifacts))
}
fn build_artifact_item(
art_pk: &str,
artifact_id: &str,
body_json: &str,
fingerprint: &str,
ttl: Option<AttributeValue>,
) -> std::collections::HashMap<String, AttributeValue> {
let mut item = std::collections::HashMap::new();
item.insert("pk".into(), AttributeValue::S(art_pk.to_string()));
item.insert("sk".into(), AttributeValue::S(artifact_id.to_string()));
item.insert(
"artifactId".into(),
AttributeValue::S(artifact_id.to_string()),
);
item.insert(
"artifactJson".into(),
AttributeValue::S(body_json.to_string()),
);
item.insert(
"fingerprint".into(),
AttributeValue::S(fingerprint.to_string()),
);
if let Some(ttl_val) = ttl {
item.insert("ttl".into(), ttl_val);
}
item
}
fn artifact_transact_items(
&self,
art_pk: &str,
plan: &super::artifacts::ReconcilePlan,
ttl: Option<&AttributeValue>,
) -> Result<Vec<aws_sdk_dynamodb::types::TransactWriteItem>, A2aStorageError> {
let mut items = Vec::with_capacity(plan.writes.len() + plan.deletes.len());
for (artifact, fingerprint) in &plan.writes {
let body = super::artifacts::artifact_to_json(artifact)?;
let item = Self::build_artifact_item(
art_pk,
&artifact.artifact_id,
&body,
fingerprint,
ttl.cloned(),
);
let put = aws_sdk_dynamodb::types::Put::builder()
.table_name(&self.config.events_table)
.set_item(Some(item))
.build()
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?;
items.push(
aws_sdk_dynamodb::types::TransactWriteItem::builder()
.put(put)
.build(),
);
}
for artifact_id in &plan.deletes {
let delete = aws_sdk_dynamodb::types::Delete::builder()
.table_name(&self.config.events_table)
.key("pk", AttributeValue::S(art_pk.to_string()))
.key("sk", AttributeValue::S(artifact_id.to_string()))
.build()
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?;
items.push(
aws_sdk_dynamodb::types::TransactWriteItem::builder()
.delete(delete)
.build(),
);
}
Ok(items)
}
fn check_artifact_transaction_cap(
artifact_item_count: usize,
other_items: usize,
) -> Result<(), A2aStorageError> {
const MAX: usize = 100;
if artifact_item_count + other_items > MAX {
return Err(A2aStorageError::DatabaseError(format!(
"artifact write exceeds DynamoDB transaction cap: {artifact_item_count} artifact items + {other_items} other items > {MAX}"
)));
}
Ok(())
}
async fn query_artifacts(
&self,
tenant: &str,
task_id: &str,
) -> Result<std::collections::HashMap<String, turul_a2a_proto::Artifact>, A2aStorageError> {
let art_pk = Self::artifact_pk(tenant, task_id);
let mut bodies = std::collections::HashMap::new();
let mut last_key: Option<std::collections::HashMap<String, AttributeValue>> = None;
loop {
let mut query = self
.client
.query()
.table_name(&self.config.events_table)
.key_condition_expression("pk = :pk")
.expression_attribute_values(":pk", AttributeValue::S(art_pk.clone()));
if let Some(start) = last_key.take() {
query = query.set_exclusive_start_key(Some(start));
}
let out = query
.send()
.await
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?;
for item in out.items.unwrap_or_default() {
if let (Some(id), Some(body)) = (
item.get("artifactId").and_then(|v| v.as_s().ok()),
item.get("artifactJson").and_then(|v| v.as_s().ok()),
) {
bodies.insert(id.clone(), super::artifacts::artifact_from_json(body)?);
}
}
match out.last_evaluated_key {
Some(k) if !k.is_empty() => last_key = Some(k),
_ => break,
}
}
Ok(bodies)
}
async fn delete_artifact_partition(
&self,
tenant: &str,
task_id: &str,
) -> Result<(), A2aStorageError> {
let art_pk = Self::artifact_pk(tenant, task_id);
let mut sks: Vec<String> = Vec::new();
let mut last_key: Option<std::collections::HashMap<String, AttributeValue>> = None;
loop {
let mut query = self
.client
.query()
.table_name(&self.config.events_table)
.key_condition_expression("pk = :pk")
.projection_expression("sk")
.expression_attribute_values(":pk", AttributeValue::S(art_pk.clone()));
if let Some(start) = last_key.take() {
query = query.set_exclusive_start_key(Some(start));
}
let out = query
.send()
.await
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?;
for item in out.items.unwrap_or_default() {
if let Some(sk) = item.get("sk").and_then(|v| v.as_s().ok()) {
sks.push(sk.clone());
}
}
match out.last_evaluated_key {
Some(k) if !k.is_empty() => last_key = Some(k),
_ => break,
}
}
for chunk in sks.chunks(25) {
let mut requests: Vec<aws_sdk_dynamodb::types::WriteRequest> = chunk
.iter()
.map(|sk| {
let key = std::collections::HashMap::from([
("pk".to_string(), AttributeValue::S(art_pk.clone())),
("sk".to_string(), AttributeValue::S(sk.clone())),
]);
aws_sdk_dynamodb::types::WriteRequest::builder()
.delete_request(
aws_sdk_dynamodb::types::DeleteRequest::builder()
.set_key(Some(key))
.build()
.expect("delete request key set"),
)
.build()
})
.collect();
while !requests.is_empty() {
let out = self
.client
.batch_write_item()
.request_items(self.config.events_table.clone(), requests.clone())
.send()
.await
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?;
requests = out
.unprocessed_items
.and_then(|mut m| m.remove(&self.config.events_table))
.unwrap_or_default();
}
}
Ok(())
}
fn stored_manifest(
item: &std::collections::HashMap<String, AttributeValue>,
) -> Result<&str, A2aStorageError> {
item.get("artifactManifest")
.and_then(|v| v.as_s().ok())
.map(String::as_str)
.ok_or_else(|| A2aStorageError::DatabaseError("Missing artifact_manifest".into()))
}
async fn fetch_stored_manifest(
&self,
tenant: &str,
task_id: &str,
) -> Result<(Vec<super::artifacts::ManifestEntry>, bool), A2aStorageError> {
let pk = Self::task_key(tenant, task_id);
let out = self
.client
.get_item()
.consistent_read(true)
.table_name(&self.config.tasks_table)
.key("pk", AttributeValue::S(pk))
.projection_expression("pk, artifactManifest, statusState")
.send()
.await
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?;
match out.item {
Some(item) => {
let was_terminal = item
.get("statusState")
.and_then(|v| v.as_s().ok())
.is_some_and(|s| TERMINAL_STATES.iter().any(|t| t.wire_name() == s));
let manifest = super::artifacts::manifest_from_json(Self::stored_manifest(&item)?)?;
Ok((manifest, was_terminal))
}
None => Ok((Vec::new(), false)),
}
}
async fn rehydrate_task(
&self,
tenant: &str,
task_id: &str,
blob: &str,
manifest_json: &str,
) -> Result<Task, A2aStorageError> {
let manifest = super::artifacts::manifest_from_json(manifest_json)?;
let mut proto: turul_a2a_proto::Task = serde_json::from_str(blob)
.map_err(|e| A2aStorageError::SerializationError(e.to_string()))?;
if !manifest.is_empty() {
let bodies = self.query_artifacts(tenant, task_id).await?;
proto.artifacts = super::artifacts::rehydrate_in_manifest_order(&manifest, bodies);
}
Task::try_from(proto).map_err(A2aStorageError::TypeError)
}
pub(crate) fn event_sort_key(seq: u64) -> String {
format!("{seq:020}")
}
pub(crate) fn build_event_item(
pk: &str,
seq: u64,
event_data: &str,
ttl: Option<AttributeValue>,
) -> std::collections::HashMap<String, AttributeValue> {
let mut item = std::collections::HashMap::new();
item.insert("pk".into(), AttributeValue::S(pk.to_string()));
item.insert("sk".into(), AttributeValue::S(Self::event_sort_key(seq)));
item.insert("eventSequence".into(), AttributeValue::N(seq.to_string()));
item.insert(
"eventData".into(),
AttributeValue::S(event_data.to_string()),
);
if let Some(ttl_val) = ttl {
item.insert("ttl".into(), ttl_val);
}
item
}
fn sequence_counter_sk() -> String {
Self::event_sort_key(0)
}
pub(crate) fn build_sequence_counter_update(
events_table: &str,
pk: &str,
observed: u64,
count: u64,
ttl: Option<AttributeValue>,
) -> Result<aws_sdk_dynamodb::types::TransactWriteItem, A2aStorageError> {
let mut update = aws_sdk_dynamodb::types::Update::builder()
.table_name(events_table)
.key("pk", AttributeValue::S(pk.to_string()))
.key("sk", AttributeValue::S(Self::sequence_counter_sk()))
.expression_attribute_values(":count", AttributeValue::N(count.to_string()));
update = if observed == 0 {
update.condition_expression("attribute_not_exists(nextSequence)")
} else {
update
.condition_expression("nextSequence = :observed")
.expression_attribute_values(":observed", AttributeValue::N(observed.to_string()))
};
update = match ttl {
Some(ttl) => update
.update_expression("ADD nextSequence :count SET #ttl = :ttl")
.expression_attribute_names("#ttl", "ttl")
.expression_attribute_values(":ttl", ttl),
None => update.update_expression("ADD nextSequence :count"),
};
Ok(aws_sdk_dynamodb::types::TransactWriteItem::builder()
.update(
update
.build()
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?,
)
.build())
}
async fn read_sequence_counter(&self, pk: &str) -> Result<u64, A2aStorageError> {
let result = self
.client
.get_item()
.consistent_read(true)
.table_name(&self.config.events_table)
.key("pk", AttributeValue::S(pk.to_string()))
.key("sk", AttributeValue::S(Self::sequence_counter_sk()))
.projection_expression("nextSequence")
.send()
.await
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?;
match result.item.as_ref().and_then(|i| i.get("nextSequence")) {
Some(AttributeValue::N(n)) => n
.parse::<u64>()
.map_err(|e| A2aStorageError::DatabaseError(e.to_string())),
_ => Ok(0),
}
}
fn sequence_counter_ttl_seconds(event_ttl_seconds: u64, task_ttl_seconds: u64) -> Option<u64> {
if event_ttl_seconds == 0 || task_ttl_seconds == 0 {
None
} else {
Some(event_ttl_seconds.max(task_ttl_seconds))
}
}
async fn allocate_event_sequences(
&self,
pk: &str,
count: usize,
) -> Result<EventSequenceAllocation, A2aStorageError> {
let observed = self.read_sequence_counter(pk).await?;
let counter_write = if count == 0 {
None
} else {
let ttl = Self::sequence_counter_ttl_seconds(
self.config.event_ttl_seconds,
self.config.task_ttl_seconds,
)
.and_then(Self::ttl_epoch);
Some(Self::build_sequence_counter_update(
&self.config.events_table,
pk,
observed,
count as u64,
ttl,
)?)
};
Ok(EventSequenceAllocation {
sequences: EventSequenceAllocation::sequences_after(observed, count),
observed,
counter_write,
})
}
fn event_transact_items(
&self,
pk: &str,
sequences: &[u64],
events: &[StreamEvent],
) -> Result<Vec<aws_sdk_dynamodb::types::TransactWriteItem>, A2aStorageError> {
let ttl = Self::ttl_epoch(self.config.event_ttl_seconds);
sequences
.iter()
.zip(events)
.map(|(&seq, event)| {
let event_data = serde_json::to_string(event)
.map_err(|e| A2aStorageError::SerializationError(e.to_string()))?;
let put = aws_sdk_dynamodb::types::Put::builder()
.table_name(&self.config.events_table)
.set_item(Some(Self::build_event_item(
pk,
seq,
&event_data,
ttl.clone(),
)))
.condition_expression("attribute_not_exists(pk) AND attribute_not_exists(sk)")
.build()
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?;
Ok(aws_sdk_dynamodb::types::TransactWriteItem::builder()
.put(put)
.build())
})
.collect()
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn build_task_item(
pk: &str,
tenant: &str,
task_id: &str,
owner: &str,
context_id: &str,
state_str: &str,
task_json: &str,
artifact_manifest: &str,
latest_event_sequence: Option<u64>,
ttl: Option<AttributeValue>,
) -> std::collections::HashMap<String, AttributeValue> {
let mut item = std::collections::HashMap::new();
item.insert("pk".into(), AttributeValue::S(pk.to_string()));
item.insert("tenant".into(), AttributeValue::S(tenant.to_string()));
item.insert("taskId".into(), AttributeValue::S(task_id.to_string()));
item.insert("owner".into(), AttributeValue::S(owner.to_string()));
item.insert(
"contextId".into(),
AttributeValue::S(context_id.to_string()),
);
item.insert(
"statusState".into(),
AttributeValue::S(state_str.to_string()),
);
item.insert("taskJson".into(), AttributeValue::S(task_json.to_string()));
item.insert(
"artifactManifest".into(),
AttributeValue::S(artifact_manifest.to_string()),
);
item.insert("updatedAt".into(), AttributeValue::S(Self::now_iso()));
if let Some(seq) = latest_event_sequence {
item.insert(
"latestEventSequence".into(),
AttributeValue::N(seq.to_string()),
);
}
if let Some(ttl_val) = ttl {
item.insert("ttl".into(), ttl_val);
}
item
}
fn now_iso() -> String {
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default();
format!("{:010}.{:03}", now.as_secs(), now.subsec_millis())
}
fn now_epoch() -> u64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
}
fn ttl_epoch(ttl_seconds: u64) -> Option<AttributeValue> {
if ttl_seconds == 0 {
return None;
}
Some(AttributeValue::N(
(Self::now_epoch() + ttl_seconds).to_string(),
))
}
fn task_ttl(&self, state_str: &str) -> Option<AttributeValue> {
turul_a2a_types::state_machine::TERMINAL_STATES
.iter()
.any(|s| s.wire_name() == state_str)
.then(|| Self::ttl_epoch(self.config.task_ttl_seconds))
.flatten()
}
fn task_to_json(task: &Task) -> Result<String, A2aStorageError> {
serde_json::to_string(task).map_err(|e| A2aStorageError::SerializationError(e.to_string()))
}
fn task_from_json(json: &str) -> Result<Task, A2aStorageError> {
let proto: turul_a2a_proto::Task = serde_json::from_str(json)
.map_err(|e| A2aStorageError::SerializationError(e.to_string()))?;
Task::try_from(proto).map_err(A2aStorageError::TypeError)
}
fn status_state_str(task: &Task) -> &'static str {
task.status()
.and_then(|s| s.state().ok())
.map(TaskState::wire_name)
.unwrap_or_default()
}
}
#[async_trait]
impl A2aTaskStorage for DynamoDbA2aStorage {
fn backend_name(&self) -> &'static str {
"dynamodb"
}
async fn create_task(
&self,
tenant: &str,
owner: &str,
mut task: Task,
) -> Result<Task, A2aStorageError> {
super::status_time::stamp_status(&mut task, &mut []);
let pk = Self::task_key(tenant, task.id());
let (blob, artifacts) = Self::split_task(&task)?;
let state_str = Self::status_state_str(&task);
let plan = super::artifacts::reconcile(&artifacts, &[])?;
let manifest_json = super::artifacts::manifest_to_json(&plan.manifest)?;
let ttl = self.task_ttl(state_str);
let item = Self::build_task_item(
&pk,
tenant,
task.id(),
owner,
task.context_id(),
state_str,
&blob,
&manifest_json,
None,
ttl.clone(),
);
if plan.writes.is_empty() {
self.client
.put_item()
.table_name(&self.config.tasks_table)
.set_item(Some(item))
.send()
.await
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?;
} else {
let art_pk = Self::artifact_pk(tenant, task.id());
let artifact_items = self.artifact_transact_items(&art_pk, &plan, ttl.as_ref())?;
Self::check_artifact_transaction_cap(artifact_items.len(), 1)?;
let task_put = aws_sdk_dynamodb::types::Put::builder()
.table_name(&self.config.tasks_table)
.set_item(Some(item))
.build()
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?;
let mut transact = vec![
aws_sdk_dynamodb::types::TransactWriteItem::builder()
.put(task_put)
.build(),
];
transact.extend(artifact_items);
self.client
.transact_write_items()
.set_transact_items(Some(transact))
.send()
.await
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?;
}
Ok(task)
}
async fn get_task(
&self,
tenant: &str,
task_id: &str,
owner: &str,
history_length: Option<i32>,
) -> Result<Option<Task>, A2aStorageError> {
let pk = Self::task_key(tenant, task_id);
let result = self
.client
.get_item()
.consistent_read(true)
.table_name(&self.config.tasks_table)
.key("pk", AttributeValue::S(pk))
.send()
.await
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?;
let item = match result.item {
Some(item) => item,
None => return Ok(None),
};
let stored_owner = item
.get("owner")
.and_then(|v| v.as_s().ok())
.unwrap_or(&String::new())
.clone();
if stored_owner != owner {
return Ok(None);
}
let json = item
.get("taskJson")
.and_then(|v| v.as_s().ok())
.ok_or_else(|| A2aStorageError::DatabaseError("Missing task_json".into()))?;
let task = self
.rehydrate_task(tenant, task_id, json, Self::stored_manifest(&item)?)
.await?;
Ok(Some(super::trim_history(task, history_length)))
}
async fn update_task(
&self,
tenant: &str,
owner: &str,
mut task: Task,
) -> Result<(), A2aStorageError> {
super::status_time::stamp_status(&mut task, &mut []);
let pk = Self::task_key(tenant, task.id());
let (blob, artifacts) = Self::split_task(&task)?;
let state_str = Self::status_state_str(&task);
let (stored, was_terminal) = self.fetch_stored_manifest(tenant, task.id()).await?;
let plan = super::artifacts::reconcile(&artifacts, &stored)?;
let manifest_json = super::artifacts::manifest_to_json(&plan.manifest)?;
let ttl = self.task_ttl(state_str);
let item = Self::build_task_item(
&pk,
tenant,
task.id(),
owner,
task.context_id(),
state_str,
&blob,
&manifest_json,
None,
ttl.clone(),
);
let artifact_items = if plan.writes.is_empty() && plan.deletes.is_empty() {
Vec::new()
} else {
let art_pk = Self::artifact_pk(tenant, task.id());
self.artifact_transact_items(&art_pk, &plan, ttl.as_ref())?
};
let result: Result<(), String> = if artifact_items.is_empty() {
self.client
.put_item()
.table_name(&self.config.tasks_table)
.set_item(Some(item))
.condition_expression("attribute_exists(pk) AND #o = :expected_owner")
.expression_attribute_names("#o", "owner")
.expression_attribute_values(
":expected_owner",
AttributeValue::S(owner.to_string()),
)
.send()
.await
.map(|_| ())
.map_err(|e| format!("{e:?}"))
} else {
Self::check_artifact_transaction_cap(artifact_items.len(), 1)?;
let task_put = aws_sdk_dynamodb::types::Put::builder()
.table_name(&self.config.tasks_table)
.set_item(Some(item))
.condition_expression("attribute_exists(pk) AND #o = :expected_owner")
.expression_attribute_names("#o", "owner")
.expression_attribute_values(
":expected_owner",
AttributeValue::S(owner.to_string()),
)
.build()
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?;
let mut transact = vec![
aws_sdk_dynamodb::types::TransactWriteItem::builder()
.put(task_put)
.build(),
];
transact.extend(artifact_items);
self.client
.transact_write_items()
.set_transact_items(Some(transact))
.send()
.await
.map(|_| ())
.map_err(|e| format!("{e:?}"))
};
match result {
Ok(()) => {
if !was_terminal {
self.stamp_child_ttl(tenant, task.id(), ttl.as_ref()).await;
}
Ok(())
}
Err(err_str) => {
if err_str.contains("ConditionalCheckFailed") {
Err(A2aStorageError::OwnerMismatch {
task_id: task.id().to_string(),
})
} else {
Err(A2aStorageError::DatabaseError(err_str))
}
}
}
}
async fn delete_task(
&self,
tenant: &str,
task_id: &str,
owner: &str,
) -> Result<bool, A2aStorageError> {
let existing = self.get_task(tenant, task_id, owner, Some(0)).await?;
if existing.is_none() {
return Ok(false);
}
self.delete_artifact_partition(tenant, task_id).await?;
let pk = Self::task_key(tenant, task_id);
self.client
.delete_item()
.table_name(&self.config.tasks_table)
.key("pk", AttributeValue::S(pk))
.send()
.await
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?;
if let Err(e) = self.delete_lease_item(tenant, task_id).await {
tracing::warn!(
target: "turul_a2a::lease_orphan_cleanup_error",
%tenant, %task_id, error = %e,
"failed to delete executor lease of a deleted task"
);
}
Ok(true)
}
async fn list_tasks(&self, filter: TaskFilter) -> Result<TaskListPage, A2aStorageError> {
let tenant = filter.tenant.as_deref().unwrap_or("");
let owner = filter.owner.as_deref().unwrap_or("");
let page_size = filter.resolved_page_size()?;
let cursor = filter
.page_token
.as_deref()
.map(super::status_time::decode_cursor)
.transpose()?;
let mut expr = String::from("tenant = :t AND #o = :o");
if filter.context_id.is_some() {
expr.push_str(" AND contextId = :ctx");
}
if filter.status.is_some() {
expr.push_str(" AND statusState = :st");
}
let mut items = Vec::new();
let mut start_key = None;
loop {
let mut scan = self
.client
.scan()
.table_name(&self.config.tasks_table)
.filter_expression(&expr)
.expression_attribute_names("#o", "owner")
.expression_attribute_values(":t", AttributeValue::S(tenant.to_string()))
.expression_attribute_values(":o", AttributeValue::S(owner.to_string()))
.set_exclusive_start_key(start_key);
if let Some(ref ctx) = filter.context_id {
scan = scan.expression_attribute_values(":ctx", AttributeValue::S(ctx.clone()));
}
if let Some(status) = filter.status {
scan = scan.expression_attribute_values(
":st",
AttributeValue::S(status.wire_name().to_string()),
);
}
let out = scan
.send()
.await
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?;
items.extend(out.items.unwrap_or_default());
start_key = out.last_evaluated_key;
if start_key.is_none() {
break;
}
}
let mut tasks_with_keys: Vec<(Task, String, &str)> = Vec::with_capacity(items.len());
for item in &items {
let Some(task) = item
.get("taskJson")
.and_then(|v| v.as_s().ok())
.and_then(|json| Self::task_from_json(json).ok())
else {
continue;
};
if !filter.admits_status_time(&task) {
continue;
}
let manifest_json = Self::stored_manifest(item)?;
let key = super::status_time::sort_key(&task);
tasks_with_keys.push((task, key, manifest_json));
}
tasks_with_keys.sort_by(|(a_task, a_key, _), (b_task, b_key, _)| {
b_key.cmp(a_key).then_with(|| b_task.id().cmp(a_task.id()))
});
let total_size = tasks_with_keys.len() as i32;
let start_idx = match cursor {
Some(cursor) => tasks_with_keys
.iter()
.position(|(t, key, _)| (key.as_str(), t.id()) < cursor)
.unwrap_or(tasks_with_keys.len()),
None => 0,
};
let include_artifacts = filter.include_artifacts.unwrap_or(false);
let end_idx = (start_idx + page_size as usize).min(tasks_with_keys.len());
let mut page_tasks: Vec<Task> = Vec::with_capacity(end_idx - start_idx);
for (task, _, manifest_json) in &tasks_with_keys[start_idx..end_idx] {
let task = if include_artifacts {
let blob = Self::task_to_json(task)?;
self.rehydrate_task(tenant, task.id(), &blob, manifest_json)
.await?
} else {
task.clone()
};
page_tasks.push(super::trim_history(task, filter.history_length));
}
let next_page_token = if end_idx < tasks_with_keys.len() {
let (last, key, _) = &tasks_with_keys[end_idx - 1];
super::status_time::encode_cursor(key, last.id())
} else {
String::new()
};
Ok(TaskListPage {
tasks: page_tasks,
next_page_token,
page_size,
total_size,
})
}
async fn update_task_status(
&self,
tenant: &str,
task_id: &str,
owner: &str,
new_status: TaskStatus,
) -> Result<Task, A2aStorageError> {
let pk = Self::task_key(tenant, task_id);
let out = self
.client
.get_item()
.consistent_read(true)
.table_name(&self.config.tasks_table)
.key("pk", AttributeValue::S(pk.clone()))
.send()
.await
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?;
let item = out
.item
.ok_or_else(|| A2aStorageError::TaskNotFound(task_id.to_string()))?;
let stored_owner = item
.get("owner")
.and_then(|v| v.as_s().ok())
.cloned()
.unwrap_or_default();
if stored_owner != owner {
return Err(A2aStorageError::TaskNotFound(task_id.to_string()));
}
let blob = item
.get("taskJson")
.and_then(|v| v.as_s().ok())
.ok_or_else(|| A2aStorageError::DatabaseError("Missing task_json".into()))?;
let manifest_json = Self::stored_manifest(&item)?;
let task = Self::task_from_json(blob)?;
let current_state = task
.status()
.ok_or_else(|| A2aStorageError::TaskNotFound(task_id.to_string()))?
.state()
.map_err(A2aStorageError::TypeError)?;
let new_state = new_status.state().map_err(A2aStorageError::TypeError)?;
turul_a2a_types::state_machine::validate_transition(current_state, new_state).map_err(
|e| match e {
turul_a2a_types::A2aTypeError::InvalidTransition { current, requested } => {
A2aStorageError::InvalidTransition {
task_id: task_id.to_string(),
current,
requested,
}
}
turul_a2a_types::A2aTypeError::TerminalState(s) => A2aStorageError::TerminalState {
task_id: task_id.to_string(),
state: s,
},
other => A2aStorageError::TypeError(other),
},
)?;
let mut proto = task.as_proto().clone();
proto.status = Some(new_status.into_proto());
let mut updated = Task::try_from(proto).map_err(A2aStorageError::TypeError)?;
super::status_time::stamp_status(&mut updated, &mut []);
let new_blob = Self::task_to_json(&updated)?;
let state_str = Self::status_state_str(&updated);
let mut update = self
.client
.update_item()
.table_name(&self.config.tasks_table)
.key("pk", AttributeValue::S(pk))
.condition_expression("attribute_exists(pk) AND #o = :owner")
.expression_attribute_names("#o", "owner")
.expression_attribute_values(":owner", AttributeValue::S(owner.to_string()))
.expression_attribute_values(":blob", AttributeValue::S(new_blob.clone()))
.expression_attribute_values(":state", AttributeValue::S(state_str.to_string()))
.expression_attribute_values(":ts", AttributeValue::S(Self::now_iso()));
let ttl = self.task_ttl(state_str);
update = if let Some(ttl) = ttl.clone() {
update
.update_expression(
"SET taskJson = :blob, statusState = :state, updatedAt = :ts, #ttl = :ttl",
)
.expression_attribute_names("#ttl", "ttl")
.expression_attribute_values(":ttl", ttl)
} else {
update.update_expression("SET taskJson = :blob, statusState = :state, updatedAt = :ts")
};
update
.send()
.await
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?;
self.stamp_child_ttl(tenant, task_id, ttl.as_ref()).await;
self.rehydrate_task(tenant, task_id, &new_blob, manifest_json)
.await
}
async fn append_message(
&self,
tenant: &str,
task_id: &str,
owner: &str,
message: Message,
) -> Result<(), A2aStorageError> {
let task = self
.get_task(tenant, task_id, owner, None)
.await?
.ok_or_else(|| A2aStorageError::TaskNotFound(task_id.to_string()))?;
let mut proto = task.into_proto();
proto.history.push(message.into_proto());
let updated = Task::try_from(proto).map_err(A2aStorageError::TypeError)?;
self.update_task(tenant, owner, updated).await
}
async fn append_artifact(
&self,
tenant: &str,
task_id: &str,
owner: &str,
artifact: Artifact,
append: bool,
_last_chunk: bool,
) -> Result<(), A2aStorageError> {
let task = self
.get_task(tenant, task_id, owner, None)
.await?
.ok_or_else(|| A2aStorageError::TaskNotFound(task_id.to_string()))?;
let mut proto = task.into_proto();
if append
&& let Some(existing) = proto
.artifacts
.iter_mut()
.find(|a| a.artifact_id == artifact.as_proto().artifact_id)
{
existing.parts.extend(artifact.into_proto().parts);
let updated = Task::try_from(proto).map_err(A2aStorageError::TypeError)?;
return self.update_task(tenant, owner, updated).await;
}
proto.artifacts.push(artifact.into_proto());
let updated = Task::try_from(proto).map_err(A2aStorageError::TypeError)?;
self.update_task(tenant, owner, updated).await
}
async fn task_count(&self) -> Result<usize, A2aStorageError> {
let result = self
.client
.scan()
.table_name(&self.config.tasks_table)
.filter_expression("attribute_exists(taskJson)")
.select(aws_sdk_dynamodb::types::Select::Count)
.send()
.await
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?;
Ok(result.count as usize)
}
async fn maintenance(&self) -> Result<(), A2aStorageError> {
Ok(())
}
async fn set_cancel_requested(
&self,
tenant: &str,
task_id: &str,
owner: &str,
) -> Result<(), A2aStorageError> {
let pk = Self::task_key(tenant, task_id);
let result = self
.client
.update_item()
.table_name(&self.config.tasks_table)
.key("pk", AttributeValue::S(pk.clone()))
.update_expression("SET cancelRequested = :true")
.condition_expression(format!(
"attribute_exists(pk) AND #owner = :owner \
AND (attribute_not_exists(statusState) \
OR {})",
*TERMINAL_STATE_CONDITION
))
.expression_attribute_names("#owner", "owner")
.set_expression_attribute_values(Some(terminal_condition_values([
(":true", AttributeValue::Bool(true)),
(":owner", AttributeValue::S(owner.to_string())),
])))
.send()
.await;
match result {
Ok(_) => Ok(()),
Err(err) => {
let err_dbg = format!("{err:?}");
if err_dbg.contains("ConditionalCheckFailed") {
match self.get_task(tenant, task_id, owner, Some(0)).await {
Ok(Some(t)) => {
let state = t.status().and_then(|s| s.state().ok());
if let Some(s) = state
&& turul_a2a_types::state_machine::is_terminal(s)
{
return Err(A2aStorageError::TerminalState {
task_id: task_id.to_string(),
state: s,
});
}
Err(A2aStorageError::TaskNotFound(task_id.to_string()))
}
Ok(None) => Err(A2aStorageError::TaskNotFound(task_id.to_string())),
Err(_) => Err(A2aStorageError::DatabaseError(err_dbg)),
}
} else {
Err(A2aStorageError::DatabaseError(err_dbg))
}
}
}
}
}
#[async_trait]
impl crate::storage::A2aCancellationSupervisor for DynamoDbA2aStorage {
fn backend_name(&self) -> &'static str {
"dynamodb"
}
async fn supervisor_get_cancel_requested(
&self,
tenant: &str,
task_id: &str,
) -> Result<bool, A2aStorageError> {
let pk = Self::task_key(tenant, task_id);
let result = self
.client
.get_item()
.table_name(&self.config.tasks_table)
.key("pk", AttributeValue::S(pk))
.projection_expression("cancelRequested, statusState")
.send()
.await
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?;
let Some(item) = result.item else {
return Ok(false);
};
let state_str = item
.get("statusState")
.and_then(|v| v.as_s().ok())
.map(String::as_str);
let is_terminal = state_str
.and_then(TaskState::from_wire_name)
.is_some_and(|s| s.is_terminal());
if is_terminal {
return Ok(false);
}
let marker = item
.get("cancelRequested")
.and_then(|v| v.as_bool().ok().copied())
.unwrap_or(false);
Ok(marker)
}
async fn supervisor_list_cancel_requested(
&self,
tenant: &str,
task_ids: &[String],
) -> Result<Vec<String>, A2aStorageError> {
let mut hits = Vec::new();
for chunk in task_ids.chunks(100) {
let keys: Vec<_> = chunk
.iter()
.map(|tid| {
let mut m = std::collections::HashMap::new();
m.insert(
"pk".to_string(),
AttributeValue::S(Self::task_key(tenant, tid)),
);
m
})
.collect();
let keys_and_attrs = aws_sdk_dynamodb::types::KeysAndAttributes::builder()
.set_keys(Some(keys))
.projection_expression("pk, cancelRequested, statusState")
.build()
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?;
let resp = self
.client
.batch_get_item()
.request_items(self.config.tasks_table.clone(), keys_and_attrs)
.send()
.await
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?;
if let Some(responses) = resp.responses
&& let Some(items) = responses.get(&self.config.tasks_table)
{
for item in items {
let pk = item
.get("pk")
.and_then(|v| v.as_s().ok())
.cloned()
.unwrap_or_default();
let state_str = item
.get("statusState")
.and_then(|v| v.as_s().ok())
.map(String::as_str);
let is_terminal = state_str
.and_then(TaskState::from_wire_name)
.is_some_and(|s| s.is_terminal());
if is_terminal {
continue;
}
let marker = item
.get("cancelRequested")
.and_then(|v| v.as_bool().ok().copied())
.unwrap_or(false);
if !marker {
continue;
}
if let Some(task_id) = pk.split_once('#').map(|(_, t)| t.to_string()) {
hits.push(task_id);
}
}
}
}
Ok(hits)
}
async fn supervisor_renew_executor_leases(
&self,
tenant: &str,
task_ids: &[String],
expires_at_micros: Option<i64>,
) -> Result<(), A2aStorageError> {
let mut first_error = None;
for chunk in task_ids.chunks(32) {
let results = futures::future::join_all(chunk.iter().map(|task_id| async move {
match expires_at_micros {
Some(expiry) => self.write_lease_item(tenant, task_id, expiry).await,
None => self.delete_lease_item(tenant, task_id).await,
}
}))
.await;
for result in results {
if let Err(e) = result
&& first_error.is_none()
{
first_error = Some(e);
}
}
}
first_error.map_or(Ok(()), Err)
}
async fn supervisor_list_expired_executor_leases(
&self,
now_micros: i64,
) -> Result<Vec<crate::storage::SupervisedTask>, A2aStorageError> {
let db_err = |e: &dyn std::fmt::Debug| A2aStorageError::DatabaseError(format!("{e:?}"));
let mut expired: Vec<(String, String, AttributeValue)> = Vec::new();
let mut start_key = None;
loop {
let page = self
.client
.scan()
.table_name(&self.config.tasks_table)
.consistent_read(true)
.filter_expression("leaseExpiresAt < :now")
.expression_attribute_values(":now", AttributeValue::N(now_micros.to_string()))
.projection_expression("tenant, taskId, leaseExpiresAt")
.set_exclusive_start_key(start_key)
.send()
.await
.map_err(|e| db_err(&e))?;
expired.extend(page.items().iter().filter_map(|item| {
let tenant = item.get("tenant")?.as_s().ok()?;
let task_id = item.get("taskId")?.as_s().ok()?;
let seen = item.get("leaseExpiresAt")?;
Some((tenant.clone(), task_id.clone(), seen.clone()))
}));
match page.last_evaluated_key {
Some(k) => start_key = Some(k),
None => break,
}
}
let mut out = Vec::new();
for chunk in expired.chunks(100) {
let mut held_keys: std::collections::HashSet<(String, String)> =
std::collections::HashSet::new();
let keys = chunk
.iter()
.map(|(tenant, task_id, _)| {
HashMap::from([(
"pk".to_string(),
AttributeValue::S(Self::task_key(tenant, task_id)),
)])
})
.collect();
let mut request = Some(
aws_sdk_dynamodb::types::KeysAndAttributes::builder()
.set_keys(Some(keys))
.consistent_read(true)
.projection_expression("tenant, taskId, #o, contextId, statusState")
.expression_attribute_names("#o", "owner")
.build()
.map_err(|e| db_err(&e))?,
);
while let Some(keys) = request.take() {
let resp = self
.client
.batch_get_item()
.request_items(self.config.tasks_table.clone(), keys)
.send()
.await
.map_err(|e| db_err(&e))?;
for item in resp
.responses()
.and_then(|r| r.get(&self.config.tasks_table))
.into_iter()
.flatten()
{
let attr = |name: &str| {
item.get(name)
.and_then(|v| v.as_s().ok())
.cloned()
.ok_or_else(|| {
A2aStorageError::DatabaseError(format!(
"task item missing string attribute {name}"
))
})
};
let parsed = (|| {
Ok::<_, A2aStorageError>((
attr("tenant")?,
attr("taskId")?,
attr("owner")?,
attr("contextId")?,
attr("statusState")?,
))
})();
let (tenant, task_id, owner, context_id, state) = match parsed {
Ok(p) => p,
Err(e) => {
tracing::warn!(
target: "turul_a2a::executor_lease_reap_error",
error = %e,
"skipping malformed task item in expired-lease listing"
);
continue;
}
};
let held = TaskState::from_wire_name(&state).is_some_and(|s| {
turul_a2a_types::state_machine::EXECUTOR_HELD_STATES.contains(&s)
});
if held {
held_keys.insert((tenant.clone(), task_id.clone()));
out.push(crate::storage::SupervisedTask::new(
tenant, task_id, owner, context_id,
));
}
}
request = resp
.unprocessed_keys
.and_then(|mut u| u.remove(&self.config.tasks_table));
}
for (tenant, task_id, seen) in chunk.iter().filter(|(tenant, task_id, _)| {
!held_keys.contains(&(tenant.clone(), task_id.clone()))
}) {
if let Err(e) = self
.delete_lease_item_if_expiry(tenant, task_id, seen)
.await
{
tracing::warn!(
target: "turul_a2a::lease_orphan_cleanup_error",
tenant = %tenant, task_id = %task_id, error = %e,
"failed to delete stale executor lease"
);
}
}
}
Ok(out)
}
async fn supervisor_list_idle_client_held_tasks(
&self,
status_before_micros: i64,
limit: usize,
) -> Result<Vec<crate::storage::SupervisedTask>, A2aStorageError> {
let db_err = |e: &dyn std::fmt::Debug| A2aStorageError::DatabaseError(format!("{e:?}"));
let before = super::status_time::key_of_micros(status_before_micros);
let held = turul_a2a_types::state_machine::CLIENT_HELD_STATES;
let placeholders = (0..held.len())
.map(|i| format!(":held{i}"))
.collect::<Vec<_>>()
.join(", ");
let values: HashMap<String, AttributeValue> = held
.iter()
.enumerate()
.map(|(i, s)| {
(
format!(":held{i}"),
AttributeValue::S(s.wire_name().to_string()),
)
})
.collect();
let mut out = Vec::new();
let mut start_key = None;
loop {
let page = self
.client
.scan()
.table_name(&self.config.tasks_table)
.filter_expression(format!("statusState IN ({placeholders})"))
.set_expression_attribute_values(Some(values.clone()))
.projection_expression("tenant, taskId, #o, contextId, taskJson")
.expression_attribute_names("#o", "owner")
.set_exclusive_start_key(start_key)
.send()
.await
.map_err(|e| db_err(&e))?;
for item in page.items() {
let attr = |name: &str| item.get(name).and_then(|v| v.as_s().ok());
let (Some(tenant), Some(task_id), Some(owner), Some(context_id), Some(json)) = (
attr("tenant"),
attr("taskId"),
attr("owner"),
attr("contextId"),
attr("taskJson"),
) else {
continue;
};
let task = match Self::task_from_json(json) {
Ok(task) => task,
Err(e) => {
tracing::warn!(
target: "turul_a2a::idle_task_sweep_error",
tenant = %tenant, task_id = %task_id, error = %e,
"skipping malformed task item in idle-task listing"
);
continue;
}
};
if super::status_time::sort_key(&task) < before {
out.push(crate::storage::SupervisedTask::new(
tenant.clone(),
task_id.clone(),
owner.clone(),
context_id.clone(),
));
if out.len() >= limit {
return Ok(out);
}
}
}
match page.last_evaluated_key {
Some(k) => start_key = Some(k),
None => break,
}
}
Ok(out)
}
}
impl DynamoDbA2aStorage {
async fn write_lease_item(
&self,
tenant: &str,
task_id: &str,
expires_at_micros: i64,
) -> Result<(), A2aStorageError> {
self.client
.put_item()
.table_name(&self.config.tasks_table)
.item("pk", AttributeValue::S(Self::lease_pk(tenant, task_id)))
.item("tenant", AttributeValue::S(tenant.to_string()))
.item("taskId", AttributeValue::S(task_id.to_string()))
.item(
"leaseExpiresAt",
AttributeValue::N(expires_at_micros.to_string()),
)
.send()
.await
.map(|_| ())
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))
}
async fn delete_lease_item_if_expiry(
&self,
tenant: &str,
task_id: &str,
seen: &AttributeValue,
) -> Result<(), A2aStorageError> {
match self
.client
.delete_item()
.table_name(&self.config.tasks_table)
.key("pk", AttributeValue::S(Self::lease_pk(tenant, task_id)))
.condition_expression("leaseExpiresAt = :seen")
.expression_attribute_values(":seen", seen.clone())
.send()
.await
{
Ok(_) => Ok(()),
Err(e)
if e.as_service_error()
.is_some_and(|se| se.is_conditional_check_failed_exception()) =>
{
Ok(())
}
Err(e) => Err(A2aStorageError::DatabaseError(format!("{e:?}"))),
}
}
async fn delete_lease_item(&self, tenant: &str, task_id: &str) -> Result<(), A2aStorageError> {
self.client
.delete_item()
.table_name(&self.config.tasks_table)
.key("pk", AttributeValue::S(Self::lease_pk(tenant, task_id)))
.send()
.await
.map(|_| ())
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))
}
}
#[async_trait]
impl A2aPushNotificationStorage for DynamoDbA2aStorage {
fn backend_name(&self) -> &'static str {
"dynamodb"
}
async fn create_config(
&self,
tenant: &str,
mut config: turul_a2a_proto::TaskPushNotificationConfig,
) -> Result<turul_a2a_proto::TaskPushNotificationConfig, A2aStorageError> {
if config.id.is_empty() {
config.id = uuid::Uuid::now_v7().to_string();
}
let config_pk = format!("{tenant}#{}#{}", config.task_id, config.id);
let task_pk = Self::task_key(tenant, &config.task_id);
let json = serde_json::to_string(&config)
.map_err(|e| A2aStorageError::SerializationError(e.to_string()))?;
const MAX_ATTEMPTS: u32 = 5;
let backoffs_ms: [u64; 4] = [10, 50, 250, 1000];
for attempt in 0..MAX_ATTEMPTS {
let seq_read =
query_latest_event_sequence(&self.client, &self.config.tasks_table, &task_pk)
.await?;
let task_item = self
.client
.get_item()
.table_name(&self.config.tasks_table)
.key("pk", AttributeValue::S(task_pk.clone()))
.projection_expression("pk, statusState")
.send()
.await
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?
.item;
let task_exists = task_item.is_some();
let ttl = task_item
.as_ref()
.and_then(|item| item.get("statusState")?.as_s().ok())
.and_then(|state| self.task_ttl(state));
let condition_check = if task_exists {
aws_sdk_dynamodb::types::ConditionCheck::builder()
.table_name(&self.config.tasks_table)
.key("pk", AttributeValue::S(task_pk.clone()))
.condition_expression(
"latestEventSequence = :seq OR \
(attribute_not_exists(latestEventSequence) AND :seq = :zero)",
)
.expression_attribute_values(":seq", AttributeValue::N(seq_read.to_string()))
.expression_attribute_values(":zero", AttributeValue::N("0".into()))
.build()
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?
} else {
aws_sdk_dynamodb::types::ConditionCheck::builder()
.table_name(&self.config.tasks_table)
.key("pk", AttributeValue::S(task_pk.clone()))
.condition_expression("attribute_not_exists(pk)")
.build()
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?
};
let mut config_put = aws_sdk_dynamodb::types::Put::builder()
.table_name(&self.config.push_configs_table)
.item("pk", AttributeValue::S(config_pk.clone()))
.item("tenant", AttributeValue::S(tenant.to_string()))
.item("taskId", AttributeValue::S(config.task_id.clone()))
.item("configId", AttributeValue::S(config.id.clone()))
.item("configJson", AttributeValue::S(json.clone()))
.item(
"registeredAfterEventSequence",
AttributeValue::N(seq_read.to_string()),
);
if let Some(ttl) = ttl {
config_put = config_put.item("ttl", ttl);
}
let config_put = config_put
.build()
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?;
let items = vec![
aws_sdk_dynamodb::types::TransactWriteItem::builder()
.condition_check(condition_check)
.build(),
aws_sdk_dynamodb::types::TransactWriteItem::builder()
.put(config_put)
.build(),
];
let send_result = self
.client
.transact_write_items()
.set_transact_items(Some(items))
.send()
.await;
match send_result {
Ok(_) => return Ok(config),
Err(e) => {
let err_dbg = format!("{e:?}");
let is_cas_loss = err_dbg.contains("TransactionCanceledException")
&& err_dbg.contains("ConditionalCheckFailed");
if is_cas_loss && attempt + 1 < MAX_ATTEMPTS {
let ms = backoffs_ms[attempt as usize];
tokio::time::sleep(std::time::Duration::from_millis(ms)).await;
continue;
}
if is_cas_loss {
return Err(A2aStorageError::CreateConfigCasTimeout {
tenant: tenant.into(),
task_id: config.task_id.clone(),
});
}
return Err(A2aStorageError::DatabaseError(err_dbg));
}
}
}
Err(A2aStorageError::CreateConfigCasTimeout {
tenant: tenant.into(),
task_id: config.task_id.clone(),
})
}
async fn get_config(
&self,
tenant: &str,
task_id: &str,
config_id: &str,
) -> Result<Option<turul_a2a_proto::TaskPushNotificationConfig>, A2aStorageError> {
let pk = format!("{tenant}#{task_id}#{config_id}");
let result = self
.client
.get_item()
.table_name(&self.config.push_configs_table)
.key("pk", AttributeValue::S(pk))
.send()
.await
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?;
match result.item {
Some(item) => {
let json = item
.get("configJson")
.and_then(|v| v.as_s().ok())
.ok_or_else(|| A2aStorageError::DatabaseError("Missing config_json".into()))?;
let config = serde_json::from_str(json)
.map_err(|e| A2aStorageError::SerializationError(e.to_string()))?;
Ok(Some(config))
}
None => Ok(None),
}
}
async fn list_configs(
&self,
tenant: &str,
task_id: &str,
page_token: Option<&str>,
page_size: Option<i32>,
) -> Result<PushConfigListPage, A2aStorageError> {
let page_size = super::filter::push_config_page_size(page_size);
let result = self
.client
.scan()
.table_name(&self.config.push_configs_table)
.filter_expression("tenant = :t AND taskId = :tid")
.expression_attribute_values(":t", AttributeValue::S(tenant.to_string()))
.expression_attribute_values(":tid", AttributeValue::S(task_id.to_string()))
.send()
.await
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?;
let items = result.items.unwrap_or_default();
let mut configs: Vec<turul_a2a_proto::TaskPushNotificationConfig> = items
.iter()
.filter_map(|item| {
item.get("configJson")
.and_then(|v| v.as_s().ok())
.and_then(|json| serde_json::from_str(json).ok())
})
.collect();
configs.sort_by(|a, b| a.id.cmp(&b.id));
let start_idx = if let Some(token) = page_token {
configs
.iter()
.position(|c| c.id.as_str() > token)
.unwrap_or(configs.len())
} else {
0
};
let end_idx = (start_idx + page_size as usize).min(configs.len());
let page_configs = configs[start_idx..end_idx].to_vec();
let next_page_token = if end_idx < configs.len() {
page_configs
.last()
.map(|c| c.id.clone())
.unwrap_or_default()
} else {
String::new()
};
Ok(PushConfigListPage {
configs: page_configs,
next_page_token,
})
}
async fn delete_config(
&self,
tenant: &str,
task_id: &str,
config_id: &str,
) -> Result<(), A2aStorageError> {
let pk = format!("{tenant}#{task_id}#{config_id}");
self.client
.delete_item()
.table_name(&self.config.push_configs_table)
.key("pk", AttributeValue::S(pk))
.send()
.await
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?;
Ok(())
}
async fn list_configs_eligible_at_event(
&self,
tenant: &str,
task_id: &str,
event_sequence: u64,
page_token: Option<&str>,
page_size: Option<i32>,
) -> Result<PushConfigListPage, A2aStorageError> {
let page_size = super::filter::push_config_page_size(page_size);
let result = self
.client
.scan()
.table_name(&self.config.push_configs_table)
.filter_expression(
"tenant = :t AND taskId = :tid AND registeredAfterEventSequence < :seq",
)
.expression_attribute_values(":t", AttributeValue::S(tenant.to_string()))
.expression_attribute_values(":tid", AttributeValue::S(task_id.to_string()))
.expression_attribute_values(":seq", AttributeValue::N(event_sequence.to_string()))
.send()
.await
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?;
let items = result.items.unwrap_or_default();
let mut configs: Vec<turul_a2a_proto::TaskPushNotificationConfig> = items
.iter()
.filter_map(|item| {
item.get("configJson")
.and_then(|v| v.as_s().ok())
.and_then(|json| serde_json::from_str(json).ok())
})
.collect();
configs.sort_by(|a, b| a.id.cmp(&b.id));
let start_idx = if let Some(token) = page_token {
configs
.iter()
.position(|c| c.id.as_str() > token)
.unwrap_or(configs.len())
} else {
0
};
let end_idx = (start_idx + page_size as usize).min(configs.len());
let page_configs = configs[start_idx..end_idx].to_vec();
let next_page_token = if end_idx < configs.len() {
page_configs
.last()
.map(|c| c.id.clone())
.unwrap_or_default()
} else {
String::new()
};
Ok(PushConfigListPage {
configs: page_configs,
next_page_token,
})
}
}
async fn query_latest_event_sequence(
client: &Client,
tasks_table: &str,
pk: &str,
) -> Result<u64, A2aStorageError> {
let result = client
.get_item()
.consistent_read(true)
.table_name(tasks_table)
.key("pk", AttributeValue::S(pk.to_string()))
.projection_expression("latestEventSequence")
.send()
.await
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?;
if let Some(item) = result.item
&& let Some(AttributeValue::N(n)) = item.get("latestEventSequence")
{
return n
.parse::<u64>()
.map_err(|e| A2aStorageError::DatabaseError(e.to_string()));
}
Ok(0)
}
#[async_trait]
impl A2aEventStore for DynamoDbA2aStorage {
fn backend_name(&self) -> &'static str {
"dynamodb"
}
async fn append_event(
&self,
tenant: &str,
task_id: &str,
event: StreamEvent,
) -> Result<u64, A2aStorageError> {
let pk = Self::task_key(tenant, task_id);
let events = [event];
let mut retry = SequenceCommitRetry::default();
loop {
let mut allocation = self.allocate_event_sequences(&pk, 1).await?;
let mut items = Vec::new();
let event_items = self.event_transact_items(&pk, &allocation.sequences, &events)?;
let sequence_items = allocation.attach(&mut items, event_items);
match self
.client
.transact_write_items()
.set_transact_items(Some(items))
.send()
.await
{
Ok(_) => return Ok(allocation.latest()),
Err(err)
if lost_sequence_race(cancellation_reasons(&err), sequence_items.as_ref()) =>
{
retry.backoff(&pk).await?
}
Err(err) => return Err(A2aStorageError::DatabaseError(format!("{err:?}"))),
}
}
}
async fn get_events_after(
&self,
tenant: &str,
task_id: &str,
after_sequence: u64,
) -> Result<Vec<(u64, StreamEvent)>, A2aStorageError> {
let pk = Self::task_key(tenant, task_id);
let after_sk = Self::event_sort_key(after_sequence);
let result = self
.client
.query()
.table_name(&self.config.events_table)
.key_condition_expression("pk = :pk AND sk > :sk")
.expression_attribute_values(":pk", AttributeValue::S(pk))
.expression_attribute_values(":sk", AttributeValue::S(after_sk))
.scan_index_forward(true)
.send()
.await
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?;
let mut events = Vec::new();
if let Some(items) = result.items {
for item in items {
let seq = item
.get("eventSequence")
.and_then(|v| {
if let AttributeValue::N(n) = v {
Some(n)
} else {
None
}
})
.and_then(|n| n.parse::<u64>().ok())
.ok_or_else(|| {
A2aStorageError::DatabaseError("Missing eventSequence".into())
})?;
let data = item
.get("eventData")
.and_then(|v| {
if let AttributeValue::S(s) = v {
Some(s)
} else {
None
}
})
.ok_or_else(|| A2aStorageError::DatabaseError("Missing eventData".into()))?;
let event: StreamEvent = serde_json::from_str(data)
.map_err(|e| A2aStorageError::SerializationError(e.to_string()))?;
events.push((seq, event));
}
}
Ok(events)
}
async fn latest_sequence(&self, tenant: &str, task_id: &str) -> Result<u64, A2aStorageError> {
self.read_sequence_counter(&Self::task_key(tenant, task_id))
.await
}
async fn cleanup_expired(&self) -> Result<u64, A2aStorageError> {
Ok(0)
}
}
#[async_trait]
impl A2aAtomicStore for DynamoDbA2aStorage {
fn backend_name(&self) -> &'static str {
"dynamodb"
}
fn push_dispatch_enabled(&self) -> bool {
self.push_dispatch_enabled
}
async fn create_task_with_events(
&self,
tenant: &str,
owner: &str,
mut task: Task,
mut events: Vec<StreamEvent>,
) -> Result<(Task, Vec<u64>), A2aStorageError> {
super::status_time::stamp_status(&mut task, &mut events);
let pk = Self::task_key(tenant, task.id());
let (blob, artifacts) = Self::split_task(&task)?;
let state_str = Self::status_state_str(&task);
let plan = super::artifacts::reconcile(&artifacts, &[])?;
let manifest_json = super::artifacts::manifest_to_json(&plan.manifest)?;
let art_pk = Self::artifact_pk(tenant, task.id());
let ttl = self.task_ttl(state_str);
let mut retry = SequenceCommitRetry::default();
loop {
let mut allocation = self.allocate_event_sequences(&pk, events.len()).await?;
let task_item = Self::build_task_item(
&pk,
tenant,
task.id(),
owner,
task.context_id(),
state_str,
&blob,
&manifest_json,
Some(allocation.latest()),
ttl.clone(),
);
let task_put = aws_sdk_dynamodb::types::Put::builder()
.table_name(&self.config.tasks_table)
.set_item(Some(task_item));
let mut items = vec![
aws_sdk_dynamodb::types::TransactWriteItem::builder()
.put(
task_put
.build()
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?,
)
.build(),
];
let artifact_items = self.artifact_transact_items(&art_pk, &plan, ttl.as_ref())?;
Self::check_artifact_transaction_cap(
artifact_items.len(),
1 + events.len() + usize::from(allocation.counter_write.is_some()),
)?;
items.extend(artifact_items);
let event_items = self.event_transact_items(&pk, &allocation.sequences, &events)?;
let sequence_items = allocation.attach(&mut items, event_items);
match self
.client
.transact_write_items()
.set_transact_items(Some(items))
.send()
.await
{
Ok(_) => return Ok((task, allocation.sequences)),
Err(err)
if lost_sequence_race(cancellation_reasons(&err), sequence_items.as_ref()) =>
{
retry.backoff(&pk).await?
}
Err(err) => return Err(A2aStorageError::DatabaseError(format!("{err:?}"))),
}
}
}
async fn update_task_status_with_events(
&self,
tenant: &str,
task_id: &str,
owner: &str,
new_status: TaskStatus,
mut events: Vec<StreamEvent>,
) -> Result<(Task, Vec<u64>), A2aStorageError> {
let pk = Self::task_key(tenant, task_id);
let new_state = new_status.state().map_err(A2aStorageError::TypeError)?;
let mut status_proto = new_status.into_proto();
let has_push_configs = if self.push_dispatch_enabled {
self.task_has_push_config(tenant, task_id).await?
} else {
false
};
let mut retry = SequenceCommitRetry::default();
loop {
let read = self
.client
.get_item()
.consistent_read(true)
.table_name(&self.config.tasks_table)
.key("pk", AttributeValue::S(pk.clone()))
.send()
.await
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?;
let item = read
.item
.ok_or_else(|| A2aStorageError::TaskNotFound(task_id.to_string()))?;
let stored_owner = item
.get("owner")
.and_then(|v| v.as_s().ok())
.cloned()
.unwrap_or_default();
if stored_owner != owner {
return Err(A2aStorageError::TaskNotFound(task_id.to_string()));
}
let blob = item
.get("taskJson")
.and_then(|v| v.as_s().ok())
.ok_or_else(|| A2aStorageError::DatabaseError("Missing task_json".into()))?;
let manifest_json = Self::stored_manifest(&item)?;
let prior_latest = match item.get("latestEventSequence") {
Some(AttributeValue::N(n)) => n
.parse::<u64>()
.map_err(|e| A2aStorageError::DatabaseError(e.to_string()))?,
_ => 0,
};
let task = Self::task_from_json(blob)?;
let current_state = task
.status()
.ok_or_else(|| A2aStorageError::TaskNotFound(task_id.to_string()))?
.state()
.map_err(A2aStorageError::TypeError)?;
turul_a2a_types::state_machine::validate_transition(current_state, new_state).map_err(
|e| match e {
turul_a2a_types::A2aTypeError::InvalidTransition { current, requested } => {
A2aStorageError::InvalidTransition {
task_id: task_id.to_string(),
current,
requested,
}
}
turul_a2a_types::A2aTypeError::TerminalState(s) => {
A2aStorageError::TerminalStateAlreadySet {
task_id: task_id.to_string(),
current_state: s.wire_name().to_string(),
}
}
other => A2aStorageError::TypeError(other),
},
)?;
let mut proto = task.as_proto().clone();
proto.status = Some(status_proto.clone());
let mut updated_task = Task::try_from(proto).map_err(A2aStorageError::TypeError)?;
super::status_time::stamp_status(&mut updated_task, &mut events);
if let Some(stamped) = updated_task.as_proto().status.clone() {
status_proto = stamped;
}
let task_json = Self::task_to_json(&updated_task)?;
let state_str = Self::status_state_str(&updated_task);
let ttl = self.task_ttl(state_str);
let mut allocation = self.allocate_event_sequences(&pk, events.len()).await?;
let task_item = Self::build_task_item(
&pk,
tenant,
task_id,
owner,
updated_task.context_id(),
state_str,
&task_json,
manifest_json,
Some(prior_latest.max(allocation.latest())),
ttl.clone(),
);
let mut items = vec![
aws_sdk_dynamodb::types::TransactWriteItem::builder()
.put(
aws_sdk_dynamodb::types::Put::builder()
.table_name(&self.config.tasks_table)
.set_item(Some(task_item))
.condition_expression(format!(
"#owner = :owner AND (attribute_not_exists(statusState) \
OR {})",
*TERMINAL_STATE_CONDITION
))
.expression_attribute_names("#owner", "owner")
.set_expression_attribute_values(Some(terminal_condition_values([(
":owner",
AttributeValue::S(owner.to_string()),
)])))
.build()
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?,
)
.build(),
];
let event_items = self.event_transact_items(&pk, &allocation.sequences, &events)?;
let sequence_items = allocation.attach(&mut items, event_items);
let now_micros = ddb_systime_to_micros(std::time::SystemTime::now());
let marker_ttl = Self::ttl_epoch(self.config.push_delivery_ttl_seconds);
for (&seq, event) in allocation.sequences.iter().zip(&events) {
if self.push_dispatch_enabled
&& has_push_configs
&& event.is_terminal()
&& matches!(event, StreamEvent::StatusUpdate { .. })
{
let marker_sk = seq.to_string();
let mut marker_put = aws_sdk_dynamodb::types::Put::builder()
.table_name(&self.config.push_pending_dispatches_table)
.item("pk", AttributeValue::S(pk.clone()))
.item("sk", AttributeValue::S(marker_sk))
.item("tenant", AttributeValue::S(tenant.to_string()))
.item("taskId", AttributeValue::S(task_id.to_string()))
.item("eventSequence", AttributeValue::N(seq.to_string()))
.item("owner", AttributeValue::S(owner.to_string()))
.item(
"recordedAtMicros",
AttributeValue::N(now_micros.to_string()),
);
if let Some(ttl) = marker_ttl.clone() {
marker_put = marker_put.item("ttl", ttl);
}
items.push(
aws_sdk_dynamodb::types::TransactWriteItem::builder()
.put(
marker_put.build().map_err(|e| {
A2aStorageError::DatabaseError(format!("{e:?}"))
})?,
)
.build(),
);
}
}
let err = match self
.client
.transact_write_items()
.set_transact_items(Some(items))
.send()
.await
{
Ok(_) => {
self.stamp_child_ttl(tenant, task_id, ttl.as_ref()).await;
let rehydrated = self
.rehydrate_task(tenant, task_id, &task_json, manifest_json)
.await?;
return Ok((rehydrated, allocation.sequences));
}
Err(err)
if lost_sequence_race(cancellation_reasons(&err), sequence_items.as_ref()) =>
{
retry.backoff(&pk).await?;
continue;
}
Err(err) => err,
};
let err_dbg = format!("{err:?}");
if err_dbg.contains("TransactionCanceledException")
|| err_dbg.contains("ConditionalCheckFailed")
{
match self.get_task(tenant, task_id, owner, Some(0)).await {
Ok(Some(current_task)) => {
let probe_state = current_task.status().and_then(|s| s.state().ok());
if let Some(s) = probe_state
&& turul_a2a_types::state_machine::is_terminal(s)
{
return Err(A2aStorageError::TerminalStateAlreadySet {
task_id: task_id.to_string(),
current_state: s.wire_name().to_string(),
});
}
}
Ok(None) => {
return Err(A2aStorageError::TaskNotFound(task_id.to_string()));
}
Err(_) => {
}
}
}
return Err(A2aStorageError::DatabaseError(err_dbg));
}
}
async fn update_task_with_events(
&self,
tenant: &str,
owner: &str,
mut task: Task,
mut events: Vec<StreamEvent>,
) -> Result<Vec<u64>, A2aStorageError> {
super::status_time::stamp_status(&mut task, &mut events);
let pk = Self::task_key(tenant, task.id());
let (blob, artifacts) = Self::split_task(&task)?;
let state_str = Self::status_state_str(&task);
let art_pk = Self::artifact_pk(tenant, task.id());
let task_id = task.id().to_string();
let ttl = self.task_ttl(state_str);
let mut retry = SequenceCommitRetry::default();
loop {
let (stored, _) = self.fetch_stored_manifest(tenant, &task_id).await?;
let plan = super::artifacts::reconcile(&artifacts, &stored)?;
let manifest_json = super::artifacts::manifest_to_json(&plan.manifest)?;
let prior_latest =
query_latest_event_sequence(&self.client, &self.config.tasks_table, &pk).await?;
let mut allocation = self.allocate_event_sequences(&pk, events.len()).await?;
let task_item = Self::build_task_item(
&pk,
tenant,
&task_id,
owner,
task.context_id(),
state_str,
&blob,
&manifest_json,
Some(prior_latest.max(allocation.latest())),
ttl.clone(),
);
let task_put = aws_sdk_dynamodb::types::Put::builder()
.table_name(&self.config.tasks_table)
.set_item(Some(task_item))
.condition_expression(format!(
"#owner = :owner AND (attribute_not_exists(statusState) \
OR {})",
*TERMINAL_STATE_CONDITION
))
.expression_attribute_names("#owner", "owner")
.set_expression_attribute_values(Some(terminal_condition_values([(
":owner",
AttributeValue::S(owner.to_string()),
)])));
let mut items = vec![
aws_sdk_dynamodb::types::TransactWriteItem::builder()
.put(
task_put
.build()
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?,
)
.build(),
];
let artifact_items = self.artifact_transact_items(&art_pk, &plan, ttl.as_ref())?;
Self::check_artifact_transaction_cap(
artifact_items.len(),
1 + events.len() + usize::from(allocation.counter_write.is_some()),
)?;
items.extend(artifact_items);
let event_items = self.event_transact_items(&pk, &allocation.sequences, &events)?;
let sequence_items = allocation.attach(&mut items, event_items);
let err = match self
.client
.transact_write_items()
.set_transact_items(Some(items))
.send()
.await
{
Ok(_) => {
self.stamp_child_ttl(tenant, &task_id, ttl.as_ref()).await;
return Ok(allocation.sequences);
}
Err(err)
if lost_sequence_race(cancellation_reasons(&err), sequence_items.as_ref()) =>
{
retry.backoff(&pk).await?;
continue;
}
Err(err) => err,
};
let err_dbg = format!("{err:?}");
if err_dbg.contains("TransactionCanceledException")
|| err_dbg.contains("ConditionalCheckFailed")
{
match self.get_task(tenant, &task_id, owner, Some(0)).await {
Ok(Some(current_task)) => {
let probe_state = current_task.status().and_then(|s| s.state().ok());
if let Some(s) = probe_state
&& turul_a2a_types::state_machine::is_terminal(s)
{
return Err(A2aStorageError::TerminalStateAlreadySet {
task_id,
current_state: s.wire_name().to_string(),
});
}
}
Ok(None) => {
return Err(A2aStorageError::TaskNotFound(task_id));
}
Err(_) => {}
}
}
return Err(A2aStorageError::DatabaseError(err_dbg));
}
}
}
impl DynamoDbA2aStorage {
fn push_delivery_pk(tenant: &str, task_id: &str, event_sequence: u64) -> String {
format!("{tenant}#{task_id}#{event_sequence}")
}
}
fn ddb_systime_to_micros(t: std::time::SystemTime) -> i64 {
t.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_micros() as i64)
.unwrap_or(0)
}
fn ddb_micros_to_systime(m: i64) -> std::time::SystemTime {
std::time::UNIX_EPOCH + std::time::Duration::from_micros(m.max(0) as u64)
}
fn ddb_claim_status_to_str(s: ClaimStatus) -> &'static str {
match s {
ClaimStatus::Pending => "Pending",
ClaimStatus::Attempting => "Attempting",
ClaimStatus::Succeeded => "Succeeded",
ClaimStatus::GaveUp => "GaveUp",
ClaimStatus::Abandoned => "Abandoned",
}
}
fn ddb_claim_status_from_str(s: &str) -> Result<ClaimStatus, A2aStorageError> {
match s {
"Pending" => Ok(ClaimStatus::Pending),
"Attempting" => Ok(ClaimStatus::Attempting),
"Succeeded" => Ok(ClaimStatus::Succeeded),
"GaveUp" => Ok(ClaimStatus::GaveUp),
"Abandoned" => Ok(ClaimStatus::Abandoned),
other => Err(A2aStorageError::DatabaseError(format!(
"unknown claim status: {other}"
))),
}
}
#[derive(serde::Serialize, serde::Deserialize)]
#[serde(tag = "t", content = "s")]
enum DdbErrorClassWire {
NetworkError,
Timeout,
HttpError4xx(u16),
HttpError5xx(u16),
HttpError429,
SSRFBlocked,
PayloadTooLarge,
TlsRejected,
}
impl From<DeliveryErrorClass> for DdbErrorClassWire {
fn from(c: DeliveryErrorClass) -> Self {
match c {
DeliveryErrorClass::NetworkError => Self::NetworkError,
DeliveryErrorClass::Timeout => Self::Timeout,
DeliveryErrorClass::HttpError4xx { status } => Self::HttpError4xx(status),
DeliveryErrorClass::HttpError5xx { status } => Self::HttpError5xx(status),
DeliveryErrorClass::HttpError429 => Self::HttpError429,
DeliveryErrorClass::SSRFBlocked => Self::SSRFBlocked,
DeliveryErrorClass::PayloadTooLarge => Self::PayloadTooLarge,
DeliveryErrorClass::TlsRejected => Self::TlsRejected,
}
}
}
impl From<DdbErrorClassWire> for DeliveryErrorClass {
fn from(w: DdbErrorClassWire) -> Self {
match w {
DdbErrorClassWire::NetworkError => DeliveryErrorClass::NetworkError,
DdbErrorClassWire::Timeout => DeliveryErrorClass::Timeout,
DdbErrorClassWire::HttpError4xx(s) => DeliveryErrorClass::HttpError4xx { status: s },
DdbErrorClassWire::HttpError5xx(s) => DeliveryErrorClass::HttpError5xx { status: s },
DdbErrorClassWire::HttpError429 => DeliveryErrorClass::HttpError429,
DdbErrorClassWire::SSRFBlocked => DeliveryErrorClass::SSRFBlocked,
DdbErrorClassWire::PayloadTooLarge => DeliveryErrorClass::PayloadTooLarge,
DdbErrorClassWire::TlsRejected => DeliveryErrorClass::TlsRejected,
}
}
}
fn ddb_error_class_to_json(c: DeliveryErrorClass) -> String {
serde_json::to_string(&DdbErrorClassWire::from(c)).unwrap_or_else(|_| "{}".into())
}
fn ddb_error_class_from_json(s: &str) -> Option<DeliveryErrorClass> {
serde_json::from_str::<DdbErrorClassWire>(s)
.ok()
.map(Into::into)
}
fn ddb_gave_up_reason_to_str(r: GaveUpReason) -> &'static str {
match r {
GaveUpReason::MaxAttemptsExhausted => "MaxAttemptsExhausted",
GaveUpReason::NonRetryableHttpStatus => "NonRetryableHttpStatus",
GaveUpReason::SsrfBlocked => "SsrfBlocked",
GaveUpReason::PayloadTooLarge => "PayloadTooLarge",
GaveUpReason::TlsRejected => "TlsRejected",
}
}
fn ddb_abandoned_reason_to_str(r: AbandonedReason) -> &'static str {
match r {
AbandonedReason::ConfigDeleted => "ConfigDeleted",
AbandonedReason::TaskDeleted => "TaskDeleted",
AbandonedReason::NonHttpsUrlInProduction => "NonHttpsUrlInProduction",
}
}
#[async_trait]
impl A2aPushDeliveryStore for DynamoDbA2aStorage {
fn backend_name(&self) -> &'static str {
"dynamodb"
}
async fn claim_delivery(
&self,
tenant: &str,
task_id: &str,
event_sequence: u64,
config_id: &str,
claimant: &str,
owner: &str,
claim_expiry: std::time::Duration,
) -> Result<DeliveryClaim, A2aStorageError> {
let now = std::time::SystemTime::now();
let now_micros = ddb_systime_to_micros(now);
let expires_micros = ddb_systime_to_micros(now + claim_expiry);
let pk = Self::push_delivery_pk(tenant, task_id, event_sequence);
let sk = config_id.to_string();
let existing = self
.client
.get_item()
.table_name(&self.config.push_deliveries_table)
.key("pk", AttributeValue::S(pk.clone()))
.key("sk", AttributeValue::S(sk.clone()))
.send()
.await
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?;
if let Some(item) = existing.item {
let prev_expires = item
.get("expiresAtMicros")
.and_then(|v| {
if let AttributeValue::N(n) = v {
n.parse::<i64>().ok()
} else {
None
}
})
.unwrap_or(0);
let prev_status_s = item
.get("status")
.and_then(|v| {
if let AttributeValue::S(s) = v {
Some(s.as_str())
} else {
None
}
})
.unwrap_or("Pending");
let prev_status = ddb_claim_status_from_str(prev_status_s)?;
let is_terminal = matches!(
prev_status,
ClaimStatus::Succeeded | ClaimStatus::GaveUp | ClaimStatus::Abandoned
);
let still_live = prev_expires > now_micros;
if is_terminal || still_live {
return Err(A2aStorageError::ClaimAlreadyHeld {
tenant: tenant.to_string(),
task_id: task_id.to_string(),
event_sequence,
config_id: config_id.to_string(),
});
}
let prev_gen = item
.get("generation")
.and_then(|v| {
if let AttributeValue::N(n) = v {
n.parse::<i64>().ok()
} else {
None
}
})
.unwrap_or(0);
let prev_count = item
.get("deliveryAttemptCount")
.and_then(|v| {
if let AttributeValue::N(n) = v {
n.parse::<i64>().ok()
} else {
None
}
})
.unwrap_or(0);
let prev_owner = item
.get("owner")
.and_then(|v| {
if let AttributeValue::S(s) = v {
Some(s.clone())
} else {
None
}
})
.unwrap_or_default();
let new_gen = prev_gen + 1;
let update = self
.client
.update_item()
.table_name(&self.config.push_deliveries_table)
.key("pk", AttributeValue::S(pk.clone()))
.key("sk", AttributeValue::S(sk.clone()))
.update_expression(
"SET claimant = :claimant, generation = :newgen, \
claimedAtMicros = :claimed, expiresAtMicros = :expires, \
#status = :pending \
REMOVE gaveUpAtMicros, gaveUpReason, abandonedReason",
)
.condition_expression(
"generation = :prevgen \
AND (#status = :pending OR #status = :attempting) \
AND expiresAtMicros < :nowguard",
)
.expression_attribute_names("#status", "status")
.expression_attribute_values(":claimant", AttributeValue::S(claimant.to_string()))
.expression_attribute_values(":newgen", AttributeValue::N(new_gen.to_string()))
.expression_attribute_values(":prevgen", AttributeValue::N(prev_gen.to_string()))
.expression_attribute_values(":claimed", AttributeValue::N(now_micros.to_string()))
.expression_attribute_values(
":expires",
AttributeValue::N(expires_micros.to_string()),
)
.expression_attribute_values(":pending", AttributeValue::S("Pending".into()))
.expression_attribute_values(":attempting", AttributeValue::S("Attempting".into()))
.expression_attribute_values(
":nowguard",
AttributeValue::N(now_micros.to_string()),
);
let update = if let Some(ttl) = Self::ttl_epoch(self.config.push_delivery_ttl_seconds) {
update
.update_expression(
"SET claimant = :claimant, generation = :newgen, \
claimedAtMicros = :claimed, expiresAtMicros = :expires, \
#status = :pending, #ttl = :ttl \
REMOVE gaveUpAtMicros, gaveUpReason, abandonedReason",
)
.expression_attribute_names("#ttl", "ttl")
.expression_attribute_values(":ttl", ttl)
} else {
update
};
let res = update.send().await;
match res {
Ok(_) => {
return Ok(DeliveryClaim {
claimant: claimant.to_string(),
owner: prev_owner,
generation: new_gen as u64,
claimed_at: now,
delivery_attempt_count: prev_count as u32,
status: ClaimStatus::Pending,
});
}
Err(e) => {
let err_dbg = format!("{e:?}");
if err_dbg.contains("ConditionalCheckFailed") {
return Err(A2aStorageError::ClaimAlreadyHeld {
tenant: tenant.to_string(),
task_id: task_id.to_string(),
event_sequence,
config_id: config_id.to_string(),
});
}
return Err(A2aStorageError::DatabaseError(err_dbg));
}
}
}
let mut put = self
.client
.put_item()
.table_name(&self.config.push_deliveries_table)
.item("pk", AttributeValue::S(pk))
.item("sk", AttributeValue::S(sk))
.item("tenant", AttributeValue::S(tenant.to_string()))
.item("taskId", AttributeValue::S(task_id.to_string()))
.item(
"eventSequence",
AttributeValue::N(event_sequence.to_string()),
)
.item("configId", AttributeValue::S(config_id.to_string()))
.item("claimant", AttributeValue::S(claimant.to_string()))
.item("owner", AttributeValue::S(owner.to_string()))
.item("generation", AttributeValue::N("1".into()))
.item("claimedAtMicros", AttributeValue::N(now_micros.to_string()))
.item(
"expiresAtMicros",
AttributeValue::N(expires_micros.to_string()),
)
.item("deliveryAttemptCount", AttributeValue::N("0".into()))
.item("status", AttributeValue::S("Pending".into()))
.condition_expression("attribute_not_exists(pk) AND attribute_not_exists(sk)");
if let Some(ttl) = Self::ttl_epoch(self.config.push_delivery_ttl_seconds) {
put = put.item("ttl", ttl);
}
match put.send().await {
Ok(_) => Ok(DeliveryClaim {
claimant: claimant.to_string(),
owner: owner.to_string(),
generation: 1,
claimed_at: now,
delivery_attempt_count: 0,
status: ClaimStatus::Pending,
}),
Err(e) => {
let err_dbg = format!("{e:?}");
if err_dbg.contains("ConditionalCheckFailed") {
Err(A2aStorageError::ClaimAlreadyHeld {
tenant: tenant.to_string(),
task_id: task_id.to_string(),
event_sequence,
config_id: config_id.to_string(),
})
} else {
Err(A2aStorageError::DatabaseError(err_dbg))
}
}
}
}
async fn record_attempt_started(
&self,
tenant: &str,
task_id: &str,
event_sequence: u64,
config_id: &str,
claimant: &str,
claim_generation: u64,
) -> Result<u32, A2aStorageError> {
let now_micros = ddb_systime_to_micros(std::time::SystemTime::now());
let pk = Self::push_delivery_pk(tenant, task_id, event_sequence);
let sk = config_id.to_string();
let res = self
.client
.update_item()
.table_name(&self.config.push_deliveries_table)
.key("pk", AttributeValue::S(pk))
.key("sk", AttributeValue::S(sk))
.update_expression(
"SET deliveryAttemptCount = deliveryAttemptCount + :one, \
#status = :attempting, \
firstAttemptedAtMicros = if_not_exists(firstAttemptedAtMicros, :now), \
lastAttemptedAtMicros = :now",
)
.condition_expression(
"claimant = :claimant AND generation = :gen \
AND (#status = :pending OR #status = :attempting)",
)
.expression_attribute_names("#status", "status")
.expression_attribute_values(":one", AttributeValue::N("1".into()))
.expression_attribute_values(":pending", AttributeValue::S("Pending".into()))
.expression_attribute_values(":attempting", AttributeValue::S("Attempting".into()))
.expression_attribute_values(":now", AttributeValue::N(now_micros.to_string()))
.expression_attribute_values(":claimant", AttributeValue::S(claimant.to_string()))
.expression_attribute_values(":gen", AttributeValue::N(claim_generation.to_string()))
.return_values(aws_sdk_dynamodb::types::ReturnValue::AllNew)
.send()
.await;
match res {
Ok(out) => {
let count = out
.attributes
.as_ref()
.and_then(|m| m.get("deliveryAttemptCount"))
.and_then(|v| {
if let AttributeValue::N(n) = v {
n.parse::<i64>().ok()
} else {
None
}
})
.unwrap_or(0);
Ok(count.max(0) as u32)
}
Err(e) => {
let err_dbg = format!("{e:?}");
if err_dbg.contains("ConditionalCheckFailed") {
Err(A2aStorageError::StaleDeliveryClaim {
tenant: tenant.to_string(),
task_id: task_id.to_string(),
event_sequence,
config_id: config_id.to_string(),
})
} else {
Err(A2aStorageError::DatabaseError(err_dbg))
}
}
}
}
async fn record_delivery_outcome(
&self,
tenant: &str,
task_id: &str,
event_sequence: u64,
config_id: &str,
claimant: &str,
claim_generation: u64,
outcome: DeliveryOutcome,
) -> Result<(), A2aStorageError> {
let now_micros = ddb_systime_to_micros(std::time::SystemTime::now());
let pk = Self::push_delivery_pk(tenant, task_id, event_sequence);
let sk = config_id.to_string();
let existing = self
.client
.get_item()
.table_name(&self.config.push_deliveries_table)
.key("pk", AttributeValue::S(pk.clone()))
.key("sk", AttributeValue::S(sk.clone()))
.send()
.await
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?;
let cur_status = existing
.item
.as_ref()
.and_then(|m| m.get("status"))
.and_then(|v| {
if let AttributeValue::S(s) = v {
Some(s.as_str())
} else {
None
}
})
.and_then(|s| ddb_claim_status_from_str(s).ok());
if matches!(
cur_status,
Some(ClaimStatus::Succeeded) | Some(ClaimStatus::GaveUp) | Some(ClaimStatus::Abandoned)
) {
return Ok(());
}
let builder = self
.client
.update_item()
.table_name(&self.config.push_deliveries_table)
.key("pk", AttributeValue::S(pk))
.key("sk", AttributeValue::S(sk))
.condition_expression(
"claimant = :claimant AND generation = :gen \
AND (#status = :pending OR #status = :attempting)",
)
.expression_attribute_names("#status", "status")
.expression_attribute_values(":claimant", AttributeValue::S(claimant.to_string()))
.expression_attribute_values(":gen", AttributeValue::N(claim_generation.to_string()))
.expression_attribute_values(":pending", AttributeValue::S("Pending".into()))
.expression_attribute_values(":attempting", AttributeValue::S("Attempting".into()));
let res = match outcome {
DeliveryOutcome::Succeeded { http_status } => {
builder
.update_expression(
"SET #status = :st, lastHttpStatus = :hs, \
firstAttemptedAtMicros = if_not_exists(firstAttemptedAtMicros, :now), \
lastAttemptedAtMicros = :now",
)
.expression_attribute_names("#status", "status")
.expression_attribute_values(":st", AttributeValue::S("Succeeded".into()))
.expression_attribute_values(":hs", AttributeValue::N(http_status.to_string()))
.expression_attribute_values(":now", AttributeValue::N(now_micros.to_string()))
.send()
.await
}
DeliveryOutcome::Retry {
http_status,
error_class,
next_attempt_at: _,
} => {
let mut b = builder
.update_expression(
"SET #status = :st, lastErrorClass = :ec, lastAttemptedAtMicros = :now",
)
.expression_attribute_names("#status", "status")
.expression_attribute_values(":st", AttributeValue::S("Attempting".into()))
.expression_attribute_values(
":ec",
AttributeValue::S(ddb_error_class_to_json(error_class)),
)
.expression_attribute_values(":now", AttributeValue::N(now_micros.to_string()));
if let Some(hs) = http_status {
b = b
.update_expression(
"SET #status = :st, lastErrorClass = :ec, lastAttemptedAtMicros = :now, lastHttpStatus = :hs",
)
.expression_attribute_values(":hs", AttributeValue::N(hs.to_string()));
}
b.send().await
}
DeliveryOutcome::GaveUp {
reason,
last_error_class,
last_http_status,
retained_until,
} => {
if matches!(cur_status, Some(ClaimStatus::GaveUp)) {
return Ok(());
}
let retained_until_epoch = retained_until
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_secs();
let mut b = builder
.update_expression(
"SET #status = :st, gaveUpAtMicros = :now, \
gaveUpReason = :gr, lastErrorClass = :ec, #ttl = :ttl, \
firstAttemptedAtMicros = if_not_exists(firstAttemptedAtMicros, :claimedGuard), \
lastAttemptedAtMicros = if_not_exists(lastAttemptedAtMicros, :now)",
)
.expression_attribute_names("#status", "status")
.expression_attribute_names("#ttl", "ttl")
.expression_attribute_values(
":ttl",
AttributeValue::N(retained_until_epoch.to_string()),
)
.expression_attribute_values(":st", AttributeValue::S("GaveUp".into()))
.expression_attribute_values(":now", AttributeValue::N(now_micros.to_string()))
.expression_attribute_values(
":gr",
AttributeValue::S(ddb_gave_up_reason_to_str(reason).into()),
)
.expression_attribute_values(
":ec",
AttributeValue::S(ddb_error_class_to_json(last_error_class)),
)
.expression_attribute_values(
":claimedGuard",
AttributeValue::N(now_micros.to_string()),
);
if let Some(hs) = last_http_status {
b = b
.update_expression(
"SET #status = :st, gaveUpAtMicros = :now, \
gaveUpReason = :gr, lastErrorClass = :ec, lastHttpStatus = :hs, #ttl = :ttl, \
firstAttemptedAtMicros = if_not_exists(firstAttemptedAtMicros, :claimedGuard), \
lastAttemptedAtMicros = if_not_exists(lastAttemptedAtMicros, :now)",
)
.expression_attribute_values(":hs", AttributeValue::N(hs.to_string()));
}
b.send().await
}
DeliveryOutcome::Abandoned { reason } => {
if matches!(cur_status, Some(ClaimStatus::Abandoned)) {
return Ok(());
}
builder
.update_expression("SET #status = :st, abandonedReason = :ar")
.expression_attribute_names("#status", "status")
.expression_attribute_values(":st", AttributeValue::S("Abandoned".into()))
.expression_attribute_values(
":ar",
AttributeValue::S(ddb_abandoned_reason_to_str(reason).into()),
)
.send()
.await
}
};
match res {
Ok(_) => {
let _ = ddb_claim_status_to_str(ClaimStatus::Pending);
Ok(())
}
Err(e) => {
let err_dbg = format!("{e:?}");
if err_dbg.contains("ConditionalCheckFailed") {
Err(A2aStorageError::StaleDeliveryClaim {
tenant: tenant.to_string(),
task_id: task_id.to_string(),
event_sequence,
config_id: config_id.to_string(),
})
} else {
Err(A2aStorageError::DatabaseError(err_dbg))
}
}
}
}
async fn sweep_expired_claims(&self) -> Result<u64, A2aStorageError> {
let now_micros = ddb_systime_to_micros(std::time::SystemTime::now());
let mut count: u64 = 0;
let mut last_key = None;
loop {
let mut scan = self
.client
.scan()
.table_name(&self.config.push_deliveries_table)
.filter_expression(
"expiresAtMicros < :now AND (#status = :pending OR #status = :attempting)",
)
.expression_attribute_names("#status", "status")
.expression_attribute_values(":now", AttributeValue::N(now_micros.to_string()))
.expression_attribute_values(":pending", AttributeValue::S("Pending".into()))
.expression_attribute_values(":attempting", AttributeValue::S("Attempting".into()));
if let Some(k) = last_key.take() {
scan = scan.set_exclusive_start_key(Some(k));
}
let out = scan
.send()
.await
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?;
count += out.count as u64;
last_key = out.last_evaluated_key;
if last_key.is_none() {
break;
}
}
Ok(count)
}
async fn list_reclaimable_claims(
&self,
limit: usize,
) -> Result<Vec<crate::push::claim::ReclaimableClaim>, A2aStorageError> {
let now_micros = ddb_systime_to_micros(std::time::SystemTime::now());
let mut out: Vec<crate::push::claim::ReclaimableClaim> = Vec::new();
let mut last_key = None;
while out.len() < limit {
let mut scan = self
.client
.scan()
.table_name(&self.config.push_deliveries_table)
.filter_expression(
"expiresAtMicros < :now AND (#status = :pending OR #status = :attempting)",
)
.expression_attribute_names("#status", "status")
.expression_attribute_values(":now", AttributeValue::N(now_micros.to_string()))
.expression_attribute_values(":pending", AttributeValue::S("Pending".into()))
.expression_attribute_values(":attempting", AttributeValue::S("Attempting".into()));
if let Some(k) = last_key.take() {
scan = scan.set_exclusive_start_key(Some(k));
}
let page = scan
.send()
.await
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?;
if let Some(items) = page.items {
for item in items {
if out.len() >= limit {
break;
}
let tenant = item
.get("tenant")
.and_then(|v| {
if let AttributeValue::S(s) = v {
Some(s.clone())
} else {
None
}
})
.unwrap_or_default();
let owner = item
.get("owner")
.and_then(|v| {
if let AttributeValue::S(s) = v {
Some(s.clone())
} else {
None
}
})
.unwrap_or_default();
let task_id = item
.get("taskId")
.and_then(|v| {
if let AttributeValue::S(s) = v {
Some(s.clone())
} else {
None
}
})
.unwrap_or_default();
let event_sequence = item
.get("eventSequence")
.and_then(|v| {
if let AttributeValue::N(n) = v {
n.parse::<u64>().ok()
} else {
None
}
})
.unwrap_or(0);
let config_id = item
.get("configId")
.and_then(|v| {
if let AttributeValue::S(s) = v {
Some(s.clone())
} else {
None
}
})
.unwrap_or_default();
out.push(crate::push::claim::ReclaimableClaim {
tenant,
owner,
task_id,
event_sequence,
config_id,
});
}
}
last_key = page.last_evaluated_key;
if last_key.is_none() {
break;
}
}
Ok(out)
}
async fn record_pending_dispatch(
&self,
tenant: &str,
owner: &str,
task_id: &str,
event_sequence: u64,
) -> Result<(), A2aStorageError> {
let now_micros = ddb_systime_to_micros(std::time::SystemTime::now());
let pk = format!("{tenant}#{task_id}");
let sk = event_sequence.to_string();
let mut put = self
.client
.put_item()
.table_name(&self.config.push_pending_dispatches_table)
.item("pk", AttributeValue::S(pk))
.item("sk", AttributeValue::S(sk))
.item("tenant", AttributeValue::S(tenant.to_string()))
.item("taskId", AttributeValue::S(task_id.to_string()))
.item(
"eventSequence",
AttributeValue::N(event_sequence.to_string()),
)
.item("owner", AttributeValue::S(owner.to_string()))
.item(
"recordedAtMicros",
AttributeValue::N(now_micros.to_string()),
);
if let Some(ttl) = Self::ttl_epoch(self.config.push_delivery_ttl_seconds) {
put = put.item("ttl", ttl);
}
put.send()
.await
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?;
Ok(())
}
async fn delete_pending_dispatch(
&self,
tenant: &str,
task_id: &str,
event_sequence: u64,
) -> Result<(), A2aStorageError> {
let pk = format!("{tenant}#{task_id}");
let sk = event_sequence.to_string();
self.client
.delete_item()
.table_name(&self.config.push_pending_dispatches_table)
.key("pk", AttributeValue::S(pk))
.key("sk", AttributeValue::S(sk))
.send()
.await
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?;
Ok(())
}
async fn list_stale_pending_dispatches(
&self,
older_than_recorded_at: std::time::SystemTime,
limit: usize,
) -> Result<Vec<crate::push::claim::PendingDispatch>, A2aStorageError> {
let cutoff_micros = ddb_systime_to_micros(older_than_recorded_at);
let mut out: Vec<crate::push::claim::PendingDispatch> = Vec::new();
let mut last_key = None;
while out.len() < limit {
let mut scan = self
.client
.scan()
.table_name(&self.config.push_pending_dispatches_table)
.filter_expression("recordedAtMicros <= :cutoff")
.expression_attribute_values(
":cutoff",
AttributeValue::N(cutoff_micros.to_string()),
);
if let Some(k) = last_key.take() {
scan = scan.set_exclusive_start_key(Some(k));
}
let page = scan
.send()
.await
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?;
if let Some(items) = page.items {
for item in items {
if out.len() >= limit {
break;
}
let tenant = item
.get("tenant")
.and_then(|v| {
if let AttributeValue::S(s) = v {
Some(s.clone())
} else {
None
}
})
.unwrap_or_default();
let owner = item
.get("owner")
.and_then(|v| {
if let AttributeValue::S(s) = v {
Some(s.clone())
} else {
None
}
})
.unwrap_or_default();
let task_id = item
.get("taskId")
.and_then(|v| {
if let AttributeValue::S(s) = v {
Some(s.clone())
} else {
None
}
})
.unwrap_or_default();
let event_sequence = item
.get("eventSequence")
.and_then(|v| {
if let AttributeValue::N(n) = v {
n.parse::<u64>().ok()
} else {
None
}
})
.unwrap_or(0);
let recorded_at_micros = item
.get("recordedAtMicros")
.and_then(|v| {
if let AttributeValue::N(n) = v {
n.parse::<i64>().ok()
} else {
None
}
})
.unwrap_or(0);
out.push(crate::push::claim::PendingDispatch {
tenant,
owner,
task_id,
event_sequence,
recorded_at: ddb_micros_to_systime(recorded_at_micros),
});
}
}
last_key = page.last_evaluated_key;
if last_key.is_none() {
break;
}
}
Ok(out)
}
async fn list_failed_deliveries(
&self,
tenant: &str,
since: std::time::SystemTime,
limit: usize,
) -> Result<Vec<FailedDelivery>, A2aStorageError> {
let since_micros = ddb_systime_to_micros(since);
let mut rows: Vec<FailedDelivery> = Vec::new();
let mut last_key = None;
loop {
let mut scan = self
.client
.scan()
.table_name(&self.config.push_deliveries_table)
.filter_expression("tenant = :t AND #status = :st AND gaveUpAtMicros >= :since")
.expression_attribute_names("#status", "status")
.expression_attribute_values(":t", AttributeValue::S(tenant.to_string()))
.expression_attribute_values(":st", AttributeValue::S("GaveUp".into()))
.expression_attribute_values(":since", AttributeValue::N(since_micros.to_string()));
if let Some(k) = last_key.take() {
scan = scan.set_exclusive_start_key(Some(k));
}
let out = scan
.send()
.await
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?;
if let Some(items) = out.items {
for item in items {
let task_id = item
.get("taskId")
.and_then(|v| {
if let AttributeValue::S(s) = v {
Some(s.clone())
} else {
None
}
})
.unwrap_or_default();
let config_id = item
.get("configId")
.and_then(|v| {
if let AttributeValue::S(s) = v {
Some(s.clone())
} else {
None
}
})
.unwrap_or_default();
let event_sequence = item
.get("eventSequence")
.and_then(|v| {
if let AttributeValue::N(n) = v {
n.parse::<u64>().ok()
} else {
None
}
})
.unwrap_or(0);
let claimed = item
.get("claimedAtMicros")
.and_then(|v| {
if let AttributeValue::N(n) = v {
n.parse::<i64>().ok()
} else {
None
}
})
.unwrap_or(0);
let first = item.get("firstAttemptedAtMicros").and_then(|v| {
if let AttributeValue::N(n) = v {
n.parse::<i64>().ok()
} else {
None
}
});
let last = item.get("lastAttemptedAtMicros").and_then(|v| {
if let AttributeValue::N(n) = v {
n.parse::<i64>().ok()
} else {
None
}
});
let gave_up = item
.get("gaveUpAtMicros")
.and_then(|v| {
if let AttributeValue::N(n) = v {
n.parse::<i64>().ok()
} else {
None
}
})
.unwrap_or(claimed);
let attempt_count = item
.get("deliveryAttemptCount")
.and_then(|v| {
if let AttributeValue::N(n) = v {
n.parse::<i64>().ok()
} else {
None
}
})
.unwrap_or(0);
let http_status = item.get("lastHttpStatus").and_then(|v| {
if let AttributeValue::N(n) = v {
n.parse::<u16>().ok()
} else {
None
}
});
let err_class_s = item.get("lastErrorClass").and_then(|v| {
if let AttributeValue::S(s) = v {
Some(s.as_str())
} else {
None
}
});
let last_error_class = err_class_s
.and_then(ddb_error_class_from_json)
.unwrap_or(DeliveryErrorClass::NetworkError);
rows.push(FailedDelivery {
task_id,
config_id,
event_sequence,
first_attempted_at: ddb_micros_to_systime(first.unwrap_or(claimed)),
last_attempted_at: ddb_micros_to_systime(last.unwrap_or(gave_up)),
gave_up_at: ddb_micros_to_systime(gave_up),
delivery_attempt_count: attempt_count.max(0) as u32,
last_http_status: http_status,
last_error_class,
});
}
}
last_key = out.last_evaluated_key;
if last_key.is_none() {
break;
}
}
rows.sort_by_key(|r| std::cmp::Reverse(r.gave_up_at));
rows.truncate(limit);
Ok(rows)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::storage::parity_tests;
use aws_sdk_dynamodb::types::{
AttributeDefinition, KeySchemaElement, KeyType, ScalarAttributeType,
};
async fn create_test_table(client: &Client, table_name: &str) {
let result = client
.create_table()
.table_name(table_name)
.key_schema(
KeySchemaElement::builder()
.attribute_name("pk")
.key_type(KeyType::Hash)
.build()
.unwrap(),
)
.attribute_definitions(
AttributeDefinition::builder()
.attribute_name("pk")
.attribute_type(ScalarAttributeType::S)
.build()
.unwrap(),
)
.billing_mode(aws_sdk_dynamodb::types::BillingMode::PayPerRequest)
.send()
.await;
if let Err(e) = &result {
let err_debug = format!("{e:?}");
if !err_debug.contains("ResourceInUse") && !err_debug.contains("Table already exists") {
panic!("Failed to create table {table_name}: {err_debug}");
}
}
for _ in 0..30 {
let desc = client.describe_table().table_name(table_name).send().await;
if let Ok(resp) = desc
&& let Some(table) = resp.table
&& table.table_status == Some(aws_sdk_dynamodb::types::TableStatus::Active)
{
return;
}
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
}
panic!("Table {table_name} did not become ACTIVE in time");
}
const TEST_TASKS_TABLE: &str = "test_a2a_tasks";
const TEST_PUSH_TABLE: &str = "test_a2a_push_configs";
const TEST_EVENTS_TABLE: &str = "test_a2a_task_events";
const TEST_PUSH_DELIVERIES_TABLE: &str = "test_a2a_push_deliveries";
const TEST_PENDING_DISPATCHES_TABLE: &str = "test_a2a_push_pending_dispatches";
async fn create_push_deliveries_table(client: &Client, table_name: &str) {
let result = client
.create_table()
.table_name(table_name)
.key_schema(
KeySchemaElement::builder()
.attribute_name("pk")
.key_type(KeyType::Hash)
.build()
.unwrap(),
)
.key_schema(
KeySchemaElement::builder()
.attribute_name("sk")
.key_type(KeyType::Range)
.build()
.unwrap(),
)
.attribute_definitions(
AttributeDefinition::builder()
.attribute_name("pk")
.attribute_type(ScalarAttributeType::S)
.build()
.unwrap(),
)
.attribute_definitions(
AttributeDefinition::builder()
.attribute_name("sk")
.attribute_type(ScalarAttributeType::S)
.build()
.unwrap(),
)
.billing_mode(aws_sdk_dynamodb::types::BillingMode::PayPerRequest)
.send()
.await;
if let Err(e) = &result {
let err_debug = format!("{e:?}");
if !err_debug.contains("ResourceInUse") && !err_debug.contains("Table already exists") {
panic!("Failed to create push-deliveries table {table_name}: {err_debug}");
}
}
for _ in 0..30 {
let desc = client.describe_table().table_name(table_name).send().await;
if let Ok(resp) = desc
&& let Some(table) = resp.table
&& table.table_status == Some(aws_sdk_dynamodb::types::TableStatus::Active)
{
return;
}
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
}
panic!("Push-deliveries table {table_name} did not become ACTIVE in time");
}
async fn create_events_table(client: &Client, table_name: &str) {
let result = client
.create_table()
.table_name(table_name)
.key_schema(
KeySchemaElement::builder()
.attribute_name("pk")
.key_type(KeyType::Hash)
.build()
.unwrap(),
)
.key_schema(
KeySchemaElement::builder()
.attribute_name("sk")
.key_type(KeyType::Range)
.build()
.unwrap(),
)
.attribute_definitions(
AttributeDefinition::builder()
.attribute_name("pk")
.attribute_type(ScalarAttributeType::S)
.build()
.unwrap(),
)
.attribute_definitions(
AttributeDefinition::builder()
.attribute_name("sk")
.attribute_type(ScalarAttributeType::S)
.build()
.unwrap(),
)
.billing_mode(aws_sdk_dynamodb::types::BillingMode::PayPerRequest)
.send()
.await;
if let Err(e) = &result {
let err_debug = format!("{e:?}");
if !err_debug.contains("ResourceInUse") && !err_debug.contains("Table already exists") {
panic!("Failed to create events table {table_name}: {err_debug}");
}
}
for _ in 0..30 {
let desc = client.describe_table().table_name(table_name).send().await;
if let Ok(resp) = desc
&& let Some(table) = resp.table
&& table.table_status == Some(aws_sdk_dynamodb::types::TableStatus::Active)
{
return;
}
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
}
panic!("Events table {table_name} did not become ACTIVE in time");
}
async fn storage() -> Option<TestStorage> {
if std::env::var("A2A_DYNAMODB_TESTS").ok().as_deref() != Some("1") {
return None;
}
static TABLES_CREATED: std::sync::atomic::AtomicBool =
std::sync::atomic::AtomicBool::new(false);
let loader = aws_config::defaults(aws_config::BehaviorVersion::latest());
let loader = match std::env::var("A2A_DYNAMODB_ENDPOINT") {
Ok(ep) if !ep.is_empty() => loader.endpoint_url(ep),
_ => loader,
};
let config = loader.load().await;
let client = Client::new(&config);
if !TABLES_CREATED.swap(true, std::sync::atomic::Ordering::SeqCst) {
create_test_table(&client, TEST_TASKS_TABLE).await;
create_test_table(&client, TEST_PUSH_TABLE).await;
create_events_table(&client, TEST_EVENTS_TABLE).await;
create_push_deliveries_table(&client, TEST_PUSH_DELIVERIES_TABLE).await;
create_push_deliveries_table(&client, TEST_PENDING_DISPATCHES_TABLE).await;
}
let storage = DynamoDbA2aStorage::from_client(
client,
DynamoDbConfig {
tasks_table: TEST_TASKS_TABLE.into(),
push_configs_table: TEST_PUSH_TABLE.into(),
events_table: TEST_EVENTS_TABLE.into(),
push_deliveries_table: TEST_PUSH_DELIVERIES_TABLE.into(),
push_pending_dispatches_table: TEST_PENDING_DISPATCHES_TABLE.into(),
..DynamoDbConfig::default()
},
);
Some(TestStorage {
inner: storage,
tenant_prefix: uuid::Uuid::now_v7().to_string(),
})
}
macro_rules! skip_unless_dynamodb_env {
($storage_binding:ident) => {
let $storage_binding = match storage().await {
Some(s) => s,
None => {
eprintln!(
"SKIP dynamodb live test: set A2A_DYNAMODB_TESTS=1 \
(and optionally A2A_DYNAMODB_ENDPOINT=http://localhost:8000 \
for Local DynamoDB) to enable."
);
return;
}
};
};
}
struct TestStorage {
inner: DynamoDbA2aStorage,
tenant_prefix: String,
}
impl TestStorage {
fn scoped_tenant(&self, tenant: &str) -> String {
format!("{}_{}", self.tenant_prefix, tenant)
}
fn with_push_dispatch_enabled(mut self, enabled: bool) -> Self {
self.inner = self.inner.with_push_dispatch_enabled(enabled);
self
}
}
#[async_trait]
impl A2aTaskStorage for TestStorage {
fn backend_name(&self) -> &'static str {
"dynamodb-test"
}
async fn create_task(
&self,
tenant: &str,
owner: &str,
task: Task,
) -> Result<Task, A2aStorageError> {
self.inner
.create_task(&self.scoped_tenant(tenant), owner, task)
.await
}
async fn get_task(
&self,
tenant: &str,
task_id: &str,
owner: &str,
history_length: Option<i32>,
) -> Result<Option<Task>, A2aStorageError> {
self.inner
.get_task(&self.scoped_tenant(tenant), task_id, owner, history_length)
.await
}
async fn update_task(
&self,
tenant: &str,
owner: &str,
task: Task,
) -> Result<(), A2aStorageError> {
self.inner
.update_task(&self.scoped_tenant(tenant), owner, task)
.await
}
async fn delete_task(
&self,
tenant: &str,
task_id: &str,
owner: &str,
) -> Result<bool, A2aStorageError> {
self.inner
.delete_task(&self.scoped_tenant(tenant), task_id, owner)
.await
}
async fn list_tasks(
&self,
mut filter: TaskFilter,
) -> Result<TaskListPage, A2aStorageError> {
filter.tenant = filter.tenant.map(|t| self.scoped_tenant(&t));
self.inner.list_tasks(filter).await
}
async fn update_task_status(
&self,
tenant: &str,
task_id: &str,
owner: &str,
new_status: TaskStatus,
) -> Result<Task, A2aStorageError> {
self.inner
.update_task_status(&self.scoped_tenant(tenant), task_id, owner, new_status)
.await
}
async fn append_message(
&self,
tenant: &str,
task_id: &str,
owner: &str,
message: Message,
) -> Result<(), A2aStorageError> {
self.inner
.append_message(&self.scoped_tenant(tenant), task_id, owner, message)
.await
}
async fn append_artifact(
&self,
tenant: &str,
task_id: &str,
owner: &str,
artifact: Artifact,
append: bool,
last_chunk: bool,
) -> Result<(), A2aStorageError> {
self.inner
.append_artifact(
&self.scoped_tenant(tenant),
task_id,
owner,
artifact,
append,
last_chunk,
)
.await
}
async fn task_count(&self) -> Result<usize, A2aStorageError> {
let result = self
.inner
.client
.scan()
.table_name(&self.inner.config.tasks_table)
.filter_expression("begins_with(pk, :prefix)")
.expression_attribute_values(
":prefix",
AttributeValue::S(format!("{}_", self.tenant_prefix)),
)
.select(aws_sdk_dynamodb::types::Select::Count)
.send()
.await
.map_err(|e| A2aStorageError::DatabaseError(format!("{e:?}")))?;
Ok(result.count as usize)
}
async fn maintenance(&self) -> Result<(), A2aStorageError> {
self.inner.maintenance().await
}
async fn set_cancel_requested(
&self,
tenant: &str,
task_id: &str,
owner: &str,
) -> Result<(), A2aStorageError> {
self.inner
.set_cancel_requested(&self.scoped_tenant(tenant), task_id, owner)
.await
}
}
#[async_trait]
impl crate::storage::A2aCancellationSupervisor for TestStorage {
fn backend_name(&self) -> &'static str {
"dynamodb-test"
}
async fn supervisor_get_cancel_requested(
&self,
tenant: &str,
task_id: &str,
) -> Result<bool, A2aStorageError> {
<DynamoDbA2aStorage as crate::storage::A2aCancellationSupervisor>::supervisor_get_cancel_requested(
&self.inner, &self.scoped_tenant(tenant), task_id,
).await
}
async fn supervisor_list_cancel_requested(
&self,
tenant: &str,
task_ids: &[String],
) -> Result<Vec<String>, A2aStorageError> {
<DynamoDbA2aStorage as crate::storage::A2aCancellationSupervisor>::supervisor_list_cancel_requested(
&self.inner, &self.scoped_tenant(tenant), task_ids,
).await
}
async fn supervisor_renew_executor_leases(
&self,
tenant: &str,
task_ids: &[String],
expires_at_micros: Option<i64>,
) -> Result<(), A2aStorageError> {
<DynamoDbA2aStorage as crate::storage::A2aCancellationSupervisor>::supervisor_renew_executor_leases(
&self.inner, &self.scoped_tenant(tenant), task_ids, expires_at_micros,
).await
}
async fn supervisor_list_expired_executor_leases(
&self,
now_micros: i64,
) -> Result<Vec<crate::storage::SupervisedTask>, A2aStorageError> {
let prefix = format!("{}_", self.tenant_prefix);
let all = <DynamoDbA2aStorage as crate::storage::A2aCancellationSupervisor>::supervisor_list_expired_executor_leases(
&self.inner, now_micros,
).await?;
Ok(all
.into_iter()
.filter_map(|mut t| {
t.tenant = t.tenant.strip_prefix(&prefix)?.to_string();
Some(t)
})
.collect())
}
async fn supervisor_list_idle_client_held_tasks(
&self,
status_before_micros: i64,
limit: usize,
) -> Result<Vec<crate::storage::SupervisedTask>, A2aStorageError> {
let prefix = format!("{}_", self.tenant_prefix);
let all = <DynamoDbA2aStorage as crate::storage::A2aCancellationSupervisor>::supervisor_list_idle_client_held_tasks(
&self.inner, status_before_micros, usize::MAX,
).await?;
Ok(all
.into_iter()
.filter_map(|mut t| {
t.tenant = t.tenant.strip_prefix(&prefix)?.to_string();
Some(t)
})
.take(limit)
.collect())
}
}
#[async_trait]
impl A2aPushNotificationStorage for TestStorage {
fn backend_name(&self) -> &'static str {
"dynamodb-test"
}
async fn create_config(
&self,
tenant: &str,
config: turul_a2a_proto::TaskPushNotificationConfig,
) -> Result<turul_a2a_proto::TaskPushNotificationConfig, A2aStorageError> {
self.inner
.create_config(&self.scoped_tenant(tenant), config)
.await
}
async fn get_config(
&self,
tenant: &str,
task_id: &str,
config_id: &str,
) -> Result<Option<turul_a2a_proto::TaskPushNotificationConfig>, A2aStorageError> {
self.inner
.get_config(&self.scoped_tenant(tenant), task_id, config_id)
.await
}
async fn list_configs(
&self,
tenant: &str,
task_id: &str,
page_token: Option<&str>,
page_size: Option<i32>,
) -> Result<PushConfigListPage, A2aStorageError> {
self.inner
.list_configs(&self.scoped_tenant(tenant), task_id, page_token, page_size)
.await
}
async fn delete_config(
&self,
tenant: &str,
task_id: &str,
config_id: &str,
) -> Result<(), A2aStorageError> {
self.inner
.delete_config(&self.scoped_tenant(tenant), task_id, config_id)
.await
}
async fn list_configs_eligible_at_event(
&self,
tenant: &str,
task_id: &str,
event_sequence: u64,
page_token: Option<&str>,
page_size: Option<i32>,
) -> Result<PushConfigListPage, A2aStorageError> {
self.inner
.list_configs_eligible_at_event(
&self.scoped_tenant(tenant),
task_id,
event_sequence,
page_token,
page_size,
)
.await
}
}
#[async_trait]
impl A2aEventStore for TestStorage {
fn backend_name(&self) -> &'static str {
"dynamodb-test"
}
async fn append_event(
&self,
tenant: &str,
task_id: &str,
event: StreamEvent,
) -> Result<u64, A2aStorageError> {
self.inner
.append_event(&self.scoped_tenant(tenant), task_id, event)
.await
}
async fn get_events_after(
&self,
tenant: &str,
task_id: &str,
after_sequence: u64,
) -> Result<Vec<(u64, StreamEvent)>, A2aStorageError> {
self.inner
.get_events_after(&self.scoped_tenant(tenant), task_id, after_sequence)
.await
}
async fn latest_sequence(
&self,
tenant: &str,
task_id: &str,
) -> Result<u64, A2aStorageError> {
self.inner
.latest_sequence(&self.scoped_tenant(tenant), task_id)
.await
}
async fn cleanup_expired(&self) -> Result<u64, A2aStorageError> {
self.inner.cleanup_expired().await
}
}
#[async_trait]
impl A2aAtomicStore for TestStorage {
fn backend_name(&self) -> &'static str {
"dynamodb-test"
}
fn push_dispatch_enabled(&self) -> bool {
self.inner.push_dispatch_enabled()
}
async fn create_task_with_events(
&self,
tenant: &str,
owner: &str,
task: Task,
events: Vec<StreamEvent>,
) -> Result<(Task, Vec<u64>), A2aStorageError> {
self.inner
.create_task_with_events(&self.scoped_tenant(tenant), owner, task, events)
.await
}
async fn update_task_status_with_events(
&self,
tenant: &str,
task_id: &str,
owner: &str,
new_status: TaskStatus,
events: Vec<StreamEvent>,
) -> Result<(Task, Vec<u64>), A2aStorageError> {
self.inner
.update_task_status_with_events(
&self.scoped_tenant(tenant),
task_id,
owner,
new_status,
events,
)
.await
}
async fn update_task_with_events(
&self,
tenant: &str,
owner: &str,
task: Task,
events: Vec<StreamEvent>,
) -> Result<Vec<u64>, A2aStorageError> {
self.inner
.update_task_with_events(&self.scoped_tenant(tenant), owner, task, events)
.await
}
}
#[async_trait]
impl A2aPushDeliveryStore for TestStorage {
fn backend_name(&self) -> &'static str {
"dynamodb-test"
}
async fn claim_delivery(
&self,
tenant: &str,
task_id: &str,
event_sequence: u64,
config_id: &str,
claimant: &str,
owner: &str,
claim_expiry: std::time::Duration,
) -> Result<DeliveryClaim, A2aStorageError> {
self.inner
.claim_delivery(
&self.scoped_tenant(tenant),
task_id,
event_sequence,
config_id,
claimant,
owner,
claim_expiry,
)
.await
}
async fn record_attempt_started(
&self,
tenant: &str,
task_id: &str,
event_sequence: u64,
config_id: &str,
claimant: &str,
claim_generation: u64,
) -> Result<u32, A2aStorageError> {
self.inner
.record_attempt_started(
&self.scoped_tenant(tenant),
task_id,
event_sequence,
config_id,
claimant,
claim_generation,
)
.await
}
async fn record_delivery_outcome(
&self,
tenant: &str,
task_id: &str,
event_sequence: u64,
config_id: &str,
claimant: &str,
claim_generation: u64,
outcome: DeliveryOutcome,
) -> Result<(), A2aStorageError> {
self.inner
.record_delivery_outcome(
&self.scoped_tenant(tenant),
task_id,
event_sequence,
config_id,
claimant,
claim_generation,
outcome,
)
.await
}
async fn sweep_expired_claims(&self) -> Result<u64, A2aStorageError> {
self.inner.sweep_expired_claims().await
}
async fn list_reclaimable_claims(
&self,
limit: usize,
) -> Result<Vec<crate::push::claim::ReclaimableClaim>, A2aStorageError> {
let prefix = format!("{}_", self.tenant_prefix);
let rows = self.inner.list_reclaimable_claims(usize::MAX).await?;
Ok(rows
.into_iter()
.filter_map(|mut row| {
row.tenant = row.tenant.strip_prefix(&prefix)?.to_string();
Some(row)
})
.take(limit)
.collect())
}
async fn record_pending_dispatch(
&self,
tenant: &str,
owner: &str,
task_id: &str,
event_sequence: u64,
) -> Result<(), A2aStorageError> {
self.inner
.record_pending_dispatch(
&self.scoped_tenant(tenant),
owner,
task_id,
event_sequence,
)
.await
}
async fn delete_pending_dispatch(
&self,
tenant: &str,
task_id: &str,
event_sequence: u64,
) -> Result<(), A2aStorageError> {
self.inner
.delete_pending_dispatch(&self.scoped_tenant(tenant), task_id, event_sequence)
.await
}
async fn list_stale_pending_dispatches(
&self,
older_than_recorded_at: std::time::SystemTime,
limit: usize,
) -> Result<Vec<crate::push::claim::PendingDispatch>, A2aStorageError> {
self.inner
.list_stale_pending_dispatches(older_than_recorded_at, limit)
.await
}
async fn list_failed_deliveries(
&self,
tenant: &str,
since: std::time::SystemTime,
limit: usize,
) -> Result<Vec<FailedDelivery>, A2aStorageError> {
self.inner
.list_failed_deliveries(&self.scoped_tenant(tenant), since, limit)
.await
}
}
#[tokio::test]
async fn test_create_and_retrieve() {
skip_unless_dynamodb_env!(s);
parity_tests::test_create_and_retrieve(&s).await;
}
#[tokio::test]
async fn test_state_machine_enforcement() {
skip_unless_dynamodb_env!(s);
parity_tests::test_state_machine_enforcement(&s).await;
}
#[tokio::test]
async fn test_terminal_state_rejection() {
skip_unless_dynamodb_env!(s);
parity_tests::test_terminal_state_rejection(&s).await;
}
#[tokio::test]
async fn test_tenant_isolation() {
skip_unless_dynamodb_env!(s);
parity_tests::test_tenant_isolation(&s).await;
}
#[tokio::test]
async fn test_owner_isolation() {
skip_unless_dynamodb_env!(s);
parity_tests::test_owner_isolation(&s).await;
}
#[tokio::test]
async fn test_history_length() {
skip_unless_dynamodb_env!(s);
parity_tests::test_history_length(&s).await;
}
#[tokio::test]
async fn test_list_pagination() {
skip_unless_dynamodb_env!(s);
parity_tests::test_list_pagination(&s).await;
}
#[tokio::test]
async fn test_list_pagination_exact_multiple() {
skip_unless_dynamodb_env!(s);
parity_tests::test_list_pagination_exact_multiple(&s).await;
}
#[tokio::test]
async fn test_list_page_size_bounds() {
skip_unless_dynamodb_env!(s);
parity_tests::test_list_page_size_bounds(&s).await;
}
#[tokio::test]
async fn test_list_status_timestamp_after() {
skip_unless_dynamodb_env!(s);
parity_tests::test_list_status_timestamp_after(&s).await;
}
#[tokio::test]
async fn test_list_filter_by_status() {
skip_unless_dynamodb_env!(s);
parity_tests::test_list_filter_by_status(&s).await;
}
#[tokio::test]
async fn test_list_filter_by_context_id() {
skip_unless_dynamodb_env!(s);
parity_tests::test_list_filter_by_context_id(&s).await;
}
#[tokio::test]
async fn test_append_message() {
skip_unless_dynamodb_env!(s);
parity_tests::test_append_message(&s).await;
}
#[tokio::test]
async fn test_append_artifact() {
skip_unless_dynamodb_env!(s);
parity_tests::test_append_artifact(&s).await;
}
#[tokio::test]
async fn test_task_count() {
skip_unless_dynamodb_env!(s);
parity_tests::test_task_count(&s).await;
}
#[tokio::test]
async fn test_owner_isolation_mutations() {
skip_unless_dynamodb_env!(s);
parity_tests::test_owner_isolation_mutations(&s).await;
}
#[tokio::test]
async fn test_artifact_chunk_semantics() {
skip_unless_dynamodb_env!(s);
parity_tests::test_artifact_chunk_semantics(&s).await;
}
#[tokio::test]
async fn test_artifact_separation_roundtrip() {
skip_unless_dynamodb_env!(s);
parity_tests::test_artifact_separation_roundtrip(&s).await;
}
#[tokio::test]
async fn test_artifact_chunks_survive_status_transition() {
skip_unless_dynamodb_env!(s);
parity_tests::test_artifact_chunks_survive_status_transition(&s).await;
}
#[tokio::test]
async fn test_artifact_removal_via_full_task_write() {
skip_unless_dynamodb_env!(s);
parity_tests::test_artifact_removal_via_full_task_write(&s).await;
}
#[tokio::test]
async fn test_list_orders_by_status_timestamp() {
skip_unless_dynamodb_env!(s);
parity_tests::test_list_orders_by_status_timestamp(&s).await;
}
#[tokio::test]
async fn test_list_include_artifacts_branches() {
skip_unless_dynamodb_env!(s);
parity_tests::test_list_include_artifacts_branches(&s).await;
}
#[tokio::test]
async fn test_create_task_with_artifacts() {
skip_unless_dynamodb_env!(s);
parity_tests::test_create_task_with_artifacts(&s).await;
}
#[tokio::test]
async fn test_delete_task_removes_artifact_records() {
skip_unless_dynamodb_env!(s);
parity_tests::test_delete_task_removes_artifact_records(&s).await;
}
async fn get_task_item(
s: &TestStorage,
tenant: &str,
task_id: &str,
) -> std::collections::HashMap<String, AttributeValue> {
s.inner
.client
.get_item()
.consistent_read(true)
.table_name(&s.inner.config.tasks_table)
.key(
"pk",
AttributeValue::S(DynamoDbA2aStorage::task_key(tenant, task_id)),
)
.send()
.await
.unwrap()
.item
.unwrap()
}
fn epoch_attr(item: &std::collections::HashMap<String, AttributeValue>, name: &str) -> u64 {
item.get(name)
.and_then(|v| v.as_n().ok())
.and_then(|n| n.parse().ok())
.unwrap_or_else(|| panic!("item has no numeric `{name}` attribute: {item:?}"))
}
async fn artifact_item(
s: &TestStorage,
tenant: &str,
task_id: &str,
artifact_id: &str,
) -> std::collections::HashMap<String, AttributeValue> {
s.inner
.client
.get_item()
.consistent_read(true)
.table_name(&s.inner.config.events_table)
.key(
"pk",
AttributeValue::S(DynamoDbA2aStorage::artifact_pk(tenant, task_id)),
)
.key("sk", AttributeValue::S(artifact_id.to_string()))
.send()
.await
.unwrap()
.item
.expect("artifact item")
}
async fn create_push_config(
s: &TestStorage,
tenant: &str,
task_id: &str,
) -> std::collections::HashMap<String, AttributeValue> {
let created = A2aPushNotificationStorage::create_config(
&s.inner,
tenant,
turul_a2a_proto::TaskPushNotificationConfig {
tenant: String::new(),
id: String::new(),
task_id: task_id.to_string(),
url: "https://example.com/webhook".to_string(),
token: String::new(),
authentication: None,
},
)
.await
.unwrap();
s.inner
.client
.get_item()
.consistent_read(true)
.table_name(&s.inner.config.push_configs_table)
.key(
"pk",
AttributeValue::S(format!("{tenant}#{task_id}#{}", created.id)),
)
.send()
.await
.unwrap()
.item
.expect("push config item")
}
#[tokio::test]
async fn test_native_ttl_attributes_on_tasks_and_events() {
skip_unless_dynamodb_env!(s);
let tenant = s.scoped_tenant("default");
let before = DynamoDbA2aStorage::now_epoch();
s.inner
.create_task_with_events(
&tenant,
"owner",
Task::new("ddb-ttl-1", TaskStatus::new(TaskState::Submitted))
.with_context_id("ctx"),
vec![StreamEvent::StatusUpdate {
status_update: crate::streaming::StatusUpdatePayload {
task_id: "ddb-ttl-1".into(),
context_id: "ctx".into(),
status: serde_json::json!({"state": "TASK_STATE_SUBMITTED"}),
},
}],
)
.await
.unwrap();
assert!(
!get_task_item(&s, &tenant, "ddb-ttl-1")
.await
.contains_key("ttl"),
"a created task has no ttl"
);
s.inner
.update_task_status(
&tenant,
"ddb-ttl-1",
"owner",
TaskStatus::new(TaskState::Working),
)
.await
.unwrap();
s.inner
.append_artifact(
&tenant,
"ddb-ttl-1",
"owner",
Artifact::new("art-1", vec![turul_a2a_types::Part::text("body")]),
false,
true,
)
.await
.unwrap();
assert!(
!get_task_item(&s, &tenant, "ddb-ttl-1")
.await
.contains_key("ttl"),
"a WORKING task, including after an artifact write, has no ttl"
);
assert!(
!artifact_item(&s, &tenant, "ddb-ttl-1", "art-1")
.await
.contains_key("ttl"),
"an artifact of a live task has no ttl"
);
let live_config = create_push_config(&s, &tenant, "ddb-ttl-1").await;
assert!(
!live_config.contains_key("ttl"),
"a push config of a live task has no ttl"
);
s.inner
.update_task_status(
&tenant,
"ddb-ttl-1",
"owner",
TaskStatus::new(TaskState::Completed),
)
.await
.unwrap();
let after = DynamoDbA2aStorage::now_epoch();
let task_ttl = s.inner.config.task_ttl_seconds;
let event_ttl = s.inner.config.event_ttl_seconds;
assert!(task_ttl > 0 && event_ttl > 0, "default config enables TTL");
let task_item = get_task_item(&s, &tenant, "ddb-ttl-1").await;
let ttl = epoch_attr(&task_item, "ttl");
assert!(
(before + task_ttl..=after + task_ttl).contains(&ttl),
"terminal task ttl {ttl} not within now + task_ttl_seconds"
);
assert_eq!(
task_item.get("ttl"),
artifact_item(&s, &tenant, "ddb-ttl-1", "art-1")
.await
.get("ttl"),
"the terminal write stamps the task ttl onto its artifact items"
);
let stamped_config = s
.inner
.client
.get_item()
.consistent_read(true)
.table_name(&s.inner.config.push_configs_table)
.key("pk", live_config["pk"].clone())
.send()
.await
.unwrap()
.item
.unwrap();
assert_eq!(
task_item.get("ttl"),
stamped_config.get("ttl"),
"the terminal write stamps the task ttl onto its push configs"
);
let late_config = create_push_config(&s, &tenant, "ddb-ttl-1").await;
let late_ttl = epoch_attr(&late_config, "ttl");
assert!(
(before + task_ttl..=DynamoDbA2aStorage::now_epoch() + task_ttl).contains(&late_ttl),
"a push config created on a terminal task carries a ttl at write"
);
let event_item = s
.inner
.client
.get_item()
.consistent_read(true)
.table_name(&s.inner.config.events_table)
.key(
"pk",
AttributeValue::S(DynamoDbA2aStorage::task_key(&tenant, "ddb-ttl-1")),
)
.key(
"sk",
AttributeValue::S(DynamoDbA2aStorage::event_sort_key(1)),
)
.send()
.await
.unwrap()
.item
.expect("event item for sequence 1");
let ttl = epoch_attr(&event_item, "ttl");
assert!(
(before + event_ttl..=after + event_ttl).contains(&ttl),
"event ttl {ttl} not within now + event_ttl_seconds"
);
}
async fn lease_item(
s: &TestStorage,
tenant: &str,
task_id: &str,
) -> Option<std::collections::HashMap<String, AttributeValue>> {
s.inner
.client
.get_item()
.consistent_read(true)
.table_name(&s.inner.config.tasks_table)
.key(
"pk",
AttributeValue::S(DynamoDbA2aStorage::lease_pk(tenant, task_id)),
)
.send()
.await
.unwrap()
.item
}
async fn renew(s: &TestStorage, tenant: &str, ids: &[String], expiry: Option<i64>) {
crate::storage::A2aCancellationSupervisor::supervisor_renew_executor_leases(
&s.inner, tenant, ids, expiry,
)
.await
.unwrap();
}
#[tokio::test]
async fn test_lease_is_a_separate_item_ignored_by_task_reads() {
skip_unless_dynamodb_env!(s);
let tenant = s.scoped_tenant("default");
s.inner
.create_task_with_events(
&tenant,
"owner",
Task::new("ddb-lease-1", TaskStatus::new(TaskState::Working))
.with_context_id("ctx"),
vec![],
)
.await
.unwrap();
let ids = ["ddb-lease-1".to_string(), "ddb-lease-missing".to_string()];
renew(&s, &tenant, &ids, Some(1234)).await;
let lease = lease_item(&s, &tenant, "ddb-lease-1").await.unwrap();
assert_eq!(lease["leaseExpiresAt"], AttributeValue::N("1234".into()));
assert_eq!(lease["tenant"], AttributeValue::S(tenant.clone()));
assert_eq!(lease["taskId"], AttributeValue::S("ddb-lease-1".into()));
assert!(!lease.contains_key("ttl"), "a lease item has no ttl");
assert!(
!get_task_item(&s, &tenant, "ddb-lease-1")
.await
.contains_key("leaseExpiresAt"),
"the task item carries no lease"
);
assert!(
s.inner
.get_task(&tenant, "ddb-lease-missing", "owner", None)
.await
.unwrap()
.is_none(),
"renewing an unknown id creates no task"
);
let page = s
.inner
.list_tasks(TaskFilter {
tenant: Some(tenant.clone()),
owner: Some("owner".into()),
..Default::default()
})
.await
.unwrap();
assert_eq!(page.tasks.len(), 1);
assert_eq!(page.total_size, 1);
assert!(
s.inner
.get_task(&tenant, "ddb-lease-1", "owner", None)
.await
.unwrap()
.is_some()
);
let count_with_lease = s.inner.task_count().await.unwrap();
let listed =
crate::storage::A2aCancellationSupervisor::supervisor_list_expired_executor_leases(
&s.inner,
i64::MAX,
)
.await
.unwrap();
assert!(
listed
.iter()
.any(|t| t.tenant == tenant && t.task_id == "ddb-lease-1")
);
assert!(
!listed
.iter()
.any(|t| t.tenant == tenant && t.task_id == "ddb-lease-missing"),
"an unknown id is never listed"
);
assert!(
lease_item(&s, &tenant, "ddb-lease-missing").await.is_none(),
"listing removes the orphan lease item"
);
renew(&s, &tenant, &ids[..1], None).await;
assert!(lease_item(&s, &tenant, "ddb-lease-1").await.is_none());
assert_eq!(
s.inner.task_count().await.unwrap(),
count_with_lease,
"a lease item is not counted as a task"
);
renew(&s, &tenant, &ids, None).await;
}
#[tokio::test]
async fn test_expired_lease_listing_deletes_lease_of_terminal_task() {
skip_unless_dynamodb_env!(s);
let tenant = s.scoped_tenant("default");
for (id, state) in [
("ddb-lease-done", TaskState::Completed),
("ddb-lease-held", TaskState::Working),
] {
s.inner
.create_task(
&tenant,
"owner",
Task::new(id, TaskStatus::new(state)).with_context_id("ctx"),
)
.await
.unwrap();
renew(&s, &tenant, &[id.to_string()], Some(1)).await;
}
let listed =
crate::storage::A2aCancellationSupervisor::supervisor_list_expired_executor_leases(
&s.inner,
i64::MAX,
)
.await
.unwrap();
assert!(
listed
.iter()
.all(|t| !(t.tenant == tenant && t.task_id == "ddb-lease-done"))
);
assert!(
listed
.iter()
.any(|t| t.tenant == tenant && t.task_id == "ddb-lease-held")
);
assert!(
lease_item(&s, &tenant, "ddb-lease-done").await.is_none(),
"the lease of a terminal task is deleted"
);
assert!(
lease_item(&s, &tenant, "ddb-lease-held").await.is_some(),
"the lease of an executor-held task stays"
);
renew(&s, &tenant, &["ddb-lease-held".to_string()], None).await;
}
#[tokio::test]
async fn test_stale_lease_delete_is_conditional_on_the_seen_expiry() {
skip_unless_dynamodb_env!(s);
let tenant = s.scoped_tenant("default");
let ids = ["ddb-lease-cond".to_string()];
renew(&s, &tenant, &ids, Some(2)).await;
s.inner
.delete_lease_item_if_expiry(&tenant, &ids[0], &AttributeValue::N("1".into()))
.await
.unwrap();
assert_eq!(
lease_item(&s, &tenant, &ids[0]).await.unwrap()["leaseExpiresAt"],
AttributeValue::N("2".into()),
"a lease whose expiry changed is kept"
);
s.inner
.delete_lease_item_if_expiry(&tenant, &ids[0], &AttributeValue::N("2".into()))
.await
.unwrap();
assert!(lease_item(&s, &tenant, &ids[0]).await.is_none());
}
#[tokio::test]
async fn test_delete_task_removes_lease_item() {
skip_unless_dynamodb_env!(s);
let tenant = s.scoped_tenant("default");
s.inner
.create_task(
&tenant,
"owner",
Task::new("ddb-lease-del", TaskStatus::new(TaskState::Working))
.with_context_id("ctx"),
)
.await
.unwrap();
renew(&s, &tenant, &["ddb-lease-del".to_string()], Some(1)).await;
assert!(
s.inner
.delete_task(&tenant, "ddb-lease-del", "owner")
.await
.unwrap()
);
assert!(lease_item(&s, &tenant, "ddb-lease-del").await.is_none());
}
#[tokio::test]
async fn test_lease_renewal_concurrent_with_task_writes() {
skip_unless_dynamodb_env!(s);
let tenant = s.scoped_tenant("default");
s.inner
.create_task(
&tenant,
"owner",
Task::new("ddb-lease-race", TaskStatus::new(TaskState::Working))
.with_context_id("ctx"),
)
.await
.unwrap();
let writer = {
let inner = s.inner.clone();
let tenant = tenant.clone();
tokio::spawn(async move {
for i in 0..40 {
inner
.append_artifact(
&tenant,
"ddb-lease-race",
"owner",
Artifact::new(
format!("art-{i}"),
vec![turul_a2a_types::Part::text("x")],
),
false,
true,
)
.await
.unwrap();
}
})
};
let ids = ["ddb-lease-race".to_string()];
for expiry in 1..=40 {
renew(&s, &tenant, &ids, Some(expiry)).await;
}
writer.await.unwrap();
let lease = lease_item(&s, &tenant, "ddb-lease-race").await.unwrap();
assert_eq!(lease["leaseExpiresAt"], AttributeValue::N("40".into()));
assert_eq!(
s.inner
.get_task(&tenant, "ddb-lease-race", "owner", None)
.await
.unwrap()
.unwrap()
.as_proto()
.artifacts
.len(),
40
);
}
#[tokio::test]
async fn test_every_terminal_path_stamps_child_ttl() {
skip_unless_dynamodb_env!(s);
let tenant = s.scoped_tenant("default");
for (task_id, path) in [
("ddb-child-status", 0),
("ddb-child-status-events", 1),
("ddb-child-update", 2),
("ddb-child-update-events", 3),
] {
let mut task =
Task::new(task_id, TaskStatus::new(TaskState::Working)).with_context_id("ctx");
task.push_text_artifact("art", "Result", "body");
s.inner.create_task(&tenant, "owner", task).await.unwrap();
let config = create_push_config(&s, &tenant, task_id).await;
assert!(!config.contains_key("ttl"));
assert!(
!artifact_item(&s, &tenant, task_id, "art")
.await
.contains_key("ttl")
);
let mut done = s
.inner
.get_task(&tenant, task_id, "owner", None)
.await
.unwrap()
.unwrap();
let completed = TaskStatus::new(TaskState::Completed);
match path {
0 => {
s.inner
.update_task_status(&tenant, task_id, "owner", completed)
.await
.unwrap();
}
1 => {
s.inner
.update_task_status_with_events(
&tenant,
task_id,
"owner",
completed,
vec![],
)
.await
.unwrap();
}
_ => {
done.set_status(completed);
if path == 2 {
s.inner.update_task(&tenant, "owner", done).await.unwrap();
} else {
s.inner
.update_task_with_events(&tenant, "owner", done, vec![])
.await
.unwrap();
}
}
}
let task_ttl = get_task_item(&s, &tenant, task_id).await["ttl"].clone();
assert_eq!(
artifact_item(&s, &tenant, task_id, "art").await["ttl"],
task_ttl,
"{task_id}: artifact ttl"
);
let config_item = s
.inner
.client
.get_item()
.consistent_read(true)
.table_name(&s.inner.config.push_configs_table)
.key("pk", config["pk"].clone())
.send()
.await
.unwrap()
.item
.unwrap();
assert_eq!(config_item["ttl"], task_ttl, "{task_id}: push config ttl");
}
}
#[tokio::test]
async fn test_child_ttl_is_stamped_only_on_the_terminal_transition() {
skip_unless_dynamodb_env!(s);
let tenant = s.scoped_tenant("default");
let task_id = "ddb-child-once";
let mut task =
Task::new(task_id, TaskStatus::new(TaskState::Working)).with_context_id("ctx");
task.push_text_artifact("art", "Result", "body");
s.inner.create_task(&tenant, "owner", task).await.unwrap();
let mut done = s
.inner
.get_task(&tenant, task_id, "owner", None)
.await
.unwrap()
.unwrap();
done.set_status(TaskStatus::new(TaskState::Completed));
s.inner
.update_task(&tenant, "owner", done.clone())
.await
.unwrap();
assert!(
artifact_item(&s, &tenant, task_id, "art")
.await
.contains_key("ttl"),
"the transition stamps the child"
);
s.inner
.client
.update_item()
.table_name(&s.inner.config.events_table)
.key(
"pk",
AttributeValue::S(DynamoDbA2aStorage::artifact_pk(&tenant, task_id)),
)
.key("sk", AttributeValue::S("art".into()))
.update_expression("REMOVE #ttl")
.expression_attribute_names("#ttl", "ttl")
.send()
.await
.unwrap();
s.inner.update_task(&tenant, "owner", done).await.unwrap();
assert!(
!artifact_item(&s, &tenant, task_id, "art")
.await
.contains_key("ttl"),
"a write to an already-terminal task does not stamp again"
);
}
#[tokio::test]
async fn test_gave_up_delivery_ttl_is_retention_end() {
skip_unless_dynamodb_env!(s);
let tenant = s.scoped_tenant("default");
let claim = s
.inner
.claim_delivery(
&tenant,
"ddb-ret-1",
1,
"cfg-ret",
"A",
"owner",
std::time::Duration::from_secs(60),
)
.await
.unwrap();
let retention_end = DynamoDbA2aStorage::now_epoch() + 90_000;
s.inner
.record_delivery_outcome(
&tenant,
"ddb-ret-1",
1,
"cfg-ret",
"A",
claim.generation,
DeliveryOutcome::GaveUp {
reason: GaveUpReason::MaxAttemptsExhausted,
last_error_class: DeliveryErrorClass::Timeout,
last_http_status: None,
retained_until: std::time::UNIX_EPOCH
+ std::time::Duration::from_secs(retention_end),
},
)
.await
.unwrap();
let item = s
.inner
.client
.get_item()
.consistent_read(true)
.table_name(&s.inner.config.push_deliveries_table)
.key(
"pk",
AttributeValue::S(DynamoDbA2aStorage::push_delivery_pk(
&tenant,
"ddb-ret-1",
1,
)),
)
.key("sk", AttributeValue::S("cfg-ret".into()))
.send()
.await
.unwrap()
.item
.expect("claim item");
assert_eq!(epoch_attr(&item, "ttl"), retention_end);
}
#[tokio::test]
async fn test_persisted_state_is_wire_name() {
skip_unless_dynamodb_env!(s);
let tenant = s.scoped_tenant("default");
s.inner
.create_task(
&tenant,
"owner",
Task::new("ddb-wire-1", TaskStatus::new(TaskState::Submitted))
.with_context_id("ctx"),
)
.await
.unwrap();
let state = |item: &std::collections::HashMap<String, AttributeValue>| {
item.get("statusState")
.and_then(|v| v.as_s().ok())
.cloned()
.unwrap()
};
assert_eq!(
state(&get_task_item(&s, &tenant, "ddb-wire-1").await),
"TASK_STATE_SUBMITTED"
);
for next in [TaskState::Working, TaskState::Completed] {
s.inner
.update_task_status(&tenant, "ddb-wire-1", "owner", TaskStatus::new(next))
.await
.unwrap();
}
assert_eq!(
state(&get_task_item(&s, &tenant, "ddb-wire-1").await),
"TASK_STATE_COMPLETED"
);
assert!(matches!(
s.inner
.set_cancel_requested(&tenant, "ddb-wire-1", "owner")
.await,
Err(A2aStorageError::TerminalState {
state: TaskState::Completed,
..
})
));
}
#[tokio::test]
async fn test_status_transition_does_not_rewrite_artifact_item() {
skip_unless_dynamodb_env!(s);
let tenant = s.scoped_tenant("default");
s.inner
.create_task(
&tenant,
"owner",
Task::new("ddb-shape-1", TaskStatus::new(TaskState::Submitted))
.with_context_id("ctx"),
)
.await
.unwrap();
s.inner
.append_artifact(
&tenant,
"ddb-shape-1",
"owner",
Artifact::new("art-1", vec![turul_a2a_types::Part::text("body")]),
false,
false,
)
.await
.unwrap();
let task_pk = DynamoDbA2aStorage::task_key(&tenant, "ddb-shape-1");
let task_item = s
.inner
.client
.get_item()
.table_name(&s.inner.config.tasks_table)
.key("pk", AttributeValue::S(task_pk))
.send()
.await
.unwrap()
.item
.unwrap();
let blob = task_item
.get("taskJson")
.and_then(|v| v.as_s().ok())
.unwrap();
let blob_task: turul_a2a_proto::Task = serde_json::from_str(blob).unwrap();
assert!(blob_task.artifacts.is_empty());
assert!(task_item.contains_key("artifactManifest"));
let art_pk = DynamoDbA2aStorage::artifact_pk(&tenant, "ddb-shape-1");
let art_item = s
.inner
.client
.get_item()
.table_name(&s.inner.config.events_table)
.key("pk", AttributeValue::S(art_pk.clone()))
.key("sk", AttributeValue::S("art-1".into()))
.send()
.await
.unwrap()
.item
.unwrap();
let fp_before = art_item
.get("fingerprint")
.and_then(|v| v.as_s().ok())
.unwrap()
.clone();
s.inner
.update_task_status(
&tenant,
"ddb-shape-1",
"owner",
TaskStatus::new(TaskState::Working),
)
.await
.unwrap();
let art_after = s
.inner
.client
.get_item()
.table_name(&s.inner.config.events_table)
.key("pk", AttributeValue::S(art_pk))
.key("sk", AttributeValue::S("art-1".into()))
.send()
.await
.unwrap()
.item
.unwrap();
let fp_after = art_after
.get("fingerprint")
.and_then(|v| v.as_s().ok())
.unwrap()
.clone();
assert_eq!(
fp_before, fp_after,
"status transition must not rewrite artifact item"
);
}
#[tokio::test]
async fn test_task_item_without_manifest_is_storage_error() {
skip_unless_dynamodb_env!(s);
let tenant = s.scoped_tenant("default");
let mut task = Task::new("ddb-no-manifest-1", TaskStatus::new(TaskState::Submitted))
.with_context_id("ctx");
task.push_text_artifact("art-inline", "Result", "inline body");
let mut item = DynamoDbA2aStorage::build_task_item(
&DynamoDbA2aStorage::task_key(&tenant, "ddb-no-manifest-1"),
&tenant,
"ddb-no-manifest-1",
"owner",
"ctx",
TaskState::Submitted.wire_name(),
&DynamoDbA2aStorage::task_to_json(&task).unwrap(),
"[]",
None,
None,
);
item.remove("artifactManifest");
s.inner
.client
.put_item()
.table_name(&s.inner.config.tasks_table)
.set_item(Some(item))
.send()
.await
.unwrap();
assert!(
s.inner
.get_task(&tenant, "ddb-no-manifest-1", "owner", None)
.await
.is_err(),
"a manifest-less item must not read as a task"
);
assert!(
s.inner
.update_task_status(
&tenant,
"ddb-no-manifest-1",
"owner",
TaskStatus::new(TaskState::Working),
)
.await
.is_err(),
"a status transition must not proceed on a manifest-less item"
);
assert!(
s.inner.update_task(&tenant, "owner", task).await.is_err(),
"a full-task write must not proceed on a manifest-less item"
);
}
#[tokio::test]
async fn test_push_config_crud() {
skip_unless_dynamodb_env!(s);
parity_tests::test_push_config_crud(&s).await;
}
#[tokio::test]
async fn test_push_config_idempotent_delete() {
skip_unless_dynamodb_env!(s);
parity_tests::test_push_config_idempotent_delete(&s).await;
}
#[tokio::test]
async fn test_push_config_tenant_isolation() {
skip_unless_dynamodb_env!(s);
parity_tests::test_push_config_tenant_isolation(&s).await;
}
#[tokio::test]
async fn test_push_config_list_pagination() {
skip_unless_dynamodb_env!(s);
parity_tests::test_push_config_list_pagination(&s).await;
}
#[tokio::test]
async fn test_push_config_list_pagination_exact_multiple() {
skip_unless_dynamodb_env!(s);
parity_tests::test_push_config_list_pagination_exact_multiple(&s).await;
}
#[tokio::test]
async fn test_event_append_and_retrieve() {
skip_unless_dynamodb_env!(s);
parity_tests::test_event_append_and_retrieve(&s).await;
}
#[tokio::test]
async fn test_event_monotonic_ordering() {
skip_unless_dynamodb_env!(s);
parity_tests::test_event_monotonic_ordering(&s).await;
}
#[tokio::test]
async fn test_event_per_task_isolation() {
skip_unless_dynamodb_env!(s);
parity_tests::test_event_per_task_isolation(&s).await;
}
#[tokio::test]
async fn test_event_tenant_isolation() {
skip_unless_dynamodb_env!(s);
parity_tests::test_event_tenant_isolation(&s).await;
}
#[tokio::test]
async fn test_event_latest_sequence() {
skip_unless_dynamodb_env!(s);
parity_tests::test_event_latest_sequence(&s).await;
}
#[tokio::test]
async fn test_event_empty_task() {
skip_unless_dynamodb_env!(s);
parity_tests::test_event_empty_task(&s).await;
}
#[tokio::test]
async fn test_atomic_create_task_with_events() {
skip_unless_dynamodb_env!(s);
parity_tests::test_atomic_create_task_with_events(&s, &s, &s).await;
}
#[tokio::test]
async fn test_atomic_concurrent_sequence_integrity() {
skip_unless_dynamodb_env!(s);
parity_tests::test_atomic_concurrent_sequence_integrity(std::sync::Arc::new(s)).await;
}
#[tokio::test]
async fn test_atomic_update_status_with_events() {
skip_unless_dynamodb_env!(s);
parity_tests::test_atomic_update_status_with_events(&s, &s, &s).await;
}
#[tokio::test]
async fn test_read_your_writes_across_traits() {
skip_unless_dynamodb_env!(s);
parity_tests::test_read_your_writes_across_traits(&s, &s).await;
}
#[tokio::test]
async fn test_atomic_status_rejects_invalid_transition() {
skip_unless_dynamodb_env!(s);
parity_tests::test_atomic_status_rejects_invalid_transition(&s, &s, &s).await;
}
#[tokio::test]
async fn test_atomic_update_task_with_events() {
skip_unless_dynamodb_env!(s);
parity_tests::test_atomic_update_task_with_events(&s, &s, &s).await;
}
#[tokio::test]
async fn test_atomic_terminal_with_new_artifact_in_one_call() {
skip_unless_dynamodb_env!(s);
parity_tests::test_atomic_terminal_with_new_artifact_in_one_call(&s, &s, &s).await;
}
#[tokio::test]
async fn test_terminal_cas_single_winner_on_concurrent_terminals() {
skip_unless_dynamodb_env!(s);
let s = std::sync::Arc::new(s);
parity_tests::test_terminal_cas_single_winner_on_concurrent_terminals(
s.clone(),
s.clone(),
s,
)
.await;
}
#[tokio::test]
async fn test_terminal_cas_single_winner_from_submitted_includes_rejected() {
skip_unless_dynamodb_env!(s);
let s = std::sync::Arc::new(s);
parity_tests::test_terminal_cas_single_winner_from_submitted_includes_rejected(
s.clone(),
s.clone(),
s,
)
.await;
}
#[tokio::test]
async fn test_terminal_cas_rejects_sequential_second_terminal() {
skip_unless_dynamodb_env!(s);
parity_tests::test_terminal_cas_rejects_sequential_second_terminal(&s, &s, &s).await;
}
#[tokio::test]
async fn test_update_task_with_events_rejects_terminal_already_set() {
skip_unless_dynamodb_env!(s);
parity_tests::test_update_task_with_events_rejects_terminal_already_set(&s, &s, &s).await;
}
#[tokio::test]
async fn test_invalid_transition_distinct_from_terminal_already_set() {
skip_unless_dynamodb_env!(s);
parity_tests::test_invalid_transition_distinct_from_terminal_already_set(&s, &s).await;
}
#[tokio::test]
async fn test_cleanup_is_noop_for_engine_native_ttl() {
skip_unless_dynamodb_env!(s);
parity_tests::test_cleanup_is_noop_for_engine_native_ttl(&s, &s, &s).await;
}
#[tokio::test]
async fn test_cancel_marker_roundtrip() {
skip_unless_dynamodb_env!(s);
parity_tests::test_cancel_marker_roundtrip(&s, &s).await;
}
#[tokio::test]
async fn test_supervisor_list_cancel_requested_parity() {
skip_unless_dynamodb_env!(s);
parity_tests::test_supervisor_list_cancel_requested_parity(&s, &s, &s).await;
}
#[tokio::test]
async fn test_supervisor_executor_lease_parity() {
skip_unless_dynamodb_env!(s);
parity_tests::test_supervisor_executor_lease_parity(&s, &s, &s).await;
}
#[tokio::test]
async fn test_supervisor_idle_client_held_parity() {
skip_unless_dynamodb_env!(s);
parity_tests::test_supervisor_idle_client_held_parity(&s, &s, &s).await;
}
#[tokio::test]
async fn test_atomic_owner_isolation() {
skip_unless_dynamodb_env!(s);
parity_tests::test_atomic_owner_isolation(&s, &s).await;
}
#[tokio::test]
async fn test_atomic_tenant_isolation() {
skip_unless_dynamodb_env!(s);
parity_tests::test_atomic_tenant_isolation(&s, &s, &s).await;
}
#[tokio::test]
async fn test_atomic_create_with_empty_events() {
skip_unless_dynamodb_env!(s);
parity_tests::test_atomic_create_with_empty_events(&s, &s, &s).await;
}
#[tokio::test]
async fn test_atomic_sequence_continuity() {
skip_unless_dynamodb_env!(s);
parity_tests::test_atomic_sequence_continuity(&s, &s).await;
}
#[test]
fn event_sort_key_is_twenty_digit_zero_padded_decimal() {
assert_eq!(
DynamoDbA2aStorage::event_sort_key(0),
"00000000000000000000"
);
assert_eq!(
DynamoDbA2aStorage::event_sort_key(1),
"00000000000000000001"
);
assert_eq!(
DynamoDbA2aStorage::event_sort_key(42),
"00000000000000000042"
);
let max = DynamoDbA2aStorage::event_sort_key(u64::MAX);
assert_eq!(max.len(), 20);
assert_eq!(max, "18446744073709551615");
}
#[test]
fn event_sort_key_lex_order_matches_numeric_order() {
assert!(DynamoDbA2aStorage::event_sort_key(1) < DynamoDbA2aStorage::event_sort_key(2));
assert!(DynamoDbA2aStorage::event_sort_key(9) < DynamoDbA2aStorage::event_sort_key(10));
assert!(DynamoDbA2aStorage::event_sort_key(99) < DynamoDbA2aStorage::event_sort_key(100));
assert!(
DynamoDbA2aStorage::event_sort_key(999_999)
< DynamoDbA2aStorage::event_sort_key(1_000_000)
);
let mut numeric: Vec<u64> = (0u64..200).collect();
let mut lex: Vec<String> = numeric
.iter()
.copied()
.map(DynamoDbA2aStorage::event_sort_key)
.collect();
lex.sort();
numeric.sort();
let numeric_as_keys: Vec<String> = numeric
.into_iter()
.map(DynamoDbA2aStorage::event_sort_key)
.collect();
assert_eq!(lex, numeric_as_keys);
}
fn assert_required_event_attributes(
item: &std::collections::HashMap<String, AttributeValue>,
expected_pk: &str,
expected_seq: u64,
expected_data: &str,
) {
match item.get("pk") {
Some(AttributeValue::S(v)) => assert_eq!(v, expected_pk),
other => panic!("missing/wrong pk attribute: {other:?}"),
}
match item.get("sk") {
Some(AttributeValue::S(v)) => {
assert_eq!(v, &format!("{expected_seq:020}"));
assert_eq!(v.len(), 20, "sk must be fixed-width 20 digits");
}
other => panic!("missing/wrong sk attribute: {other:?}"),
}
match item.get("eventSequence") {
Some(AttributeValue::N(v)) => {
assert_eq!(v, &expected_seq.to_string());
}
other => panic!("missing/wrong eventSequence attribute: {other:?}"),
}
match item.get("eventData") {
Some(AttributeValue::S(v)) => assert_eq!(v, expected_data),
other => panic!("missing/wrong eventData attribute: {other:?}"),
}
}
#[test]
fn build_event_item_has_required_attributes_for_append_event_path() {
let item = DynamoDbA2aStorage::build_event_item(
"tenant-a#task-1",
7,
"{\"statusUpdate\":{}}",
None,
);
assert_required_event_attributes(&item, "tenant-a#task-1", 7, "{\"statusUpdate\":{}}");
assert!(
!item.contains_key("ttl"),
"TTL must be absent when not configured"
);
}
#[test]
fn build_event_item_has_required_attributes_for_create_task_path() {
let item = DynamoDbA2aStorage::build_event_item(
"tenant-b#task-2",
1,
"{\"statusUpdate\":{\"state\":\"TASK_STATE_SUBMITTED\"}}",
None,
);
assert_required_event_attributes(
&item,
"tenant-b#task-2",
1,
"{\"statusUpdate\":{\"state\":\"TASK_STATE_SUBMITTED\"}}",
);
}
#[test]
fn build_event_item_has_required_attributes_for_update_status_path() {
let big_seq = 4_000_000_000u64;
let item = DynamoDbA2aStorage::build_event_item(
"tenant-c#task-3",
big_seq,
"{\"statusUpdate\":{\"state\":\"TASK_STATE_COMPLETED\"}}",
None,
);
assert_required_event_attributes(
&item,
"tenant-c#task-3",
big_seq,
"{\"statusUpdate\":{\"state\":\"TASK_STATE_COMPLETED\"}}",
);
}
#[test]
fn build_event_item_has_required_attributes_for_update_task_path() {
let item = DynamoDbA2aStorage::build_event_item(
"tenant-d#task-4",
42,
"{\"artifactUpdate\":{\"append\":true}}",
None,
);
assert_required_event_attributes(
&item,
"tenant-d#task-4",
42,
"{\"artifactUpdate\":{\"append\":true}}",
);
}
#[test]
fn build_event_item_includes_ttl_when_provided() {
let ttl_attr = AttributeValue::N("1234567890".into());
let item = DynamoDbA2aStorage::build_event_item("t#x", 1, "{}", Some(ttl_attr.clone()));
assert_required_event_attributes(&item, "t#x", 1, "{}");
match item.get("ttl") {
Some(got) => assert_eq!(got, &ttl_attr),
None => panic!("ttl attribute should be present when provided"),
}
}
#[allow(clippy::too_many_arguments)]
fn assert_required_task_attributes(
item: &std::collections::HashMap<String, AttributeValue>,
expected_pk: &str,
expected_tenant: &str,
expected_task_id: &str,
expected_owner: &str,
expected_context_id: &str,
expected_state: &str,
expected_json: &str,
) {
let s = |k: &str| match item.get(k) {
Some(AttributeValue::S(v)) => v.clone(),
other => panic!("missing/wrong {k} attribute: {other:?}"),
};
assert_eq!(s("pk"), expected_pk);
assert_eq!(s("tenant"), expected_tenant);
assert_eq!(s("taskId"), expected_task_id);
assert_eq!(s("owner"), expected_owner);
assert_eq!(s("contextId"), expected_context_id);
assert_eq!(s("statusState"), expected_state);
assert_eq!(s("taskJson"), expected_json);
assert!(item.contains_key("updatedAt"), "updatedAt must be present");
}
#[test]
fn build_task_item_persists_ttl_tenant_and_taskid() {
let ttl_attr = AttributeValue::N("1700000000".into());
let item = DynamoDbA2aStorage::build_task_item(
"tenant-a#task-1",
"tenant-a",
"task-1",
"owner-1",
"ctx-1",
"Working",
"{\"id\":\"task-1\"}",
"[]",
Some(5),
Some(ttl_attr.clone()),
);
assert_required_task_attributes(
&item,
"tenant-a#task-1",
"tenant-a",
"task-1",
"owner-1",
"ctx-1",
"Working",
"{\"id\":\"task-1\"}",
);
assert_eq!(item.get("ttl"), Some(&ttl_attr), "ttl must be persisted");
assert_eq!(
item.get("artifactManifest"),
Some(&AttributeValue::S("[]".into())),
"artifactManifest must be persisted"
);
match item.get("latestEventSequence") {
Some(AttributeValue::N(v)) => assert_eq!(v, "5"),
other => panic!("missing/wrong latestEventSequence: {other:?}"),
}
}
#[test]
fn build_task_item_omits_optional_attrs_when_absent() {
let item = DynamoDbA2aStorage::build_task_item(
"t#x",
"t",
"x",
"o",
"c",
"Submitted",
"{}",
"[]",
None,
None,
);
assert!(
!item.contains_key("ttl"),
"ttl must be absent when not configured"
);
assert!(
!item.contains_key("latestEventSequence"),
"latestEventSequence must be absent on non-event write paths"
);
assert!(item.contains_key("tenant"));
assert!(item.contains_key("taskId"));
}
#[test]
fn default_task_ttl_is_one_day() {
assert_eq!(DynamoDbConfig::default().task_ttl_seconds, 24 * 3600);
}
#[test]
fn sequence_counter_sorts_below_every_event_row() {
let counter = DynamoDbA2aStorage::sequence_counter_sk();
assert!(counter < DynamoDbA2aStorage::event_sort_key(1));
assert!(counter <= DynamoDbA2aStorage::event_sort_key(0));
}
fn counter_update(
observed: u64,
count: u64,
ttl: Option<AttributeValue>,
) -> aws_sdk_dynamodb::types::Update {
DynamoDbA2aStorage::build_sequence_counter_update("events", "t#task", observed, count, ttl)
.unwrap()
.update()
.cloned()
.expect("counter write is an Update")
}
#[test]
fn counter_update_adds_count_on_the_counter_key() {
let update = counter_update(4, 3, None);
assert_eq!(update.table_name(), "events");
assert_eq!(
update.key().get("pk"),
Some(&AttributeValue::S("t#task".into()))
);
assert_eq!(
update.key().get("sk"),
Some(&AttributeValue::S(DynamoDbA2aStorage::sequence_counter_sk()))
);
assert_eq!(update.update_expression(), "ADD nextSequence :count");
let values = update.expression_attribute_values().unwrap();
assert_eq!(values.get(":count"), Some(&AttributeValue::N("3".into())));
}
#[test]
fn counter_update_for_first_events_requires_absent_counter() {
let update = counter_update(0, 1, None);
assert_eq!(
update.condition_expression(),
Some("attribute_not_exists(nextSequence)")
);
assert!(
!update
.expression_attribute_values()
.unwrap()
.contains_key(":observed")
);
}
#[test]
fn counter_update_requires_the_observed_value() {
let update = counter_update(7, 2, None);
assert_eq!(
update.condition_expression(),
Some("nextSequence = :observed")
);
assert_eq!(
update
.expression_attribute_values()
.unwrap()
.get(":observed"),
Some(&AttributeValue::N("7".into()))
);
}
#[test]
fn counter_update_refreshes_ttl_when_configured() {
let update = counter_update(1, 1, Some(AttributeValue::N("123".into())));
assert_eq!(
update.update_expression(),
"ADD nextSequence :count SET #ttl = :ttl"
);
assert_eq!(
update.expression_attribute_names().unwrap().get("#ttl"),
Some(&"ttl".to_string())
);
assert_eq!(
update.expression_attribute_values().unwrap().get(":ttl"),
Some(&AttributeValue::N("123".into()))
);
}
#[test]
fn allocation_numbers_events_after_the_observed_counter() {
assert_eq!(EventSequenceAllocation::sequences_after(0, 2), vec![1, 2]);
assert_eq!(
EventSequenceAllocation::sequences_after(5, 3),
vec![6, 7, 8]
);
assert!(EventSequenceAllocation::sequences_after(9, 0).is_empty());
let with_events = EventSequenceAllocation {
observed: 5,
sequences: vec![6, 7],
counter_write: None,
};
assert_eq!(with_events.latest(), 7);
let without_events = EventSequenceAllocation {
observed: 5,
sequences: vec![],
counter_write: None,
};
assert_eq!(without_events.latest(), 5);
}
fn reasons(codes: &[&str]) -> Vec<aws_sdk_dynamodb::types::CancellationReason> {
codes
.iter()
.map(|c| {
aws_sdk_dynamodb::types::CancellationReason::builder()
.code(*c)
.build()
})
.collect()
}
fn items(events: std::ops::Range<usize>, counter: usize) -> SequenceItems {
SequenceItems { events, counter }
}
#[test]
fn counter_condition_failure_alone_is_a_lost_race() {
let r = reasons(&["None", "None", "ConditionalCheckFailed"]);
assert!(lost_sequence_race(&r, Some(&items(1..2, 2))));
}
#[test]
fn transaction_conflict_is_a_lost_race() {
let r = reasons(&["TransactionConflict", "None", "None"]);
assert!(lost_sequence_race(&r, Some(&items(1..2, 2))));
let r = reasons(&["None", "None", "TransactionConflict"]);
assert!(lost_sequence_race(&r, Some(&items(1..2, 2))));
}
#[test]
fn task_condition_failure_is_not_a_lost_race() {
let r = reasons(&["ConditionalCheckFailed", "None", "ConditionalCheckFailed"]);
assert!(!lost_sequence_race(&r, Some(&items(1..2, 2))));
let r = reasons(&["ConditionalCheckFailed", "None", "None"]);
assert!(!lost_sequence_race(&r, Some(&items(1..2, 2))));
}
#[test]
fn event_key_collision_is_not_a_lost_race() {
let r = reasons(&["None", "ConditionalCheckFailed", "None"]);
assert!(!lost_sequence_race(&r, Some(&items(1..2, 2))));
}
#[test]
fn event_key_and_counter_failure_is_a_lost_race() {
let r = reasons(&["ConditionalCheckFailed", "ConditionalCheckFailed"]);
assert!(lost_sequence_race(&r, Some(&items(0..1, 1))));
let r = reasons(&["None", "ConditionalCheckFailed", "ConditionalCheckFailed"]);
assert!(lost_sequence_race(&r, Some(&items(1..2, 2))));
}
#[test]
fn marker_condition_failure_is_not_a_lost_race() {
let r = reasons(&[
"None",
"None",
"ConditionalCheckFailed",
"ConditionalCheckFailed",
]);
assert!(!lost_sequence_race(&r, Some(&items(1..2, 2))));
}
#[test]
fn transaction_without_counter_never_retries() {
let r = reasons(&["TransactionConflict"]);
assert!(!lost_sequence_race(&r, None));
assert!(!lost_sequence_race(&[], Some(&items(0..0, 0))));
}
#[test]
fn attach_records_event_and_counter_positions() {
let put = || {
aws_sdk_dynamodb::types::TransactWriteItem::builder()
.condition_check(
aws_sdk_dynamodb::types::ConditionCheck::builder()
.table_name("t")
.key("pk", AttributeValue::S("p".into()))
.condition_expression("attribute_exists(pk)")
.build()
.unwrap(),
)
.build()
};
let mut allocation = EventSequenceAllocation {
observed: 0,
sequences: vec![1, 2],
counter_write: Some(put()),
};
let mut items = vec![put()];
let positions = allocation.attach(&mut items, vec![put(), put()]).unwrap();
assert_eq!(positions.events, 1..3);
assert_eq!(positions.counter, 3);
assert_eq!(items.len(), 4);
let mut empty = EventSequenceAllocation {
observed: 4,
sequences: vec![],
counter_write: None,
};
assert!(empty.attach(&mut items, vec![]).is_none());
}
#[test]
fn sequence_counter_never_expires_when_events_or_task_never_expire() {
assert_eq!(
DynamoDbA2aStorage::sequence_counter_ttl_seconds(0, 3600),
None
);
assert_eq!(
DynamoDbA2aStorage::sequence_counter_ttl_seconds(3600, 0),
None
);
assert_eq!(DynamoDbA2aStorage::sequence_counter_ttl_seconds(0, 0), None);
assert_eq!(
DynamoDbA2aStorage::sequence_counter_ttl_seconds(60, 3600),
Some(3600)
);
assert_eq!(
DynamoDbA2aStorage::sequence_counter_ttl_seconds(7200, 3600),
Some(7200)
);
}
#[tokio::test]
async fn test_push_claim_is_exclusive() {
skip_unless_dynamodb_env!(s);
parity_tests::test_push_claim_is_exclusive(&s).await;
}
#[tokio::test]
async fn test_push_claim_expired_is_reclaimable() {
skip_unless_dynamodb_env!(s);
parity_tests::test_push_claim_expired_is_reclaimable(&s).await;
}
#[tokio::test]
async fn test_push_outcome_fenced_to_current_claim() {
skip_unless_dynamodb_env!(s);
parity_tests::test_push_outcome_fenced_to_current_claim(&s).await;
}
#[tokio::test]
async fn test_push_claim_terminal_succeeded_blocks_reclaim() {
skip_unless_dynamodb_env!(s);
parity_tests::test_push_claim_terminal_succeeded_blocks_reclaim(&s).await;
}
#[tokio::test]
async fn test_push_claim_terminal_gaveup_blocks_reclaim() {
skip_unless_dynamodb_env!(s);
parity_tests::test_push_claim_terminal_gaveup_blocks_reclaim(&s).await;
}
#[tokio::test]
async fn test_push_claim_terminal_abandoned_blocks_reclaim_and_not_listed() {
skip_unless_dynamodb_env!(s);
parity_tests::test_push_claim_terminal_abandoned_blocks_reclaim_and_not_listed(&s).await;
}
#[tokio::test]
async fn test_push_attempt_started_advances_count_and_status() {
skip_unless_dynamodb_env!(s);
parity_tests::test_push_attempt_started_advances_count_and_status(&s).await;
}
#[tokio::test]
async fn test_push_attempt_started_is_fenced() {
skip_unless_dynamodb_env!(s);
parity_tests::test_push_attempt_started_is_fenced(&s).await;
}
#[tokio::test]
async fn test_push_retry_outcome_keeps_claim_open() {
skip_unless_dynamodb_env!(s);
parity_tests::test_push_retry_outcome_keeps_claim_open(&s).await;
}
#[tokio::test]
async fn test_push_outcome_idempotent_on_terminal() {
skip_unless_dynamodb_env!(s);
parity_tests::test_push_outcome_idempotent_on_terminal(&s).await;
}
#[tokio::test]
async fn test_push_sweep_counts_expired_nonterminal_and_preserves_status() {
skip_unless_dynamodb_env!(s);
parity_tests::test_push_sweep_counts_expired_nonterminal_and_preserves_status(&s).await;
}
#[tokio::test]
async fn test_push_list_reclaimable_filters_and_returns_identity() {
skip_unless_dynamodb_env!(s);
parity_tests::test_push_list_reclaimable_filters_and_returns_identity(&s).await;
}
#[tokio::test]
async fn test_push_list_failed_filters_and_orders() {
skip_unless_dynamodb_env!(s);
parity_tests::test_push_list_failed_filters_and_orders(&s).await;
}
#[tokio::test]
async fn test_push_list_failed_is_tenant_scoped() {
skip_unless_dynamodb_env!(s);
parity_tests::test_push_list_failed_is_tenant_scoped(&s).await;
}
#[tokio::test]
async fn test_push_failed_delivery_diagnostics_roundtrip() {
skip_unless_dynamodb_env!(s);
parity_tests::test_push_failed_delivery_diagnostics_roundtrip(&s).await;
}
#[tokio::test]
async fn test_push_attempt_started_rejected_after_terminal() {
skip_unless_dynamodb_env!(s);
parity_tests::test_push_attempt_started_rejected_after_terminal(&s).await;
}
#[tokio::test]
async fn test_push_outcome_does_not_overwrite_terminal() {
skip_unless_dynamodb_env!(s);
parity_tests::test_push_outcome_does_not_overwrite_terminal(&s).await;
}
#[tokio::test]
async fn test_push_concurrent_claim_race() {
skip_unless_dynamodb_env!(s);
parity_tests::test_push_concurrent_claim_race(std::sync::Arc::new(s)).await;
}
#[tokio::test]
async fn test_atomic_marker_written_for_terminal_status() {
skip_unless_dynamodb_env!(s);
let s = s.with_push_dispatch_enabled(true);
parity_tests::test_atomic_marker_written_for_terminal_status(&s, &s, &s, &s).await;
}
#[tokio::test]
async fn test_atomic_marker_skipped_for_non_terminal_status() {
skip_unless_dynamodb_env!(s);
let s = s.with_push_dispatch_enabled(true);
parity_tests::test_atomic_marker_skipped_for_non_terminal_status(&s, &s, &s).await;
}
#[tokio::test]
async fn test_atomic_marker_skipped_for_artifact_event() {
skip_unless_dynamodb_env!(s);
let s = s.with_push_dispatch_enabled(true);
parity_tests::test_atomic_marker_skipped_for_artifact_event(&s, &s, &s, &s).await;
}
#[tokio::test]
async fn test_atomic_marker_absent_when_opt_in_off() {
skip_unless_dynamodb_env!(s);
parity_tests::test_atomic_marker_absent_when_opt_in_off(&s, &s, &s).await;
}
#[tokio::test]
async fn test_config_registered_at_or_after_event_not_eligible() {
skip_unless_dynamodb_env!(s);
parity_tests::test_config_registered_at_or_after_event_not_eligible(&s, &s, &s).await;
}
#[tokio::test]
async fn test_late_create_config_stamps_advanced_sequence() {
skip_unless_dynamodb_env!(s);
parity_tests::test_late_create_config_stamps_advanced_sequence(&s, &s, &s).await;
}
}