use std::collections::{BTreeMap, HashMap};
use std::time::Duration;
use aws_sdk_dynamodb::Client;
use aws_sdk_dynamodb::error::{ProvideErrorMetadata, SdkError};
use aws_sdk_dynamodb::types::{AttributeValue, KeysAndAttributes, WriteRequest};
use base64::Engine as _;
use serde_json::{Map, Value};
use tracing::Instrument as _;
use crate::{Deadline, RuntimeError};
pub type Item = HashMap<String, AttributeValue>;
pub const MAX_BATCH_WRITE_REQUESTS: usize = 25;
pub const MAX_BATCH_GET_KEYS: usize = 100;
const FIRST_BACKOFF: Duration = Duration::from_millis(25);
async fn back_off(retry: usize, deadline: Deadline) {
let pause = FIRST_BACKOFF * (1 << (retry - 1).min(6));
tokio::time::sleep(pause.min(deadline.remaining())).await;
}
#[must_use]
pub fn span(operation: &str, table: &str) -> tracing::Span {
tracing::info_span!(
"dynamodb",
"otel.name" = %format_args!("DynamoDB.{operation}"),
"otel.kind" = "CLIENT",
"db.system.name" = "aws.dynamodb",
"db.operation.name" = operation,
"aws.dynamodb.table_names" = table,
)
}
#[must_use]
pub fn is_conditional_failure<E, R>(error: &SdkError<E, R>) -> bool
where
E: ProvideErrorMetadata,
{
error
.as_service_error()
.and_then(ProvideErrorMetadata::code)
== Some("ConditionalCheckFailedException")
}
pub fn to_object(mut item: Item, without: &[&str]) -> Result<Map<String, Value>, RuntimeError> {
for name in without {
item.remove(*name);
}
serde_dynamo::aws_sdk_dynamodb_1::from_item(item)
.map_err(|error| RuntimeError::other("converting an item to JSON", error))
}
pub fn encode_cursor(key: Item) -> Result<String, RuntimeError> {
let key: BTreeMap<String, Value> = serde_dynamo::aws_sdk_dynamodb_1::from_item(key)
.map_err(|error| RuntimeError::other("encoding a page token", error))?;
let json = Value::Object(key.into_iter().collect()).to_string();
Ok(base64::engine::general_purpose::STANDARD.encode(json))
}
#[must_use]
pub fn decode_cursor(token: &str) -> Option<Item> {
let bytes = base64::engine::general_purpose::STANDARD
.decode(token.as_bytes())
.ok()?;
let key: Map<String, Value> = serde_json::from_slice(&bytes).ok()?;
serde_dynamo::aws_sdk_dynamodb_1::to_item(key).ok()
}
pub struct CursorSecret(ring::hmac::Key);
impl CursorSecret {
#[must_use]
pub fn new(secret: &[u8]) -> Self {
Self(ring::hmac::Key::new(ring::hmac::HMAC_SHA256, secret))
}
}
impl std::fmt::Debug for CursorSecret {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("CursorSecret(..)")
}
}
const URL_SAFE: base64::engine::GeneralPurpose = base64::engine::general_purpose::URL_SAFE_NO_PAD;
pub fn encode_cursor_signed(key: Item, secret: &CursorSecret) -> Result<String, RuntimeError> {
let key: BTreeMap<String, Value> = serde_dynamo::aws_sdk_dynamodb_1::from_item(key)
.map_err(|error| RuntimeError::other("encoding a page token", error))?;
let json = Value::Object(key.into_iter().collect()).to_string();
let tag = ring::hmac::sign(&secret.0, json.as_bytes());
Ok(format!(
"{}.{}",
URL_SAFE.encode(&json),
URL_SAFE.encode(tag.as_ref())
))
}
#[must_use]
pub fn decode_cursor_signed(token: &str, secret: &CursorSecret) -> Option<Item> {
let (payload, tag) = token.split_once('.')?;
let json = URL_SAFE.decode(payload).ok()?;
let tag = URL_SAFE.decode(tag).ok()?;
ring::hmac::verify(&secret.0, &json, &tag).ok()?;
let key: Map<String, Value> = serde_json::from_slice(&json).ok()?;
serde_dynamo::aws_sdk_dynamodb_1::to_item(key).ok()
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub struct PageLimits {
pub max_items: usize,
pub max_pages: usize,
}
impl PageLimits {
#[must_use]
pub const fn new(max_items: usize, max_pages: usize) -> Self {
Self {
max_items,
max_pages,
}
}
}
#[derive(Debug, Clone, PartialEq)]
#[non_exhaustive]
pub struct Page {
pub items: Vec<Item>,
pub next: Option<Item>,
pub stopped_by: Option<Stop>,
}
impl Page {
#[must_use]
pub fn is_complete(&self) -> bool {
self.next.is_none() && self.stopped_by.is_none()
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
#[non_exhaustive]
pub enum Stop {
Items,
Pages,
Deadline,
}
struct PageResponse {
items: Vec<Item>,
next: Option<Item>,
}
fn page_limit(own: Option<i32>, remaining: usize) -> i32 {
let cap = i32::try_from(remaining).unwrap_or(i32::MAX).max(1);
own.map_or(cap, |own| own.min(cap))
}
async fn read_bounded<F, Fut>(
limits: PageLimits,
deadline: Deadline,
mut fetch: F,
) -> Result<Page, RuntimeError>
where
F: FnMut(Option<Item>, usize, usize) -> Fut,
Fut: std::future::Future<Output = Result<PageResponse, RuntimeError>>,
{
let mut items: Vec<Item> = Vec::new();
let mut start: Option<Item> = None;
if limits.max_items == 0 {
return Ok(Page {
items,
next: None,
stopped_by: Some(Stop::Items),
});
}
for page in 0..limits.max_pages {
if deadline.is_expired() {
return Ok(Page {
items,
next: start,
stopped_by: Some(Stop::Deadline),
});
}
let remaining = limits.max_items.saturating_sub(items.len());
let response = match deadline.run(fetch(start.clone(), page, remaining)).await {
Ok(response) => response?,
Err(_) => {
return Ok(Page {
items,
next: start,
stopped_by: Some(Stop::Deadline),
});
}
};
items.extend(response.items);
start = response.next;
if start.is_none() && items.len() <= limits.max_items {
return Ok(Page {
items,
next: None,
stopped_by: None,
});
}
if items.len() >= limits.max_items {
items.truncate(limits.max_items);
return Ok(Page {
items,
next: start,
stopped_by: Some(Stop::Items),
});
}
}
Ok(Page {
items,
next: start,
stopped_by: Some(Stop::Pages),
})
}
pub async fn query_bounded<B>(
limits: PageLimits,
deadline: Deadline,
mut build: B,
) -> Result<Page, RuntimeError>
where
B: FnMut(Option<Item>) -> aws_sdk_dynamodb::operation::query::builders::QueryFluentBuilder,
{
read_bounded(limits, deadline, |start, page, remaining| {
let request = build(start);
let limit = page_limit(*request.get_limit(), remaining);
let request = request.limit(limit);
let table = request.get_table_name().clone().unwrap_or_default();
async move {
let response = request
.send()
.instrument(span("Query", &table))
.await
.map_err(|error| RuntimeError::other(format!("query page {page}"), error))?;
Ok(PageResponse {
items: response.items.unwrap_or_default(),
next: response.last_evaluated_key,
})
}
})
.await
}
pub async fn scan_bounded<B>(
limits: PageLimits,
deadline: Deadline,
mut build: B,
) -> Result<Page, RuntimeError>
where
B: FnMut(Option<Item>) -> aws_sdk_dynamodb::operation::scan::builders::ScanFluentBuilder,
{
read_bounded(limits, deadline, |start, page, remaining| {
let request = build(start);
let limit = page_limit(*request.get_limit(), remaining);
let request = request.limit(limit);
let table = request.get_table_name().clone().unwrap_or_default();
async move {
let response = request
.send()
.instrument(span("Scan", &table))
.await
.map_err(|error| RuntimeError::other(format!("scan page {page}"), error))?;
Ok(PageResponse {
items: response.items.unwrap_or_default(),
next: response.last_evaluated_key,
})
}
})
.await
}
#[derive(Debug, Clone, Default, PartialEq)]
#[non_exhaustive]
pub struct BatchWriteOutcome {
pub written: usize,
pub unprocessed: Vec<WriteRequest>,
pub attempts: usize,
}
impl BatchWriteOutcome {
#[must_use]
pub fn is_complete(&self) -> bool {
self.unprocessed.is_empty()
}
}
pub async fn batch_write(
client: &Client,
table: &str,
requests: Vec<WriteRequest>,
max_attempts: usize,
deadline: Deadline,
) -> Result<BatchWriteOutcome, RuntimeError> {
let total = requests.len();
let mut outcome = BatchWriteOutcome::default();
let mut pending: Vec<_> = requests;
let mut retry = 0;
while !pending.is_empty() && outcome.attempts < max_attempts {
if retry > 0 {
back_off(retry, deadline).await;
}
if deadline.is_expired() {
break;
}
outcome.attempts += 1;
let chunk: Vec<_> = pending
.drain(..pending.len().min(MAX_BATCH_WRITE_REQUESTS))
.collect();
let sent = chunk.len();
let response = deadline
.run(
client
.batch_write_item()
.request_items(table, chunk)
.send()
.instrument(span("BatchWriteItem", table)),
)
.await?
.map_err(|error| RuntimeError::other(format!("batch write to {table}"), error))?;
let returned = response
.unprocessed_items
.unwrap_or_default()
.remove(table)
.unwrap_or_default();
outcome.written += sent - returned.len();
retry = if returned.is_empty() { 0 } else { retry + 1 };
let mut next = returned;
next.extend(pending);
pending = next;
}
outcome.unprocessed = pending;
debug_assert!(outcome.written + outcome.unprocessed.len() <= total);
Ok(outcome)
}
#[derive(Debug, Clone, Default, PartialEq)]
#[non_exhaustive]
pub struct BatchGetOutcome {
pub items: Vec<Item>,
pub unprocessed: Vec<Item>,
}
impl BatchGetOutcome {
#[must_use]
pub fn is_complete(&self) -> bool {
self.unprocessed.is_empty()
}
}
pub async fn batch_get<F>(
client: &Client,
table: &str,
keys: Vec<Item>,
max_attempts: usize,
deadline: Deadline,
configure: F,
) -> Result<BatchGetOutcome, RuntimeError>
where
F: Fn(
aws_sdk_dynamodb::types::builders::KeysAndAttributesBuilder,
) -> aws_sdk_dynamodb::types::builders::KeysAndAttributesBuilder,
{
let mut read = BatchGetOutcome::default();
let mut batches = keys.chunks(MAX_BATCH_GET_KEYS).map(<[Item]>::to_vec);
while let Some(mut pending) = batches.next() {
for attempt in 0..max_attempts {
if pending.is_empty() {
break;
}
if attempt > 0 {
back_off(attempt, deadline).await;
}
if deadline.is_expired() {
break;
}
let request = configure(KeysAndAttributes::builder().set_keys(Some(pending)))
.build()
.map_err(|error| RuntimeError::other("building a batch read", error))?;
let mut response = deadline
.run(
client
.batch_get_item()
.request_items(table, request)
.send()
.instrument(span("BatchGetItem", table)),
)
.await?
.map_err(|error| RuntimeError::other(format!("batch read from {table}"), error))?;
read.items.extend(
response
.responses
.as_mut()
.and_then(|responses| responses.remove(table))
.unwrap_or_default(),
);
pending = response
.unprocessed_keys
.unwrap_or_default()
.remove(table)
.map(|unprocessed| unprocessed.keys)
.unwrap_or_default();
}
if !pending.is_empty() {
read.unprocessed.extend(pending);
read.unprocessed.extend(batches.flatten());
break;
}
}
Ok(read)
}