use std::future::Future;
use serde::Serialize;
use serde::de::DeserializeOwned;
use crate::BoxError;
mod preview;
pub use preview::{PreviewConfig, PreviewConfigBuilder, PreviewField, PreviewMode, build_preview};
pub trait Serdes<T>: Send + Sync + 'static {
fn serialize(
&self,
value: T,
context: SerdesContext,
) -> impl Future<Output = Result<String, BoxError>> + Send;
fn deserialize(
&self,
wire: String,
context: SerdesContext,
) -> impl Future<Output = Result<T, BoxError>> + Send;
}
impl<T, S> Serdes<T> for std::sync::Arc<S>
where
S: Serdes<T> + ?Sized,
{
fn serialize(
&self,
value: T,
context: SerdesContext,
) -> impl Future<Output = Result<String, BoxError>> + Send {
self.as_ref().serialize(value, context)
}
fn deserialize(
&self,
wire: String,
context: SerdesContext,
) -> impl Future<Output = Result<T, BoxError>> + Send {
self.as_ref().deserialize(wire, context)
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct JsonSerdes;
impl<T> Serdes<T> for JsonSerdes
where
T: Serialize + DeserializeOwned + Send + 'static,
{
fn serialize(
&self,
value: T,
_context: SerdesContext,
) -> impl Future<Output = Result<String, BoxError>> + Send {
std::future::ready(serde_json::to_string(&value).map_err(Into::into))
}
fn deserialize(
&self,
wire: String,
_context: SerdesContext,
) -> impl Future<Output = Result<T, BoxError>> + Send {
std::future::ready(serde_json::from_str(&wire).map_err(Into::into))
}
}
pub trait PayloadSerializer: Send + Sync + 'static {
fn serialize_payload<T>(
&self,
value: T,
context: SerdesContext,
) -> impl Future<Output = Result<String, BoxError>> + Send
where
T: Serialize + Send + 'static;
}
impl PayloadSerializer for JsonSerdes {
fn serialize_payload<T>(
&self,
value: T,
_context: SerdesContext,
) -> impl Future<Output = Result<String, BoxError>> + Send
where
T: Serialize + Send + 'static,
{
std::future::ready(serde_json::to_string(&value).map_err(Into::into))
}
}
impl<S> PayloadSerializer for std::sync::Arc<S>
where
S: PayloadSerializer + ?Sized,
{
fn serialize_payload<T>(
&self,
value: T,
context: SerdesContext,
) -> impl Future<Output = Result<String, BoxError>> + Send
where
T: Serialize + Send + 'static,
{
self.as_ref().serialize_payload(value, context)
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct PassthroughSerdes;
impl Serdes<String> for PassthroughSerdes {
fn serialize(
&self,
value: String,
_context: SerdesContext,
) -> impl Future<Output = Result<String, BoxError>> + Send {
std::future::ready(Ok(value))
}
fn deserialize(
&self,
wire: String,
_context: SerdesContext,
) -> impl Future<Output = Result<String, BoxError>> + Send {
std::future::ready(Ok(wire))
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum PayloadOrigin {
SdkRecord,
External,
}
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct SerdesContext {
operation_id: String,
durable_execution_arn: String,
origin: PayloadOrigin,
}
impl SerdesContext {
#[must_use]
pub fn new(operation_id: impl Into<String>, durable_execution_arn: impl Into<String>) -> Self {
Self {
operation_id: operation_id.into(),
durable_execution_arn: durable_execution_arn.into(),
origin: PayloadOrigin::SdkRecord,
}
}
pub fn operation_id(&self) -> &str {
&self.operation_id
}
pub fn durable_execution_arn(&self) -> &str {
&self.durable_execution_arn
}
pub fn origin(&self) -> PayloadOrigin {
self.origin
}
#[must_use]
pub fn with_origin(mut self, origin: PayloadOrigin) -> Self {
self.origin = origin;
self
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum FileSystemSerdesMode {
Always,
Overflow,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum FileSystemPathEncoding {
Uri,
Hash,
}
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct FileSystemSerdesConfig {
pub storage_mode: FileSystemSerdesMode,
pub path_encoding: FileSystemPathEncoding,
pub overflow_threshold_bytes: usize,
preview: Option<PreviewFn>,
}
type PreviewHook = dyn Fn(&str) -> Result<Option<String>, BoxError> + Send + Sync;
#[derive(Clone)]
struct PreviewFn(std::sync::Arc<PreviewHook>);
impl std::fmt::Debug for PreviewFn {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("PreviewFn")
}
}
impl Default for FileSystemSerdesConfig {
fn default() -> Self {
Self {
storage_mode: FileSystemSerdesMode::Always,
path_encoding: FileSystemPathEncoding::Uri,
overflow_threshold_bytes: (256 * 1024) - 1024,
preview: None,
}
}
}
impl FileSystemSerdesConfig {
pub fn builder() -> FileSystemSerdesConfigBuilder {
FileSystemSerdesConfigBuilder::default()
}
}
#[derive(Debug, Default)]
#[must_use = "builders do nothing unless .build() is called"]
#[non_exhaustive]
pub struct FileSystemSerdesConfigBuilder {
storage_mode: Option<FileSystemSerdesMode>,
path_encoding: Option<FileSystemPathEncoding>,
overflow_threshold_bytes: Option<usize>,
preview: Option<PreviewFn>,
}
impl FileSystemSerdesConfigBuilder {
pub fn storage_mode(mut self, mode: FileSystemSerdesMode) -> Self {
self.storage_mode = Some(mode);
self
}
pub fn path_encoding(mut self, encoding: FileSystemPathEncoding) -> Self {
self.path_encoding = Some(encoding);
self
}
pub fn overflow_threshold_bytes(mut self, bytes: usize) -> Self {
self.overflow_threshold_bytes = Some(bytes);
self
}
pub fn preview<F>(mut self, f: F) -> Self
where
F: Fn(&str) -> Result<Option<String>, BoxError> + Send + Sync + 'static,
{
self.preview = Some(PreviewFn(std::sync::Arc::new(f)));
self
}
#[must_use]
pub fn build(self) -> FileSystemSerdesConfig {
let defaults = FileSystemSerdesConfig::default();
FileSystemSerdesConfig {
storage_mode: self.storage_mode.unwrap_or(defaults.storage_mode),
path_encoding: self.path_encoding.unwrap_or(defaults.path_encoding),
overflow_threshold_bytes: self
.overflow_threshold_bytes
.unwrap_or(defaults.overflow_threshold_bytes),
preview: self.preview,
}
}
}
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct FileSystemSerdes {
base_path: String,
config: FileSystemSerdesConfig,
}
impl FileSystemSerdes {
#[must_use]
pub fn new(base_path: impl Into<String>) -> Self {
Self {
base_path: base_path.into(),
config: FileSystemSerdesConfig::default(),
}
}
#[must_use]
pub fn with_config(base_path: impl Into<String>, config: FileSystemSerdesConfig) -> Self {
Self {
base_path: base_path.into(),
config,
}
}
fn serialize_sync<T: Serialize>(
&self,
value: &T,
context: &SerdesContext,
) -> Result<String, BoxError> {
let json_str = serde_json::to_string(value).map_err(|e| -> BoxError {
Box::new(FileSystemSerdesError::with_source(
"failed to render value as JSON",
e,
))
})?;
match self.config.storage_mode {
FileSystemSerdesMode::Always => self.write_file_envelope(&json_str, context),
FileSystemSerdesMode::Overflow => {
let inline = inline_envelope(&json_str);
if inline.len() > self.config.overflow_threshold_bytes {
self.write_file_envelope(&json_str, context)
} else {
Ok(inline)
}
}
}
}
fn write_file_envelope(
&self,
json_str: &str,
context: &SerdesContext,
) -> Result<String, BoxError> {
let preview = match &self.config.preview {
Some(PreviewFn(hook)) => match hook(json_str) {
Ok(Some(text)) => match preview::parse_ordered(&text) {
Some(node @ preview::Node::Object(_)) => Some(preview::render_compact(&node)),
_ => {
return Err(Box::new(FileSystemSerdesError::new(
"preview must be a JSON object",
)));
}
},
Ok(None) => None,
Err(e) => {
return Err(Box::new(FileSystemSerdesError::with_source(
"preview function failed",
e,
)));
}
},
None => None,
};
let relative_path = self.write_to_file(json_str, context)?;
Ok(file_envelope(&relative_path, preview.as_deref()))
}
fn deserialize_sync<T: DeserializeOwned>(
&self,
envelope: &str,
context: &SerdesContext,
) -> Result<T, BoxError> {
#[derive(serde::Deserialize)]
struct EnvelopeMarker {
version: u64,
kind: String,
}
#[derive(serde::Deserialize)]
struct Envelope<'a> {
#[serde(rename = "__aws_durable_serdes")]
marker: Option<EnvelopeMarker>,
#[serde(borrow)]
value: Option<&'a serde_json::value::RawValue>,
#[serde(borrow)]
file: Option<std::borrow::Cow<'a, str>>,
#[serde(borrow)]
data: Option<std::borrow::Cow<'a, str>>,
}
if context.origin() == PayloadOrigin::External {
return serde_json::from_str(envelope).map_err(|e| -> BoxError {
Box::new(FileSystemSerdesError::with_source(
"invalid external payload JSON",
e,
))
});
}
let parsed: Envelope<'_> = serde_json::from_str(envelope).map_err(|e| -> BoxError {
Box::new(FileSystemSerdesError::with_source(
"invalid envelope JSON",
e,
))
})?;
if let Some(marker) = parsed.marker {
if marker.version != ENVELOPE_VERSION {
return Err(Box::new(FileSystemSerdesError::new(format!(
"unsupported envelope version {} (this SDK reads version {ENVELOPE_VERSION})",
marker.version
))));
}
return match marker.kind.as_str() {
"inline" => {
let value = parsed.value.ok_or_else(|| -> BoxError {
Box::new(FileSystemSerdesError::new(
"inline envelope is missing its 'value' field".to_owned(),
))
})?;
serde_json::from_str(value.get()).map_err(|e| -> BoxError {
Box::new(FileSystemSerdesError::with_source(
"invalid inline data in envelope",
e,
))
})
}
"file" => {
let relative = parsed.file.ok_or_else(|| -> BoxError {
Box::new(FileSystemSerdesError::new(
"file envelope is missing its 'file' field".to_owned(),
))
})?;
self.read_relative_file(&relative)
}
other => Err(Box::new(FileSystemSerdesError::new(format!(
"unsupported envelope kind '{other}'"
)))),
};
}
if let Some(file_path) = parsed.file.as_deref() {
self.read_contained_file(std::path::Path::new(file_path), file_path)
} else if let Some(data) = parsed.data {
serde_json::from_str(&data).map_err(|e| -> BoxError {
Box::new(FileSystemSerdesError::with_source(
"invalid inline data in envelope",
e,
))
})
} else {
Err(Box::new(FileSystemSerdesError::new(
"envelope carries no '__aws_durable_serdes' marker and no unversioned 'file' or 'data' field"
.to_owned(),
)))
}
}
fn read_relative_file<T: DeserializeOwned>(&self, relative: &str) -> Result<T, BoxError> {
let rel = std::path::Path::new(relative);
if rel.is_absolute() {
return Err(Box::new(FileSystemSerdesError::new(format!(
"file reference '{relative}' must be relative to base_path"
))));
}
let joined = format!("{}/{relative}", self.base_path);
let cleaned_base = path_clean(&self.base_path);
let cleaned_joined = path_clean(&joined);
if !std::path::Path::new(&cleaned_joined).starts_with(std::path::Path::new(&cleaned_base)) {
return Err(Box::new(FileSystemSerdesError::new(format!(
"file reference '{relative}' escapes base_path '{}'",
self.base_path
))));
}
self.read_contained_file(std::path::Path::new(&joined), relative)
}
fn read_contained_file<T: DeserializeOwned>(
&self,
path: &std::path::Path,
display_path: &str,
) -> Result<T, BoxError> {
let canonical_base = std::fs::canonicalize(&self.base_path).map_err(|e| {
filesystem_call_failure(
format!("failed to canonicalize base_path '{}'", self.base_path),
e,
)
})?;
let canonical_file = std::fs::canonicalize(path).map_err(|e| {
filesystem_call_failure(format!("failed to resolve file '{display_path}'"), e)
})?;
if !canonical_file.starts_with(&canonical_base) {
return Err(Box::new(FileSystemSerdesError::new(format!(
"file reference '{display_path}' escapes base_path '{}'",
self.base_path
))));
}
let contents = std::fs::read_to_string(&canonical_file).map_err(|e| {
filesystem_call_failure(format!("failed to read file '{display_path}'"), e)
})?;
serde_json::from_str(&contents).map_err(|e| -> BoxError {
Box::new(FileSystemSerdesError::with_source(
format!("invalid JSON in file '{display_path}'"),
e,
))
})
}
fn write_to_file(
&self,
json_str: &str,
context: &SerdesContext,
) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
let relative_dir = self.resolve_execution_dir(context)?;
let dir_path = format!("{}/{relative_dir}", self.base_path);
std::fs::create_dir_all(&dir_path).map_err(|e| {
filesystem_call_failure(format!("failed to create directory '{dir_path}'"), e)
})?;
self.assert_canonical_containment(&dir_path)?;
let encoded_op = encode_segment(&context.operation_id, self.config.path_encoding);
let file_name =
publish_unique_payload_file(&dir_path, &encoded_op, json_str, unique_file_suffix)?;
Ok(format!("{relative_dir}/{file_name}"))
}
fn resolve_execution_dir(
&self,
context: &SerdesContext,
) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
let relative_dir = match self.config.path_encoding {
FileSystemPathEncoding::Uri => {
if let Some(parts) = parse_durable_execution_arn(&context.durable_execution_arn) {
let enc_fn = encode_segment(&parts.function_name, self.config.path_encoding);
let enc_exec = encode_segment(&parts.execution_name, self.config.path_encoding);
let enc_inv = encode_segment(&parts.invocation_id, self.config.path_encoding);
format!("{enc_fn}/{enc_exec}/{enc_inv}")
} else {
percent_encode(&context.durable_execution_arn)
}
}
FileSystemPathEncoding::Hash => sha256_hex(&context.durable_execution_arn),
};
let cleaned_base = path_clean(&self.base_path);
let cleaned_dir = path_clean(&format!("{}/{relative_dir}", self.base_path));
if !std::path::Path::new(&cleaned_dir).starts_with(std::path::Path::new(&cleaned_base)) {
return Err(Box::new(FileSystemSerdesError::new(format!(
"resolved path '{}' escapes base_path '{}'",
cleaned_dir, self.base_path
))));
}
Ok(relative_dir)
}
fn assert_canonical_containment(
&self,
dir_path: &str,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let canonical_dir = std::fs::canonicalize(dir_path).map_err(|e| {
filesystem_call_failure(format!("failed to canonicalize directory '{dir_path}'"), e)
})?;
let canonical_base = std::fs::canonicalize(&self.base_path).map_err(|e| {
filesystem_call_failure(
format!("failed to canonicalize base_path '{}'", self.base_path),
e,
)
})?;
if !canonical_dir.starts_with(&canonical_base) {
return Err(Box::new(FileSystemSerdesError::new(format!(
"canonical path '{}' escapes base_path '{}' (symlink redirection)",
canonical_dir.display(),
canonical_base.display()
))));
}
Ok(())
}
}
impl<T> Serdes<T> for FileSystemSerdes
where
T: Serialize + DeserializeOwned + Send + 'static,
{
fn serialize(
&self,
value: T,
context: SerdesContext,
) -> impl Future<Output = Result<String, BoxError>> + Send {
let this = self.clone();
async move {
crate::task::run_blocking(move || this.serialize_sync(&value, &context))
.await?
.into_value()
}
}
fn deserialize(
&self,
wire: String,
context: SerdesContext,
) -> impl Future<Output = Result<T, BoxError>> + Send {
let this = self.clone();
async move {
crate::task::run_blocking(move || this.deserialize_sync(&wire, &context))
.await?
.into_value()
}
}
}
#[derive(Debug)]
#[non_exhaustive]
pub struct FileSystemSerdesError {
context: String,
source: Option<crate::error::Source>,
}
impl FileSystemSerdesError {
#[must_use]
pub fn new(context: impl Into<String>) -> Self {
Self {
context: context.into(),
source: None,
}
}
pub(crate) fn with_source(
context: impl Into<String>,
source: impl Into<crate::error::Source>,
) -> Self {
Self {
context: context.into(),
source: Some(source.into()),
}
}
}
impl std::fmt::Display for FileSystemSerdesError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
struct Frame<'a>(&'a str);
impl std::fmt::Display for Frame<'_> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "filesystem serdes error: {}", self.0)
}
}
if f.alternate() {
crate::error::write_chain(f, &Frame(&self.context), std::error::Error::source(self))
} else {
write!(f, "{}", Frame(&self.context))
}
}
}
impl std::error::Error for FileSystemSerdesError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
self.source
.as_deref()
.map(|s| s as &(dyn std::error::Error + 'static))
}
}
#[derive(Debug)]
#[non_exhaustive]
pub struct RetryableSerdesError {
source: BoxError,
}
impl RetryableSerdesError {
#[must_use]
pub fn new(source: impl Into<BoxError>) -> Self {
Self {
source: source.into(),
}
}
}
impl std::fmt::Display for RetryableSerdesError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
const FRAME: &str = "retryable serdes failure";
if f.alternate() {
crate::error::write_chain(f, &FRAME, std::error::Error::source(self))
} else {
f.write_str(FRAME)
}
}
}
impl std::error::Error for RetryableSerdesError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
Some(&*self.source)
}
}
pub(crate) fn is_retryable_serdes_error(err: &(dyn std::error::Error + 'static)) -> bool {
let mut current = Some(err);
while let Some(e) = current {
if e.downcast_ref::<RetryableSerdesError>().is_some() {
return true;
}
current = e.source();
}
false
}
fn filesystem_call_failure(context: String, source: impl Into<crate::error::Source>) -> BoxError {
Box::new(RetryableSerdesError::new(
FileSystemSerdesError::with_source(context, source),
))
}
const ENVELOPE_VERSION: u64 = 1;
fn inline_envelope(json_str: &str) -> String {
format!(
r#"{{"__aws_durable_serdes":{{"version":{ENVELOPE_VERSION},"kind":"inline"}},"value":{json_str}}}"#
)
}
fn file_envelope(relative_path: &str, preview: Option<&str>) -> String {
let escaped = json_escape_string(relative_path);
match preview {
Some(preview) => format!(
r#"{{"__aws_durable_serdes":{{"version":{ENVELOPE_VERSION},"kind":"file"}},"file":"{escaped}","preview":{preview}}}"#
),
None => format!(
r#"{{"__aws_durable_serdes":{{"version":{ENVELOPE_VERSION},"kind":"file"}},"file":"{escaped}"}}"#
),
}
}
const MAX_PUBLISH_ATTEMPTS: u32 = 16;
fn publish_unique_payload_file(
dir_path: &str,
encoded_op: &str,
json_str: &str,
mut candidate_suffix: impl FnMut() -> String,
) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
use std::io::Write as _;
for _ in 0..MAX_PUBLISH_ATTEMPTS {
let file_name = format!("{encoded_op}-{}.json", candidate_suffix());
let file_path = format!("{dir_path}/{file_name}");
let staging_path = format!("{file_path}.{}.tmp", unique_file_suffix());
let mut staging = match std::fs::OpenOptions::new()
.write(true)
.create_new(true)
.open(&staging_path)
{
Ok(f) => f,
Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue,
Err(e) => {
return Err(filesystem_call_failure(
format!("failed to write file '{staging_path}'"),
e,
));
}
};
if let Err(e) = staging.write_all(json_str.as_bytes()) {
drop(staging);
let _ = std::fs::remove_file(&staging_path);
return Err(filesystem_call_failure(
format!("failed to write file '{staging_path}'"),
e,
));
}
drop(staging);
let linked = std::fs::hard_link(&staging_path, &file_path);
let _ = std::fs::remove_file(&staging_path);
match linked {
Ok(()) => return Ok(file_name),
Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {}
Err(e) => {
return Err(filesystem_call_failure(
format!("failed to move file into place at '{file_path}'"),
e,
));
}
}
}
Err(Box::new(RetryableSerdesError::new(
FileSystemSerdesError::new(format!(
"failed to publish a unique payload file under '{dir_path}' after {MAX_PUBLISH_ATTEMPTS} attempts"
)),
)))
}
fn unique_file_suffix() -> String {
use std::sync::atomic::{AtomicU64, Ordering};
static COUNTER: AtomicU64 = AtomicU64::new(0);
let nanos = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_or(0, |d| d.as_nanos());
let count = COUNTER.fetch_add(1, Ordering::Relaxed);
format!("{nanos:x}-{:x}-{count:x}", std::process::id())
}
struct DurableExecutionArnParts {
function_name: String,
execution_name: String,
invocation_id: String,
}
fn parse_durable_execution_arn(arn: &str) -> Option<DurableExecutionArnParts> {
let (prefix, exec_suffix) = arn.split_once("/durable-execution/")?;
let function_name = prefix.rsplit(':').nth(1)?;
if function_name.is_empty() {
return None;
}
let (execution_name, invocation_id) = exec_suffix.split_once('/')?;
Some(DurableExecutionArnParts {
function_name: function_name.to_owned(),
execution_name: execution_name.to_owned(),
invocation_id: invocation_id.to_owned(),
})
}
fn percent_encode(input: &str) -> String {
let mut encoded = String::with_capacity(input.len());
for byte in input.bytes() {
if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~') {
encoded.push(byte as char);
} else {
use std::fmt::Write;
let _ = write!(encoded, "%{byte:02X}");
}
}
encoded
}
fn encode_segment(value: &str, encoding: FileSystemPathEncoding) -> String {
match encoding {
FileSystemPathEncoding::Hash => sha256_hex(value),
FileSystemPathEncoding::Uri => {
let encoded = percent_encode(value);
if encoded == "." || encoded == ".." {
format!("%2E{}", &encoded[1..])
} else {
encoded
}
}
}
}
fn sha256_hex(input: &str) -> String {
use sha2::Digest;
let hash = sha2::Sha256::digest(input.as_bytes());
let mut hex = String::with_capacity(64);
for byte in hash {
use std::fmt::Write;
let _ = write!(hex, "{byte:02x}");
}
hex
}
fn json_escape_string(input: &str) -> String {
let mut escaped = String::with_capacity(input.len());
for ch in input.chars() {
match ch {
'\\' => escaped.push_str("\\\\"),
'"' => escaped.push_str("\\\""),
'\n' => escaped.push_str("\\n"),
'\r' => escaped.push_str("\\r"),
'\t' => escaped.push_str("\\t"),
c => escaped.push(c),
}
}
escaped
}
fn path_clean(path: &str) -> String {
use std::path::{Component, Path};
let mut components: Vec<&std::ffi::OsStr> = Vec::new();
let p = Path::new(path);
let mut has_root = false;
for component in p.components() {
match component {
Component::RootDir => {
has_root = true;
components.clear();
}
Component::CurDir => {}
Component::ParentDir => {
components.pop();
}
Component::Normal(c) => {
components.push(c);
}
Component::Prefix(prefix) => {
components.clear();
components.push(prefix.as_os_str());
}
}
}
let joined: std::path::PathBuf = components.iter().collect();
if has_root {
format!("/{}", joined.to_string_lossy())
} else {
joined.to_string_lossy().to_string()
}
}
#[cfg(test)]
pub(crate) mod test_support {
use super::{PayloadSerializer, Serdes, SerdesContext};
use serde::Serialize;
use serde::de::DeserializeOwned;
use std::sync::{Arc, Mutex};
type BoxError = Box<dyn std::error::Error + Send + Sync>;
#[derive(Debug)]
pub(crate) struct HexEnvelopeSerdes;
impl<T> Serdes<T> for HexEnvelopeSerdes
where
T: Serialize + DeserializeOwned + Send + 'static,
{
#[expect(clippy::unused_async_trait_impl)]
async fn serialize(&self, value: T, _context: SerdesContext) -> Result<String, BoxError> {
Ok(hex_envelope(&serde_json::to_string(&value)?))
}
#[expect(clippy::unused_async_trait_impl)]
async fn deserialize(&self, wire: String, _context: SerdesContext) -> Result<T, BoxError> {
Ok(serde_json::from_str(&hex_decode(&wire)?)?)
}
}
pub(crate) fn hex_envelope(json_str: &str) -> String {
use std::fmt::Write;
let mut out = String::with_capacity(5 + json_str.len() * 2);
out.push_str("HEX1:");
for byte in json_str.as_bytes() {
let _ = write!(out, "{byte:02x}");
}
out
}
pub(crate) fn hex_envelope_of(value: &serde_json::Value) -> String {
hex_envelope(&value.to_string())
}
fn hex_decode(payload: &str) -> Result<String, BoxError> {
let hex = payload
.strip_prefix("HEX1:")
.ok_or_else(|| -> BoxError { "missing HEX1: marker".into() })?;
if hex.len() % 2 != 0 {
return Err("odd-length hex body".into());
}
let mut bytes = Vec::with_capacity(hex.len() / 2);
for pair in hex.as_bytes().chunks(2) {
let s = std::str::from_utf8(pair)?;
bytes.push(u8::from_str_radix(s, 16)?);
}
Ok(String::from_utf8(bytes)?)
}
#[derive(Debug, Clone, Default)]
pub(crate) struct RecordingSerdes {
serialize_inputs: Arc<Mutex<Vec<serde_json::Value>>>,
deserialize_outputs: Arc<Mutex<Vec<serde_json::Value>>>,
}
impl RecordingSerdes {
pub(crate) fn new() -> Self {
Self::default()
}
pub(crate) fn serialize_inputs(&self) -> Vec<serde_json::Value> {
self.serialize_inputs
.lock()
.map(|seen| seen.clone())
.unwrap_or_default()
}
pub(crate) fn distinct_serialize_inputs(&self) -> Vec<serde_json::Value> {
let mut distinct: Vec<serde_json::Value> = Vec::new();
for value in self.serialize_inputs() {
if !distinct.contains(&value) {
distinct.push(value);
}
}
distinct
}
pub(crate) fn deserialize_outputs(&self) -> Vec<serde_json::Value> {
self.deserialize_outputs
.lock()
.map(|seen| seen.clone())
.unwrap_or_default()
}
}
impl<T> Serdes<T> for RecordingSerdes
where
T: Serialize + DeserializeOwned + Send + 'static,
{
#[expect(clippy::unused_async_trait_impl)]
async fn serialize(&self, value: T, _context: SerdesContext) -> Result<String, BoxError> {
let json = serde_json::to_string(&value)?;
if let Ok(mut seen) = self.serialize_inputs.lock() {
seen.push(serde_json::from_str(&json)?);
}
Ok(hex_envelope(&json))
}
#[expect(clippy::unused_async_trait_impl)]
async fn deserialize(&self, wire: String, _context: SerdesContext) -> Result<T, BoxError> {
let json = hex_decode(&wire)?;
if let Ok(mut seen) = self.deserialize_outputs.lock() {
seen.push(serde_json::from_str(&json)?);
}
Ok(serde_json::from_str(&json)?)
}
}
impl PayloadSerializer for RecordingSerdes {
#[expect(clippy::unused_async_trait_impl)]
async fn serialize_payload<T>(
&self,
value: T,
_context: SerdesContext,
) -> Result<String, BoxError>
where
T: Serialize + Send + 'static,
{
let json = serde_json::to_string(&value)?;
if let Ok(mut seen) = self.serialize_inputs.lock() {
seen.push(serde_json::from_str(&json)?);
}
Ok(hex_envelope(&json))
}
}
}
#[cfg(test)]
#[expect(clippy::indexing_slicing)] mod tests {
use super::*;
#[derive(Serialize)]
struct SerializeOnly {
amount: u32,
}
#[tokio::test]
async fn json_payload_serializer_matches_json_serdes() {
let context = SerdesContext::new("invoke-1", "arn:test");
let value = serde_json::json!({"b": [1, 2], "a": "x"});
let payload = JsonSerdes
.serialize_payload(value.clone(), context.clone())
.await
.unwrap();
let serdes = Serdes::serialize(&JsonSerdes, value, context.clone())
.await
.unwrap();
assert_eq!(payload, serdes);
let only = JsonSerdes
.serialize_payload(SerializeOnly { amount: 100 }, context)
.await
.unwrap();
assert_eq!(only, r#"{"amount":100}"#);
}
#[tokio::test]
async fn arc_forwards_payload_serializer() {
let context = SerdesContext::new("invoke-1", "arn:test");
let shared = std::sync::Arc::new(JsonSerdes);
let forwarded = shared
.serialize_payload(SerializeOnly { amount: 7 }, context.clone())
.await
.unwrap();
let direct = JsonSerdes
.serialize_payload(SerializeOnly { amount: 7 }, context)
.await
.unwrap();
assert_eq!(forwarded, direct);
}
#[test]
fn percent_encode_basic() {
assert_eq!(percent_encode("hello"), "hello");
assert_eq!(percent_encode("a/b:c"), "a%2Fb%3Ac");
assert_eq!(percent_encode("step-1"), "step-1");
}
#[test]
fn sha256_hex_known_answer() {
assert_eq!(
sha256_hex("hello"),
"2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824"
);
}
#[test]
fn parse_valid_durable_execution_arn() {
let arn = "arn:aws:lambda:us-east-1:123456789012:function:my-fn:1/durable-execution/exec-abc/inv-123";
let parts = parse_durable_execution_arn(arn).expect("should parse");
assert_eq!(parts.function_name, "my-fn");
assert_eq!(parts.execution_name, "exec-abc");
assert_eq!(parts.invocation_id, "inv-123");
}
#[test]
fn parse_invalid_arn_returns_none() {
assert!(parse_durable_execution_arn("not-an-arn").is_none());
assert!(
parse_durable_execution_arn("arn:aws:lambda:us-east-1:123:function:fn:1").is_none()
);
}
fn count_files(dir: &std::path::Path) -> usize {
let Ok(entries) = std::fs::read_dir(dir) else {
return 0;
};
entries
.flatten()
.map(|entry| {
let path = entry.path();
if path.is_dir() { count_files(&path) } else { 1 }
})
.sum()
}
fn fs_context() -> SerdesContext {
SerdesContext::new(
"step-1".to_owned(),
"arn:aws:lambda:us-east-1:123:function:my-fn:1/durable-execution/exec-1/inv-1"
.to_owned(),
)
}
#[test]
#[expect(clippy::disallowed_methods)]
fn dropped_filesystem_serdes_write_finishes_before_idle() {
let tmp = std::env::temp_dir().join("fs_serdes_test_dropped_write");
let _ = std::fs::remove_dir_all(&tmp);
let runtime = tokio::runtime::Builder::new_current_thread()
.max_blocking_threads(1)
.build()
.expect("runtime");
runtime.block_on(async {
let (release, wait_for_release) = std::sync::mpsc::channel::<()>();
let occupier = tokio::task::spawn_blocking(move || {
let _ = wait_for_release.recv();
});
let coordinator = crate::coordinator::InvocationCoordinator::new();
let tasks = std::sync::Arc::clone(coordinator.tasks());
let serdes = FileSystemSerdes::new(tmp.to_string_lossy().to_string());
let mut write = Box::pin(Serdes::<serde_json::Value>::serialize(
&serdes,
serde_json::json!({"value": 1}),
fs_context(),
));
let polled = crate::task::test_support::with_current_coordinator(&coordinator, || {
write
.as_mut()
.poll(&mut std::task::Context::from_waker(std::task::Waker::noop()))
});
assert!(
polled.is_pending(),
"the write waits for the blocking thread"
);
drop(write);
assert_eq!(tasks.live(), 1, "the queued write is counted");
release.send(()).expect("the occupier waits");
occupier.await.expect("the occupier returns");
tasks.wait_idle().await;
assert_eq!(tasks.live(), 0);
});
assert_eq!(count_files(&tmp), 1, "the payload file was written");
let _ = std::fs::remove_dir_all(&tmp);
}
#[test]
fn filesystem_serdes_without_runtime_is_unmarked() {
let tmp = std::env::temp_dir().join("fs_serdes_test_no_runtime");
let serdes = FileSystemSerdes::new(tmp.to_string_lossy().to_string());
let polled = crate::task::test_support::poll_once_without_runtime(move || async move {
Serdes::<serde_json::Value>::serialize(&serdes, serde_json::json!(1), fs_context())
.await
});
let std::task::Poll::Ready(Err(err)) = polled else {
unreachable!("expected an immediate error, got {polled:?}");
};
assert!(
!is_retryable_serdes_error(err.as_ref()),
"a missing runtime is not a transient failure: {err}"
);
assert_eq!(err.to_string(), crate::task::NO_RUNTIME_MESSAGE);
}
#[tokio::test]
#[expect(clippy::disallowed_methods)]
async fn filesystem_serdes_blocking_panic_resumes_payload() {
#[derive(serde::Deserialize)]
struct PanicsOnSerialize;
impl Serialize for PanicsOnSerialize {
#[expect(clippy::panic)] fn serialize<S: serde::Serializer>(&self, _: S) -> Result<S::Ok, S::Error> {
panic!("blocking boom")
}
}
let tmp = std::env::temp_dir().join("fs_serdes_test_blocking_panic");
let _ = std::fs::remove_dir_all(&tmp);
let serdes = FileSystemSerdes::new(tmp.to_string_lossy().to_string());
let join_err = tokio::spawn(async move {
Serdes::<PanicsOnSerialize>::serialize(&serdes, PanicsOnSerialize, fs_context()).await
})
.await
.expect_err("the blocking panic resumes on the awaiting task");
assert!(join_err.is_panic(), "a panic, not a cancellation");
let payload = join_err.into_panic();
assert_eq!(payload.downcast_ref::<&str>(), Some(&"blocking boom"));
assert_eq!(count_files(&tmp), 0, "no payload file was written");
}
#[test]
fn filesystem_serdes_always_mode_round_trip() {
let tmp = std::env::temp_dir().join("fs_serdes_test_always");
let _ = std::fs::remove_dir_all(&tmp);
let serdes = FileSystemSerdes::new(tmp.to_string_lossy().to_string());
let ctx = SerdesContext::new(
"step-1".to_owned(),
"arn:aws:lambda:us-east-1:123:function:my-fn:1/durable-execution/exec-1/inv-1"
.to_owned(),
);
let input = serde_json::json!({"value": 42, "name": "test"});
let envelope = serdes
.serialize_sync(&input, &ctx)
.expect("serialize should succeed");
assert!(envelope.contains(r#""file""#), "envelope: {envelope}");
assert!(!envelope.contains(r#""data""#), "envelope: {envelope}");
let output = serdes
.deserialize_sync::<serde_json::Value>(&envelope, &ctx)
.expect("deserialize should succeed");
assert_eq!(output, input);
let _ = std::fs::remove_dir_all(&tmp);
}
#[test]
fn publish_never_clobbers_on_candidate_name_collision() {
let tmp = std::env::temp_dir().join(format!(
"fs_serdes_publish_collision_{}_{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.expect("clock after epoch")
.as_nanos()
));
std::fs::create_dir_all(&tmp).expect("create temp dir");
let dir = tmp.to_string_lossy().into_owned();
let name_a = publish_unique_payload_file(&dir, "op", r#""writer-a""#, || "dup".to_owned())
.expect("writer A publishes");
assert_eq!(name_a, "op-dup.json");
let mut draws = vec!["fresh".to_owned(), "dup".to_owned()];
let name_b = publish_unique_payload_file(&dir, "op", r#""writer-b""#, move || {
draws.pop().expect("enough draws")
})
.expect("writer B retries under a fresh name");
assert_eq!(name_b, "op-fresh.json");
assert_eq!(
std::fs::read_to_string(tmp.join("op-dup.json")).expect("read A"),
r#""writer-a""#,
"an existing published file must never be overwritten"
);
assert_eq!(
std::fs::read_to_string(tmp.join("op-fresh.json")).expect("read B"),
r#""writer-b""#
);
let err = publish_unique_payload_file(&dir, "op", r#""writer-c""#, || "dup".to_owned())
.expect_err("exhausted retries must fail");
assert!(is_retryable_serdes_error(&*err), "got: {err:#}");
let message = crate::error::chain_string(&*err);
assert!(message.contains("attempts"), "got: {message}");
assert_eq!(
std::fs::read_to_string(tmp.join("op-dup.json")).expect("read A again"),
r#""writer-a""#
);
let leftovers: Vec<_> = std::fs::read_dir(&tmp)
.expect("list dir")
.filter_map(Result::ok)
.filter(|e| e.file_name().to_string_lossy().ends_with(".tmp"))
.collect();
assert!(leftovers.is_empty(), "staging files leaked: {leftovers:?}");
let _ = std::fs::remove_dir_all(&tmp);
}
#[test]
fn filesystem_serdes_overflow_mode_small_inline() {
let tmp = std::env::temp_dir().join("fs_serdes_test_overflow_small");
let _ = std::fs::remove_dir_all(&tmp);
let config = FileSystemSerdesConfig::builder()
.storage_mode(FileSystemSerdesMode::Overflow)
.overflow_threshold_bytes(1024)
.build();
let serdes = FileSystemSerdes::with_config(tmp.to_string_lossy().to_string(), config);
let ctx = SerdesContext::new(
"step-2".to_owned(),
"arn:aws:lambda:us-east-1:123:function:fn:1/durable-execution/e/i".to_owned(),
);
let small = serde_json::json!({"x": 1});
let envelope = serdes
.serialize_sync(&small, &ctx)
.expect("serialize should succeed");
assert!(
envelope.contains(r#""__aws_durable_serdes""#),
"envelope: {envelope}"
);
assert!(envelope.contains(r#""value""#), "envelope: {envelope}");
assert!(!envelope.contains(r#""file""#), "envelope: {envelope}");
let output = serdes
.deserialize_sync::<serde_json::Value>(&envelope, &ctx)
.expect("deserialize should succeed");
assert_eq!(output, small);
let _ = std::fs::remove_dir_all(&tmp);
}
#[test]
fn filesystem_serdes_overflow_mode_large_to_file() {
let tmp = std::env::temp_dir().join("fs_serdes_test_overflow_large");
let _ = std::fs::remove_dir_all(&tmp);
let config = FileSystemSerdesConfig::builder()
.storage_mode(FileSystemSerdesMode::Overflow)
.overflow_threshold_bytes(50) .build();
let serdes = FileSystemSerdes::with_config(tmp.to_string_lossy().to_string(), config);
let ctx = SerdesContext::new(
"step-3".to_owned(),
"arn:aws:lambda:us-east-1:123:function:fn:1/durable-execution/e/i".to_owned(),
);
let large =
serde_json::json!({"big": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"});
let envelope = serdes
.serialize_sync(&large, &ctx)
.expect("serialize should succeed");
assert!(envelope.contains(r#""file""#), "envelope: {envelope}");
let output = serdes
.deserialize_sync::<serde_json::Value>(&envelope, &ctx)
.expect("deserialize should succeed");
assert_eq!(output, large);
let _ = std::fs::remove_dir_all(&tmp);
}
#[test]
fn filesystem_serdes_hash_encoding() {
let tmp = std::env::temp_dir().join("fs_serdes_test_hash");
let _ = std::fs::remove_dir_all(&tmp);
let config = FileSystemSerdesConfig::builder()
.path_encoding(FileSystemPathEncoding::Hash)
.build();
let serdes = FileSystemSerdes::with_config(tmp.to_string_lossy().to_string(), config);
let ctx = SerdesContext::new(
"step/with:special".to_owned(),
"some-weird-arn/with/slashes".to_owned(),
);
let value = serde_json::json!("hello");
let envelope = serdes
.serialize_sync(&value, &ctx)
.expect("serialize should succeed");
let parsed: serde_json::Value = serde_json::from_str(&envelope).unwrap();
let file_path = parsed.get("file").and_then(|v| v.as_str()).unwrap();
let segments: Vec<&str> = file_path.split('/').collect();
let dir_segment = segments.get(segments.len() - 2).unwrap();
assert_eq!(dir_segment.len(), 64, "dir hash: {dir_segment}");
let output = serdes
.deserialize_sync::<serde_json::Value>(&envelope, &ctx)
.expect("deserialize should succeed");
assert_eq!(output, value);
let _ = std::fs::remove_dir_all(&tmp);
}
#[test]
fn filesystem_serdes_error_on_missing_file() {
let tmp = std::env::temp_dir().join("fs_serdes_test_missing_file");
let _ = std::fs::remove_dir_all(&tmp);
std::fs::create_dir_all(&tmp).unwrap();
let serdes = FileSystemSerdes::new(tmp.to_string_lossy().to_string());
let ctx = SerdesContext::new("x".to_owned(), "y".to_owned());
let envelope = file_envelope("does-not-exist.json", None);
let result = serdes.deserialize_sync::<serde_json::Value>(&envelope, &ctx);
assert!(result.is_err());
let err_msg = crate::error::chain_string(&*result.unwrap_err());
assert!(err_msg.contains("failed to resolve file"), "err: {err_msg}");
let unversioned = format!(
r#"{{"file":"{}/does-not-exist.json"}}"#,
tmp.to_string_lossy()
);
let result = serdes.deserialize_sync::<serde_json::Value>(&unversioned, &ctx);
assert!(result.is_err());
let err_msg = crate::error::chain_string(&*result.unwrap_err());
assert!(err_msg.contains("failed to resolve file"), "err: {err_msg}");
let _ = std::fs::remove_dir_all(&tmp);
}
#[test]
fn filesystem_serdes_pointer_format_stability() {
let tmp = std::env::temp_dir().join("fs_serdes_test_format");
let _ = std::fs::remove_dir_all(&tmp);
let serdes = FileSystemSerdes::new(tmp.to_string_lossy().to_string());
let ctx = SerdesContext::new(
"op-1".to_owned(),
"arn:aws:lambda:us-east-1:123:function:fn:1/durable-execution/exec/inv".to_owned(),
);
let envelope = serdes
.serialize_sync(&serde_json::json!(42), &ctx)
.expect("serialize");
let parsed: serde_json::Value = serde_json::from_str(&envelope).unwrap();
assert!(parsed.is_object());
assert!(parsed.get("file").is_some());
assert!(parsed.get("file").unwrap().is_string());
let path = parsed.get("file").and_then(|v| v.as_str()).unwrap();
#[expect(clippy::case_sensitive_file_extension_comparisons)]
let has_json_ext = path.ends_with(".json");
assert!(has_json_ext, "path: {path}");
let _ = std::fs::remove_dir_all(&tmp);
}
#[test]
fn filesystem_serdes_uri_encoding_dir_structure() {
let tmp = std::env::temp_dir().join("fs_serdes_test_uri_dir");
let _ = std::fs::remove_dir_all(&tmp);
let serdes = FileSystemSerdes::new(tmp.to_string_lossy().to_string());
let ctx = SerdesContext::new(
"step-1".to_owned(),
"arn:aws:lambda:us-west-2:999:function:my-function:42/durable-execution/my-exec/my-inv"
.to_owned(),
);
let envelope = serdes
.serialize_sync(&serde_json::json!("data"), &ctx)
.expect("serialize");
let parsed: serde_json::Value = serde_json::from_str(&envelope).unwrap();
let path = parsed.get("file").and_then(|v| v.as_str()).unwrap();
assert!(
!std::path::Path::new(path).is_absolute(),
"path must be relative to base_path: {path}"
);
assert!(path.starts_with("my-function/"), "path: {path}");
assert!(path.contains("my-exec"), "path: {path}");
assert!(path.contains("my-inv"), "path: {path}");
let file_name = path.rsplit('/').next().unwrap();
assert!(file_name.starts_with("step-1-"), "file name: {file_name}");
#[expect(clippy::case_sensitive_file_extension_comparisons)]
let correct_suffix = file_name.ends_with(".json");
assert!(correct_suffix, "file name: {file_name}");
let _ = std::fs::remove_dir_all(&tmp);
}
#[test]
fn deserialize_reads_js_python_inline_data_envelopes() {
let serdes = FileSystemSerdes::new("/unused");
let ctx = SerdesContext::new("op", "arn:test");
let object = serdes
.deserialize_sync::<serde_json::Value>(r#"{"data":"{\"value\":42}"}"#, &ctx)
.expect("an object in a data envelope decodes");
assert_eq!(object, serde_json::json!({"value": 42}));
let string = serdes
.deserialize_sync::<serde_json::Value>(r#"{"data":"\"hello\""}"#, &ctx)
.expect("a string in a data envelope decodes");
assert_eq!(string, serde_json::json!("hello"));
}
#[test]
fn deserialize_rejects_rust_only_inline_envelopes() {
let serdes = FileSystemSerdes::new("/unused");
let ctx = SerdesContext::new("op", "arn:test");
let unquoted = serdes
.deserialize_sync::<serde_json::Value>(r#"{"data":{"value":42}}"#, &ctx)
.expect_err("an unquoted data envelope is rejected");
assert!(
crate::error::chain_string(&*unquoted).contains("invalid envelope JSON"),
"{unquoted}"
);
for envelope in [r#"{"raw":"not json at all"}"#, r#"{"value":42}"#] {
let err = serdes
.deserialize_sync::<serde_json::Value>(envelope, &ctx)
.expect_err("the envelope is rejected");
assert!(
crate::error::chain_string(&*err).contains(
"envelope carries no '__aws_durable_serdes' marker and no unversioned 'file' or 'data' field"
),
"{envelope}: {err}"
);
}
}
#[test]
fn overflow_mode_never_writes_raw_envelope() {
let tmp = std::env::temp_dir().join("fs_serdes_test_no_raw");
let _ = std::fs::remove_dir_all(&tmp);
let config = FileSystemSerdesConfig::builder()
.storage_mode(FileSystemSerdesMode::Overflow)
.overflow_threshold_bytes(4096)
.build();
let serdes = FileSystemSerdes::with_config(tmp.to_string_lossy().to_string(), config);
let ctx = SerdesContext::new("op", "arn:test");
let value = serde_json::json!("plain text");
let envelope = serdes.serialize_sync(&value, &ctx).expect("serialize");
assert_eq!(
envelope,
r#"{"__aws_durable_serdes":{"version":1,"kind":"inline"},"value":"plain text"}"#
);
assert_eq!(
serdes
.deserialize_sync::<serde_json::Value>(&envelope, &ctx)
.expect("deserialize"),
value
);
let _ = std::fs::remove_dir_all(&tmp);
}
#[test]
fn filesystem_serdes_replay_reads_back() {
let tmp = std::env::temp_dir().join("fs_serdes_test_replay");
let _ = std::fs::remove_dir_all(&tmp);
let base = tmp.to_string_lossy().to_string();
let ctx = SerdesContext::new(
"replay-op".to_owned(),
"arn:aws:lambda:us-east-1:123:function:fn:1/durable-execution/e1/i1".to_owned(),
);
let serdes1 = FileSystemSerdes::new(base.clone());
let input = serde_json::json!({"items": [1, 2, 3]});
let envelope = serdes1.serialize_sync(&input, &ctx).expect("serialize");
let serdes2 = FileSystemSerdes::new(base);
let output = serdes2
.deserialize_sync::<serde_json::Value>(&envelope, &ctx)
.expect("deserialize on replay");
assert_eq!(output, input);
let _ = std::fs::remove_dir_all(&tmp);
}
#[test]
fn json_escape_string_handles_special_chars() {
assert_eq!(json_escape_string(r#"a\b"c"#), r#"a\\b\"c"#);
assert_eq!(json_escape_string("line\nnewline"), r"line\nnewline");
assert_eq!(json_escape_string("tab\there"), r"tab\there");
assert_eq!(
json_escape_string("/mnt/efs/data.json"),
"/mnt/efs/data.json"
);
}
#[test]
fn filesystem_serdes_envelope_with_special_path_chars() {
let tmp = std::env::temp_dir().join("fs_serdes_test_escape");
let _ = std::fs::remove_dir_all(&tmp);
let serdes = FileSystemSerdes::new(tmp.to_string_lossy().to_string());
let ctx = SerdesContext::new(
"step-1".to_owned(),
"arn:aws:lambda:us-east-1:123:function:fn:1/durable-execution/exec/inv".to_owned(),
);
let input = serde_json::json!({"value": "test"});
let envelope = serdes
.serialize_sync(&input, &ctx)
.expect("serialize should succeed");
let parsed: serde_json::Value =
serde_json::from_str(&envelope).expect("envelope must be valid JSON");
assert!(parsed.get("file").is_some());
let output = serdes
.deserialize_sync::<serde_json::Value>(&envelope, &ctx)
.expect("deserialize should succeed");
assert_eq!(output, input);
let _ = std::fs::remove_dir_all(&tmp);
}
#[tokio::test]
async fn serialize_via_trait_writes_file() {
let tmp = std::env::temp_dir().join("fs_serdes_trait_ctx_ser");
let _ = std::fs::remove_dir_all(&tmp);
let base = tmp.to_string_lossy().to_string();
let serdes = FileSystemSerdes::new(base.clone());
let ctx = SerdesContext::new(
"step-result-1",
"arn:aws:lambda:us-east-1:123:function:fn:1/durable-execution/exec-1/inv-1",
);
let input = serde_json::json!({"total": 42});
let envelope = serdes
.serialize(input, ctx)
.await
.expect("Serdes::serialize should succeed");
assert!(
envelope.contains("\"file\""),
"envelope should be a file pointer: {envelope}"
);
assert!(
!envelope.contains("\"data\""),
"ALWAYS mode should write to file"
);
let expected_dir = format!("{base}/fn/exec-1/inv-1");
assert!(
std::path::Path::new(&expected_dir).exists(),
"execution dir should be created: {expected_dir}"
);
let _ = std::fs::remove_dir_all(&tmp);
}
#[tokio::test]
async fn deserialize_via_trait_reads_file() {
let tmp = std::env::temp_dir().join("fs_serdes_trait_ctx_deser");
let _ = std::fs::remove_dir_all(&tmp);
let base = tmp.to_string_lossy().to_string();
let serdes = FileSystemSerdes::new(base.clone());
let ctx = SerdesContext::new(
"op-deser-1",
"arn:aws:lambda:us-east-1:123:function:fn:1/durable-execution/exec-1/inv-1",
);
let input = serde_json::json!({"key": "value"});
let envelope = serdes
.serialize(input.clone(), ctx.clone())
.await
.expect("serialize");
let output: serde_json::Value = serdes
.deserialize(envelope, ctx)
.await
.expect("Serdes::deserialize should succeed");
assert_eq!(output, input);
let _ = std::fs::remove_dir_all(&tmp);
}
#[tokio::test]
async fn hex_envelope_probe_serdes_round_trips() {
use super::test_support::{HexEnvelopeSerdes, hex_envelope_of};
let ctx = SerdesContext::new("op-1", "arn:test");
let value = serde_json::json!({"label": "a\"b\\c\nd ☃", "nested": [[1, -2], []]});
let wire = HexEnvelopeSerdes
.serialize(value.clone(), ctx.clone())
.await
.expect("serialize");
assert_eq!(wire, hex_envelope_of(&value));
assert!(serde_json::from_str::<serde_json::Value>(&wire).is_err());
let back: serde_json::Value = HexEnvelopeSerdes
.deserialize(wire, ctx.clone())
.await
.expect("deserialize");
assert_eq!(back, value);
assert!(
Serdes::<serde_json::Value>::deserialize(&HexEnvelopeSerdes, value.to_string(), ctx)
.await
.is_err()
);
}
#[tokio::test]
async fn passthrough_serdes_is_identity() {
let ctx = SerdesContext::new("op-1", "arn:test");
let wire = PassthroughSerdes
.serialize("approved".to_owned(), ctx.clone())
.await
.expect("serialize");
assert_eq!(wire, "approved");
let back = PassthroughSerdes
.deserialize("not json {".to_owned(), ctx)
.await
.expect("deserialize");
assert_eq!(back, "not json {");
}
#[tokio::test]
async fn json_serdes_is_plain_json() {
let ctx = SerdesContext::new("op-1", "arn:test");
let value = serde_json::json!({"a": [1, "two"], "b": null});
let wire = JsonSerdes
.serialize(value.clone(), ctx.clone())
.await
.expect("serialize");
assert_eq!(wire, value.to_string());
let back: serde_json::Value = JsonSerdes
.deserialize(wire, ctx)
.await
.expect("deserialize");
assert_eq!(back, value);
}
#[tokio::test]
async fn custom_serdes_receives_the_typed_value() {
struct UpperSerdes;
impl Serdes<String> for UpperSerdes {
#[expect(clippy::unused_async_trait_impl)]
async fn serialize(
&self,
value: String,
_context: SerdesContext,
) -> Result<String, BoxError> {
Ok(value.to_uppercase())
}
#[expect(clippy::unused_async_trait_impl)]
async fn deserialize(
&self,
wire: String,
_context: SerdesContext,
) -> Result<String, BoxError> {
Ok(wire.to_lowercase())
}
}
let ctx = SerdesContext::new("op-1", "arn:test");
let wire = UpperSerdes
.serialize("hello".to_owned(), ctx.clone())
.await
.expect("serialize");
assert_eq!(wire, "HELLO");
let back = UpperSerdes
.deserialize("WORLD".to_owned(), ctx)
.await
.expect("deserialize");
assert_eq!(back, "world");
}
#[tokio::test]
async fn i128_survives_custom_serdes_round_trip() {
struct DecimalSerdes;
impl Serdes<i128> for DecimalSerdes {
#[expect(clippy::unused_async_trait_impl)]
async fn serialize(
&self,
value: i128,
_context: SerdesContext,
) -> Result<String, BoxError> {
Ok(value.to_string())
}
#[expect(clippy::unused_async_trait_impl)]
async fn deserialize(
&self,
wire: String,
_context: SerdesContext,
) -> Result<i128, BoxError> {
Ok(wire.parse()?)
}
}
let ctx = SerdesContext::new("op-1", "arn:test");
let value = 7_i128 << 100;
let wire = DecimalSerdes
.serialize(value, ctx.clone())
.await
.expect("serialize");
let back = DecimalSerdes
.deserialize(wire, ctx)
.await
.expect("deserialize");
assert_eq!(back, value);
}
#[tokio::test]
async fn struct_field_declaration_order_survives_custom_serdes() {
#[derive(Debug, PartialEq, serde::Serialize, serde::Deserialize)]
struct Ordered {
zebra: u8,
mango: u8,
apple: u8,
}
struct PassThrough;
impl Serdes<Ordered> for PassThrough {
#[expect(clippy::unused_async_trait_impl)]
async fn serialize(
&self,
value: Ordered,
_context: SerdesContext,
) -> Result<String, BoxError> {
Ok(serde_json::to_string(&value)?)
}
#[expect(clippy::unused_async_trait_impl)]
async fn deserialize(
&self,
wire: String,
_context: SerdesContext,
) -> Result<Ordered, BoxError> {
Ok(serde_json::from_str(&wire)?)
}
}
let ctx = SerdesContext::new("op-1", "arn:test");
let value = Ordered {
zebra: 1,
mango: 2,
apple: 3,
};
let wire = PassThrough
.serialize(value, ctx.clone())
.await
.expect("serialize");
assert_eq!(wire, r#"{"zebra":1,"mango":2,"apple":3}"#);
let back = PassThrough
.deserialize(wire, ctx)
.await
.expect("deserialize");
assert_eq!(
back,
Ordered {
zebra: 1,
mango: 2,
apple: 3
}
);
}
#[tokio::test]
async fn arc_serdes_shared_across_output_types() {
let shared = std::sync::Arc::new(JsonSerdes);
let ctx = SerdesContext::new("op-1", "arn:test");
let s = Serdes::<String>::serialize(&shared, "x".to_owned(), ctx.clone())
.await
.expect("string serialize");
assert_eq!(s, "\"x\"");
let n = Serdes::<u32>::serialize(&shared, 7_u32, ctx.clone())
.await
.expect("u32 serialize");
assert_eq!(n, "7");
let back: u32 = shared
.deserialize("7".to_owned(), ctx)
.await
.expect("u32 deserialize");
assert_eq!(back, 7);
}
#[tokio::test]
async fn send_not_sync_value_round_trips_through_filesystem() {
#[derive(Debug, Default, PartialEq, serde::Serialize, serde::Deserialize)]
struct NotSync {
value: String,
#[serde(skip)]
cell: std::cell::Cell<u8>,
}
fn assert_send<F: Send>(f: F) -> F {
f
}
let tmp = std::env::temp_dir().join("fs_serdes_not_sync");
let _ = std::fs::remove_dir_all(&tmp);
let serdes = FileSystemSerdes::new(tmp.to_string_lossy().to_string());
let ctx = SerdesContext::new(
"op-1",
"arn:aws:lambda:us-east-1:123:function:fn:1/durable-execution/e/i",
);
let input = NotSync {
value: "send-not-sync".to_owned(),
cell: std::cell::Cell::new(3),
};
let envelope = assert_send(serdes.serialize(input, ctx.clone()))
.await
.expect("serialize");
let back: NotSync = assert_send(serdes.deserialize(envelope, ctx))
.await
.expect("deserialize");
assert_eq!(back.value, "send-not-sync");
let _ = std::fs::remove_dir_all(&tmp);
}
fn traversal_arn(function_name: &str, execution_name: &str, invocation_id: &str) -> String {
format!(
"arn:aws:lambda:us-east-1:123456789012:function:{function_name}:1/durable-execution/{execution_name}/{invocation_id}"
)
}
#[test]
fn traversal_in_function_name_is_contained() {
let tmp = std::env::temp_dir().join("fs_serdes_traversal_fn");
let _ = std::fs::remove_dir_all(&tmp);
std::fs::create_dir_all(&tmp).unwrap();
let serdes = FileSystemSerdes::new(tmp.to_string_lossy().to_string());
let arn = traversal_arn("../../etc", "exec-1", "inv-1");
let ctx = SerdesContext::new("step-1".to_owned(), arn);
let input = serde_json::json!({"data": "test"});
let result = serdes.serialize_sync(&input, &ctx);
match result {
Ok(envelope) => {
let parsed: serde_json::Value = serde_json::from_str(&envelope).unwrap();
let file_path = parsed["file"].as_str().unwrap();
let canonical_file =
std::fs::canonicalize(tmp.join(file_path)).expect("file should exist");
let canonical_base = std::fs::canonicalize(&tmp).unwrap();
assert!(
canonical_file.starts_with(&canonical_base),
"file {canonical_file:?} is outside base {canonical_base:?}"
);
}
Err(e) => {
assert!(
e.to_string().contains("escapes base_path"),
"unexpected error: {e}"
);
}
}
let _ = std::fs::remove_dir_all(&tmp);
}
#[test]
fn traversal_in_execution_name_is_contained() {
let tmp = std::env::temp_dir().join("fs_serdes_traversal_exec");
let _ = std::fs::remove_dir_all(&tmp);
std::fs::create_dir_all(&tmp).unwrap();
let serdes = FileSystemSerdes::new(tmp.to_string_lossy().to_string());
let arn = traversal_arn("my-fn", "../../etc", "inv-1");
let ctx = SerdesContext::new("step-1".to_owned(), arn);
let input = serde_json::json!({"data": "test"});
let result = serdes.serialize_sync(&input, &ctx);
match result {
Ok(envelope) => {
let parsed: serde_json::Value = serde_json::from_str(&envelope).unwrap();
let file_path = parsed["file"].as_str().unwrap();
let canonical_file =
std::fs::canonicalize(tmp.join(file_path)).expect("file should exist");
let canonical_base = std::fs::canonicalize(&tmp).unwrap();
assert!(
canonical_file.starts_with(&canonical_base),
"file {canonical_file:?} is outside base {canonical_base:?}"
);
}
Err(e) => {
assert!(
e.to_string().contains("escapes base_path"),
"unexpected error: {e}"
);
}
}
let _ = std::fs::remove_dir_all(&tmp);
}
#[test]
fn traversal_in_invocation_id_is_contained() {
let tmp = std::env::temp_dir().join("fs_serdes_traversal_inv");
let _ = std::fs::remove_dir_all(&tmp);
std::fs::create_dir_all(&tmp).unwrap();
let serdes = FileSystemSerdes::new(tmp.to_string_lossy().to_string());
let arn = traversal_arn("my-fn", "exec-1", "../../etc");
let ctx = SerdesContext::new("step-1".to_owned(), arn);
let input = serde_json::json!({"data": "test"});
let result = serdes.serialize_sync(&input, &ctx);
match result {
Ok(envelope) => {
let parsed: serde_json::Value = serde_json::from_str(&envelope).unwrap();
let file_path = parsed["file"].as_str().unwrap();
let canonical_file =
std::fs::canonicalize(tmp.join(file_path)).expect("file should exist");
let canonical_base = std::fs::canonicalize(&tmp).unwrap();
assert!(
canonical_file.starts_with(&canonical_base),
"file {canonical_file:?} is outside base {canonical_base:?}"
);
}
Err(e) => {
assert!(
e.to_string().contains("escapes base_path"),
"unexpected error: {e}"
);
}
}
let _ = std::fs::remove_dir_all(&tmp);
}
#[test]
fn encoded_components_round_trip() {
let tmp = std::env::temp_dir().join("fs_serdes_encoded_rt");
let _ = std::fs::remove_dir_all(&tmp);
std::fs::create_dir_all(&tmp).unwrap();
let serdes = FileSystemSerdes::new(tmp.to_string_lossy().to_string());
let arn = traversal_arn("fn/special:chars", "exec name+here", "inv/id");
let ctx = SerdesContext::new("step-1".to_owned(), arn);
let input = serde_json::json!({"round": "trip", "value": 99});
let envelope = serdes
.serialize_sync(&input, &ctx)
.expect("serialize should succeed with encoded components");
let output = serdes
.deserialize_sync::<serde_json::Value>(&envelope, &ctx)
.expect("deserialize should find the file written by serialize");
assert_eq!(output, input);
let _ = std::fs::remove_dir_all(&tmp);
}
#[test]
fn bare_dotdot_in_each_component_is_contained() {
let tmp = std::env::temp_dir().join("fs_serdes_bare_dotdot");
let _ = std::fs::remove_dir_all(&tmp);
std::fs::create_dir_all(&tmp).unwrap();
let serdes = FileSystemSerdes::new(tmp.to_string_lossy().to_string());
let arn_fn = traversal_arn("..", "exec-1", "inv-1");
let ctx_fn = SerdesContext::new("step-1".to_owned(), arn_fn);
let input = serde_json::json!({"test": "bare_dotdot"});
match serdes.serialize_sync(&input, &ctx_fn) {
Ok(envelope) => {
let parsed: serde_json::Value = serde_json::from_str(&envelope).unwrap();
let file_path = parsed["file"].as_str().unwrap();
let canonical_file =
std::fs::canonicalize(tmp.join(file_path)).expect("file should exist");
let canonical_base = std::fs::canonicalize(&tmp).unwrap();
assert!(
canonical_file.starts_with(&canonical_base),
"bare '..' in function_name: file {canonical_file:?} escapes base {canonical_base:?}"
);
}
Err(e) => {
assert!(
e.to_string().contains("escapes base_path"),
"unexpected error for bare '..' in function_name: {e}"
);
}
}
let arn_exec = traversal_arn("my-fn", "..", "inv-1");
let ctx_exec = SerdesContext::new("step-1".to_owned(), arn_exec);
match serdes.serialize_sync(&input, &ctx_exec) {
Ok(envelope) => {
let parsed: serde_json::Value = serde_json::from_str(&envelope).unwrap();
let file_path = parsed["file"].as_str().unwrap();
let canonical_file =
std::fs::canonicalize(tmp.join(file_path)).expect("file should exist");
let canonical_base = std::fs::canonicalize(&tmp).unwrap();
assert!(
canonical_file.starts_with(&canonical_base),
"bare '..' in execution_name: file {canonical_file:?} escapes base {canonical_base:?}"
);
}
Err(e) => {
assert!(
e.to_string().contains("escapes base_path"),
"unexpected error for bare '..' in execution_name: {e}"
);
}
}
let arn_inv = traversal_arn("my-fn", "exec-1", "..");
let ctx_inv = SerdesContext::new("step-1".to_owned(), arn_inv);
match serdes.serialize_sync(&input, &ctx_inv) {
Ok(envelope) => {
let parsed: serde_json::Value = serde_json::from_str(&envelope).unwrap();
let file_path = parsed["file"].as_str().unwrap();
let canonical_file =
std::fs::canonicalize(tmp.join(file_path)).expect("file should exist");
let canonical_base = std::fs::canonicalize(&tmp).unwrap();
assert!(
canonical_file.starts_with(&canonical_base),
"bare '..' in invocation_id: file {canonical_file:?} escapes base {canonical_base:?}"
);
}
Err(e) => {
assert!(
e.to_string().contains("escapes base_path"),
"unexpected error for bare '..' in invocation_id: {e}"
);
}
}
let _ = std::fs::remove_dir_all(&tmp);
}
#[cfg(unix)]
#[test]
fn symlink_under_base_path_rejected() {
let root = std::env::temp_dir().join("fs_serdes_symlink_escape");
let _ = std::fs::remove_dir_all(&root);
let base = root.join("base");
let outside = root.join("outside");
std::fs::create_dir_all(&base).unwrap();
std::fs::create_dir_all(&outside).unwrap();
std::os::unix::fs::symlink(&outside, base.join("my-fn")).unwrap();
let serdes = FileSystemSerdes::new(base.to_string_lossy().to_string());
let ctx = SerdesContext::new(
"step-1".to_owned(),
traversal_arn("my-fn", "exec-1", "inv-1"),
);
let result = serdes.serialize_sync(&serde_json::json!({"data": "test"}), &ctx);
let err = result.expect_err("symlink escape must be rejected");
assert!(
err.to_string().contains("escapes base_path"),
"unexpected error: {err}"
);
let redirected = outside.join("exec-1").join("inv-1");
if redirected.exists() {
let files: Vec<_> = std::fs::read_dir(&redirected)
.unwrap()
.filter_map(Result::ok)
.filter(|e| e.path().is_file())
.collect();
assert!(
files.is_empty(),
"no file may be written outside base_path: {files:?}"
);
}
let _ = std::fs::remove_dir_all(&root);
}
#[cfg(unix)]
#[test]
fn symlinked_base_path_itself_still_works() {
let root = std::env::temp_dir().join("fs_serdes_symlink_base");
let _ = std::fs::remove_dir_all(&root);
let real_base = root.join("real-base");
std::fs::create_dir_all(&real_base).unwrap();
let link_base = root.join("link-base");
std::os::unix::fs::symlink(&real_base, &link_base).unwrap();
let serdes = FileSystemSerdes::new(link_base.to_string_lossy().to_string());
let ctx = SerdesContext::new(
"step-1".to_owned(),
traversal_arn("my-fn", "exec-1", "inv-1"),
);
let envelope = serdes
.serialize_sync(&serde_json::json!({"data": "test"}), &ctx)
.expect("write through a symlinked base path must succeed");
let parsed: serde_json::Value = serde_json::from_str(&envelope).unwrap();
let file_path = parsed["file"].as_str().unwrap();
let canonical_file =
std::fs::canonicalize(link_base.join(file_path)).expect("file should exist");
let canonical_base = std::fs::canonicalize(&real_base).unwrap();
assert!(
canonical_file.starts_with(&canonical_base),
"file {canonical_file:?} must be inside base {canonical_base:?}"
);
let _ = std::fs::remove_dir_all(&root);
}
#[test]
fn serialize_writes_unique_immutable_files() {
let tmp = std::env::temp_dir().join("fs_serdes_unique_files");
let _ = std::fs::remove_dir_all(&tmp);
let serdes = FileSystemSerdes::new(tmp.to_string_lossy().to_string());
let ctx = SerdesContext::new(
"step-1",
"arn:aws:lambda:us-east-1:123:function:fn:1/durable-execution/e/i",
);
let first = serdes
.serialize_sync(&serde_json::json!(1), &ctx)
.expect("first serialize");
let second = serdes
.serialize_sync(&serde_json::json!(2), &ctx)
.expect("second serialize");
let first_path = serde_json::from_str::<serde_json::Value>(&first).unwrap()["file"]
.as_str()
.unwrap()
.to_owned();
let second_path = serde_json::from_str::<serde_json::Value>(&second).unwrap()["file"]
.as_str()
.unwrap()
.to_owned();
assert_ne!(
first_path, second_path,
"a second serialize of the same operation must not reuse the path"
);
let replayed: serde_json::Value = serdes
.deserialize_sync(&first, &ctx)
.expect("first envelope must stay readable");
assert_eq!(replayed, serde_json::json!(1));
let second_read: serde_json::Value =
serdes.deserialize_sync(&second, &ctx).expect("second");
assert_eq!(second_read, serde_json::json!(2));
let _ = std::fs::remove_dir_all(&tmp);
}
#[test]
fn serialize_leaves_no_staging_files_behind() {
fn assert_no_tmp(dir: &std::path::Path) {
for entry in std::fs::read_dir(dir).unwrap().filter_map(Result::ok) {
let path = entry.path();
if path.is_dir() {
assert_no_tmp(&path);
} else {
assert!(
path.extension().is_none_or(|ext| ext != "tmp"),
"staging file left behind: {path:?}"
);
}
}
}
let tmp = std::env::temp_dir().join("fs_serdes_no_staging");
let _ = std::fs::remove_dir_all(&tmp);
let serdes = FileSystemSerdes::new(tmp.to_string_lossy().to_string());
let ctx = SerdesContext::new(
"step-1",
"arn:aws:lambda:us-east-1:123:function:fn:1/durable-execution/e/i",
);
serdes
.serialize_sync(&serde_json::json!({"k": "v"}), &ctx)
.expect("serialize");
assert_no_tmp(&tmp);
let _ = std::fs::remove_dir_all(&tmp);
}
#[test]
fn external_origin_never_resolves_file_references() {
let tmp = std::env::temp_dir().join("fs_serdes_external_no_file");
let _ = std::fs::remove_dir_all(&tmp);
std::fs::create_dir_all(&tmp).unwrap();
let secret = tmp.join("secret.json");
std::fs::write(&secret, r#""the file contents""#).unwrap();
let serdes = FileSystemSerdes::new(tmp.to_string_lossy().to_string());
let ctx = SerdesContext::new("cb-1", "arn:test").with_origin(PayloadOrigin::External);
let unversioned = format!(r#"{{"file":"{}"}}"#, secret.to_string_lossy());
let out: serde_json::Value = serdes
.deserialize_sync(&unversioned, &ctx)
.expect("external payload parses as its own JSON");
assert_eq!(
out,
serde_json::json!({"file": secret.to_string_lossy()}),
"the payload must come back as data, not as the file's contents"
);
let spoofed = file_envelope("secret.json", None);
let out: serde_json::Value = serdes
.deserialize_sync(&spoofed, &ctx)
.expect("external payload parses as its own JSON");
assert!(
out.get("__aws_durable_serdes").is_some(),
"spoofed marker must come back as plain data: {out}"
);
assert_ne!(out, serde_json::json!("the file contents"));
let _ = std::fs::remove_dir_all(&tmp);
}
#[test]
fn external_inline_payload_with_file_key_parses_as_value() {
let serdes = FileSystemSerdes::new("/unused-base");
let ctx = SerdesContext::new("cb-1", "arn:test").with_origin(PayloadOrigin::External);
let payload = r#"{"file": "/tmp/result.json", "data": {"status": "ready"}}"#;
let out: serde_json::Value = serdes
.deserialize_sync(payload, &ctx)
.expect("must parse as the value");
assert_eq!(
out,
serde_json::json!({"file": "/tmp/result.json", "data": {"status": "ready"}})
);
}
#[test]
fn inline_value_containing_file_key_round_trips() {
let tmp = std::env::temp_dir().join("fs_serdes_inline_file_key");
let _ = std::fs::remove_dir_all(&tmp);
let config = FileSystemSerdesConfig::builder()
.storage_mode(FileSystemSerdesMode::Overflow)
.overflow_threshold_bytes(4096)
.build();
let serdes = FileSystemSerdes::with_config(tmp.to_string_lossy().to_string(), config);
let ctx = SerdesContext::new("op", "arn:test");
let value = serde_json::json!({"file": "/tmp/result.json", "data": {"status": "ready"}});
let envelope = serdes.serialize_sync(&value, &ctx).expect("serialize");
assert!(
envelope.contains(r#""__aws_durable_serdes""#),
"inline envelope must carry the marker: {envelope}"
);
let out: serde_json::Value = serdes
.deserialize_sync(&envelope, &ctx)
.expect("deserialize");
assert_eq!(
out, value,
"the value's own 'file' key must not be read as a file reference"
);
let _ = std::fs::remove_dir_all(&tmp);
}
#[test]
fn file_reference_escaping_base_rejected() {
let tmp = std::env::temp_dir().join("fs_serdes_ref_escape");
let _ = std::fs::remove_dir_all(&tmp);
std::fs::create_dir_all(&tmp).unwrap();
let serdes = FileSystemSerdes::new(tmp.to_string_lossy().to_string());
let ctx = SerdesContext::new("op", "arn:test");
let absolute = file_envelope("/etc/hostname", None);
let err = serdes
.deserialize_sync::<serde_json::Value>(&absolute, &ctx)
.unwrap_err();
assert!(
err.to_string().contains("must be relative"),
"unexpected error: {err}"
);
let escape = file_envelope("../outside.json", None);
let err = serdes
.deserialize_sync::<serde_json::Value>(&escape, &ctx)
.unwrap_err();
assert!(
err.to_string().contains("escapes base_path"),
"unexpected error: {err}"
);
let _ = std::fs::remove_dir_all(&tmp);
}
#[test]
fn unversioned_file_envelope_strictly_contained() {
let tmp = std::env::temp_dir().join("fs_serdes_unversioned_containment");
let _ = std::fs::remove_dir_all(&tmp);
std::fs::create_dir_all(&tmp).unwrap();
let serdes = FileSystemSerdes::new(tmp.to_string_lossy().to_string());
let ctx = SerdesContext::new("op", "arn:test");
let inside = tmp.join("unversioned-op.json");
std::fs::write(&inside, r#"{"unversioned": true}"#).unwrap();
let unversioned_ok = format!(r#"{{"file":"{}"}}"#, inside.to_string_lossy());
let out: serde_json::Value = serdes
.deserialize_sync(&unversioned_ok, &ctx)
.expect("an unversioned record under base_path is readable");
assert_eq!(out, serde_json::json!({"unversioned": true}));
let outside_dir = std::env::temp_dir().join("fs_serdes_unversioned_outside");
let _ = std::fs::remove_dir_all(&outside_dir);
std::fs::create_dir_all(&outside_dir).unwrap();
let outside = outside_dir.join("outside.json");
std::fs::write(&outside, r#""outside""#).unwrap();
let unversioned_escape = format!(r#"{{"file":"{}"}}"#, outside.to_string_lossy());
let err = serdes
.deserialize_sync::<serde_json::Value>(&unversioned_escape, &ctx)
.unwrap_err();
assert!(
err.to_string().contains("escapes base_path"),
"unexpected error: {err}"
);
let _ = std::fs::remove_dir_all(&tmp);
let _ = std::fs::remove_dir_all(&outside_dir);
}
#[cfg(unix)]
#[test]
fn file_read_through_symlink_escape_rejected() {
let root = std::env::temp_dir().join("fs_serdes_read_symlink");
let _ = std::fs::remove_dir_all(&root);
let base = root.join("base");
let outside = root.join("outside");
std::fs::create_dir_all(&base).unwrap();
std::fs::create_dir_all(&outside).unwrap();
std::fs::write(outside.join("target.json"), r#""escaped""#).unwrap();
std::os::unix::fs::symlink(outside.join("target.json"), base.join("link.json")).unwrap();
let serdes = FileSystemSerdes::new(base.to_string_lossy().to_string());
let ctx = SerdesContext::new("op", "arn:test");
let envelope = file_envelope("link.json", None);
let err = serdes
.deserialize_sync::<serde_json::Value>(&envelope, &ctx)
.unwrap_err();
assert!(
err.to_string().contains("escapes base_path"),
"unexpected error: {err}"
);
let _ = std::fs::remove_dir_all(&root);
}
#[test]
fn unknown_envelope_version_rejected() {
let serdes = FileSystemSerdes::new("/unused-base");
let ctx = SerdesContext::new("op", "arn:test");
let future = r#"{"__aws_durable_serdes":{"version":2,"kind":"inline"},"value":1}"#;
let err = serdes
.deserialize_sync::<serde_json::Value>(future, &ctx)
.unwrap_err();
assert!(
err.to_string().contains("unsupported envelope version 2"),
"unexpected error: {err}"
);
}
#[test]
fn unknown_envelope_kind_rejected() {
let serdes = FileSystemSerdes::new("/unused-base");
let ctx = SerdesContext::new("op", "arn:test");
let chunked = r#"{"__aws_durable_serdes":{"version":1,"kind":"chunked"},"value":1}"#;
let err = serdes
.deserialize_sync::<serde_json::Value>(chunked, &ctx)
.unwrap_err();
assert!(
err.to_string()
.contains("unsupported envelope kind 'chunked'"),
"unexpected error: {err}"
);
}
#[test]
fn retryable_serdes_error_displays_one_frame() {
let err = RetryableSerdesError::new("boom");
assert_eq!(err.to_string(), "retryable serdes failure");
assert_eq!(format!("{err:#}"), "retryable serdes failure: boom");
let source = std::error::Error::source(&err).expect("the marker has a source");
assert_eq!(source.to_string(), "boom");
}
#[derive(Debug)]
struct Wrapping(BoxError);
impl std::fmt::Display for Wrapping {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("wrapping")
}
}
impl std::error::Error for Wrapping {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
Some(&*self.0)
}
}
#[test]
fn is_retryable_serdes_error_walks_the_chain() {
let bare: BoxError = Box::new(RetryableSerdesError::new("boom"));
assert!(is_retryable_serdes_error(&*bare));
let nested: BoxError = Box::new(Wrapping(Box::new(Wrapping(Box::new(
RetryableSerdesError::new("boom"),
)))));
assert!(is_retryable_serdes_error(&*nested));
let plain: BoxError = "plain".into();
assert!(!is_retryable_serdes_error(&*plain));
}
fn scratch_path(label: &str) -> std::path::PathBuf {
std::env::temp_dir().join(format!(
"fs_serdes_test_{label}_{}_{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.expect("clock after epoch")
.as_nanos()
))
}
#[test]
fn filesystem_io_failures_are_marked() {
let file_base = scratch_path("io_file_base");
std::fs::write(&file_base, "not a directory").expect("create the file");
let serdes = FileSystemSerdes::with_config(
file_base.to_string_lossy().into_owned(),
FileSystemSerdesConfig::builder()
.storage_mode(FileSystemSerdesMode::Always)
.build(),
);
let ctx = SerdesContext::new("op", "arn:test");
let err = serdes
.serialize_sync(&"value", &ctx)
.expect_err("create_dir_all under a file fails");
assert!(is_retryable_serdes_error(&*err), "got: {err:#}");
let _ = std::fs::remove_file(&file_base);
let dir_base = scratch_path("io_dir_base");
std::fs::create_dir_all(&dir_base).expect("create the base dir");
let serdes = FileSystemSerdes::new(dir_base.to_string_lossy().into_owned());
let err = serdes
.deserialize_sync::<serde_json::Value>(&file_envelope("missing.json", None), &ctx)
.expect_err("reading a missing file fails");
assert!(is_retryable_serdes_error(&*err), "got: {err:#}");
let _ = std::fs::remove_dir_all(&dir_base);
}
#[test]
fn filesystem_format_and_containment_failures_are_permanent() {
let dir_base = scratch_path("permanent_base");
std::fs::create_dir_all(&dir_base).expect("create the base dir");
let serdes = FileSystemSerdes::new(dir_base.to_string_lossy().into_owned());
let ctx = SerdesContext::new("op", "arn:test");
let unsupported_version =
r#"{"__aws_durable_serdes":{"version":2,"kind":"inline"},"value":1}"#;
let escaping = file_envelope("../escape.json", None);
for wire in ["{not json", unsupported_version, escaping.as_str()] {
let err = serdes
.deserialize_sync::<serde_json::Value>(wire, &ctx)
.expect_err("the record is rejected");
assert!(
!is_retryable_serdes_error(&*err),
"{wire} must fail permanently, got: {err:#}"
);
}
let _ = std::fs::remove_dir_all(&dir_base);
}
fn id_and_masked_secret() -> PreviewConfig {
PreviewConfig::builder(PreviewMode::ExcludeAll)
.include("id")
.mask("secret")
.build()
}
fn secret_value() -> serde_json::Value {
serde_json::json!({"id": "abc", "secret": "s3cr3t", "other": "ignored"})
}
fn with_preview(
base: &std::path::Path,
builder: FileSystemSerdesConfigBuilder,
) -> FileSystemSerdes {
FileSystemSerdes::with_config(base.to_string_lossy().into_owned(), builder.build())
}
fn envelope_json(envelope: &str) -> serde_json::Value {
serde_json::from_str(envelope).expect("the envelope is JSON")
}
#[test]
fn always_mode_writes_preview_beside_file() {
let base = scratch_path("preview_always");
let config = id_and_masked_secret();
let serdes = with_preview(
&base,
FileSystemSerdesConfig::builder()
.storage_mode(FileSystemSerdesMode::Always)
.preview(move |json| Ok(build_preview(json, &config))),
);
let ctx = fs_context();
let envelope = serdes
.serialize_sync(&secret_value(), &ctx)
.expect("serialize");
let parsed = envelope_json(&envelope);
assert_eq!(parsed["__aws_durable_serdes"]["kind"], "file", "{envelope}");
assert_eq!(
parsed["preview"],
serde_json::json!({"id": "abc", "secret": "***"}),
"{envelope}"
);
let output: serde_json::Value = serdes
.deserialize_sync(&envelope, &ctx)
.expect("deserialize");
assert_eq!(output, secret_value());
let _ = std::fs::remove_dir_all(&base);
}
#[test]
fn overflow_mode_writes_preview_when_value_overflows() {
let base = scratch_path("preview_overflow");
let config = id_and_masked_secret();
let serdes = with_preview(
&base,
FileSystemSerdesConfig::builder()
.storage_mode(FileSystemSerdesMode::Overflow)
.overflow_threshold_bytes(10)
.preview(move |json| Ok(build_preview(json, &config))),
);
let ctx = fs_context();
let envelope = serdes
.serialize_sync(&secret_value(), &ctx)
.expect("serialize");
let parsed = envelope_json(&envelope);
assert_eq!(parsed["__aws_durable_serdes"]["kind"], "file", "{envelope}");
assert_eq!(
parsed["preview"],
serde_json::json!({"id": "abc", "secret": "***"}),
"{envelope}"
);
assert_eq!(count_files(&base), 1);
let _ = std::fs::remove_dir_all(&base);
}
#[test]
fn overflow_inline_value_has_no_preview_and_skips_the_hook() {
let base = scratch_path("preview_inline");
let calls = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
let hook_calls = std::sync::Arc::clone(&calls);
let serdes = with_preview(
&base,
FileSystemSerdesConfig::builder()
.storage_mode(FileSystemSerdesMode::Overflow)
.preview(move |_json| {
hook_calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
Ok(Some(r#"{"id":"abc"}"#.to_owned()))
}),
);
let envelope = serdes
.serialize_sync(&secret_value(), &fs_context())
.expect("serialize");
let parsed = envelope_json(&envelope);
assert_eq!(
parsed["__aws_durable_serdes"]["kind"], "inline",
"{envelope}"
);
assert!(parsed.get("preview").is_none(), "{envelope}");
assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 0);
let _ = std::fs::remove_dir_all(&base);
}
#[test]
fn preview_none_writes_envelope_without_preview() {
let base = scratch_path("preview_none");
let serdes = with_preview(
&base,
FileSystemSerdesConfig::builder().preview(|_json| Ok(None)),
);
let envelope = serdes
.serialize_sync(&secret_value(), &fs_context())
.expect("serialize");
let parsed = envelope_json(&envelope);
let file = parsed["file"].as_str().expect("the envelope names a file");
assert_eq!(envelope, file_envelope(file, None));
let _ = std::fs::remove_dir_all(&base);
}
#[test]
fn preview_that_is_not_an_object_fails_serialize() {
for text in ["[1]", "nope"] {
let base = scratch_path("preview_not_object");
let serdes = with_preview(
&base,
FileSystemSerdesConfig::builder().preview(move |_json| Ok(Some(text.to_owned()))),
);
let err = serdes
.serialize_sync(&secret_value(), &fs_context())
.expect_err("the preview is rejected");
assert!(
err.to_string().contains("preview must be a JSON object"),
"{text}: {err:#}"
);
assert!(!is_retryable_serdes_error(&*err), "{text}: {err:#}");
assert_eq!(count_files(&base), 0, "{text}: no payload file");
let _ = std::fs::remove_dir_all(&base);
}
}
#[test]
fn deserialize_ignores_preview() {
let base = scratch_path("preview_read");
std::fs::create_dir_all(&base).expect("create the base dir");
let stored = base.join("stored.json");
std::fs::write(&stored, r#"{"id":"abc","secret":"s3cr3t"}"#).expect("write the payload");
let serdes = FileSystemSerdes::new(base.to_string_lossy().into_owned());
let ctx = fs_context();
let expected = serde_json::json!({"id": "abc", "secret": "s3cr3t"});
let versioned = file_envelope("stored.json", Some(r#"{"id":"abc"}"#));
let output: serde_json::Value = serdes
.deserialize_sync(&versioned, &ctx)
.expect("the versioned envelope reads");
assert_eq!(output, expected);
let unversioned = format!(
r#"{{"file":"{}","preview":{{"id":"abc"}}}}"#,
json_escape_string(&stored.to_string_lossy())
);
let output: serde_json::Value = serdes
.deserialize_sync(&unversioned, &ctx)
.expect("the JS and Python envelope reads");
assert_eq!(output, expected);
let _ = std::fs::remove_dir_all(&base);
}
#[test]
fn preview_error_is_permanent_serialize_failure() {
let base = scratch_path("preview_error");
let serdes = with_preview(
&base,
FileSystemSerdesConfig::builder().preview(|_json| Err("boom".into())),
);
let err = serdes
.serialize_sync(&secret_value(), &fs_context())
.expect_err("the hook fails");
let message = format!("{err:#}");
assert!(message.contains("preview function failed"), "{message}");
assert!(message.contains("boom"), "{message}");
assert!(!is_retryable_serdes_error(&*err), "{message}");
assert_eq!(count_files(&base), 0, "no payload file");
let _ = std::fs::remove_dir_all(&base);
}
#[test]
fn preview_hook_key_order_kept_in_envelope() {
let base = scratch_path("preview_order");
let serdes = with_preview(
&base,
FileSystemSerdesConfig::builder()
.storage_mode(FileSystemSerdesMode::Always)
.preview(|_json| Ok(Some("{ \"b\": 1, \"a\": 2 }".to_owned()))),
);
let envelope = serdes
.serialize_sync(&secret_value(), &fs_context())
.expect("serialize");
assert!(
envelope.contains(r#""preview":{"b":1,"a":2}"#),
"{envelope}"
);
let _ = std::fs::remove_dir_all(&base);
}
#[test]
fn preview_hook_receives_the_value_json() {
let base = scratch_path("preview_input");
let seen = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
let hook_seen = std::sync::Arc::clone(&seen);
let serdes = with_preview(
&base,
FileSystemSerdesConfig::builder().preview(move |json| {
hook_seen.lock().expect("lock").push(json.to_owned());
Ok(None)
}),
);
serdes
.serialize_sync(&secret_value(), &fs_context())
.expect("serialize");
assert_eq!(
*seen.lock().expect("lock"),
vec![serde_json::to_string(&secret_value()).expect("render")]
);
let _ = std::fs::remove_dir_all(&base);
}
#[test]
fn preview_config_keeps_clone_debug_and_default() {
assert!(FileSystemSerdesConfig::default().preview.is_none());
assert!(FileSystemSerdesConfig::builder().build().preview.is_none());
let config = FileSystemSerdesConfig::builder()
.preview(|_json| Ok(None))
.build();
let copy = config.clone();
assert!(copy.preview.is_some());
let debug = format!("{copy:?}");
assert!(debug.contains("preview: Some(PreviewFn)"), "{debug}");
}
}