use std::str::FromStr as _;
use lambda_runtime::tracing::subscriber;
use subscriber::Layer as _;
use subscriber::filter::{EnvFilter, LevelFilter};
use subscriber::fmt::format::{FormatEvent, FormatFields as _, JsonFields, Writer};
use subscriber::fmt::{FmtContext, FormattedFields, MakeWriter};
use subscriber::layer::SubscriberExt as _;
use tracing::Id;
use tracing::span::Attributes;
pub(crate) use crate::observability::field_names as fields;
fn record_optional(span: &tracing::Span, tenant_id: Option<&str>, operation_name: Option<&str>) {
if let Some(tenant_id) = tenant_id {
span.record(fields::TENANT_ID, tenant_id);
}
if let Some(operation_name) = operation_name {
span.record(fields::OPERATION_NAME, operation_name);
}
}
#[must_use]
pub(crate) fn operation_span(
execution_arn: &str,
request_id: &str,
tenant_id: Option<&str>,
operation_id: &str,
operation_name: Option<&str>,
attempt: u32,
is_replay: bool,
) -> tracing::Span {
let span = tracing::info_span!(
"durable_operation",
{ fields::EXECUTION_ARN } = execution_arn,
{ fields::REQUEST_ID } = request_id,
{ fields::TENANT_ID } = tracing::field::Empty,
{ fields::OPERATION_ID } = operation_id,
{ fields::OPERATION_NAME } = tracing::field::Empty,
{ fields::ATTEMPT } = attempt,
{ fields::IS_REPLAY } = is_replay,
);
record_optional(&span, tenant_id, operation_name);
span
}
#[must_use]
pub(crate) fn execution_span(
execution_arn: &str,
request_id: &str,
tenant_id: Option<&str>,
is_replay: bool,
) -> tracing::Span {
let span = tracing::info_span!(
"durable_execution",
{ fields::EXECUTION_ARN } = execution_arn,
{ fields::REQUEST_ID } = request_id,
{ fields::TENANT_ID } = tracing::field::Empty,
{ fields::IS_REPLAY } = is_replay,
);
record_optional(&span, tenant_id, None);
span
}
#[must_use]
pub(crate) fn scoped_execution_span(
execution_arn: &str,
request_id: &str,
tenant_id: Option<&str>,
operation_id: &str,
operation_name: Option<&str>,
is_replay: bool,
) -> tracing::Span {
let span = tracing::info_span!(
"durable_execution",
{ fields::EXECUTION_ARN } = execution_arn,
{ fields::REQUEST_ID } = request_id,
{ fields::TENANT_ID } = tracing::field::Empty,
{ fields::OPERATION_ID } = operation_id,
{ fields::OPERATION_NAME } = tracing::field::Empty,
{ fields::IS_REPLAY } = is_replay,
);
record_optional(&span, tenant_id, operation_name);
span
}
use crate::observability::{TARGET, event_names};
#[derive(Debug, Clone, Copy)]
pub(crate) struct OperationIdentity<'a> {
pub(crate) execution_arn: &'a str,
pub(crate) request_id: &'a str,
pub(crate) operation_id: &'a str,
pub(crate) operation_name: Option<&'a str>,
pub(crate) operation_type: &'a str,
pub(crate) operation_sub_type: Option<&'a str>,
pub(crate) attempt: u32,
}
fn operation_type_str(op_type: &aws_sdk_lambda::types::OperationType) -> &'static str {
use aws_sdk_lambda::types::OperationType;
match *op_type {
OperationType::Step => "Step",
OperationType::Wait => "Wait",
OperationType::Context => "Context",
OperationType::Callback => "Callback",
OperationType::ChainedInvoke => "ChainedInvoke",
OperationType::Execution => "Execution",
_ => "Unknown",
}
}
#[derive(Debug)]
pub(crate) struct PendingTransitionEvent {
operation_id: String,
operation_name: Option<String>,
operation_type: &'static str,
operation_sub_type: Option<String>,
attempt: u32,
action: aws_sdk_lambda::types::OperationAction,
delay_seconds: i32,
error: String,
span: tracing::Span,
}
impl PendingTransitionEvent {
pub(crate) fn capture(update: &aws_sdk_lambda::types::OperationUpdate, attempt: u32) -> Self {
Self {
span: tracing::Span::current(),
operation_id: update.id().to_owned(),
operation_name: update.name().map(str::to_owned),
operation_type: operation_type_str(update.r#type()),
operation_sub_type: update.sub_type().map(str::to_owned),
attempt,
action: update.action().clone(),
delay_seconds: update
.step_options()
.and_then(aws_sdk_lambda::types::StepOptions::next_attempt_delay_seconds)
.unwrap_or_default(),
error: update
.error()
.and_then(|e| e.error_message())
.unwrap_or_default()
.to_owned(),
}
}
pub(crate) fn emit(&self, execution_arn: &str, request_id: &str) {
use aws_sdk_lambda::types::OperationAction;
let op = OperationIdentity {
execution_arn,
request_id,
operation_id: &self.operation_id,
operation_name: self.operation_name.as_deref(),
operation_type: self.operation_type,
operation_sub_type: self.operation_sub_type.as_deref(),
attempt: self.attempt,
};
self.span.in_scope(|| match self.action {
OperationAction::Start => operation_started_event(&op),
OperationAction::Succeed => operation_succeeded_event(&op),
OperationAction::Fail => operation_failed_event(&op, &self.error),
OperationAction::Retry => {
operation_retry_scheduled_event(&op, self.delay_seconds, &self.error);
}
_ => {}
});
}
}
fn operation_started_event(op: &OperationIdentity<'_>) {
tracing::event!(
target: TARGET,
tracing::Level::DEBUG,
{ fields::EXECUTION_ARN } = op.execution_arn,
{ fields::REQUEST_ID } = op.request_id,
{ fields::OPERATION_ID } = op.operation_id,
{ fields::OPERATION_NAME } = op.operation_name,
{ fields::OPERATION_TYPE } = op.operation_type,
{ fields::OPERATION_SUB_TYPE } = op.operation_sub_type,
{ fields::ATTEMPT } = op.attempt,
{ fields::IS_REPLAY } = false,
"{}",
event_names::OPERATION_STARTED,
);
}
fn operation_succeeded_event(op: &OperationIdentity<'_>) {
tracing::event!(
target: TARGET,
tracing::Level::DEBUG,
{ fields::EXECUTION_ARN } = op.execution_arn,
{ fields::REQUEST_ID } = op.request_id,
{ fields::OPERATION_ID } = op.operation_id,
{ fields::OPERATION_NAME } = op.operation_name,
{ fields::OPERATION_TYPE } = op.operation_type,
{ fields::OPERATION_SUB_TYPE } = op.operation_sub_type,
{ fields::ATTEMPT } = op.attempt,
{ fields::IS_REPLAY } = false,
"{}",
event_names::OPERATION_SUCCEEDED,
);
}
fn operation_failed_event(op: &OperationIdentity<'_>, error: &str) {
tracing::event!(
target: TARGET,
tracing::Level::DEBUG,
{ fields::EXECUTION_ARN } = op.execution_arn,
{ fields::REQUEST_ID } = op.request_id,
{ fields::OPERATION_ID } = op.operation_id,
{ fields::OPERATION_NAME } = op.operation_name,
{ fields::OPERATION_TYPE } = op.operation_type,
{ fields::OPERATION_SUB_TYPE } = op.operation_sub_type,
{ fields::ATTEMPT } = op.attempt,
{ fields::IS_REPLAY } = false,
{ fields::ERROR } = error,
"{}",
event_names::OPERATION_FAILED,
);
}
fn operation_retry_scheduled_event(op: &OperationIdentity<'_>, delay_seconds: i32, error: &str) {
tracing::event!(
target: TARGET,
tracing::Level::DEBUG,
{ fields::EXECUTION_ARN } = op.execution_arn,
{ fields::REQUEST_ID } = op.request_id,
{ fields::OPERATION_ID } = op.operation_id,
{ fields::OPERATION_NAME } = op.operation_name,
{ fields::OPERATION_TYPE } = op.operation_type,
{ fields::OPERATION_SUB_TYPE } = op.operation_sub_type,
{ fields::ATTEMPT } = op.attempt,
{ fields::IS_REPLAY } = false,
{ fields::DELAY_SECONDS } = delay_seconds,
{ fields::ERROR } = error,
"{}",
event_names::OPERATION_RETRY_SCHEDULED,
);
}
pub(crate) fn operation_replayed_event(op: &OperationIdentity<'_>) {
tracing::event!(
target: TARGET,
tracing::Level::DEBUG,
{ fields::EXECUTION_ARN } = op.execution_arn,
{ fields::REQUEST_ID } = op.request_id,
{ fields::OPERATION_ID } = op.operation_id,
{ fields::OPERATION_NAME } = op.operation_name,
{ fields::OPERATION_TYPE } = op.operation_type,
{ fields::OPERATION_SUB_TYPE } = op.operation_sub_type,
{ fields::ATTEMPT } = op.attempt,
{ fields::IS_REPLAY } = true,
"{}",
event_names::OPERATION_REPLAYED,
);
}
pub(crate) fn invocation_begin_event(is_replaying: bool, execution_arn: &str, request_id: &str) {
if is_replaying {
execution_resumed_event(execution_arn, request_id);
} else {
execution_started_event(execution_arn, request_id);
}
}
pub(crate) fn execution_started_event(execution_arn: &str, request_id: &str) {
tracing::event!(
target: TARGET,
tracing::Level::DEBUG,
{ fields::EXECUTION_ARN } = execution_arn,
{ fields::REQUEST_ID } = request_id,
"{}",
event_names::EXECUTION_STARTED,
);
}
pub(crate) fn execution_resumed_event(execution_arn: &str, request_id: &str) {
tracing::event!(
target: TARGET,
tracing::Level::DEBUG,
{ fields::EXECUTION_ARN } = execution_arn,
{ fields::REQUEST_ID } = request_id,
"{}",
event_names::EXECUTION_RESUMED,
);
}
pub(crate) fn execution_suspended_event(execution_arn: &str, request_id: &str) {
tracing::event!(
target: TARGET,
tracing::Level::DEBUG,
{ fields::EXECUTION_ARN } = execution_arn,
{ fields::REQUEST_ID } = request_id,
"{}",
event_names::EXECUTION_SUSPENDED,
);
}
struct ReplayVisitor {
is_replay: Option<bool>,
}
impl tracing::field::Visit for ReplayVisitor {
fn record_bool(&mut self, field: &tracing::field::Field, value: bool) {
if field.name() == fields::IS_REPLAY {
self.is_replay = Some(value);
}
}
fn record_debug(&mut self, _field: &tracing::field::Field, _value: &dyn std::fmt::Debug) {}
}
pub(crate) fn is_replay_event(attrs: &Attributes<'_>) -> bool {
let mut visitor = ReplayVisitor { is_replay: None };
attrs.record(&mut visitor);
visitor.is_replay == Some(true)
}
fn replay_flag_in_record(values: &tracing::span::Record<'_>) -> Option<bool> {
let mut visitor = ReplayVisitor { is_replay: None };
values.record(&mut visitor);
visitor.is_replay
}
#[cfg(test)]
#[derive(Debug)]
pub(crate) struct ReplayTracker {
replay_spans: std::sync::RwLock<std::collections::HashSet<u64>>,
}
#[cfg(test)]
impl ReplayTracker {
pub(crate) fn new() -> Self {
Self {
replay_spans: std::sync::RwLock::new(std::collections::HashSet::new()),
}
}
pub(crate) fn is_replay(&self, id: u64) -> bool {
self.replay_spans.read().is_ok_and(|set| set.contains(&id))
}
}
#[derive(Debug, Clone, Default)]
#[non_exhaustive]
pub struct ReplayFilterLayer;
impl ReplayFilterLayer {
#[must_use]
pub fn new() -> Self {
Self
}
}
impl<S> subscriber::layer::Filter<S> for ReplayFilterLayer
where
S: tracing::Subscriber + for<'lookup> subscriber::registry::LookupSpan<'lookup>,
{
fn enabled(
&self,
_meta: &tracing::Metadata<'_>,
_ctx: &subscriber::layer::Context<'_, S>,
) -> bool {
true
}
fn event_enabled(
&self,
event: &tracing::Event<'_>,
ctx: &subscriber::layer::Context<'_, S>,
) -> bool {
let mut visitor = ReplayVisitor { is_replay: None };
event.record(&mut visitor);
if let Some(is_replay) = visitor.is_replay {
return !is_replay;
}
if let Some(scope) = ctx.event_scope(event) {
for span in scope {
let extensions = span.extensions();
let Some(fields) = extensions.get::<ReplaySpanFields>() else {
continue;
};
if fields.is_replay {
return false;
}
if fields.kind == SdkSpan::Execution {
break;
}
}
}
true
}
fn on_new_span(&self, attrs: &Attributes<'_>, id: &Id, ctx: subscriber::layer::Context<'_, S>) {
if let Some(kind) = sdk_span(attrs.metadata())
&& let Some(span) = ctx.span(id)
{
span.extensions_mut().replace(ReplaySpanFields {
kind,
is_replay: is_replay_event(attrs),
});
}
}
fn on_record(
&self,
id: &Id,
values: &tracing::span::Record<'_>,
ctx: subscriber::layer::Context<'_, S>,
) {
if let Some(is_replay) = replay_flag_in_record(values)
&& let Some(span) = ctx.span(id)
&& let Some(fields) = span.extensions_mut().get_mut::<ReplaySpanFields>()
{
fields.is_replay = is_replay;
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum SdkSpan {
Execution,
Operation,
}
const SPAN_TARGET: &str = module_path!();
fn sdk_span(meta: &tracing::Metadata<'_>) -> Option<SdkSpan> {
if !meta.is_span() || meta.target() != SPAN_TARGET {
return None;
}
match meta.name() {
crate::observability::span_names::EXECUTION => Some(SdkSpan::Execution),
crate::observability::span_names::OPERATION => Some(SdkSpan::Operation),
_ => None,
}
}
#[derive(Debug)]
struct ReplaySpanFields {
kind: SdkSpan,
is_replay: bool,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct LogSettings {
pub(crate) level: String,
pub(crate) directives: String,
pub(crate) format: String,
}
impl LogSettings {
pub(crate) fn from_env() -> Self {
Self::from_lookup(|name| std::env::var(name).ok())
}
pub(crate) fn from_lookup(lookup: impl Fn(&str) -> Option<String>) -> Self {
let directives = lookup("RUST_LOG");
let level = lookup("AWS_LAMBDA_LOG_LEVEL")
.or_else(|| directives.clone())
.unwrap_or_else(|| "INFO".to_owned());
Self {
level,
directives: directives.unwrap_or_default(),
format: lookup("AWS_LAMBDA_LOG_FORMAT").unwrap_or_default(),
}
}
}
struct DefaultFilter<F> {
env: F,
replay: ReplayFilterLayer,
}
impl<S, F> subscriber::layer::Filter<S> for DefaultFilter<F>
where
S: tracing::Subscriber + for<'lookup> subscriber::registry::LookupSpan<'lookup>,
F: subscriber::layer::Filter<S>,
{
fn enabled(
&self,
meta: &tracing::Metadata<'_>,
cx: &subscriber::layer::Context<'_, S>,
) -> bool {
sdk_span(meta).is_some() || self.env.enabled(meta, cx)
}
fn callsite_enabled(
&self,
meta: &'static tracing::Metadata<'static>,
) -> tracing::subscriber::Interest {
let env = self.env.callsite_enabled(meta);
if sdk_span(meta).is_none() && env.is_never() {
tracing::subscriber::Interest::never()
} else {
tracing::subscriber::Interest::sometimes()
}
}
fn event_enabled(
&self,
event: &tracing::Event<'_>,
cx: &subscriber::layer::Context<'_, S>,
) -> bool {
subscriber::layer::Filter::<S>::event_enabled(&self.replay, event, cx)
}
fn max_level_hint(&self) -> Option<LevelFilter> {
self.env
.max_level_hint()
.map(|hint| hint.max(LevelFilter::INFO))
}
fn on_new_span(&self, attrs: &Attributes<'_>, id: &Id, ctx: subscriber::layer::Context<'_, S>) {
self.env.on_new_span(attrs, id, ctx.clone());
subscriber::layer::Filter::<S>::on_new_span(&self.replay, attrs, id, ctx);
}
fn on_record(
&self,
id: &Id,
values: &tracing::span::Record<'_>,
ctx: subscriber::layer::Context<'_, S>,
) {
self.env.on_record(id, values, ctx.clone());
subscriber::layer::Filter::<S>::on_record(&self.replay, id, values, ctx);
}
fn on_enter(&self, id: &Id, ctx: subscriber::layer::Context<'_, S>) {
self.env.on_enter(id, ctx);
}
fn on_exit(&self, id: &Id, ctx: subscriber::layer::Context<'_, S>) {
self.env.on_exit(id, ctx);
}
fn on_close(&self, id: Id, ctx: subscriber::layer::Context<'_, S>) {
self.env.on_close(id, ctx);
}
}
pub(crate) fn default_subscriber<W>(
level: &str,
directives: &str,
format: &str,
writer: W,
) -> Box<dyn tracing::Subscriber + Send + Sync>
where
W: for<'w> MakeWriter<'w> + Send + Sync + 'static,
{
let level = LevelFilter::from_str(level).unwrap_or(LevelFilter::INFO);
let filter = DefaultFilter {
env: EnvFilter::builder()
.with_default_directive(level.into())
.parse_lossy(directives),
replay: ReplayFilterLayer::new(),
};
let registry = subscriber::registry();
if format.eq_ignore_ascii_case("json") {
let layer = subscriber::fmt::layer()
.fmt_fields(JsonFields::new())
.event_format(TopLevelJson)
.with_writer(writer);
Box::new(registry.with(layer.with_filter(filter)))
} else {
let layer = subscriber::fmt::layer()
.fmt_fields(TextFields::default())
.with_target(false)
.without_time()
.with_writer(writer)
.and_then(SpanFieldsInPlace);
Box::new(registry.with(layer.with_filter(filter)))
}
}
#[derive(Debug, Default)]
struct TextFields(subscriber::fmt::format::DefaultFields);
impl<'w> subscriber::fmt::format::FormatFields<'w> for TextFields {
fn format_fields<R: subscriber::field::RecordFields>(
&self,
writer: Writer<'w>,
fields: R,
) -> std::fmt::Result {
self.0.format_fields(writer, fields)
}
fn add_fields(
&self,
_current: &'w mut FormattedFields<Self>,
_fields: &tracing::span::Record<'_>,
) -> std::fmt::Result {
Ok(())
}
}
struct TextSpanFields(Vec<(&'static str, String)>);
struct SpanFieldsInPlace;
impl<S> subscriber::Layer<S> for SpanFieldsInPlace
where
S: tracing::Subscriber + for<'a> subscriber::registry::LookupSpan<'a>,
{
#[expect(clippy::expect_used)]
fn on_new_span(&self, attrs: &Attributes<'_>, id: &Id, ctx: subscriber::layer::Context<'_, S>) {
let span = ctx.span(id).expect("the registry stores the new span");
let mut extensions = span.extensions_mut();
let Some(formatted) = extensions.get_mut::<FormattedFields<TextFields>>() else {
return;
};
let mut fields = Vec::new();
let mut texts = FieldTexts {
formatted,
fields: &mut fields,
result: Ok(()),
};
attrs.record(&mut texts);
if texts.result.is_ok() {
extensions.insert(TextSpanFields(fields));
}
}
#[expect(clippy::expect_used)]
fn on_record(
&self,
id: &Id,
values: &tracing::span::Record<'_>,
ctx: subscriber::layer::Context<'_, S>,
) {
let span = ctx.span(id).expect("the registry stores the recorded span");
let mut extensions = span.extensions_mut();
let Some(TextSpanFields(mut fields)) = extensions.remove::<TextSpanFields>() else {
return;
};
let formatted = extensions
.get_mut::<FormattedFields<TextFields>>()
.expect("the span's texts are stored beside its formatted fields");
values.record(&mut FieldTexts {
formatted,
fields: &mut fields,
result: Ok(()),
});
let joined: Vec<&str> = fields.iter().map(|(_, text)| text.as_str()).collect();
formatted.fields = joined.join(" ");
extensions.insert(TextSpanFields(fields));
}
}
struct FieldTexts<'a> {
formatted: &'a mut FormattedFields<TextFields>,
fields: &'a mut Vec<(&'static str, String)>,
result: std::fmt::Result,
}
impl FieldTexts<'_> {
fn store(
&mut self,
field: &tracing::field::Field,
write: impl FnOnce(&mut subscriber::fmt::format::DefaultVisitor<'_>),
) {
use subscriber::field::VisitOutput as _;
let saved = std::mem::take(&mut self.formatted.fields);
let mut visitor =
subscriber::fmt::format::DefaultVisitor::new(self.formatted.as_writer(), true);
write(&mut visitor);
let result = visitor.finish();
let text = std::mem::replace(&mut self.formatted.fields, saved);
if let Err(error) = result {
self.result = Err(error);
return;
}
match self
.fields
.iter_mut()
.find(|(name, _)| *name == field.name())
{
Some((_, old)) => *old = text,
None => self.fields.push((field.name(), text)),
}
}
}
impl tracing::field::Visit for FieldTexts<'_> {
fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
self.store(field, |v| v.record_str(field, value));
}
fn record_error(
&mut self,
field: &tracing::field::Field,
value: &(dyn std::error::Error + 'static),
) {
self.store(field, |v| v.record_error(field, value));
}
fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
self.store(field, |v| v.record_debug(field, value));
}
}
#[derive(Debug, Clone, Copy)]
struct TopLevelJson;
const TRAILING_CORRELATION_KEYS: [&str; 5] = [
fields::EXECUTION_ARN,
fields::TENANT_ID,
fields::OPERATION_ID,
fields::OPERATION_NAME,
fields::ATTEMPT,
];
const TIMESTAMP_KEY: &str = "timestamp";
const LEVEL_KEY: &str = "level";
const MESSAGE_KEY: &str = "message";
fn is_reserved_key(key: &str) -> bool {
key == fields::REQUEST_ID
|| TRAILING_CORRELATION_KEYS.contains(&key)
|| key == TIMESTAMP_KEY
|| key == LEVEL_KEY
|| key == fields::IS_REPLAY
|| key == MESSAGE_KEY
}
type FieldMap = std::collections::BTreeMap<String, serde_json::Value>;
fn parse_fields(formatted: &str) -> Result<FieldMap, std::fmt::Error> {
serde_json::from_str(formatted).map_err(|_| std::fmt::Error)
}
struct SpanFields {
is_sdk: bool,
fields: FieldMap,
}
impl<S> FormatEvent<S, JsonFields> for TopLevelJson
where
S: tracing::Subscriber + for<'lookup> subscriber::registry::LookupSpan<'lookup>,
{
fn format_event(
&self,
ctx: &FmtContext<'_, S, JsonFields>,
mut writer: Writer<'_>,
event: &tracing::Event<'_>,
) -> std::fmt::Result {
let timestamp = js_timestamp(std::time::SystemTime::now())?;
let mut formatted = String::new();
ctx.field_format()
.format_fields(Writer::new(&mut formatted), event)?;
let event_fields = parse_fields(&formatted)?;
let mut spans = Vec::new();
if let Some(scope) = ctx.event_scope() {
for span in scope {
let extensions = span.extensions();
let fields = match extensions.get::<FormattedFields<JsonFields>>() {
Some(stored) => parse_fields(&stored.fields)?,
None => FieldMap::new(),
};
spans.push(SpanFields {
is_sdk: sdk_span(span.metadata()).is_some(),
fields,
});
}
}
let correlation = |key: &str| {
event_fields.get(key).or_else(|| {
spans
.iter()
.filter(|span| span.is_sdk)
.find_map(|span| span.fields.get(key))
})
};
let mut ordinary = std::collections::BTreeMap::new();
for source in std::iter::once(&event_fields).chain(spans.iter().map(|span| &span.fields)) {
for (key, value) in source {
if !is_reserved_key(key) {
ordinary.entry(key.as_str()).or_insert(value);
}
}
}
let empty_message = serde_json::Value::String(String::new());
let message = event_fields.get(MESSAGE_KEY).unwrap_or(&empty_message);
let level = serde_json::Value::String(event.metadata().level().as_str().to_owned());
let timestamp = serde_json::Value::String(timestamp);
let mut entries: Vec<(&str, &serde_json::Value)> = Vec::new();
if let Some(value) = correlation(fields::REQUEST_ID) {
entries.push((fields::REQUEST_ID, value));
}
entries.push((TIMESTAMP_KEY, ×tamp));
entries.push((LEVEL_KEY, &level));
for key in TRAILING_CORRELATION_KEYS {
if let Some(value) = correlation(key) {
entries.push((key, value));
}
}
entries.extend(ordinary);
entries.push((MESSAGE_KEY, message));
writer.write_char('{')?;
for (index, (key, value)) in entries.into_iter().enumerate() {
if index > 0 {
writer.write_char(',')?;
}
write!(writer, "{}:{value}", serde_json::Value::from(key))?;
}
writer.write_str("}\n")
}
}
fn js_timestamp(time: std::time::SystemTime) -> Result<String, std::fmt::Error> {
const MILLIS_PER_DAY: i128 = 86_400_000;
let millis = match time.duration_since(std::time::UNIX_EPOCH) {
Ok(after) => i128::from(after.as_secs()) * 1000 + i128::from(after.subsec_millis()),
Err(before) => {
let before = before.duration();
let nanos =
i128::from(before.as_secs()) * 1_000_000_000 + i128::from(before.subsec_nanos());
-((nanos + 999_999) / 1_000_000)
}
};
let (year, month, day) = civil_from_days(millis.div_euclid(MILLIS_PER_DAY));
if !(0..=9999).contains(&year) {
return Err(std::fmt::Error);
}
let of_day = millis.rem_euclid(MILLIS_PER_DAY);
let (hours, minutes) = (of_day / 3_600_000, of_day / 60_000 % 60);
let (seconds, subsec) = (of_day / 1000 % 60, of_day % 1000);
Ok(format!(
"{year:04}-{month:02}-{day:02}T{hours:02}:{minutes:02}:{seconds:02}.{subsec:03}Z"
))
}
fn civil_from_days(days: i128) -> (i128, i128, i128) {
let shifted = days + 719_468;
let era = shifted.div_euclid(146_097);
let day_of_era = shifted - era * 146_097;
let year_of_era =
(day_of_era - day_of_era / 1460 + day_of_era / 36_524 - day_of_era / 146_096) / 365;
let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100);
let march_month = (5 * day_of_year + 2) / 153;
let day = day_of_year - (153 * march_month + 2) / 5 + 1;
let month = if march_month < 10 {
march_month + 3
} else {
march_month - 9
};
let year = year_of_era + era * 400 + i128::from(month <= 2);
(year, month, day)
}
#[cfg(test)]
pub(crate) mod tests {
use super::*;
#[test]
fn operation_span_has_correct_fields() {
let _guard = test_tracing::install_test_subscriber(
tracing_subscriber::fmt().with_test_writer().finish(),
);
let span = operation_span(
"arn:aws:lambda:us-east-1:123:function:test:durable:abc",
"req-123",
None,
"deadbeef01234567890abcdef0123456789abcdef0123456789abcdef012345",
None,
1,
false,
);
assert!(!span.is_disabled());
}
#[test]
fn replay_tracker_marks_replay_spans() {
let tracker = ReplayTracker::new();
assert!(!tracker.is_replay(999));
}
#[test]
fn field_constants_match_cross_sdk_contract() {
assert_eq!(fields::EXECUTION_ARN, "executionArn");
assert_eq!(fields::REQUEST_ID, "requestId");
assert_eq!(fields::OPERATION_ID, "operationId");
assert_eq!(fields::ATTEMPT, "attempt");
assert_eq!(fields::IS_REPLAY, "isReplay");
}
#[test]
fn json_output_contains_cross_sdk_fields() {
use std::sync::{Arc, Mutex};
use tracing_subscriber::fmt::MakeWriter;
#[derive(Clone)]
struct BufWriter(Arc<Mutex<Vec<u8>>>);
impl std::io::Write for BufWriter {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
if let Ok(mut inner) = self.0.lock() {
inner.extend_from_slice(buf);
}
Ok(buf.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
impl<'a> MakeWriter<'a> for BufWriter {
type Writer = BufWriter;
fn make_writer(&'a self) -> Self::Writer {
self.clone()
}
}
let buffer = Arc::new(Mutex::new(Vec::new()));
let writer = BufWriter(Arc::clone(&buffer));
let subscriber = tracing_subscriber::fmt()
.json()
.flatten_event(true)
.with_writer(writer)
.with_span_list(false)
.finish();
let _guard = test_tracing::install_test_subscriber(subscriber);
let test_arn = "arn:aws:lambda:us-east-1:123456789012:function:myFunc:durable:exec-123";
let test_request_id = "req-abc-def-456";
let test_op_id = "aabbccdd00112233445566778899aabbccddeeff00112233445566778899aabb";
let span = operation_span(test_arn, test_request_id, None, test_op_id, None, 2, false);
let _entered = span.enter();
tracing::info!("Greeting step started for: TestUser");
let output = buffer.lock().map_or_else(
|_| String::new(),
|b| String::from_utf8_lossy(&b).to_string(),
);
assert!(
output.contains(test_arn),
"JSON output must contain executionArn. Got: {output}"
);
assert!(
output.contains(test_request_id),
"JSON output must contain requestId. Got: {output}"
);
assert!(
output.contains(test_op_id),
"JSON output must contain operationId. Got: {output}"
);
assert!(
output.contains("\"attempt\":2"),
"JSON output must contain attempt. Got: {output}"
);
assert!(
output.contains("\"isReplay\":false"),
"JSON output must contain isReplay. Got: {output}"
);
assert!(
output.contains("Greeting step started for: TestUser"),
"JSON output must contain the user message. Got: {output}"
);
}
#[test]
fn replay_span_marks_events_with_is_replay_true() {
use std::sync::{Arc, Mutex};
use tracing_subscriber::fmt::MakeWriter;
#[derive(Clone)]
struct BufWriter(Arc<Mutex<Vec<u8>>>);
impl std::io::Write for BufWriter {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
if let Ok(mut inner) = self.0.lock() {
inner.extend_from_slice(buf);
}
Ok(buf.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
impl<'a> MakeWriter<'a> for BufWriter {
type Writer = BufWriter;
fn make_writer(&'a self) -> Self::Writer {
self.clone()
}
}
let buffer = Arc::new(Mutex::new(Vec::new()));
let writer = BufWriter(Arc::clone(&buffer));
let subscriber = tracing_subscriber::fmt()
.json()
.flatten_event(true)
.with_writer(writer)
.with_span_list(false)
.finish();
let _guard = test_tracing::install_test_subscriber(subscriber);
let span = operation_span("arn:test", "req-1", None, "op-1", None, 1, true);
let _entered = span.enter();
tracing::info!("this event is inside a replay span");
let output = buffer.lock().map_or_else(
|_| String::new(),
|b| String::from_utf8_lossy(&b).to_string(),
);
assert!(
output.contains("\"isReplay\":true"),
"Replay span must emit isReplay=true. Got: {output}"
);
}
#[test]
fn replay_filter_suppresses_replay_events() {
use std::sync::{Arc, Mutex};
use tracing_subscriber::Layer as _;
use tracing_subscriber::fmt::MakeWriter;
use tracing_subscriber::layer::SubscriberExt;
#[derive(Clone)]
struct BufWriter(Arc<Mutex<Vec<u8>>>);
impl std::io::Write for BufWriter {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
if let Ok(mut inner) = self.0.lock() {
inner.extend_from_slice(buf);
}
Ok(buf.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
impl<'a> MakeWriter<'a> for BufWriter {
type Writer = BufWriter;
fn make_writer(&'a self) -> Self::Writer {
self.clone()
}
}
let buffer = Arc::new(Mutex::new(Vec::new()));
let writer = BufWriter(Arc::clone(&buffer));
let fmt_layer = tracing_subscriber::fmt::layer()
.json()
.flatten_event(true)
.with_span_list(false)
.with_writer(writer)
.with_filter(ReplayFilterLayer::new());
let subscriber = tracing_subscriber::registry().with(fmt_layer);
let _guard = test_tracing::install_test_subscriber(subscriber);
{
let span = operation_span("arn:test", "req-1", None, "op-1", None, 1, false);
let _entered = span.enter();
tracing::info!("live event");
}
{
let span = operation_span("arn:test", "req-1", None, "op-2", None, 1, true);
let _entered = span.enter();
tracing::info!("replay event");
}
let output = buffer.lock().map_or_else(
|_| String::new(),
|b| String::from_utf8_lossy(&b).to_string(),
);
assert!(
output.contains("live event"),
"Live events must pass through the replay filter. Got: {output}"
);
assert!(
!output.contains("replay event"),
"Replay events must be suppressed. Got: {output}"
);
}
#[test]
fn replay_filter_trusts_event_own_is_replay_field_at_boundary() {
use std::sync::{Arc, Mutex};
use tracing_subscriber::Layer as _;
use tracing_subscriber::fmt::MakeWriter;
use tracing_subscriber::layer::SubscriberExt;
#[derive(Clone)]
struct BufWriter(Arc<Mutex<Vec<u8>>>);
impl std::io::Write for BufWriter {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
if let Ok(mut inner) = self.0.lock() {
inner.extend_from_slice(buf);
}
Ok(buf.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
impl<'a> MakeWriter<'a> for BufWriter {
type Writer = BufWriter;
fn make_writer(&'a self) -> Self::Writer {
self.clone()
}
}
let buffer = Arc::new(Mutex::new(Vec::new()));
let writer = BufWriter(Arc::clone(&buffer));
let fmt_layer = tracing_subscriber::fmt::layer()
.json()
.flatten_event(true)
.with_span_list(false)
.with_writer(writer)
.with_filter(ReplayFilterLayer::new());
let subscriber = tracing_subscriber::registry().with(fmt_layer);
let _guard = test_tracing::install_test_subscriber(subscriber);
{
let span = execution_span("arn:test", "req-1", None, true);
span.record(fields::IS_REPLAY, false); let _entered = span.enter();
tracing::info!({ fields::IS_REPLAY } = true, "boundary replay hit");
}
{
let span = execution_span("arn:test", "req-1", None, true);
let _entered = span.enter();
tracing::info!({ fields::IS_REPLAY } = false, "live work under replay span");
}
let output = buffer.lock().map_or_else(
|_| String::new(),
|b| String::from_utf8_lossy(&b).to_string(),
);
assert!(
!output.contains("boundary replay hit"),
"an event carrying isReplay=true must be suppressed even when its \
span has already flipped to live. Got: {output}"
);
assert!(
output.contains("live work under replay span"),
"an event carrying isReplay=false must pass even under a span \
still marked as replaying. Got: {output}"
);
}
#[test]
fn replay_filter_allows_sdk_lifecycle_events() {
use std::sync::{Arc, Mutex};
use tracing_subscriber::Layer as _;
use tracing_subscriber::fmt::MakeWriter;
use tracing_subscriber::layer::SubscriberExt;
#[derive(Clone)]
struct BufWriter(Arc<Mutex<Vec<u8>>>);
impl std::io::Write for BufWriter {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
if let Ok(mut inner) = self.0.lock() {
inner.extend_from_slice(buf);
}
Ok(buf.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
impl<'a> MakeWriter<'a> for BufWriter {
type Writer = BufWriter;
fn make_writer(&'a self) -> Self::Writer {
self.clone()
}
}
let buffer = Arc::new(Mutex::new(Vec::new()));
let writer = BufWriter(Arc::clone(&buffer));
let fmt_layer = tracing_subscriber::fmt::layer()
.json()
.flatten_event(true)
.with_span_list(false)
.with_writer(writer)
.with_filter(ReplayFilterLayer::new());
let subscriber = tracing_subscriber::registry().with(fmt_layer);
let _guard = test_tracing::install_test_subscriber(subscriber);
tracing::info!("sdk lifecycle event");
let output = buffer.lock().map_or_else(
|_| String::new(),
|b| String::from_utf8_lossy(&b).to_string(),
);
assert!(
output.contains("sdk lifecycle event"),
"SDK lifecycle events must pass through. Got: {output}"
);
}
#[derive(Clone)]
struct CaptureWriter(std::sync::Arc<std::sync::Mutex<Vec<u8>>>);
impl std::io::Write for CaptureWriter {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
if let Ok(mut inner) = self.0.lock() {
inner.extend_from_slice(buf);
}
Ok(buf.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
impl<'a> MakeWriter<'a> for CaptureWriter {
type Writer = CaptureWriter;
fn make_writer(&'a self) -> Self::Writer {
self.clone()
}
}
#[test]
fn replay_filter_follows_dynamic_is_replay_updates() {
use std::sync::{Arc, Mutex};
use tracing_subscriber::Layer as _;
use tracing_subscriber::layer::SubscriberExt;
let buffer = Arc::new(Mutex::new(Vec::new()));
let writer = CaptureWriter(Arc::clone(&buffer));
let fmt_layer = tracing_subscriber::fmt::layer()
.json()
.flatten_event(true)
.with_span_list(false)
.with_writer(writer)
.with_filter(ReplayFilterLayer::new());
let subscriber = tracing_subscriber::registry().with(fmt_layer);
let _guard = test_tracing::install_test_subscriber(subscriber);
let span = execution_span("arn:test", "req-1", None, true);
{
let _entered = span.enter();
tracing::info!("suppressed while replaying");
span.record(fields::IS_REPLAY, false);
tracing::info!("emitted after replay ends");
span.record(fields::IS_REPLAY, true);
tracing::info!("suppressed again");
}
let output = buffer.lock().map_or_else(
|_| String::new(),
|b| String::from_utf8_lossy(&b).to_string(),
);
assert!(
!output.contains("suppressed while replaying"),
"Events while isReplay=true must be suppressed. Got: {output}"
);
assert!(
output.contains("emitted after replay ends"),
"Events after isReplay flips to false must pass. Got: {output}"
);
assert!(
!output.contains("suppressed again"),
"Events after isReplay flips back to true must be suppressed. Got: {output}"
);
}
#[tokio::test]
async fn replay_filter_suppresses_handler_level_replay_logs() {
use std::sync::{Arc, Mutex};
use std::time::Duration;
use tracing::Instrument as _;
use tracing_subscriber::Layer as _;
use tracing_subscriber::layer::SubscriberExt;
use crate::engine::{CheckpointLog, CheckpointRecord, CheckpointStatus};
let buffer = Arc::new(Mutex::new(Vec::new()));
let writer = CaptureWriter(Arc::clone(&buffer));
let fmt_layer = tracing_subscriber::fmt::layer()
.json()
.flatten_event(true)
.with_span_list(false)
.with_writer(writer)
.with_filter(ReplayFilterLayer::new());
let subscriber = tracing_subscriber::registry().with(fmt_layer);
let _guard = test_tracing::install_test_subscriber(subscriber);
let wire_key = crate::engine::compute_wire_id_public("1");
let record = CheckpointRecord {
id: wire_key.clone(),
status: CheckpointStatus::Succeeded,
result: None,
error_type: None,
error_message: None,
error_data: None,
stack_trace: None,
attempt: 0,
invoke_result: None,
invoke_error_type: None,
invoke_error_message: None,
invoke_error_data: None,
invoke_stack_trace: None,
replay_children: false,
callback_id: None,
op_type: None,
sub_type: None,
due_at: None,
op_name: None,
parent_id: None,
start_timestamp: None,
end_timestamp: None,
};
let log = Arc::new(CheckpointLog::from_records(vec![(wire_key, record)]));
let ctx = crate::DurableContext::new_root(
"arn:test".to_owned(),
lambda_runtime::Context::default(),
log,
);
let replay_span = ctx.replay_span();
async move {
tracing::info!("handler line before the wait");
let _ = ctx.wait(Duration::from_secs(5)).await;
tracing::info!("handler line after the wait");
}
.instrument(replay_span)
.await;
let output = buffer.lock().map_or_else(
|_| String::new(),
|b| String::from_utf8_lossy(&b).to_string(),
);
assert!(
!output.contains("handler line before the wait"),
"Handler-level events during replay must be suppressed. Got: {output}"
);
assert!(
output.contains("handler line after the wait"),
"Handler-level events past the high-water mark must be emitted. Got: {output}"
);
}
fn captured(buffer: &std::sync::Mutex<Vec<u8>>) -> String {
let raw = buffer.lock().map_or_else(
|_| String::new(),
|b| String::from_utf8_lossy(&b).to_string(),
);
let mut text = String::with_capacity(raw.len());
let mut chars = raw.chars();
while let Some(c) = chars.next() {
if c == '\u{1b}' {
for skipped in chars.by_ref() {
if skipped == 'm' {
break;
}
}
} else {
text.push(c);
}
}
text
}
fn count_message(output: &str, message: &str) -> usize {
output
.lines()
.filter(|line| line.split_whitespace().last() == Some(message))
.count()
}
#[derive(Clone, Copy)]
enum LogStyle {
Info,
Warn,
AppTarget,
}
fn log_line(style: LogStyle, message: &str) {
match style {
LogStyle::Info => tracing::info!("{message}"),
LogStyle::Warn => tracing::warn!("{message}"),
LogStyle::AppTarget => tracing::info!(target: "app", "{message}"),
}
}
fn run_logging_handler<S>(subscriber: S, log_audit: bool) -> usize
where
S: tracing::Subscriber + Send + Sync + 'static,
{
run_logging_handler_with(subscriber, log_audit, LogStyle::Info)
}
fn run_logging_handler_with<S>(subscriber: S, log_audit: bool, style: LogStyle) -> usize
where
S: tracing::Subscriber + Send + Sync + 'static,
{
use std::time::Duration;
{
let _guard = test_tracing::install_test_subscriber(subscriber);
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("runtime");
runtime.block_on(async move {
let result = crate::test_util::LocalRunner::new()
.run(
move |_event: serde_json::Value, ctx: crate::DurableContext| async move {
if log_audit {
tracing::info!(target: "audit", "audit-before");
}
log_line(style, "before");
ctx.wait(Duration::from_secs(1)).name("pause").await?;
log_line(style, "after");
Ok::<_, crate::BoxError>(())
},
serde_json::Value::Null,
)
.await
.expect("runner");
result.invocation_count()
})
}
}
#[test]
fn default_subscriber_writes_each_record_once_across_replay() {
let buffer = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
let subscriber = default_subscriber(
"INFO",
"",
"",
CaptureWriter(std::sync::Arc::clone(&buffer)),
);
let invocations = run_logging_handler(subscriber, false);
let output = captured(&buffer);
assert!(invocations >= 2, "the wait must suspend: {invocations}");
assert_eq!(count_message(&output, "before"), 1, "{output}");
assert_eq!(count_message(&output, "after"), 1, "{output}");
}
#[test]
fn default_text_replaces_recorded_span_fields_in_place() {
let buffer = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
let subscriber = default_subscriber(
"INFO",
"",
"",
CaptureWriter(std::sync::Arc::clone(&buffer)),
);
{
let _guard = test_tracing::install_test_subscriber(subscriber);
let span = tracing::info_span!(
"phase",
stage = "one",
stable = "keep",
late = tracing::field::Empty
);
span.record("stage", "two");
span.record("late", "set");
let _entered = span.enter();
tracing::info!("inside");
}
let output = captured(&buffer);
let line = output
.lines()
.find(|line| line.contains("inside"))
.expect("the line is written");
assert!(line.contains(r#"stage="two""#), "{line}");
assert!(!line.contains(r#"stage="one""#), "{line}");
assert_eq!(line.matches("stable").count(), 1, "{line}");
assert!(
line.find("stage").expect("stage") < line.find("stable").expect("stable"),
"{line}"
);
assert!(
line.find("stable").expect("stable") < line.find(r#"late="set""#).expect("late"),
"{line}"
);
}
struct FailingDebug {
calls: std::sync::atomic::AtomicUsize,
fail_on: &'static [usize],
}
impl FailingDebug {
fn new(fail_on: &'static [usize]) -> Self {
Self {
calls: std::sync::atomic::AtomicUsize::new(0),
fail_on,
}
}
}
impl std::fmt::Debug for FailingDebug {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let call = self
.calls
.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
+ 1;
if self.fail_on.contains(&call) {
return Err(std::fmt::Error);
}
f.write_str("ok")
}
}
fn text_line(body: impl FnOnce()) -> String {
let buffer = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
let subscriber = default_subscriber(
"INFO",
"",
"",
CaptureWriter(std::sync::Arc::clone(&buffer)),
);
{
let _guard = test_tracing::install_test_subscriber(subscriber);
body();
}
let output = captured(&buffer);
output
.lines()
.find(|line| line.contains("inside"))
.expect("the line is written")
.to_owned()
}
#[test]
fn default_text_writes_a_span_whose_fields_fail_to_format() {
let line = text_line(|| {
let value = FailingDebug::new(&[1]);
let span = tracing::info_span!("phase", flaky = ?value, stage = "one");
span.record("stage", "two");
let _entered = span.enter();
tracing::info!("inside");
});
assert!(line.contains("inside"), "{line}");
}
#[test]
fn default_text_keeps_span_fields_that_fail_to_format_alone() {
let line = text_line(|| {
let value = FailingDebug::new(&[2]);
let span = tracing::info_span!("phase", flaky = ?value, stage = "one");
span.record("stage", "two");
let _entered = span.enter();
tracing::info!("inside");
});
assert!(line.contains(r#"stage="one""#), "{line}");
assert!(!line.contains(r#"stage="two""#), "{line}");
assert!(line.contains("flaky=ok"), "{line}");
}
#[test]
fn default_text_ignores_a_recorded_value_that_fails_to_format() {
let line = text_line(|| {
let span = tracing::info_span!("phase", stage = "one", late = tracing::field::Empty);
span.record("stage", "two");
span.record("late", tracing::field::debug(FailingDebug::new(&[1])));
span.record("stage", "three");
let _entered = span.enter();
tracing::info!("inside");
});
assert!(line.contains(r#"stage="three""#), "{line}");
assert_eq!(line.matches("stage").count(), 1, "{line}");
assert!(!line.contains("late"), "{line}");
}
#[test]
fn default_subscriber_suppresses_replay_above_the_span_level() {
for level in ["WARN", "ERROR"] {
let buffer = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
let subscriber =
default_subscriber(level, "", "", CaptureWriter(std::sync::Arc::clone(&buffer)));
let invocations = run_logging_handler_with(subscriber, false, LogStyle::Warn);
let output = captured(&buffer);
assert!(invocations >= 2, "the wait must suspend: {invocations}");
let expected = usize::from(level == "WARN");
assert_eq!(
count_message(&output, "before"),
expected,
"{level}: {output}"
);
assert_eq!(
count_message(&output, "after"),
expected,
"{level}: {output}"
);
}
}
#[test]
fn default_subscriber_suppresses_replay_when_the_sdk_target_is_off() {
let buffer = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
let subscriber = default_subscriber(
"INFO",
"aws_durable_execution_sdk=off,app=info",
"",
CaptureWriter(std::sync::Arc::clone(&buffer)),
);
let invocations = run_logging_handler_with(subscriber, false, LogStyle::AppTarget);
let output = captured(&buffer);
assert!(invocations >= 2, "the wait must suspend: {invocations}");
assert_eq!(count_message(&output, "before"), 1, "{output}");
assert_eq!(count_message(&output, "after"), 1, "{output}");
}
#[test]
fn default_subscriber_applies_directives() {
let buffer = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
let subscriber = default_subscriber(
"WARN",
"app=debug,[=bogus",
"",
CaptureWriter(std::sync::Arc::clone(&buffer)),
);
{
let _guard = test_tracing::install_test_subscriber(subscriber);
tracing::debug!(target: "app", "app-line");
tracing::warn!("other-line");
};
let output = captured(&buffer);
assert_eq!(output.lines().count(), 1, "{output}");
assert_eq!(count_message(&output, "app-line"), 1, "{output}");
}
#[test]
fn default_subscriber_ignores_invalid_directives() {
let buffer = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
let subscriber = default_subscriber(
"WARN",
"[=bogus",
"",
CaptureWriter(std::sync::Arc::clone(&buffer)),
);
{
let _guard = test_tracing::install_test_subscriber(subscriber);
tracing::info!("info-line");
tracing::warn!("warn-line");
};
let output = captured(&buffer);
assert_eq!(output.lines().count(), 1, "{output}");
assert_eq!(count_message(&output, "warn-line"), 1, "{output}");
}
#[test]
fn default_subscriber_applies_span_directives() {
let buffer = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
let subscriber = default_subscriber(
"WARN",
"[outer]=debug",
"",
CaptureWriter(std::sync::Arc::clone(&buffer)),
);
{
let _guard = test_tracing::install_test_subscriber(subscriber);
tracing::info_span!("outer").in_scope(|| tracing::debug!("inside-line"));
tracing::debug!("outside-line");
};
let output = captured(&buffer);
assert_eq!(output.lines().count(), 1, "{output}");
assert_eq!(count_message(&output, "inside-line"), 1, "{output}");
}
#[test]
fn default_subscriber_tracks_span_records() {
let buffer = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
let subscriber = default_subscriber(
"WARN",
"[outer{flag=true}]=debug",
"",
CaptureWriter(std::sync::Arc::clone(&buffer)),
);
{
let _guard = test_tracing::install_test_subscriber(subscriber);
let span = tracing::info_span!("outer", flag = false);
span.in_scope(|| tracing::debug!("before-record"));
span.record("flag", true);
span.in_scope(|| tracing::debug!("after-record"));
};
let output = captured(&buffer);
assert_eq!(output.lines().count(), 1, "{output}");
assert_eq!(count_message(&output, "after-record"), 1, "{output}");
}
#[test]
fn default_subscriber_applies_span_directives_to_sdk_spans() {
for (directives, which) in [
("[durable_execution]=debug", "execution"),
("[durable_operation{attempt=1}]=debug", "operation"),
] {
let buffer = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
let subscriber = default_subscriber(
"WARN",
directives,
"",
CaptureWriter(std::sync::Arc::clone(&buffer)),
);
{
let _guard = test_tracing::install_test_subscriber(subscriber);
let span = if which == "execution" {
execution_span("arn", "request", None, false)
} else {
operation_span("arn", "request", None, "op", None, 1, false)
};
span.in_scope(|| tracing::debug!("inside-line"));
tracing::debug!("outside-line");
};
let output = captured(&buffer);
assert_eq!(output.lines().count(), 1, "{directives}: {output}");
assert_eq!(
count_message(&output, "inside-line"),
1,
"{directives}: {output}"
);
}
}
struct CloseCounter(std::sync::Arc<std::sync::atomic::AtomicUsize>);
impl<S> tracing_subscriber::layer::Filter<S> for CloseCounter {
fn enabled(
&self,
_meta: &tracing::Metadata<'_>,
_cx: &tracing_subscriber::layer::Context<'_, S>,
) -> bool {
true
}
fn on_close(&self, _id: Id, _ctx: tracing_subscriber::layer::Context<'_, S>) {
self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
}
}
#[test]
fn default_filter_forwards_span_close() {
use tracing_subscriber::Layer as _;
use tracing_subscriber::layer::SubscriberExt as _;
let closes = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
let filter = DefaultFilter {
env: CloseCounter(std::sync::Arc::clone(&closes)),
replay: ReplayFilterLayer::new(),
};
let subscriber = tracing_subscriber::registry().with(
tracing_subscriber::fmt::layer()
.with_writer(std::io::sink)
.with_filter(filter),
);
{
let _guard = test_tracing::install_test_subscriber(subscriber);
drop(tracing::info_span!("closed"));
};
assert_eq!(closes.load(std::sync::atomic::Ordering::SeqCst), 1);
}
fn lookup_in(vars: &[(&str, &str)]) -> impl Fn(&str) -> Option<String> {
let vars: Vec<(String, String)> = vars
.iter()
.map(|(name, value)| ((*name).to_owned(), (*value).to_owned()))
.collect();
move |name| {
vars.iter()
.find(|(var, _)| var == name)
.map(|(_, value)| value.clone())
}
}
fn settings(level: &str, directives: &str, format: &str) -> LogSettings {
LogSettings {
level: level.to_owned(),
directives: directives.to_owned(),
format: format.to_owned(),
}
}
#[test]
fn log_settings_default_to_info_text_and_no_directives() {
assert_eq!(
LogSettings::from_lookup(lookup_in(&[])),
settings("INFO", "", "")
);
}
#[test]
fn log_settings_take_the_level_from_aws_lambda_log_level_first() {
let lookup = lookup_in(&[("AWS_LAMBDA_LOG_LEVEL", "ERROR"), ("RUST_LOG", "debug")]);
assert_eq!(
LogSettings::from_lookup(lookup),
settings("ERROR", "debug", "")
);
}
#[test]
fn log_settings_fall_back_to_rust_log_for_the_level() {
let lookup = lookup_in(&[("RUST_LOG", "app=debug")]);
assert_eq!(
LogSettings::from_lookup(lookup),
settings("app=debug", "app=debug", "")
);
}
#[test]
fn log_settings_read_the_format() {
let lookup = lookup_in(&[("AWS_LAMBDA_LOG_FORMAT", "JSON")]);
assert_eq!(
LogSettings::from_lookup(lookup),
settings("INFO", "", "JSON")
);
}
#[test]
fn default_subscriber_json_format_writes_each_record_once_across_replay() {
let buffer = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
let subscriber = default_subscriber(
"INFO",
"",
"json",
CaptureWriter(std::sync::Arc::clone(&buffer)),
);
let invocations = run_logging_handler(subscriber, false);
let output = captured(&buffer);
assert!(invocations >= 2, "the wait must suspend: {invocations}");
let messages: Vec<String> = output
.lines()
.filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
.filter_map(|record| {
record
.pointer("/message")
.and_then(serde_json::Value::as_str)
.map(str::to_owned)
})
.collect();
let count = |message: &str| messages.iter().filter(|m| *m == message).count();
assert_eq!(count("before"), 1, "{output}");
assert_eq!(count("after"), 1, "{output}");
}
#[test]
fn default_subscriber_honors_level() {
let buffer = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
let subscriber = default_subscriber(
"WARN",
"",
"",
CaptureWriter(std::sync::Arc::clone(&buffer)),
);
{
let _guard = test_tracing::install_test_subscriber(subscriber);
tracing::info!("info-line");
tracing::warn!("warn-line");
};
let output = captured(&buffer);
assert_eq!(output.lines().count(), 1, "{output}");
assert_eq!(count_message(&output, "warn-line"), 1, "{output}");
}
#[test]
fn default_subscriber_unknown_level_falls_back_to_info() {
let buffer = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
let subscriber = default_subscriber(
"bogus",
"",
"",
CaptureWriter(std::sync::Arc::clone(&buffer)),
);
{
let _guard = test_tracing::install_test_subscriber(subscriber);
tracing::debug!("debug-line");
tracing::info!("info-line");
};
let output = captured(&buffer);
assert_eq!(output.lines().count(), 1, "{output}");
assert_eq!(count_message(&output, "info-line"), 1, "{output}");
}
#[test]
fn default_subscriber_json_format() {
for format in ["JSON", "json"] {
let buffer = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
let subscriber = default_subscriber(
"INFO",
"",
format,
CaptureWriter(std::sync::Arc::clone(&buffer)),
);
{
let _guard = test_tracing::install_test_subscriber(subscriber);
tracing::info!("json-line");
};
let output = captured(&buffer);
assert_eq!(output.lines().count(), 1, "{format}: {output}");
for line in output.lines() {
let parsed: serde_json::Value =
serde_json::from_str(line).expect("each line is JSON");
assert!(parsed.is_object(), "{format}: {line}");
}
}
}
#[test]
fn default_subscriber_text_format() {
let buffer = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
let subscriber = default_subscriber(
"INFO",
"",
"",
CaptureWriter(std::sync::Arc::clone(&buffer)),
);
{
let _guard = test_tracing::install_test_subscriber(subscriber);
tracing::info!("text-line");
};
let output = captured(&buffer);
assert_eq!(output.lines().count(), 1, "{output}");
for line in output.lines() {
assert!(
serde_json::from_str::<serde_json::Value>(line).is_err(),
"a text line must not parse as JSON: {line}"
);
}
}
#[test]
fn kept_target_survives_replay() {
use tracing_subscriber::Layer as _;
use tracing_subscriber::filter::{FilterExt as _, LevelFilter, Targets};
use tracing_subscriber::layer::SubscriberExt as _;
let buffer = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
let subscriber = tracing_subscriber::registry()
.with(
tracing_subscriber::fmt::layer()
.with_writer(CaptureWriter(std::sync::Arc::clone(&buffer)))
.with_filter(Targets::new().with_target("audit", LevelFilter::TRACE)),
)
.with(
tracing_subscriber::fmt::layer()
.with_writer(CaptureWriter(std::sync::Arc::clone(&buffer)))
.with_filter(
ReplayFilterLayer::new().and(
Targets::new()
.with_default(LevelFilter::TRACE)
.with_target("audit", LevelFilter::OFF),
),
),
);
let invocations = run_logging_handler(subscriber, true);
let output = captured(&buffer);
assert!(invocations >= 2, "the wait must suspend: {invocations}");
assert_eq!(count_message(&output, "audit-before"), 2, "{output}");
assert_eq!(count_message(&output, "before"), 1, "{output}");
}
pub(crate) fn json_span_subscriber(
buffer: &std::sync::Arc<std::sync::Mutex<Vec<u8>>>,
) -> impl tracing::Subscriber + Send + Sync + 'static {
tracing_subscriber::fmt()
.json()
.flatten_event(true)
.with_current_span(true)
.with_span_list(true)
.with_writer(CaptureWriter(std::sync::Arc::clone(buffer)))
.finish()
}
pub(crate) fn json_records(buffer: &std::sync::Mutex<Vec<u8>>) -> Vec<serde_json::Value> {
captured(buffer)
.lines()
.filter_map(|line| serde_json::from_str(line).ok())
.collect()
}
pub(crate) fn spans_of(records: &[serde_json::Value], message: &str) -> Vec<serde_json::Value> {
let record = records
.iter()
.find(|r| r.get("message").and_then(serde_json::Value::as_str) == Some(message))
.expect("the record is in the captured output");
record
.get("spans")
.and_then(serde_json::Value::as_array)
.cloned()
.unwrap_or_default()
}
pub(crate) fn has_field(
spans: &[serde_json::Value],
field: &str,
value: &serde_json::Value,
) -> bool {
spans.iter().any(|span| span.get(field) == Some(value))
}
pub(crate) fn any_field(spans: &[serde_json::Value], field: &str) -> bool {
spans.iter().any(|span| span.get(field).is_some())
}
fn run_capturing<F, Fut>(
runner: crate::test_util::LocalRunner,
handler: F,
) -> (Vec<serde_json::Value>, crate::test_util::TestResult<()>)
where
F: Fn(serde_json::Value, crate::DurableContext) -> Fut + Send + Sync + 'static,
Fut: Future<Output = Result<(), crate::BoxError>> + Send,
{
let buffer = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
let subscriber = json_span_subscriber(&buffer);
let result = {
let _guard = test_tracing::install_test_subscriber(subscriber);
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("runtime");
runtime.block_on(async move {
runner
.run(handler, serde_json::Value::Null)
.await
.expect("runner")
})
};
assert!(result.is_success(), "{:?}", result.error());
(json_records(&buffer), result)
}
fn json_str(value: &str) -> serde_json::Value {
serde_json::Value::String(value.to_owned())
}
#[test]
fn child_body_record_has_operation_id_and_name() {
let (records, result) = run_capturing(
crate::test_util::LocalRunner::new(),
|_event, ctx| async move {
ctx.run_in_child_context(|_child| async move {
tracing::info!("in-child");
Ok(())
})
.name("enrich")
.await?;
Ok(())
},
);
let child = result.operation("enrich").expect("child operation");
let spans = spans_of(&records, "in-child");
assert!(
has_field(&spans, fields::OPERATION_ID, &json_str(child.id())),
"{spans:?}"
);
assert!(
has_field(&spans, fields::OPERATION_NAME, &json_str("enrich")),
"{spans:?}"
);
}
#[test]
fn map_item_record_has_operation_id() {
let (records, result) = run_capturing(
crate::test_util::LocalRunner::new(),
|_event, ctx| async move {
ctx.map(vec![7_u32], |_child, _item, _idx| async move {
tracing::info!("in-item");
Ok(())
})
.name("items")
.await?;
Ok(())
},
);
let batch = result.operation("items").expect("map operation");
let item = result
.children(batch)
.find(|op| op.kind() == crate::OperationKind::Context)
.expect("item child");
assert_eq!(item.id(), crate::engine::compute_wire_id_public("1-1"));
let spans = spans_of(&records, "in-item");
assert!(
has_field(&spans, fields::OPERATION_ID, &json_str(item.id())),
"{spans:?}"
);
}
#[test]
fn flat_map_item_record_has_operation_id_and_name() {
type Boxed = std::pin::Pin<Box<dyn Future<Output = Result<(), crate::BoxError>> + Send>>;
fn handler(
nesting: crate::map_parallel::NestingMode,
) -> impl Fn(serde_json::Value, crate::DurableContext) -> Boxed + Send + Sync + 'static
{
move |_event, ctx| {
Box::pin(async move {
ctx.map(vec![7_u32], |_child, _item, _idx| async move {
tracing::info!("in-item");
Ok(())
})
.name("items")
.nesting(nesting)
.await?;
Ok(())
})
}
}
let (_records, normal) = run_capturing(
crate::test_util::LocalRunner::new(),
handler(crate::map_parallel::NestingMode::Normal),
);
let batch = normal.operation("items").expect("map operation");
let item = normal
.children(batch)
.find(|op| op.kind() == crate::OperationKind::Context)
.expect("item child");
assert_eq!(item.id(), crate::engine::compute_wire_id_public("1-1"));
let (records, _flat) = run_capturing(
crate::test_util::LocalRunner::new(),
handler(crate::map_parallel::NestingMode::Flat),
);
let spans = spans_of(&records, "in-item");
assert!(
has_field(&spans, fields::OPERATION_ID, &json_str(item.id())),
"{spans:?}"
);
assert!(
has_field(&spans, fields::OPERATION_NAME, &json_str("map-item-0")),
"{spans:?}"
);
}
#[test]
fn parallel_branch_record_has_operation_id_and_name() {
let (records, result) = run_capturing(
crate::test_util::LocalRunner::new(),
|_event, ctx| async move {
let branches: Vec<crate::Branch<()>> =
vec![crate::Branch::new("left", |_child| async move {
tracing::info!("in-branch");
Ok(())
})];
ctx.parallel(branches).name("fan-out").await?;
Ok(())
},
);
let branch = result.operation("left").expect("branch operation");
assert_eq!(branch.id(), crate::engine::compute_wire_id_public("1-1"));
let spans = spans_of(&records, "in-branch");
assert!(
has_field(&spans, fields::OPERATION_ID, &json_str(branch.id())),
"{spans:?}"
);
assert!(
has_field(&spans, fields::OPERATION_NAME, &json_str("left")),
"{spans:?}"
);
}
#[test]
fn condition_check_record_has_operation_id_and_attempt() {
let (records, result) = run_capturing(
crate::test_util::LocalRunner::new(),
|_event, ctx| async move {
ctx.wait_for_condition(
|_step, state: u32| async move {
tracing::info!("in-check");
Ok(state + 1)
},
0,
)
.wait_strategy_fn(|_state, _attempt| {
crate::builders::wait_for_condition::WaitDecision::complete()
})
.name("poll")
.await?;
Ok(())
},
);
let poll = result.operation("poll").expect("condition operation");
let spans = spans_of(&records, "in-check");
assert!(
has_field(&spans, fields::OPERATION_ID, &json_str(poll.id())),
"{spans:?}"
);
assert!(
has_field(&spans, fields::OPERATION_NAME, &json_str("poll")),
"{spans:?}"
);
assert!(
has_field(&spans, fields::ATTEMPT, &serde_json::json!(1)),
"{spans:?}"
);
}
#[test]
fn submitter_record_has_operation_id() {
let reply = crate::test_util::CallbackReply::success(&"approved").expect("reply");
let (records, result) = run_capturing(
crate::test_util::LocalRunner::new().callback_for("approve", reply),
|_event, ctx| async move {
let _value: String = ctx
.wait_for_callback(|_step, _callback_id| async move {
tracing::info!("in-submitter");
Ok(())
})
.name("approve")
.await?;
Ok(())
},
);
let wfcb = result
.operation("approve")
.expect("wait_for_callback operation");
let submitter = result
.children(wfcb)
.find(|op| op.kind() == crate::OperationKind::Step)
.expect("submitter step");
let spans = spans_of(&records, "in-submitter");
assert!(
has_field(&spans, fields::OPERATION_ID, &json_str(submitter.id())),
"{spans:?}"
);
}
#[test]
fn named_step_record_has_operation_name() {
let (records, _result) = run_capturing(
crate::test_util::LocalRunner::new(),
|_event, ctx| async move {
ctx.step(|_| async {
tracing::info!("in-step");
Ok(())
})
.name("fetch")
.await?;
Ok(())
},
);
let spans = spans_of(&records, "in-step");
assert!(
has_field(&spans, fields::OPERATION_NAME, &json_str("fetch")),
"{spans:?}"
);
}
#[test]
fn unnamed_step_record_has_no_operation_name() {
let (records, _result) = run_capturing(
crate::test_util::LocalRunner::new(),
|_event, ctx| async move {
ctx.step(|_| async {
tracing::info!("in-step");
Ok(())
})
.await?;
Ok(())
},
);
let spans = spans_of(&records, "in-step");
assert!(any_field(&spans, fields::OPERATION_ID), "{spans:?}");
assert!(!any_field(&spans, fields::OPERATION_NAME), "{spans:?}");
}
fn filtered_messages(run: impl FnOnce()) -> Vec<String> {
use std::sync::{Arc, Mutex};
use tracing_subscriber::Layer as _;
use tracing_subscriber::layer::SubscriberExt as _;
let buffer = Arc::new(Mutex::new(Vec::new()));
let layer = tracing_subscriber::fmt::layer()
.json()
.flatten_event(true)
.with_writer(CaptureWriter(Arc::clone(&buffer)))
.with_filter(ReplayFilterLayer::new());
{
let _guard =
test_tracing::install_test_subscriber(tracing_subscriber::registry().with(layer));
run();
}
let output = String::from_utf8_lossy(&buffer.lock().expect("not poisoned")).to_string();
output
.lines()
.filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
.filter_map(|v| v.get("message")?.as_str().map(str::to_owned))
.collect()
}
#[test]
fn replay_filter_uses_the_nearest_execution_namespace() {
let emitted = filtered_messages(|| {
let child = |parent: &tracing::Span, replay: bool| {
let _entered = parent.enter();
scoped_execution_span("arn", "req", None, "child", None, replay)
};
let parent = execution_span("arn", "req", None, true);
let live_child = child(&parent, false);
live_child.in_scope(|| {
tracing::info!("parent replay, child live");
tracing::info!(
{ fields::IS_REPLAY } = true,
"explicit replay in live child"
);
tracing::info_span!("not_sdk", isReplay = true)
.in_scope(|| tracing::info!("non-SDK replay span in live child"));
operation_span("arn", "req", None, "op", None, 1, true)
.in_scope(|| tracing::info!("replaying operation in live child"));
});
let parent = execution_span("arn", "req", None, false);
child(&parent, true).in_scope(|| {
tracing::info!("parent live, child replay");
tracing::info!(
{ fields::IS_REPLAY } = false,
"explicit live in replaying child"
);
operation_span("arn", "req", None, "op", None, 1, false)
.in_scope(|| tracing::info!("live operation in replaying child"));
tracing::info_span!("durable_execution", isReplay = false)
.in_scope(|| tracing::info!("foreign namespace in replaying child"));
});
});
assert_eq!(
emitted,
[
"parent replay, child live",
"non-SDK replay span in live child",
"explicit live in replaying child",
]
);
}
#[test]
fn replay_filter_on_two_layers_shares_span_state() {
use std::sync::{Arc, Mutex};
use tracing_subscriber::Layer as _;
use tracing_subscriber::layer::SubscriberExt as _;
let layer = |buffer: &Arc<Mutex<Vec<u8>>>| {
tracing_subscriber::fmt::layer()
.json()
.flatten_event(true)
.with_writer(CaptureWriter(Arc::clone(buffer)))
.with_filter(ReplayFilterLayer::new())
.boxed()
};
let (first, second) = (Arc::default(), Arc::default());
{
let _guard = test_tracing::install_test_subscriber(
tracing_subscriber::registry().with(vec![layer(&first), layer(&second)]),
);
let span = execution_span("arn", "req", None, true);
span.in_scope(|| tracing::info!("while replaying"));
span.record(fields::IS_REPLAY, false);
span.in_scope(|| tracing::info!("after replay"));
}
for buffer in [first, second] {
let output = String::from_utf8_lossy(&buffer.lock().expect("not poisoned")).to_string();
let messages: Vec<String> = output
.lines()
.filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
.filter_map(|v| v.get("message")?.as_str().map(str::to_owned))
.collect();
assert_eq!(messages, ["after replay"], "{output}");
}
}
#[test]
#[expect(clippy::indexing_slicing, reason = "test assertions on JSON records")]
fn child_execution_span_is_a_child_of_the_current_span() {
use std::sync::{Arc, Mutex};
use tracing_subscriber::layer::SubscriberExt as _;
let buffer = Arc::new(Mutex::new(Vec::new()));
let layer = tracing_subscriber::fmt::layer()
.json()
.flatten_event(true)
.with_writer(CaptureWriter(Arc::clone(&buffer)));
{
let _guard =
test_tracing::install_test_subscriber(tracing_subscriber::registry().with(layer));
let current = tracing::info_span!("current");
let child = current.in_scope(|| {
scoped_execution_span("arn", "req", Some("tenant"), "op-1", Some("named"), true)
});
let unnamed =
current.in_scope(|| scoped_execution_span("arn", "req", None, "op-2", None, false));
child.in_scope(|| tracing::info!("in child"));
unnamed.in_scope(|| tracing::info!("in unnamed child"));
}
let output = String::from_utf8_lossy(&buffer.lock().expect("not poisoned")).to_string();
let records: Vec<serde_json::Value> = output
.lines()
.map(|line| serde_json::from_str(line).expect("JSON"))
.collect();
let [named, unnamed] = records.as_slice() else {
unreachable!("two records: {output}");
};
let child_of = |record: &serde_json::Value| {
let spans = record["spans"].as_array().expect("a span list").clone();
let chain: Vec<&str> = spans.iter().filter_map(|s| s["name"].as_str()).collect();
assert_eq!(chain, ["current", "durable_execution"], "{record}");
spans[1].clone()
};
let child = child_of(named);
assert_eq!(child[fields::EXECUTION_ARN], "arn", "{child}");
assert_eq!(child[fields::REQUEST_ID], "req", "{child}");
assert_eq!(child[fields::TENANT_ID], "tenant", "{child}");
assert_eq!(child[fields::OPERATION_ID], "op-1", "{child}");
assert_eq!(child[fields::OPERATION_NAME], "named", "{child}");
assert_eq!(child[fields::IS_REPLAY], true, "{child}");
let child = child_of(unnamed);
assert_eq!(child[fields::OPERATION_ID], "op-2", "{child}");
assert_eq!(child[fields::IS_REPLAY], false, "{child}");
assert!(child.get(fields::TENANT_ID).is_none(), "{child}");
assert!(child.get(fields::OPERATION_NAME).is_none(), "{child}");
}
#[test]
fn replay_filter_ignores_other_spans_of_the_sdk_target() {
let emitted = filtered_messages(|| {
execution_span("arn", "req", None, false).in_scope(|| {
tracing::info_span!(target: SPAN_TARGET, "not_a_namespace", isReplay = true)
.in_scope(|| tracing::info!("sdk-target replay span in live namespace"));
let late = tracing::info_span!("not_sdk", isReplay = tracing::field::Empty);
late.record(fields::IS_REPLAY, true);
late.in_scope(|| tracing::info!("non-SDK span recorded as replaying"));
});
execution_span("arn", "req", None, true).in_scope(|| {
tracing::info_span!(target: SPAN_TARGET, "not_an_operation", isReplay = false)
.in_scope(|| tracing::info!("sdk-target live span in replaying namespace"));
});
});
assert_eq!(
emitted,
[
"sdk-target replay span in live namespace",
"non-SDK span recorded as replaying",
]
);
}
struct OnlyProbe;
impl<S> tracing_subscriber::layer::Filter<S> for OnlyProbe {
fn enabled(
&self,
meta: &tracing::Metadata<'_>,
_cx: &tracing_subscriber::layer::Context<'_, S>,
) -> bool {
meta.target() == "probe"
}
fn callsite_enabled(
&self,
meta: &'static tracing::Metadata<'static>,
) -> tracing::subscriber::Interest {
if meta.target() == "probe" {
tracing::subscriber::Interest::sometimes()
} else {
tracing::subscriber::Interest::never()
}
}
}
#[test]
fn default_filter_enables_only_the_replay_tracking_spans() {
use std::sync::{Arc, Mutex};
use tracing_subscriber::Layer as _;
use tracing_subscriber::layer::{Filter, SubscriberExt as _};
let buffer = Arc::new(Mutex::new(Vec::new()));
let filter = || DefaultFilter {
env: OnlyProbe,
replay: ReplayFilterLayer::new(),
};
let layer = tracing_subscriber::fmt::layer()
.json()
.flatten_event(true)
.with_writer(CaptureWriter(Arc::clone(&buffer)))
.with_filter(filter());
let (other, execution) = {
let _guard =
test_tracing::install_test_subscriber(tracing_subscriber::registry().with(layer));
let other = tracing::info_span!(target: SPAN_TARGET, "not_a_namespace");
other.in_scope(|| tracing::info!(target: "probe", "in other span"));
let execution = execution_span("arn", "req", None, false);
execution.in_scope(|| tracing::info!(target: "probe", "in execution span"));
(
other.metadata().expect("the registry enables every span"),
execution
.metadata()
.expect("the registry enables every span"),
)
};
let records = json_records(&buffer);
let names = |message: &str| -> Vec<String> {
spans_of(&records, message)
.iter()
.filter_map(|span| span.get("name")?.as_str().map(str::to_owned))
.collect()
};
assert!(names("in other span").is_empty(), "{records:?}");
assert_eq!(
names("in execution span"),
["durable_execution"],
"{records:?}"
);
let interest = |meta| {
<DefaultFilter<OnlyProbe> as Filter<tracing_subscriber::Registry>>::callsite_enabled(
&filter(),
meta,
)
};
assert!(interest(other).is_never());
assert!(interest(execution).is_sometimes());
}
#[derive(Debug)]
struct OrderedRecord(Vec<(String, serde_json::Value)>);
impl<'de> serde::Deserialize<'de> for OrderedRecord {
fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
struct RecordVisitor;
impl<'de> serde::de::Visitor<'de> for RecordVisitor {
type Value = OrderedRecord;
fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("a JSON object")
}
fn visit_map<A: serde::de::MapAccess<'de>>(
self,
mut map: A,
) -> Result<OrderedRecord, A::Error> {
let mut entries: Vec<(String, serde_json::Value)> = Vec::new();
while let Some((key, value)) = map.next_entry::<String, serde_json::Value>()? {
if entries.iter().any(|(seen, _)| *seen == key) {
return Err(serde::de::Error::custom(format!("repeated key {key}")));
}
entries.push((key, value));
}
Ok(OrderedRecord(entries))
}
}
deserializer.deserialize_map(RecordVisitor)
}
}
impl OrderedRecord {
fn keys(&self) -> Vec<&str> {
self.0.iter().map(|(key, _)| key.as_str()).collect()
}
fn get(&self, key: &str) -> Option<&serde_json::Value> {
self.0
.iter()
.find(|(seen, _)| seen == key)
.map(|(_, value)| value)
}
}
fn ordered_records(buffer: &std::sync::Mutex<Vec<u8>>) -> Vec<OrderedRecord> {
captured(buffer)
.lines()
.map(|line| {
serde_json::from_str(line).expect("each line is one record with unique keys")
})
.collect()
}
fn record<'a>(records: &'a [OrderedRecord], message: &str) -> &'a OrderedRecord {
records
.iter()
.find(|r| r.get("message") == Some(&json_str(message)))
.expect("the record is in the captured output")
}
fn is_js_timestamp(value: &serde_json::Value) -> bool {
let Some(text) = value.as_str() else {
return false;
};
let shape_ok = text.len() == 24
&& text.char_indices().all(|(i, c)| match i {
4 | 7 => c == '-',
10 => c == 'T',
13 | 16 => c == ':',
19 => c == '.',
23 => c == 'Z',
_ => c.is_ascii_digit(),
});
let number = |range: std::ops::Range<usize>| -> Option<u32> {
text.get(range).and_then(|digits| digits.parse().ok())
};
let (Some(year), Some(month), Some(day)) = (number(0..4), number(5..7), number(8..10))
else {
return false;
};
let (Some(hour), Some(minute), Some(second)) =
(number(11..13), number(14..16), number(17..19))
else {
return false;
};
let leap = (year % 4 == 0 && year % 100 != 0) || year % 400 == 0;
let month_days = match month {
1 | 3 | 5 | 7 | 8 | 10 | 12 => 31,
4 | 6 | 9 | 11 => 30,
2 if leap => 29,
2 => 28,
_ => return false,
};
shape_ok && (1..=month_days).contains(&day) && hour <= 23 && minute <= 59 && second <= 59
}
#[test]
fn is_js_timestamp_accepts_only_valid_times() {
for valid in [
"1970-01-01T00:00:00.000Z",
"2024-02-29T23:59:59.999Z",
"9999-12-31T23:59:59.999Z",
] {
assert!(is_js_timestamp(&json_str(valid)), "{valid}");
}
for invalid in [
"9999-99-99T99:99:99.999Z",
"2023-02-29T00:00:00.000Z",
"2024-13-01T00:00:00.000Z",
"2024-04-31T00:00:00.000Z",
"2024-01-00T00:00:00.000Z",
"2024-01-01T24:00:00.000Z",
"2024-01-01T00:60:00.000Z",
"2024-01-01T00:00:60.000Z",
"2024-01-01T00:00:00.000+00:00",
"2024-01-01 00:00:00.000Z",
] {
assert!(!is_js_timestamp(&json_str(invalid)), "{invalid}");
}
assert!(!is_js_timestamp(&serde_json::json!(0)));
}
fn default_json(
directives: &str,
) -> (
std::sync::Arc<std::sync::Mutex<Vec<u8>>>,
Box<dyn tracing::Subscriber + Send + Sync>,
) {
let buffer = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
let subscriber = default_subscriber(
"INFO",
directives,
"json",
CaptureWriter(std::sync::Arc::clone(&buffer)),
);
(buffer, subscriber)
}
fn run_default_json<F, Fut>(
runner: crate::test_util::LocalRunner,
directives: &str,
handler: F,
) -> (Vec<OrderedRecord>, crate::test_util::TestResult<()>, String)
where
F: Fn(serde_json::Value, crate::DurableContext) -> Fut + Send + Sync + 'static,
Fut: Future<Output = Result<(), crate::BoxError>> + Send,
{
let (buffer, subscriber) = default_json(directives);
let arn = std::sync::Arc::new(std::sync::Mutex::new(String::new()));
let seen = std::sync::Arc::clone(&arn);
let handler = std::sync::Arc::new(handler);
let result = {
let _guard = test_tracing::install_test_subscriber(subscriber);
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("runtime");
runtime.block_on(async move {
runner
.run(
move |event, ctx: crate::DurableContext| {
*seen.lock().expect("arn") = ctx.execution_arn().to_owned();
handler(event, ctx)
},
serde_json::Value::Null,
)
.await
.expect("runner")
})
};
assert!(result.is_success(), "{:?}", result.error());
let arn = arn.lock().expect("arn").clone();
assert_ne!(arn, "");
(ordered_records(&buffer), result, arn)
}
const OPERATION_KEYS: [&str; 7] = [
"requestId",
"timestamp",
"level",
"executionArn",
"operationId",
"operationName",
"attempt",
];
#[test]
fn default_json_step_record_has_top_level_fields() {
let (records, result, arn) = run_default_json(
crate::test_util::LocalRunner::new(),
"",
|_event, ctx| async move {
ctx.step(|_| async {
tracing::info!(order = 5_u32, "in-step");
Ok(())
})
.name("fetch")
.await?;
Ok(())
},
);
let step = result.operation("fetch").expect("step operation");
let record = record(&records, "in-step");
let mut expected = OPERATION_KEYS.to_vec();
expected.extend(["order", "message"]);
assert_eq!(record.keys(), expected, "{record:?}");
assert_eq!(
record.get("requestId"),
Some(&json_str("local-invocation-1"))
);
assert!(is_js_timestamp(record.get("timestamp").expect("timestamp")));
assert_eq!(record.get("level"), Some(&json_str("INFO")));
assert_eq!(record.get("executionArn"), Some(&json_str(&arn)));
assert_eq!(record.get("operationId"), Some(&json_str(step.id())));
assert_eq!(record.get("operationName"), Some(&json_str("fetch")));
assert_eq!(record.get("attempt"), Some(&serde_json::json!(1)));
assert_eq!(record.get("order"), Some(&serde_json::json!(5)));
assert_eq!(record.get("message"), Some(&json_str("in-step")));
}
#[test]
fn default_json_tenant_id_is_optional_and_ordered() {
let (buffer, subscriber) = default_json("");
{
let _guard = test_tracing::install_test_subscriber(subscriber);
execution_span("arn:run", "req-1", Some("tenant-1"), false)
.in_scope(|| tracing::info!("with-tenant"));
execution_span("arn:run", "req-1", None, false)
.in_scope(|| tracing::info!("without-tenant"));
}
let records = ordered_records(&buffer);
let with = record(&records, "with-tenant");
assert_eq!(
with.keys(),
[
"requestId",
"timestamp",
"level",
"executionArn",
"tenantId",
"message"
],
"{with:?}"
);
assert_eq!(with.get("tenantId"), Some(&json_str("tenant-1")));
let without = record(&records, "without-tenant");
assert_eq!(
without.keys(),
["requestId", "timestamp", "level", "executionArn", "message"],
"{without:?}"
);
}
#[test]
fn default_json_child_record_has_top_level_fields() {
let (records, result, arn) = run_default_json(
crate::test_util::LocalRunner::new(),
"",
|_event, ctx| async move {
ctx.run_in_child_context(|_child| async move {
tracing::info!("in-child");
Ok(())
})
.name("enrich")
.await?;
Ok(())
},
);
let child = result.operation("enrich").expect("child operation");
let record = record(&records, "in-child");
assert_eq!(
record.keys(),
[
"requestId",
"timestamp",
"level",
"executionArn",
"operationId",
"operationName",
"message"
],
"{record:?}"
);
assert_eq!(
record.get("requestId"),
Some(&json_str("local-invocation-1"))
);
assert_eq!(record.get("executionArn"), Some(&json_str(&arn)));
assert_eq!(record.get("operationId"), Some(&json_str(child.id())));
assert_eq!(record.get("operationName"), Some(&json_str("enrich")));
}
#[test]
fn default_json_map_item_record_has_top_level_fields() {
let (records, result, arn) = run_default_json(
crate::test_util::LocalRunner::new(),
"",
|_event, ctx| async move {
ctx.map(vec![7_u32], |_child, _item, _idx| async move {
tracing::info!("in-item");
Ok(())
})
.name("items")
.await?;
Ok(())
},
);
let batch = result.operation("items").expect("map operation");
let item = result
.children(batch)
.find(|op| op.kind() == crate::OperationKind::Context)
.expect("item child");
let record = record(&records, "in-item");
assert_eq!(
record.keys(),
[
"requestId",
"timestamp",
"level",
"executionArn",
"operationId",
"operationName",
"message"
],
"{record:?}"
);
assert_eq!(
record.get("requestId"),
Some(&json_str("local-invocation-1"))
);
assert_eq!(record.get("executionArn"), Some(&json_str(&arn)));
assert_eq!(record.get("operationId"), Some(&json_str(item.id())));
assert_eq!(record.get("operationName"), Some(&json_str("map-item-0")));
}
#[test]
fn default_json_step_in_child_uses_nearest_sdk_span() {
let (records, result, _arn) = run_default_json(
crate::test_util::LocalRunner::new(),
"",
|_event, ctx| async move {
ctx.run_in_child_context(|child| async move {
child
.step(|_| async {
tracing::info!("in-inner-step");
Ok(())
})
.name("inner")
.await?;
tracing::info!("in-outer-child");
Ok(())
})
.name("outer")
.await?;
Ok(())
},
);
let outer = result.operation("outer").expect("child operation");
let inner = result.operation("inner").expect("step operation");
let in_step = record(&records, "in-inner-step");
assert_eq!(in_step.get("operationId"), Some(&json_str(inner.id())));
assert_eq!(in_step.get("operationName"), Some(&json_str("inner")));
assert_eq!(in_step.get("attempt"), Some(&serde_json::json!(1)));
let in_child = record(&records, "in-outer-child");
assert_eq!(in_child.get("operationId"), Some(&json_str(outer.id())));
assert_eq!(in_child.get("operationName"), Some(&json_str("outer")));
assert_eq!(in_child.get("attempt"), None, "{in_child:?}");
}
fn impostor_span() -> tracing::Span {
tracing::info_span!(
"durable_operation",
requestId = "impostor-request",
operationId = "impostor-operation",
trace_id = "trace-1",
)
}
#[derive(Clone)]
struct Impostor;
impl crate::plugin::Plugin for Impostor {
fn wrap_user_function(&self, _info: &crate::plugin::OperationInfo, next: &mut dyn FnMut()) {
impostor_span().in_scope(next);
}
}
impl crate::plugin::PluginFactory for Impostor {
fn create_plugin(
&self,
_info: &crate::plugin::InvocationInfo,
) -> Result<Option<Box<dyn crate::plugin::Plugin>>, crate::BoxError> {
Ok(Some(Box::new(self.clone())))
}
}
#[test]
fn default_json_correlation_uses_only_events_and_sdk_spans() {
let options = crate::Options::builder()
.plugin(Impostor)
.build()
.expect("options");
let (records, result, _arn) = run_default_json(
crate::test_util::LocalRunner::new().options(options),
"",
|_event, ctx| async move {
ctx.step(|_| async {
tracing::info!("in-step");
tracing::info!(
requestId = "event-request",
operationId = "event-operation",
"explicit"
);
Ok(())
})
.name("fetch")
.await?;
Ok(())
},
);
let step = result.operation("fetch").expect("step operation");
let in_step = record(&records, "in-step");
assert_eq!(
in_step.get("trace_id"),
Some(&json_str("trace-1")),
"{in_step:?}"
);
assert_eq!(
in_step.get("requestId"),
Some(&json_str("local-invocation-1"))
);
assert_eq!(in_step.get("operationId"), Some(&json_str(step.id())));
let explicit = record(&records, "explicit");
assert_eq!(explicit.get("requestId"), Some(&json_str("event-request")));
assert_eq!(
explicit.get("operationId"),
Some(&json_str("event-operation"))
);
assert_eq!(explicit.get("trace_id"), Some(&json_str("trace-1")));
let (buffer, subscriber) = default_json("");
{
let _guard = test_tracing::install_test_subscriber(subscriber);
impostor_span().in_scope(|| tracing::info!("outside"));
}
let records = ordered_records(&buffer);
let outside = record(&records, "outside");
assert_eq!(
outside.keys(),
["timestamp", "level", "trace_id", "message"],
"{outside:?}"
);
}
#[test]
fn default_json_field_collisions_have_one_winner() {
let (buffer, subscriber) = default_json("");
{
let _guard = test_tracing::install_test_subscriber(subscriber);
let outer = tracing::info_span!(
"outer",
shared = "outer",
nested = "outer",
outer_only = "outer",
"timestamp" = "span-timestamp",
"level" = "span-level",
"message" = "span-message",
);
outer.in_scope(|| {
let inner = tracing::info_span!(
"inner",
shared = "inner",
nested = "inner",
isReplay = false,
);
inner.in_scope(|| {
tracing::info!(
shared = "event",
"timestamp" = "event-timestamp",
"level" = "event-level",
isReplay = false,
"fields" = "user-fields",
"span" = "user-span",
"spans" = "user-spans",
"target" = "user-target",
"name" = "user-name",
"collide"
);
});
});
}
let records = ordered_records(&buffer);
let record = record(&records, "collide");
let keys = record.keys();
assert_eq!(keys.first(), Some(&"timestamp"), "{record:?}");
assert_eq!(keys.get(1), Some(&"level"), "{record:?}");
assert_eq!(keys.last(), Some(&"message"), "{record:?}");
assert!(is_js_timestamp(record.get("timestamp").expect("timestamp")));
assert_eq!(record.get("level"), Some(&json_str("INFO")));
assert_eq!(record.get("message"), Some(&json_str("collide")));
assert_eq!(record.get("shared"), Some(&json_str("event")));
assert_eq!(record.get("nested"), Some(&json_str("inner")));
assert_eq!(
record.get("outer_only"),
Some(&json_str("outer")),
"{record:?}"
);
assert_eq!(record.get("isReplay"), None, "{record:?}");
for (key, value) in [
("fields", "user-fields"),
("span", "user-span"),
("spans", "user-spans"),
("target", "user-target"),
("name", "user-name"),
] {
assert_eq!(keys.iter().filter(|k| **k == key).count(), 1, "{key}");
assert_eq!(record.get(key), Some(&json_str(value)), "{key}");
}
assert_eq!(keys.len(), 11, "{record:?}");
}
#[test]
fn default_json_values_match_json_fields() {
#[derive(Debug)]
struct Shown;
impl std::fmt::Display for Shown {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("shown-display")
}
}
impl std::error::Error for Shown {}
let (buffer, subscriber) = default_json("");
{
let _guard = test_tracing::install_test_subscriber(subscriber);
let error: &(dyn std::error::Error + 'static) = &Shown;
tracing::info_span!(
"values",
span_u64 = u64::MAX,
span_i128 = i128::MAX,
span_display = %Shown,
span_debug = ?Shown,
)
.in_scope(|| {
tracing::info!(
event_u64 = u64::MAX,
event_i64 = i64::MIN,
event_i128 = i128::MAX,
event_u128 = u128::MAX,
display = %Shown,
debug = ?Shown,
finite = 1.5_f64,
infinite = f64::INFINITY,
nan = f64::NAN,
flag = true,
text = "plain",
bytes = &b"ab"[..],
error = error,
"values"
);
});
}
let records = ordered_records(&buffer);
let record = record(&records, "values");
let expected = [
("span_u64", serde_json::json!(u64::MAX)),
(
"span_i128",
json_str("170141183460469231731687303715884105727"),
),
("span_display", json_str("shown-display")),
("span_debug", json_str("Shown")),
("event_u64", serde_json::json!(u64::MAX)),
("event_i64", serde_json::json!(i64::MIN)),
(
"event_i128",
json_str("170141183460469231731687303715884105727"),
),
(
"event_u128",
json_str("340282366920938463463374607431768211455"),
),
("display", json_str("shown-display")),
("debug", json_str("Shown")),
("finite", serde_json::json!(1.5)),
("infinite", serde_json::Value::Null),
("nan", serde_json::Value::Null),
("flag", serde_json::json!(true)),
("text", json_str("plain")),
("bytes", serde_json::json!([97, 98])),
("error", json_str("shown-display")),
];
for (key, value) in expected {
assert_eq!(record.get(key), Some(&value), "{key}: {record:?}");
}
}
#[test]
fn default_json_message_is_last_and_defaults_to_empty() {
let (buffer, subscriber) = default_json("");
{
let _guard = test_tracing::install_test_subscriber(subscriber);
tracing::info!(marker = "no-message");
tracing::info!(message = 7_u64);
}
let records = ordered_records(&buffer);
assert_eq!(records.len(), 2, "{records:?}");
let last = |record: &OrderedRecord| record.0.last().cloned();
let first = records.first().expect("first record");
assert_eq!(first.get("marker"), Some(&json_str("no-message")));
assert_eq!(last(first), Some(("message".to_owned(), json_str(""))));
let second = records.get(1).expect("second record");
assert_eq!(
last(second),
Some(("message".to_owned(), serde_json::json!(7)))
);
}
#[test]
fn default_json_records_have_unique_keys() {
let directives = format!("info,{TARGET}=debug");
let (records, _result, arn) = run_default_json(
crate::test_util::LocalRunner::new(),
&directives,
|_event, ctx| async move {
ctx.step(|_| async { Ok(()) }).name("s").await?;
ctx.run_in_child_context(|_child| async move { Ok(()) })
.name("c")
.await?;
ctx.map(vec![1_u32], |_child, _item, _idx| async move { Ok(()) })
.name("m")
.await?;
Ok(())
},
);
let lifecycle: Vec<&OrderedRecord> = records
.iter()
.filter(|r| {
r.get("message")
.and_then(serde_json::Value::as_str)
.is_some_and(|m| m.starts_with("operation_") || m.starts_with("execution_"))
})
.collect();
assert!(
lifecycle
.iter()
.any(|r| r.get("message") == Some(&json_str("operation_started"))),
"{records:?}"
);
for record in &records {
assert_eq!(record.get("isReplay"), None, "{record:?}");
}
for record in lifecycle {
assert_eq!(
record.get("executionArn"),
Some(&json_str(&arn)),
"{record:?}"
);
assert!(record.get("requestId").is_some(), "{record:?}");
let is_operation = record
.get("message")
.and_then(serde_json::Value::as_str)
.is_some_and(|m| m.starts_with("operation_"));
if is_operation {
assert!(record.get("operationId").is_some(), "{record:?}");
}
}
}
#[test]
fn ordered_json_record_rejects_duplicate_keys() {
assert!(serde_json::from_str::<OrderedRecord>(r#"{"a":1,"b":2}"#).is_ok());
assert!(serde_json::from_str::<OrderedRecord>(r#"{"a":1,"b":2,"a":3}"#).is_err());
}
#[test]
fn js_timestamp_matches_js() {
use std::time::{Duration, UNIX_EPOCH};
let at = |secs: u64, nanos: u32| UNIX_EPOCH + Duration::new(secs, nanos);
let ok = |time| js_timestamp(time).expect("in range");
assert_eq!(ok(UNIX_EPOCH), "1970-01-01T00:00:00.000Z");
assert_eq!(
ok(at(1_700_000_000, 123_456_789)),
"2023-11-14T22:13:20.123Z"
);
assert_eq!(
ok(at(1_709_251_199, 999_999_999)),
"2024-02-29T23:59:59.999Z"
);
assert_eq!(ok(at(1_709_251_200, 0)), "2024-03-01T00:00:00.000Z");
assert_eq!(
ok(at(253_402_300_799, 999_000_000)),
"9999-12-31T23:59:59.999Z"
);
assert!(js_timestamp(at(253_402_300_800, 0)).is_err());
let before = |secs: u64, nanos: u32| UNIX_EPOCH - Duration::new(secs, nanos);
assert_eq!(ok(before(0, 1)), "1969-12-31T23:59:59.999Z");
assert_eq!(ok(before(1, 0)), "1969-12-31T23:59:59.000Z");
assert_eq!(ok(before(62_167_219_200, 0)), "0000-01-01T00:00:00.000Z");
assert!(js_timestamp(before(62_167_219_200, 1)).is_err());
}
#[test]
fn parse_fields_rejects_text_that_is_not_an_object() {
assert_eq!(
parse_fields(r#"{"a":1}"#).expect("an object").get("a"),
Some(&serde_json::json!(1))
);
assert!(parse_fields("a=1").is_err());
}
struct TamperFields {
replace: bool,
}
impl<S> tracing_subscriber::Layer<S> for TamperFields
where
S: tracing::Subscriber + for<'lookup> tracing_subscriber::registry::LookupSpan<'lookup>,
{
fn on_new_span(
&self,
_attrs: &Attributes<'_>,
id: &Id,
ctx: tracing_subscriber::layer::Context<'_, S>,
) {
if let Some(span) = ctx.span(id) {
let mut extensions = span.extensions_mut();
extensions.remove::<FormattedFields<JsonFields>>();
if self.replace {
extensions.insert(FormattedFields::<JsonFields>::new("a=1".to_owned()));
}
}
}
}
fn tampered_output(replace: bool) -> String {
use tracing_subscriber::layer::SubscriberExt as _;
let buffer = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
let subscriber = tracing_subscriber::registry()
.with(
tracing_subscriber::fmt::layer()
.fmt_fields(JsonFields::new())
.event_format(TopLevelJson)
.with_writer(CaptureWriter(std::sync::Arc::clone(&buffer))),
)
.with(TamperFields { replace });
{
let _guard = test_tracing::install_test_subscriber(subscriber);
tracing::info_span!("tampered_span", gone = 1_u32)
.in_scope(|| tracing::info!("tampered"));
}
captured(&buffer)
}
#[test]
fn top_level_json_handles_spans_without_json_fields() {
let records: Vec<OrderedRecord> = tampered_output(false)
.lines()
.map(|line| serde_json::from_str(line).expect("a record"))
.collect();
let record = record(&records, "tampered");
assert_eq!(
record.keys(),
["timestamp", "level", "message"],
"{record:?}"
);
assert_eq!(tampered_output(true), "");
}
}