#![cfg_attr(
not(test),
deny(
clippy::unwrap_used,
clippy::expect_used,
clippy::panic,
clippy::unreachable,
clippy::todo,
clippy::unimplemented,
clippy::indexing_slicing,
clippy::string_slice,
clippy::arithmetic_side_effects,
)
)]
use std::collections::{BTreeMap, VecDeque};
use std::future::Future;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use base64::Engine as _;
use base64::engine::general_purpose::STANDARD;
use serde::Serialize;
use crate::capsule::schema::{
CacheEffect, CapsuleBody, CapsuleEffects, HttpEffect, JobEffect, MailEffect, TenantEffect,
};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum EffectSeam {
Http,
Job,
Cache,
Mail,
Tenant,
Random,
Detached,
}
impl EffectSeam {
#[must_use]
pub const fn label(self) -> &'static str {
match self {
Self::Http => "outbound http",
Self::Job => "job enqueue",
Self::Cache => "cache",
Self::Mail => "mail",
Self::Tenant => "tenancy",
Self::Random => "randomness",
Self::Detached => "detached task",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum EffectDivergenceKind {
Unrecorded,
Mismatch,
Exhausted,
Unconsumed,
}
impl EffectDivergenceKind {
#[must_use]
pub const fn label(self) -> &'static str {
match self {
Self::Unrecorded => "unrecorded effect",
Self::Mismatch => "effect mismatch",
Self::Exhausted => "tape exhausted",
Self::Unconsumed => "unconsumed effects",
}
}
}
#[derive(Debug, Clone, Serialize)]
pub struct EffectDivergence {
pub seam: EffectSeam,
pub kind: EffectDivergenceKind,
pub index: usize,
pub expected: Option<String>,
pub actual: String,
pub detail: String,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum CachedValue {
Unrecorded,
Miss,
Hit(Vec<u8>),
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum EnqueueVerdict {
Queued,
Failed(String, Option<u16>),
Diverged,
}
#[cfg(any(test, feature = "mail"))]
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum MailVerdict {
Sent,
Failed(String, Option<crate::capsule::schema::MailErrorKind>),
Diverged,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum TenantVerdict {
Resolved(String),
RecordedFailure,
Unrecorded,
Unchecked,
}
#[derive(Debug)]
struct Ordered<T> {
pending: VecDeque<T>,
consumed: usize,
}
impl<T> Ordered<T> {
fn new(entries: Vec<T>) -> Self {
Self {
pending: entries.into(),
consumed: 0,
}
}
}
#[derive(Debug)]
pub struct ReplayEffects {
http: Mutex<Ordered<HttpEffect>>,
jobs: Mutex<Ordered<JobEffect>>,
mail: Mutex<Ordered<MailEffect>>,
cache: Mutex<BTreeMap<String, CacheSlot>>,
cache_writes: Mutex<Ordered<CacheWrite>>,
tenant: RecordedTenant,
tenant_read: std::sync::atomic::AtomicBool,
legacy_v3: bool,
observed: Mutex<crate::capsule::redact::RedactedValues>,
divergences: Mutex<Vec<EffectDivergence>>,
recorded: usize,
served: AtomicUsize,
scope_entered: std::sync::atomic::AtomicBool,
after_commit: AtomicUsize,
}
#[derive(Debug, Clone, PartialEq, Eq)]
enum CacheWrite {
Insert {
key: String,
value: Option<Vec<u8>>,
ttl_secs: Option<u64>,
},
Invalidate {
key: String,
error: Option<crate::capsule::schema::CacheInvalidationError>,
},
InvalidateNamespace {
namespace: String,
error: Option<crate::capsule::schema::CacheInvalidationError>,
},
Clear,
}
impl CacheWrite {
fn from_effect(effect: &CacheEffect) -> Option<Self> {
match effect {
CacheEffect::Get { .. } => None,
CacheEffect::Insert {
key,
value,
ttl_secs,
} => Some(Self::Insert {
key: key.clone(),
value: decode(value),
ttl_secs: *ttl_secs,
}),
CacheEffect::Invalidate { key, error } => Some(Self::Invalidate {
key: key.clone(),
error: error.clone(),
}),
CacheEffect::InvalidateNamespace { namespace, error } => {
Some(Self::InvalidateNamespace {
namespace: namespace.clone(),
error: error.clone(),
})
}
CacheEffect::Clear => Some(Self::Clear),
}
}
fn matches(&self, actual: &Self) -> bool {
match (self, actual) {
(
Self::Insert {
key,
value,
ttl_secs,
},
Self::Insert {
key: actual_key,
value: actual_value,
ttl_secs: actual_ttl,
},
) => {
matches_redacted(key, actual_key)
&& ttl_secs == actual_ttl
&& value.as_ref().is_none_or(|recorded| {
actual_value
.as_ref()
.is_some_and(|actual| bytes_match_redacted(recorded, actual))
})
}
(Self::Invalidate { key, .. }, Self::Invalidate { key: actual, .. })
| (
Self::InvalidateNamespace { namespace: key, .. },
Self::InvalidateNamespace {
namespace: actual, ..
},
) => matches_redacted(key, actual),
(Self::Clear, Self::Clear) => true,
_ => false,
}
}
fn describe(&self) -> String {
match self {
Self::Insert { key, .. } => key.clone(),
Self::Invalidate { key, .. } => format!("invalidate {key}"),
Self::InvalidateNamespace { namespace, .. } => {
format!("invalidate namespace {namespace}")
}
Self::Clear => "clear".to_owned(),
}
}
fn identifier(&self) -> Option<&str> {
match self {
Self::Insert { key, .. } | Self::Invalidate { key, .. } => Some(key),
Self::InvalidateNamespace { namespace, .. } => Some(namespace),
Self::Clear => None,
}
}
fn describe_withheld(&self) -> String {
match self {
Self::Insert { .. } => "a cache write (key withheld)".to_owned(),
Self::Invalidate { .. } => "invalidate (key withheld)".to_owned(),
Self::InvalidateNamespace { .. } => {
"invalidate namespace (namespace withheld)".to_owned()
}
Self::Clear => "clear".to_owned(),
}
}
}
fn in_namespace(key: &str, namespace: &str) -> bool {
if key
.strip_prefix(namespace)
.is_some_and(|rest| rest.starts_with(':'))
{
return true;
}
key.match_indices(':')
.filter_map(|(index, _)| key.get(..index))
.any(|prefix| matches_redacted(prefix, namespace))
}
#[derive(Debug, Clone, Copy)]
enum SlotUpdate<'a> {
Key(&'a str),
Namespace(&'a str),
All,
}
fn resolve_slot(cache: &BTreeMap<String, CacheSlot>, key: &str) -> Option<String> {
if cache.contains_key(key) {
return Some(key.to_owned());
}
let mut candidates = cache
.keys()
.filter(|recorded| matches_redacted(recorded, key));
match (candidates.next().cloned(), candidates.next()) {
(Some(only), None) => Some(only),
_ => None,
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
enum RecordedTenant {
None,
Resolved(String),
Failed,
}
#[derive(Debug)]
struct CacheSlot {
value: Option<Vec<u8>>,
was_read: bool,
touched: bool,
}
impl ReplayEffects {
#[must_use]
pub fn new(effects: CapsuleEffects) -> Self {
let mut cache: BTreeMap<String, CacheSlot> = BTreeMap::new();
for entry in &effects.cache {
if let CacheEffect::Get { key, value } = entry {
cache.entry(key.clone()).or_insert_with(|| CacheSlot {
value: value.as_ref().and_then(|encoded| decode(encoded)),
was_read: true,
touched: false,
});
}
}
let cache_writes: Vec<CacheWrite> = effects
.cache
.iter()
.filter_map(CacheWrite::from_effect)
.collect();
let recorded = effects
.http
.len()
.saturating_add(effects.jobs.len())
.saturating_add(effects.mail.len())
.saturating_add(cache.len())
.saturating_add(cache_writes.len());
Self {
http: Mutex::new(Ordered::new(effects.http)),
jobs: Mutex::new(Ordered::new(effects.jobs)),
mail: Mutex::new(Ordered::new(effects.mail)),
cache: Mutex::new(cache),
cache_writes: Mutex::new(Ordered::new(cache_writes)),
tenant: match effects.tenant {
None => RecordedTenant::None,
Some(TenantEffect { id: Some(id) }) => RecordedTenant::Resolved(id),
Some(TenantEffect { id: None }) => RecordedTenant::Failed,
},
legacy_v3: false,
tenant_read: std::sync::atomic::AtomicBool::new(false),
observed: Mutex::new(crate::capsule::redact::RedactedValues::default()),
divergences: Mutex::new(Vec::new()),
recorded,
served: AtomicUsize::new(0),
scope_entered: std::sync::atomic::AtomicBool::new(false),
after_commit: AtomicUsize::new(0),
}
}
#[must_use]
pub const fn for_format_version(mut self, format_version: u32) -> Self {
self.legacy_v3 = format_version < 4;
self
}
#[must_use]
pub(crate) const fn legacy_v3(&self) -> bool {
self.legacy_v3
}
pub(crate) fn mark_scope_entered(&self) {
self.scope_entered.store(true, Ordering::SeqCst);
}
#[must_use]
pub fn scope_entered(&self) -> bool {
self.scope_entered.load(Ordering::SeqCst)
}
#[must_use]
pub const fn recorded(&self) -> usize {
self.recorded
}
#[must_use]
pub fn served(&self) -> usize {
self.served.load(Ordering::SeqCst)
}
#[must_use]
pub fn divergences(&self) -> Vec<EffectDivergence> {
self.divergences
.lock()
.map(|log| log.clone())
.unwrap_or_default()
}
fn diverge(&self, divergence: EffectDivergence) {
if let Ok(mut log) = self.divergences.lock() {
log.push(divergence);
}
}
#[must_use]
pub(crate) fn next_http(&self, request: &OutboundRequest<'_>) -> Option<HttpEffect> {
let (method, raw_url) = (request.method, request.url);
self.observe_live_userinfo(raw_url);
let live = crate::capsule::redact::parser_view(raw_url);
let url = live.as_ref();
let Ok(mut seam) = self.http.lock() else {
return None;
};
let index = seam.consumed;
let Some(next) = seam.pending.front() else {
drop(seam);
let actual = describe_http(method, url);
self.diverge(EffectDivergence {
seam: EffectSeam::Http,
kind: if self.recorded_http_is_empty() {
EffectDivergenceKind::Unrecorded
} else {
EffectDivergenceKind::Exhausted
},
index,
expected: None,
actual: actual.clone(),
detail: format!(
"the replayed handler made an outbound request the capsule has no \
recording for ({actual}); it was refused rather than sent"
),
});
return None;
};
if !next.method.eq_ignore_ascii_case(method)
|| !matches_redacted(&crate::capsule::redact::parser_view(&next.url), url)
{
let expected = describe_http(&next.method, &next.url);
drop(seam);
let actual = describe_http(method, url);
self.diverge(EffectDivergence {
seam: EffectSeam::Http,
kind: EffectDivergenceKind::Mismatch,
index,
expected: Some(expected.clone()),
actual: actual.clone(),
detail: format!(
"the replayed handler called {actual}, but the capsule's next recorded \
outbound call was {expected}"
),
});
return None;
}
if !headers_match_redacted(&next.request_headers, request.headers)
|| !body_matches_redacted(&next.request_body, request.body)
{
let expected = describe_http(&next.method, &next.url);
let changed = describe_request_mismatch(next, request);
let recorded_url = next.url.clone();
drop(seam);
self.observe_url(&recorded_url, url);
let actual = describe_http(method, url);
self.diverge(EffectDivergence {
seam: EffectSeam::Http,
kind: EffectDivergenceKind::Mismatch,
index,
expected: Some(expected),
actual: actual.clone(),
detail: format!(
"the replayed handler called {actual} as the capsule recorded, but changed \
{changed}; the recorded response was not served"
),
});
return None;
}
let served = seam.pending.pop_front();
seam.consumed = seam.consumed.saturating_add(1);
drop(seam);
if let Some(recorded) = served.as_ref() {
self.observe_url(&recorded.url, url);
self.observe_headers(&recorded.request_headers, request.headers);
self.observe_body(&recorded.request_body, request.body);
}
self.served.fetch_add(1, Ordering::SeqCst);
served
}
fn recorded_http_is_empty(&self) -> bool {
self.http
.lock()
.is_ok_and(|seam| seam.consumed == 0 && seam.pending.is_empty())
}
#[must_use]
pub(crate) fn next_job(
&self,
name: &str,
payload: &serde_json::Value,
schedule: crate::job::EnqueueSchedule,
) -> EnqueueVerdict {
let Ok(mut seam) = self.jobs.lock() else {
return EnqueueVerdict::Diverged;
};
let index = seam.consumed;
let Some(next) = seam.pending.front() else {
drop(seam);
let actual = describe_job(name, payload);
self.diverge(EffectDivergence {
seam: EffectSeam::Job,
kind: EffectDivergenceKind::Unrecorded,
index,
expected: None,
actual: actual.clone(),
detail: format!(
"the replayed run enqueued a job the capsule has no recording for \
({actual}); nothing was written to a queue"
),
});
return EnqueueVerdict::Diverged;
};
let schedule_diverged = match schedule {
crate::job::EnqueueSchedule::Immediate => {
next.delay_secs.is_some_and(|delay| delay > 0)
|| next.due_at.is_some()
|| next.requested_due_at.is_some()
}
crate::job::EnqueueSchedule::After(delay) => {
let recorded = if self.legacy_v3 {
i64::try_from(delay.as_secs()).unwrap_or(i64::MAX)
} else {
crate::job::delay_seconds(delay)
};
let ran_at_once = delay.is_zero()
&& next.delay_secs.is_none()
&& next.due_at.is_none()
&& next.requested_due_at.is_none();
next.delay_secs != Some(recorded) && !ran_at_once
}
crate::job::EnqueueSchedule::At(deadline) => {
next.due_at != Some(deadline)
&& !(next.due_at.is_none() && next.requested_due_at == Some(deadline))
}
};
if next.name != name || schedule_diverged || !json_matches_redacted(&next.payload, payload)
{
let expected = describe_job(&next.name, &next.payload);
drop(seam);
let actual = describe_job(name, payload);
self.diverge(EffectDivergence {
seam: EffectSeam::Job,
kind: EffectDivergenceKind::Mismatch,
index,
expected: Some(expected.clone()),
actual: actual.clone(),
detail: format!(
"the replayed run enqueued {actual}, but the capsule's next recorded \
enqueue was {expected}"
),
});
return EnqueueVerdict::Diverged;
}
let recorded = seam.pending.pop_front();
seam.consumed = seam.consumed.saturating_add(1);
drop(seam);
if let Some(recorded) = recorded.as_ref() {
self.observe_json(&recorded.payload, payload);
}
self.served.fetch_add(1, Ordering::SeqCst);
recorded
.and_then(|recorded| Some((recorded.error?, recorded.error_status)))
.map_or(EnqueueVerdict::Queued, |(error, status)| {
EnqueueVerdict::Failed(error, status)
})
}
#[must_use]
pub(crate) fn cache_get(&self, key: &str) -> CachedValue {
let Ok(mut cache) = self.cache.lock() else {
return CachedValue::Unrecorded;
};
let recorded_key = resolve_slot(&cache, key);
let Some(slot) = recorded_key
.as_ref()
.and_then(|recorded| cache.get_mut(recorded))
else {
drop(cache);
self.diverge(EffectDivergence {
seam: EffectSeam::Cache,
kind: EffectDivergenceKind::Unrecorded,
index: 0,
expected: None,
actual: "a cache read (key withheld)".to_owned(),
detail: "the replayed run read a cache key the capsule has no recording for; \
it was served as a miss"
.to_owned(),
});
return CachedValue::Unrecorded;
};
let first_touch = !slot.touched;
slot.touched = true;
let was_read = slot.was_read;
let value = slot.value.clone();
drop(cache);
if let Some(recorded_key) = recorded_key {
self.observe_text(&recorded_key, key);
}
if !was_read {
self.diverge(EffectDivergence {
seam: EffectSeam::Cache,
kind: EffectDivergenceKind::Unrecorded,
index: 0,
expected: None,
actual: "a cache read (key withheld)".to_owned(),
detail: "the replayed run read a cache key that the recording wrote but never \
read; the run's own write was served"
.to_owned(),
});
return value.map_or(CachedValue::Miss, CachedValue::Hit);
}
if first_touch {
self.served.fetch_add(1, Ordering::SeqCst);
}
value.map_or(CachedValue::Miss, CachedValue::Hit)
}
pub(crate) fn cache_insert(&self, key: &str, value: &[u8], ttl_secs: Option<u64>) {
self.update_slots(SlotUpdate::Key(key), Some(value));
let _ = self.next_cache_write(&CacheWrite::Insert {
key: key.to_owned(),
value: Some(value.to_vec()),
ttl_secs,
});
}
pub(crate) fn cache_invalidate(
&self,
key: &str,
) -> Result<(), crate::cache::InvalidationError> {
let result = if self.legacy_v3 {
Ok(())
} else {
self.removal_result(&CacheWrite::Invalidate {
key: key.to_owned(),
error: None,
})
};
if result.is_ok() {
self.update_slots(SlotUpdate::Key(key), None);
}
result
}
pub(crate) fn cache_invalidate_namespace(
&self,
namespace: &str,
) -> Result<(), crate::cache::InvalidationError> {
let result = if self.legacy_v3 {
Ok(())
} else {
self.removal_result(&CacheWrite::InvalidateNamespace {
namespace: namespace.to_owned(),
error: None,
})
};
if result.is_ok() {
self.update_slots(SlotUpdate::Namespace(namespace), None);
}
result
}
pub(crate) fn cache_clear(&self) {
self.update_slots(SlotUpdate::All, None);
if !self.legacy_v3 {
let _ = self.next_cache_write(&CacheWrite::Clear);
}
}
pub(crate) fn cache_untyped_get(&self, key: &str) {
if !self.legacy_v3 {
let _ = self.cache_get(key);
}
}
pub(crate) fn cache_untyped_insert(&self, key: &str) {
if self.legacy_v3 {
return;
}
let _ = key;
self.diverge(EffectDivergence {
seam: EffectSeam::Cache,
kind: EffectDivergenceKind::Unrecorded,
index: 0,
expected: None,
actual: "an untyped cache write (key withheld)".to_owned(),
detail: "the replayed run wrote an untyped cache value; a capsule cannot record \
one, so the write was not applied"
.to_owned(),
});
}
pub(crate) fn cache_direct_get(&self, key: &str) {
if self.legacy_v3 {
return;
}
let _ = key;
self.diverge(EffectDivergence {
seam: EffectSeam::Cache,
kind: EffectDivergenceKind::Unrecorded,
index: 0,
expected: None,
actual: "a direct cache read (key withheld)".to_owned(),
detail: "the replayed run read the cache backend directly rather than through a cache helper; a capsule cannot record such a read, so it was a miss"
.to_owned(),
});
}
pub(crate) fn after_commit_registered(&self) {
self.after_commit.fetch_add(1, Ordering::SeqCst);
}
pub(crate) fn after_commit_dropped(&self) {
self.after_commit.fetch_sub(1, Ordering::SeqCst);
}
pub(crate) fn after_commit_pending(&self) -> bool {
self.after_commit.load(Ordering::SeqCst) > 0
}
pub(crate) fn detached_work_started(&self) {
if self.legacy_v3 {
return;
}
self.diverge(EffectDivergence {
seam: EffectSeam::Detached,
kind: EffectDivergenceKind::Unrecorded,
index: 0,
expected: None,
actual: "a detached task".to_owned(),
detail: "the replayed run started work on a detached task; the recorded run \
started none"
.to_owned(),
});
}
fn update_slots(&self, target: SlotUpdate<'_>, value: Option<&[u8]>) {
let Ok(mut cache) = self.cache.lock() else {
return;
};
match target {
SlotUpdate::Key(key) => {
if let Some(slot) =
resolve_slot(&cache, key).and_then(|recorded| cache.get_mut(&recorded))
{
slot.value = value.map(<[u8]>::to_vec);
} else if let Some(value) = value {
cache.insert(
key.to_owned(),
CacheSlot {
value: Some(value.to_vec()),
was_read: false,
touched: false,
},
);
}
}
SlotUpdate::Namespace(namespace) => {
for (key, slot) in cache.iter_mut() {
if in_namespace(key, namespace) {
slot.value = None;
}
}
}
SlotUpdate::All => {
for slot in cache.values_mut() {
slot.value = None;
}
}
}
}
fn removal_result(&self, actual: &CacheWrite) -> Result<(), crate::cache::InvalidationError> {
match self.next_cache_write(actual) {
Some(
CacheWrite::Invalidate {
error: Some(error), ..
}
| CacheWrite::InvalidateNamespace {
error: Some(error), ..
},
) => Err(crate::cache::InvalidationError::new(
error.attempts,
error.reason,
)),
Some(_) => Ok(()),
None => Err(crate::cache::InvalidationError::new(
1,
"the replayed run removed a cache entry the capsule has no recording for",
)),
}
}
fn next_cache_write(&self, actual: &CacheWrite) -> Option<CacheWrite> {
let Ok(mut seam) = self.cache_writes.lock() else {
return None;
};
let index = seam.consumed;
let Some(next) = seam.pending.front() else {
drop(seam);
let described = actual.describe_withheld();
self.diverge(EffectDivergence {
seam: EffectSeam::Cache,
kind: EffectDivergenceKind::Unrecorded,
index,
expected: None,
actual: described.clone(),
detail: format!(
"the replayed run made cache write {described:?}, which the capsule has no \
recording for"
),
});
return None;
};
if !next.matches(actual) {
let expected = next.describe();
let recorded_identifier = next.identifier().map(ToOwned::to_owned);
drop(seam);
let described = if self.observe_identifier(recorded_identifier.as_deref(), actual) {
actual.describe()
} else {
actual.describe_withheld()
};
self.diverge(EffectDivergence {
seam: EffectSeam::Cache,
kind: EffectDivergenceKind::Mismatch,
index,
expected: Some(expected.clone()),
actual: described.clone(),
detail: format!(
"the replayed run made cache write {described:?}, but the capsule's next \
recorded write was {expected:?}"
),
});
return None;
}
let recorded = seam.pending.pop_front();
seam.consumed = seam.consumed.saturating_add(1);
drop(seam);
if let Some(recorded) = recorded.as_ref() {
self.observe_write(recorded, actual);
}
self.served.fetch_add(1, Ordering::SeqCst);
recorded
}
#[cfg(any(test, feature = "mail"))]
#[must_use]
pub(crate) fn next_mail(&self, sent: &SentMail<'_>) -> MailVerdict {
let (to, subject) = (sent.to, sent.subject);
let Ok(mut seam) = self.mail.lock() else {
return MailVerdict::Diverged;
};
let index = seam.consumed;
let Some(next) = seam.pending.front() else {
drop(seam);
let actual = describe_mail(to, subject);
self.diverge(EffectDivergence {
seam: EffectSeam::Mail,
kind: EffectDivergenceKind::Unrecorded,
index,
expected: None,
actual: actual.clone(),
detail: format!(
"the replayed run sent mail the capsule has no recording for ({actual}); \
nothing was delivered"
),
});
return MailVerdict::Diverged;
};
let recipients_match = next.to.len() == to.len()
&& next
.to
.iter()
.zip(to.iter())
.all(|(recorded, actual)| matches_redacted(recorded, actual));
if !recipients_match || !matches_redacted(&next.subject, subject) {
let expected = describe_mail(&next.to, &next.subject);
drop(seam);
let actual = describe_mail(to, subject);
self.diverge(EffectDivergence {
seam: EffectSeam::Mail,
kind: EffectDivergenceKind::Mismatch,
index,
expected: Some(expected.clone()),
actual: actual.clone(),
detail: format!(
"the replayed run sent {actual}, but the capsule's next recorded send was \
{expected}"
),
});
return MailVerdict::Diverged;
}
let changed = mail_change(next, sent);
if let Some(changed) = changed {
let expected = describe_mail(&next.to, &next.subject);
let (recorded_to, recorded_subject) = (next.to.clone(), next.subject.clone());
drop(seam);
for (recorded, actual) in recorded_to.iter().zip(to.iter()) {
self.observe_text(recorded, actual);
}
self.observe_text(&recorded_subject, subject);
let actual = describe_mail(to, subject);
self.diverge(EffectDivergence {
seam: EffectSeam::Mail,
kind: EffectDivergenceKind::Mismatch,
index,
expected: Some(expected),
actual: actual.clone(),
detail: format!(
"the replayed run sent {actual} to the recipients the capsule recorded, but \
changed {changed}; nothing was delivered"
),
});
return MailVerdict::Diverged;
}
let recorded = seam.pending.pop_front();
seam.consumed = seam.consumed.saturating_add(1);
drop(seam);
if let Some(recorded) = recorded.as_ref() {
self.observe_mail(recorded, sent);
}
let error =
recorded.and_then(|recorded| recorded.error.map(|text| (text, recorded.error_kind)));
self.served.fetch_add(1, Ordering::SeqCst);
error.map_or(MailVerdict::Sent, |(text, kind)| {
MailVerdict::Failed(text, kind)
})
}
#[must_use]
pub(crate) fn tenant(&self) -> TenantVerdict {
let first_read = !self.tenant_read.swap(true, Ordering::SeqCst);
match &self.tenant {
RecordedTenant::Resolved(id) => TenantVerdict::Resolved(id.clone()),
RecordedTenant::Failed => TenantVerdict::RecordedFailure,
RecordedTenant::None if self.legacy_v3 => TenantVerdict::Unchecked,
RecordedTenant::None => {
if first_read {
self.diverge(EffectDivergence {
seam: EffectSeam::Tenant,
kind: EffectDivergenceKind::Unrecorded,
index: 0,
expected: None,
actual: "tenant lookup".to_owned(),
detail: "the replayed run resolved a tenant, which the capsule has no \
recording for; live tenant resolution was not consulted"
.to_owned(),
});
}
TenantVerdict::Unrecorded
}
}
}
pub(crate) fn tenant_resolved_after_recorded_failure(&self) {
self.diverge(EffectDivergence {
seam: EffectSeam::Tenant,
kind: EffectDivergenceKind::Mismatch,
index: 0,
expected: Some("a failed tenant lookup".to_owned()),
actual: "a resolved tenant".to_owned(),
detail: "the recorded tenant lookup failed, but the replayed run resolved a tenant; \
the run would continue under a tenant production never had"
.to_owned(),
});
}
pub(crate) fn refuse_unrecorded_egress(&self, subsystem: &str, method: &str, url: &str) {
let index = self.http.lock().map_or(0, |seam| seam.consumed);
let actual = describe_http(method, url);
self.diverge(EffectDivergence {
seam: EffectSeam::Http,
kind: EffectDivergenceKind::Unrecorded,
index,
expected: None,
actual: actual.clone(),
detail: format!(
"the replayed run's {subsystem} made {actual} outside the recorded outbound-HTTP \
seam; it was refused rather than sent"
),
});
}
#[must_use]
pub(crate) fn observed_redactions(&self) -> crate::capsule::redact::RedactedValues {
self.observed
.lock()
.map(|values| values.clone())
.unwrap_or_default()
}
fn observe_text(&self, recorded: &str, actual: &str) {
let Some(spans) = redacted_spans(recorded, actual) else {
return;
};
if spans.is_empty() {
return;
}
if let Ok(mut observed) = self.observed.lock() {
for (span, encoded) in spans {
observed.insert(span.as_bytes());
if encoded {
let decoded = form_decoded(span);
if !decoded.is_empty() && decoded != span {
observed.insert(decoded.as_bytes());
}
}
}
}
}
fn observe_url(&self, recorded: &str, actual: &str) {
let recorded = crate::capsule::redact::parser_view(recorded);
let actual = crate::capsule::redact::parser_view(actual);
let (recorded, actual) = (recorded.as_ref(), actual.as_ref());
self.observe_text(recorded, actual);
if crate::capsule::redact::url_userinfo(recorded).as_deref() == Some("[FILTERED]") {
self.observe_live_userinfo(actual);
}
}
fn observe_live_userinfo(&self, url: &str) {
if let Some(userinfo) = crate::capsule::redact::url_userinfo(url)
&& let Ok(mut observed) = self.observed.lock()
{
crate::capsule::redact::record_userinfo_components(&userinfo, &mut observed);
}
}
fn observe_headers(&self, recorded: &[(String, String)], actual: &[(String, String)]) {
for ((name, recorded), (_, actual)) in recorded.iter().zip(actual.iter()) {
self.observe_text(recorded, actual);
if recorded != actual
&& let Ok(mut observed) = self.observed.lock()
{
crate::capsule::redact::record_credential_components(
name,
actual.as_bytes(),
&mut observed,
);
}
}
}
fn observe_body(&self, recorded: &CapsuleBody, actual: &CapsuleBody) {
let (recorded, actual) = (body_bytes(recorded), body_bytes(actual));
if let (Ok(recorded), Ok(actual)) =
(std::str::from_utf8(&recorded), std::str::from_utf8(&actual))
{
self.observe_text(recorded, actual);
}
}
fn observe_json(&self, recorded: &serde_json::Value, actual: &serde_json::Value) {
use serde_json::Value;
match (recorded, actual) {
(Value::String(recorded), Value::String(actual)) => {
self.observe_text(recorded, actual);
}
(Value::String(recorded), actual) if recorded == FILTERED => {
if let Ok(mut observed) = self.observed.lock() {
crate::capsule::redact::record_masked_value(actual, &mut observed);
}
}
(Value::Object(recorded), Value::Object(actual)) => {
for (key, recorded) in recorded {
if let Some(actual) = actual.get(key) {
self.observe_json(recorded, actual);
}
}
}
(Value::Array(recorded), Value::Array(actual)) => {
for (recorded, actual) in recorded.iter().zip(actual.iter()) {
self.observe_json(recorded, actual);
}
}
_ => {}
}
}
#[cfg(any(test, feature = "mail"))]
fn observe_mail(&self, recorded: &MailEffect, sent: &SentMail<'_>) {
for (recorded, actual) in recorded.to.iter().zip(sent.to.iter()) {
self.observe_text(recorded, actual);
}
self.observe_text(&recorded.subject, sent.subject);
if let (Some(recorded), Some(actual)) = (recorded.from.as_deref(), sent.from) {
self.observe_text(recorded, actual);
}
self.observe_body(&recorded.body, sent.body);
self.observe_body(&recorded.alternate_body, sent.alternate_body);
if let (Some(recorded), Some(actual)) = (recorded.reply_to.as_deref(), sent.reply_to) {
self.observe_text(recorded, actual);
}
if let (Some(recorded), Some(actual)) =
(recorded.list_unsubscribe.as_deref(), sent.list_unsubscribe)
{
self.observe_text(recorded, actual);
}
self.observe_headers(&recorded.extra_headers, sent.extra_headers);
for (recorded, actual) in recorded.attachments.iter().zip(sent.attachments.iter()) {
self.observe_text(&recorded.filename, &actual.filename);
}
}
fn observe_identifier(&self, recorded: Option<&str>, actual: &CacheWrite) -> bool {
let (Some(recorded), Some(actual)) = (recorded, actual.identifier()) else {
return true;
};
if !recorded.contains(FILTERED) && !recorded.contains(FILTERED_URLENCODED) {
return recorded == actual;
}
if redacted_spans(recorded, actual).is_none() {
return false;
}
self.observe_text(recorded, actual);
true
}
fn observe_write(&self, recorded: &CacheWrite, actual: &CacheWrite) {
match (recorded, actual) {
(
CacheWrite::Insert { key, value, .. },
CacheWrite::Insert {
key: actual_key,
value: actual_value,
..
},
) => {
self.observe_text(key, actual_key);
if let (Some(recorded), Some(actual)) = (value, actual_value)
&& let (Ok(recorded), Ok(actual)) =
(std::str::from_utf8(recorded), std::str::from_utf8(actual))
{
self.observe_text(recorded, actual);
}
}
(CacheWrite::Invalidate { key, .. }, CacheWrite::Invalidate { key: actual, .. })
| (
CacheWrite::InvalidateNamespace { namespace: key, .. },
CacheWrite::InvalidateNamespace {
namespace: actual, ..
},
) => self.observe_text(key, actual),
_ => {}
}
}
#[must_use]
pub fn finish(&self) -> Vec<EffectDivergence> {
let mut divergences = self.divergences();
if let Ok(seam) = self.http.lock()
&& !seam.pending.is_empty()
{
divergences.push(unconsumed(
EffectSeam::Http,
seam.consumed,
seam.pending.len(),
seam.pending
.front()
.map(|next| format!("{} {}", next.method, next.url)),
));
}
if let Ok(seam) = self.jobs.lock()
&& !seam.pending.is_empty()
{
divergences.push(unconsumed(
EffectSeam::Job,
seam.consumed,
seam.pending.len(),
seam.pending.front().map(|next| next.name.clone()),
));
}
if let Ok(seam) = self.mail.lock()
&& !seam.pending.is_empty()
{
divergences.push(unconsumed(
EffectSeam::Mail,
seam.consumed,
seam.pending.len(),
seam.pending.front().map(|next| next.subject.clone()),
));
}
let recorded_tenant = match &self.tenant {
RecordedTenant::None => None,
RecordedTenant::Resolved(id) => Some(id.clone()),
RecordedTenant::Failed => Some("a failed tenant lookup".to_owned()),
};
if let Some(recorded) = recorded_tenant
&& !self.tenant_read.load(Ordering::SeqCst)
{
divergences.push(unconsumed(EffectSeam::Tenant, 0, 1, Some(recorded)));
}
if let Ok(seam) = self.cache_writes.lock()
&& !seam.pending.is_empty()
{
divergences.push(unconsumed(
EffectSeam::Cache,
seam.consumed,
seam.pending.len(),
seam.pending.front().map(CacheWrite::describe),
));
}
if let Ok(cache) = self.cache.lock() {
let missed: Vec<&String> = cache
.iter()
.filter(|(_, slot)| slot.was_read && !slot.touched)
.map(|(key, _)| key)
.collect();
if !missed.is_empty() {
divergences.push(unconsumed(
EffectSeam::Cache,
0,
missed.len(),
missed.first().map(|key| (*key).clone()),
));
}
}
divergences
}
}
pub(crate) fn random_width_divergence(
index: usize,
recorded: usize,
actual: usize,
) -> EffectDivergence {
EffectDivergence {
seam: EffectSeam::Random,
kind: EffectDivergenceKind::Mismatch,
index,
expected: Some(format!("{recorded} byte(s)")),
actual: format!("{actual} byte(s)"),
detail: format!(
"the replayed run drew {actual} random byte(s) where the recording drew {recorded}; \
the value it minted is not the one production used"
),
}
}
fn unconsumed(
seam: EffectSeam,
consumed: usize,
remaining: usize,
next: Option<String>,
) -> EffectDivergence {
let label = seam.label();
let next_text = next.unwrap_or_else(|| "?".to_owned());
EffectDivergence {
seam,
kind: EffectDivergenceKind::Unconsumed,
index: consumed,
expected: Some(next_text.clone()),
actual: String::new(),
detail: format!(
"{remaining} recorded {label} effect(s) were never asked for by the replayed run \
(the next one was {next_text}); reaching the recorded outcome without the \
recorded effects is not a reproduction"
),
}
}
fn decode(encoded: &str) -> Option<Vec<u8>> {
STANDARD.decode(encoded.as_bytes()).ok()
}
#[must_use]
pub fn body_bytes(body: &CapsuleBody) -> Vec<u8> {
match body {
CapsuleBody::Absent | CapsuleBody::Skipped { .. } => Vec::new(),
CapsuleBody::Text(text) => text.as_bytes().to_vec(),
CapsuleBody::Base64(encoded) => decode(encoded).unwrap_or_default(),
}
}
fn describe_http(method: &str, url: &str) -> String {
let path = url.split_once('?').map_or(url, |(head, _)| head);
let elided = if path.len() < url.len() { "?…" } else { "" };
let path = crate::capsule::redact::hide_url_userinfo(path);
format!("{method} {path}{elided}")
}
fn describe_job(name: &str, payload: &serde_json::Value) -> String {
match payload {
serde_json::Value::Object(map) => {
let keys: Vec<&str> = map.keys().map(String::as_str).collect();
format!("{name} {{{}}}", keys.join(", "))
}
serde_json::Value::Null => format!("{name} (null payload)"),
other => format!("{name} ({} payload)", json_kind(other)),
}
}
const fn json_kind(value: &serde_json::Value) -> &'static str {
match value {
serde_json::Value::Null => "null",
serde_json::Value::Bool(_) => "boolean",
serde_json::Value::Number(_) => "number",
serde_json::Value::String(_) => "string",
serde_json::Value::Array(_) => "array",
serde_json::Value::Object(_) => "object",
}
}
#[cfg(any(test, feature = "mail"))]
fn mail_change(next: &MailEffect, sent: &SentMail<'_>) -> Option<&'static str> {
let sender_match = sent.from.is_none_or(|actual| {
next.from
.as_deref()
.is_some_and(|recorded| matches_redacted(recorded, actual))
});
if !sender_match {
Some("its sender")
} else if !body_matches_redacted(&next.body, sent.body) {
Some("its body")
} else if !body_matches_redacted(&next.alternate_body, sent.alternate_body) {
Some("the other half of its multipart body")
} else if !optional_matches_redacted(next.reply_to.as_deref(), sent.reply_to) {
Some("its reply-to")
} else if !optional_matches_redacted(next.list_unsubscribe.as_deref(), sent.list_unsubscribe) {
Some("its list-unsubscribe header")
} else if !headers_match_redacted(&next.extra_headers, sent.extra_headers) {
Some("its headers")
} else if !attachments_match_redacted(&next.attachments, sent.attachments) {
Some("its attachments")
} else {
None
}
}
#[cfg(any(test, feature = "mail"))]
fn describe_mail(to: &[String], subject: &str) -> String {
let domains: Vec<&str> = to
.iter()
.map(|address| address.rsplit('@').next().unwrap_or("?"))
.collect();
format!(
"{subject:?} -> {} recipient(s) at {}",
to.len(),
domains.join(", ")
)
}
const FILTERED: &str = crate::log::filter::FILTERED_PLACEHOLDER;
const FILTERED_URLENCODED: &str = "%5BFILTERED%5D";
fn matches_redacted(recorded: &str, actual: &str) -> bool {
redacted_spans(recorded, actual).is_some()
}
fn form_decoded(value: &str) -> String {
url::form_urlencoded::parse(format!("={value}").as_bytes())
.next()
.map(|(_, decoded)| decoded.into_owned())
.unwrap_or_default()
}
fn redacted_spans<'a>(recorded: &str, actual: &'a str) -> Option<Vec<(&'a str, bool)>> {
if recorded == actual {
return Some(Vec::new());
}
let (segments, encoded) = split_placeholders(recorded);
if encoded.is_empty() {
return None;
}
let mut spans = Vec::new();
let mut push = |span: &'a str, encoded: bool| {
if !span.is_empty() {
spans.push((span, encoded));
}
};
let mut rest = actual;
let mut segments = segments.into_iter();
rest = rest.strip_prefix(segments.next().unwrap_or_default())?;
let mut segments = segments.zip(encoded).peekable();
while let Some((segment, encoded)) = segments.next() {
let last = segments.peek().is_none();
if segment.is_empty() {
if last {
push(rest, encoded);
rest = "";
}
} else if last {
let head = rest.strip_suffix(segment)?;
push(head, encoded);
rest = "";
} else {
let found = rest.find(segment)?;
push(rest.get(..found).unwrap_or_default(), encoded);
let after = found.saturating_add(segment.len());
rest = rest.get(after..).unwrap_or_default();
}
}
Some(spans)
}
pub(crate) fn has_adjacent_placeholders(recorded: &str) -> bool {
let (segments, _) = split_placeholders(recorded);
let between = segments.len().saturating_sub(2);
segments
.iter()
.skip(1)
.take(between)
.any(|segment| segment.is_empty())
}
fn split_placeholders(recorded: &str) -> (Vec<&str>, Vec<bool>) {
let mut segments = Vec::new();
let mut encoded = Vec::new();
let mut rest = recorded;
loop {
let next = match (rest.find(FILTERED), rest.find(FILTERED_URLENCODED)) {
(Some(raw), Some(url)) if url < raw => Some((url, FILTERED_URLENCODED, true)),
(Some(raw), _) => Some((raw, FILTERED, false)),
(None, Some(url)) => Some((url, FILTERED_URLENCODED, true)),
(None, None) => None,
};
let Some((at, placeholder, is_encoded)) = next else {
segments.push(rest);
return (segments, encoded);
};
segments.push(rest.get(..at).unwrap_or_default());
encoded.push(is_encoded);
rest = rest
.get(at.saturating_add(placeholder.len())..)
.unwrap_or_default();
}
}
pub(crate) struct OutboundRequest<'a> {
pub method: &'a str,
pub url: &'a str,
pub headers: &'a [(String, String)],
pub body: &'a CapsuleBody,
}
#[cfg(any(test, feature = "mail"))]
pub(crate) struct SentMail<'a> {
pub to: &'a [String],
pub from: Option<&'a str>,
pub subject: &'a str,
pub body: &'a CapsuleBody,
pub alternate_body: &'a CapsuleBody,
pub reply_to: Option<&'a str>,
pub list_unsubscribe: Option<&'a str>,
pub extra_headers: &'a [(String, String)],
pub attachments: &'a [crate::capsule::schema::MailAttachmentEffect],
}
fn body_matches_redacted(recorded: &CapsuleBody, actual: &CapsuleBody) -> bool {
if matches!(recorded, CapsuleBody::Skipped { .. }) {
return true;
}
match (recorded, actual) {
(CapsuleBody::Absent, CapsuleBody::Absent) => true,
(CapsuleBody::Absent, _) | (_, CapsuleBody::Absent) => false,
_ => bytes_match_redacted(&body_bytes(recorded), &body_bytes(actual)),
}
}
#[cfg(any(test, feature = "mail"))]
fn attachments_match_redacted(
recorded: &[crate::capsule::schema::MailAttachmentEffect],
actual: &[crate::capsule::schema::MailAttachmentEffect],
) -> bool {
recorded.len() == actual.len()
&& recorded
.iter()
.zip(actual.iter())
.all(|(recorded, actual)| {
matches_redacted(&recorded.filename, &actual.filename)
&& recorded.content_type == actual.content_type
&& recorded.len == actual.len
&& recorded.sha256 == actual.sha256
})
}
#[cfg(any(test, feature = "mail"))]
fn optional_matches_redacted(recorded: Option<&str>, actual: Option<&str>) -> bool {
match (recorded, actual) {
(None, None) => true,
(Some(recorded), Some(actual)) => matches_redacted(recorded, actual),
_ => false,
}
}
fn headers_match_redacted(recorded: &[(String, String)], actual: &[(String, String)]) -> bool {
recorded.len() == actual.len()
&& recorded.iter().zip(actual.iter()).all(
|((recorded_name, recorded_value), (actual_name, actual_value))| {
recorded_name.eq_ignore_ascii_case(actual_name)
&& matches_redacted(recorded_value, actual_value)
},
)
}
fn describe_request_mismatch(recorded: &HttpEffect, actual: &OutboundRequest<'_>) -> String {
let mut parts = Vec::new();
if !headers_match_redacted(&recorded.request_headers, actual.headers) {
let names: Vec<&str> = recorded
.request_headers
.iter()
.map(|(name, _)| name.as_str())
.collect();
parts.push(format!("its headers ({} recorded)", names.join(", ")));
}
if !body_matches_redacted(&recorded.request_body, actual.body) {
parts.push("its body".to_owned());
}
parts.join(" and ")
}
fn bytes_match_redacted(recorded: &[u8], actual: &[u8]) -> bool {
if recorded == actual {
return true;
}
match (std::str::from_utf8(recorded), std::str::from_utf8(actual)) {
(Ok(recorded), Ok(actual)) => matches_redacted(recorded, actual),
_ => false,
}
}
fn json_matches_redacted(recorded: &serde_json::Value, actual: &serde_json::Value) -> bool {
use serde_json::Value;
match (recorded, actual) {
(Value::String(recorded), Value::String(actual)) => matches_redacted(recorded, actual),
(Value::String(recorded), _) if recorded == FILTERED => true,
(Value::Object(recorded), Value::Object(actual)) => {
recorded.len() == actual.len()
&& recorded.iter().all(|(key, recorded_value)| {
actual.get(key).is_some_and(|actual_value| {
json_matches_redacted(recorded_value, actual_value)
})
})
}
(Value::Array(recorded), Value::Array(actual)) => {
recorded.len() == actual.len()
&& recorded
.iter()
.zip(actual.iter())
.all(|(recorded, actual)| json_matches_redacted(recorded, actual))
}
_ => recorded == actual,
}
}
tokio::task_local! {
static REPLAY_EFFECTS: Arc<ReplayEffects>;
}
pub async fn with_effect_tape<F: Future>(tape: Arc<ReplayEffects>, future: F) -> F::Output {
REPLAY_EFFECTS.scope(tape, future).await
}
#[must_use]
pub fn current_tape() -> Option<Arc<ReplayEffects>> {
REPLAY_EFFECTS.try_with(Arc::clone).ok()
}
#[must_use]
pub fn tape_active() -> bool {
REPLAY_EFFECTS
.try_with(|tape| tape.mark_scope_entered())
.is_ok()
}
#[cfg(test)]
mod tests {
use crate::capsule::effects::{
CachedValue, EffectDivergenceKind, EffectSeam, EnqueueVerdict, FILTERED, MailVerdict,
OutboundRequest, ReplayEffects, SentMail,
};
static NO_BODY: CapsuleBody = CapsuleBody::Absent;
use crate::capsule::schema::{
CacheEffect, CapsuleBody, CapsuleEffects, HttpEffect, JobEffect, MailEffect, TenantEffect,
};
fn http(method: &str, url: &str, status: u16) -> HttpEffect {
HttpEffect {
method: method.to_owned(),
url: url.to_owned(),
request_headers: Vec::new(),
request_body: CapsuleBody::Absent,
status,
response_headers: Vec::new(),
response_body: CapsuleBody::Text("{}".to_owned()),
error: None,
..Default::default()
}
}
fn call<'a>(method: &'a str, url: &'a str) -> OutboundRequest<'a> {
OutboundRequest {
method,
url,
headers: &[],
body: &CapsuleBody::Absent,
}
}
fn sent<'a>(to: &'a [String], subject: &'a str, body: &'a CapsuleBody) -> SentMail<'a> {
SentMail {
to,
from: None,
subject,
body,
alternate_body: &NO_BODY,
reply_to: None,
list_unsubscribe: None,
extra_headers: &[],
attachments: &[],
}
}
fn sent_full<'a>(
to: &'a [String],
body: &'a CapsuleBody,
alternate_body: &'a CapsuleBody,
attachments: &'a [crate::capsule::schema::MailAttachmentEffect],
) -> SentMail<'a> {
SentMail {
to,
from: Some("billing@shop.example"),
subject: "Receipt",
body,
alternate_body,
reply_to: None,
list_unsubscribe: None,
extra_headers: &[],
attachments,
}
}
#[test]
fn a_redacted_recording_still_matches_the_call_the_handler_makes() {
let tape = ReplayEffects::new(CapsuleEffects {
http: vec![http(
"GET",
"https://api.example/items?key=%5BFILTERED%5D&page=2",
200,
)],
jobs: vec![JobEffect {
name: "notify".to_owned(),
payload: serde_json::json!({"to": "a@example.com", "token": "[FILTERED]"}),
delay_secs: None,
due_at: None,
requested_due_at: None,
error: None,
error_status: None,
}],
..CapsuleEffects::default()
});
assert!(
tape.next_http(&call(
"GET",
"https://api.example/items?key=sk-live-42&page=2"
))
.is_some(),
"a masked query value must match whatever really stood there"
);
assert_eq!(
tape.next_job(
"notify",
&serde_json::json!({"to": "a@example.com", "token": "real-token"}),
crate::job::EnqueueSchedule::Immediate
),
EnqueueVerdict::Queued
);
assert!(tape.divergences().is_empty(), "{:?}", tape.divergences());
}
#[test]
fn a_wildcard_does_not_swallow_a_genuine_difference() {
let tape = ReplayEffects::new(CapsuleEffects {
http: vec![http(
"GET",
"https://api.example/items?key=%5BFILTERED%5D",
200,
)],
..CapsuleEffects::default()
});
assert!(
tape.next_http(&call("GET", "https://api.example/other?key=sk-live-42"))
.is_none()
);
assert_eq!(tape.divergences().len(), 1);
}
#[test]
fn a_masked_json_payload_still_compares_structurally() {
let tape = ReplayEffects::new(CapsuleEffects {
jobs: vec![JobEffect {
name: "notify".to_owned(),
payload: serde_json::json!({"token": "[FILTERED]"}),
delay_secs: None,
due_at: None,
requested_due_at: None,
error: None,
error_status: None,
}],
..CapsuleEffects::default()
});
assert_eq!(
tape.next_job(
"notify",
&serde_json::json!({"token": "real", "extra": 1}),
crate::job::EnqueueSchedule::Immediate
),
EnqueueVerdict::Diverged
);
assert_eq!(tape.divergences().len(), 1);
}
#[test]
fn an_outbound_call_is_served_from_the_tape_in_recorded_order() {
let tape = ReplayEffects::new(CapsuleEffects {
http: vec![
http("GET", "https://a.example/one", 200),
http("GET", "https://a.example/two", 503),
],
..CapsuleEffects::default()
});
let first = tape
.next_http(&call("GET", "https://a.example/one"))
.expect("the first recorded exchange serves the first call");
assert_eq!(first.status, 200);
let second = tape
.next_http(&call("GET", "https://a.example/two"))
.expect("the second call takes the second exchange");
assert_eq!(second.status, 503);
assert!(tape.divergences().is_empty(), "a faithful run diverges not");
}
#[test]
fn an_outbound_call_the_capsule_never_recorded_is_a_divergence_not_a_live_call() {
let tape = ReplayEffects::new(CapsuleEffects::default());
assert!(
tape.next_http(&call("POST", "https://payments.example/charge"))
.is_none(),
"an unrecorded call must never fall through to the network"
);
let divergences = tape.divergences();
assert_eq!(divergences.len(), 1);
assert_eq!(divergences[0].seam, EffectSeam::Http);
assert!(
divergences[0].detail.contains("payments.example"),
"the report must name the call: {}",
divergences[0].detail
);
}
#[test]
fn an_outbound_call_to_a_different_url_than_recorded_is_a_divergence() {
let tape = ReplayEffects::new(CapsuleEffects {
http: vec![http("GET", "https://a.example/one", 200)],
..CapsuleEffects::default()
});
assert!(
tape.next_http(&call("GET", "https://b.example/other"))
.is_none()
);
let divergences = tape.divergences();
assert_eq!(divergences.len(), 1);
assert_eq!(
divergences[0].expected.as_deref(),
Some("GET https://a.example/one")
);
}
#[test]
fn a_recorded_call_the_replay_never_made_is_a_divergence_too() {
let tape = ReplayEffects::new(CapsuleEffects {
http: vec![http("GET", "https://a.example/one", 200)],
..CapsuleEffects::default()
});
let divergences = tape.finish();
assert_eq!(divergences.len(), 1);
assert!(
divergences[0].detail.contains("never"),
"the report must say the recorded call was never made: {}",
divergences[0].detail
);
}
#[test]
fn an_enqueue_is_asserted_against_the_tape_and_never_reaches_a_queue() {
let tape = ReplayEffects::new(CapsuleEffects {
jobs: vec![JobEffect {
name: "send_receipt".to_owned(),
payload: serde_json::json!({"order": 7}),
delay_secs: None,
due_at: None,
requested_due_at: None,
error: None,
error_status: None,
}],
..CapsuleEffects::default()
});
assert_eq!(
tape.next_job(
"send_receipt",
&serde_json::json!({"order": 7}),
crate::job::EnqueueSchedule::Immediate
),
EnqueueVerdict::Queued,
"the recorded enqueue is satisfied from the tape"
);
assert!(tape.divergences().is_empty());
assert_eq!(
tape.next_job(
"send_receipt",
&serde_json::json!({"order": 8}),
crate::job::EnqueueSchedule::Immediate
),
EnqueueVerdict::Diverged,
"an enqueue the recording never made must not be satisfied"
);
assert_eq!(tape.divergences().len(), 1);
}
#[test]
fn a_rescheduled_enqueue_is_a_divergence() {
let tape = ReplayEffects::new(CapsuleEffects {
jobs: vec![JobEffect {
name: "send_receipt".to_owned(),
payload: serde_json::json!({}),
delay_secs: None,
due_at: None,
requested_due_at: None,
error: None,
error_status: None,
}],
..CapsuleEffects::default()
});
assert_eq!(
tape.next_job(
"send_receipt",
&serde_json::json!({}),
crate::job::EnqueueSchedule::After(std::time::Duration::from_secs(3600))
),
EnqueueVerdict::Diverged
);
assert_eq!(tape.divergences().len(), 1);
}
#[test]
fn a_relative_delay_is_compared_the_way_its_format_recorded_it() {
let recorded = |delay_secs: i64| CapsuleEffects {
jobs: vec![JobEffect {
name: "send_receipt".to_owned(),
payload: serde_json::json!({}),
delay_secs: Some(delay_secs),
..JobEffect::default()
}],
..CapsuleEffects::default()
};
let after = |millis: u64| {
crate::job::EnqueueSchedule::After(std::time::Duration::from_millis(millis))
};
let verdict = |tape: &ReplayEffects, millis: u64| {
tape.next_job("send_receipt", &serde_json::json!({}), after(millis))
};
assert_eq!(
verdict(&ReplayEffects::new(recorded(1)), 500),
EnqueueVerdict::Queued
);
let v3 = |secs| ReplayEffects::new(recorded(secs)).for_format_version(3);
assert_eq!(verdict(&v3(0), 500), EnqueueVerdict::Queued);
assert_eq!(verdict(&v3(5), 6_000), EnqueueVerdict::Diverged);
assert_eq!(verdict(&v3(5), 5_000), EnqueueVerdict::Queued);
}
#[test]
fn a_zero_relative_delay_matches_the_immediate_entry_it_recorded() {
let immediate = CapsuleEffects {
jobs: vec![JobEffect {
name: "send_receipt".to_owned(),
payload: serde_json::json!({}),
..JobEffect::default()
}],
..CapsuleEffects::default()
};
let after = |millis: u64| {
crate::job::EnqueueSchedule::After(std::time::Duration::from_millis(millis))
};
let tape = ReplayEffects::new(immediate.clone());
assert_eq!(
tape.next_job("send_receipt", &serde_json::json!({}), after(0)),
EnqueueVerdict::Queued
);
assert!(tape.finish().is_empty(), "{:?}", tape.finish());
let tape = ReplayEffects::new(immediate);
assert_eq!(
tape.next_job("send_receipt", &serde_json::json!({}), after(1_000)),
EnqueueVerdict::Diverged
);
}
#[test]
fn an_immediate_enqueue_does_not_satisfy_a_recorded_delayed_one() {
let delayed = JobEffect {
name: "send_receipt".to_owned(),
payload: serde_json::json!({}),
delay_secs: Some(3600),
..JobEffect::default()
};
let future = JobEffect {
due_at: Some(chrono::Utc::now() + chrono::Duration::hours(1)),
..delayed.clone()
};
for recorded in [delayed, future] {
let tape = ReplayEffects::new(CapsuleEffects {
jobs: vec![recorded],
..CapsuleEffects::default()
});
assert_eq!(
tape.next_job(
"send_receipt",
&serde_json::json!({}),
crate::job::EnqueueSchedule::Immediate
),
EnqueueVerdict::Diverged
);
}
let tape = ReplayEffects::new(CapsuleEffects {
jobs: vec![JobEffect {
name: "send_receipt".to_owned(),
payload: serde_json::json!({}),
delay_secs: Some(0),
..JobEffect::default()
}],
..CapsuleEffects::default()
});
assert_eq!(
tape.next_job(
"send_receipt",
&serde_json::json!({}),
crate::job::EnqueueSchedule::Immediate
),
EnqueueVerdict::Queued
);
}
#[test]
fn a_past_deadline_enqueue_at_matches_its_normalized_recording() {
let past = chrono::Utc::now() - chrono::Duration::hours(1);
let recorded = JobEffect {
name: "send_receipt".to_owned(),
payload: serde_json::json!({}),
requested_due_at: Some(past),
..JobEffect::default()
};
let tape = ReplayEffects::new(CapsuleEffects {
jobs: vec![recorded.clone()],
..CapsuleEffects::default()
});
assert_eq!(
tape.next_job(
"send_receipt",
&serde_json::json!({}),
crate::job::EnqueueSchedule::At(past)
),
EnqueueVerdict::Queued,
"{:?}",
tape.divergences()
);
let tape = ReplayEffects::new(CapsuleEffects {
jobs: vec![recorded.clone()],
..CapsuleEffects::default()
});
assert_eq!(
tape.next_job(
"send_receipt",
&serde_json::json!({}),
crate::job::EnqueueSchedule::Immediate
),
EnqueueVerdict::Diverged
);
let tape = ReplayEffects::new(CapsuleEffects {
jobs: vec![recorded],
..CapsuleEffects::default()
});
assert_eq!(
tape.next_job(
"send_receipt",
&serde_json::json!({}),
crate::job::EnqueueSchedule::At(past - chrono::Duration::hours(1))
),
EnqueueVerdict::Diverged
);
}
#[test]
fn a_v3_capsule_replays_with_the_v3_rules() {
let tape = ReplayEffects::new(CapsuleEffects::default()).for_format_version(3);
assert_eq!(tape.cache_invalidate("k"), Ok(()));
tape.cache_clear();
assert_eq!(tape.cache_invalidate_namespace("ns"), Ok(()));
tape.cache_untyped_insert("k");
tape.cache_untyped_get("k");
tape.cache_direct_get("k");
assert_eq!(tape.tenant(), super::TenantVerdict::Unchecked);
assert!(tape.finish().is_empty(), "{:?}", tape.finish());
}
#[test]
fn a_dropped_read_is_owed_even_when_the_key_was_written() {
let tape = ReplayEffects::new(CapsuleEffects {
cache: vec![
CacheEffect::Get {
key: "widgets".to_owned(),
value: Some(base64_of(b"41")),
},
CacheEffect::Invalidate {
key: "widgets".to_owned(),
error: None,
},
],
..CapsuleEffects::default()
});
assert_eq!(tape.cache_invalidate("widgets"), Ok(()));
let divergences = tape.finish();
assert_eq!(divergences.len(), 1, "{divergences:?}");
assert_eq!(divergences[0].kind, EffectDivergenceKind::Unconsumed);
}
#[test]
fn a_masked_namespace_matches_a_key_with_a_colon_in_its_suffix() {
assert!(super::in_namespace(
"tenant:[FILTERED]:user:42",
"tenant:secret"
));
assert!(super::in_namespace("ns:k", "ns"));
assert!(!super::in_namespace("other:k", "ns"));
}
#[test]
fn a_namespace_removal_clears_a_masked_recorded_key() {
let tape = ReplayEffects::new(CapsuleEffects {
cache: vec![
CacheEffect::Get {
key: "t:[FILTERED]:x".to_owned(),
value: Some(base64_of(b"41")),
},
CacheEffect::InvalidateNamespace {
namespace: "t:[FILTERED]".to_owned(),
error: None,
},
],
..CapsuleEffects::default()
});
assert_eq!(tape.cache_get("t:acme:x"), CachedValue::Hit(b"41".to_vec()));
assert_eq!(tape.cache_invalidate_namespace("t:acme"), Ok(()));
assert_eq!(tape.cache_get("t:acme:x"), CachedValue::Miss);
assert!(tape.finish().is_empty(), "{:?}", tape.finish());
}
#[test]
fn a_credential_header_token_is_observed() {
let tape = ReplayEffects::new(CapsuleEffects {
http: vec![HttpEffect {
request_headers: vec![("authorization".to_owned(), "[FILTERED]".to_owned())],
..http("POST", "https://api.example/charge", 502)
}],
..CapsuleEffects::default()
});
let headers = vec![(
"authorization".to_owned(),
"Bearer sk-live-42-long-token".to_owned(),
)];
let body = CapsuleBody::Absent;
let served = tape.next_http(&OutboundRequest {
method: "POST",
url: "https://api.example/charge",
headers: &headers,
body: &body,
});
assert!(served.is_some());
let masked = crate::capsule::redact::mask_echoes(
"upstream rejected sk-live-42-long-token",
&tape.observed_redactions(),
);
assert!(!masked.contains("sk-live-42"), "{masked}");
}
#[test]
fn a_url_encoded_masked_value_is_observed_decoded() {
let tape = ReplayEffects::new(CapsuleEffects {
http: vec![http(
"GET",
"https://api.example/x?token=%5BFILTERED%5D",
502,
)],
..CapsuleEffects::default()
});
let body = CapsuleBody::Absent;
let served = tape.next_http(&OutboundRequest {
method: "GET",
url: "https://api.example/x?token=sk%2Flive+42",
headers: &[],
body: &body,
});
assert!(served.is_some());
let masked = crate::capsule::redact::mask_echoes(
"upstream rejected sk/live 42",
&tape.observed_redactions(),
);
assert!(!masked.contains("sk/live"), "{masked}");
}
#[test]
fn a_cache_write_mismatch_does_not_print_a_masked_key() {
let recorded = |key: &str| CapsuleEffects {
cache: vec![CacheEffect::Invalidate {
key: key.to_owned(),
error: None,
}],
..CapsuleEffects::default()
};
let tape = ReplayEffects::new(recorded("user:[FILTERED]"));
let _ = tape.cache_invalidate_namespace("user:sk-live-42");
assert!(tape.observed_redactions().contains(b"sk-live-42"));
let tape = ReplayEffects::new(recorded("tok:[FILTERED]:x"));
let _ = tape.cache_invalidate_namespace("other:sk-live-42");
let divergences = tape.divergences();
assert_eq!(divergences.len(), 1, "{divergences:?}");
assert!(!divergences[0].actual.contains("sk-live-42"));
assert!(!divergences[0].detail.contains("sk-live-42"));
let tape = ReplayEffects::new(CapsuleEffects::default());
let _ = tape.cache_invalidate("token:sk-live-42");
let divergences = tape.divergences();
assert_eq!(divergences.len(), 1, "{divergences:?}");
assert!(!divergences[0].actual.contains("sk-live-42"));
assert!(!divergences[0].detail.contains("sk-live-42"));
}
#[test]
fn a_mismatched_invalidation_prints_no_live_key() {
let tape = ReplayEffects::new(CapsuleEffects {
cache: vec![CacheEffect::Invalidate {
key: "user:7".to_owned(),
error: None,
}],
..CapsuleEffects::default()
});
let _ = tape.cache_invalidate("token:sk-live-42");
let divergences = tape.divergences();
assert_eq!(divergences.len(), 1, "{divergences:?}");
assert!(!divergences[0].actual.contains("sk-live-42"));
assert!(!divergences[0].detail.contains("sk-live-42"));
}
#[test]
fn a_request_mismatch_learns_the_masked_url_value() {
let mut recorded = http("POST", "https://api.example/users/[FILTERED]/charge", 200);
recorded.request_body = CapsuleBody::Text("amount=10".to_owned());
let tape = ReplayEffects::new(CapsuleEffects {
http: vec![recorded],
..CapsuleEffects::default()
});
let body = CapsuleBody::Text("amount=99".to_owned());
let served = tape.next_http(&OutboundRequest {
method: "POST",
url: "https://api.example/users/sk-live-42/charge",
headers: &[],
body: &body,
});
assert!(served.is_none(), "the body changed");
assert_eq!(tape.divergences().len(), 1);
assert!(tape.observed_redactions().contains(b"sk-live-42"));
}
#[test]
fn a_mail_content_mismatch_learns_the_masked_envelope() {
let tape = ReplayEffects::new(CapsuleEffects {
mail: vec![MailEffect {
to: vec!["[FILTERED]@shop.example".to_owned()],
subject: "Code [FILTERED]".to_owned(),
body: CapsuleBody::Text("hi".to_owned()),
error: None,
..Default::default()
}],
..CapsuleEffects::default()
});
let to = ["sk-live-42@shop.example".to_owned()];
let body = CapsuleBody::Text("bye".to_owned());
assert_eq!(
tape.next_mail(&sent(&to, "Code 991827", &body)),
MailVerdict::Diverged
);
let learned = tape.observed_redactions();
assert!(learned.contains(b"sk-live-42"));
assert!(learned.contains(b"991827"));
}
#[test]
fn a_read_back_of_an_unrecorded_write_prints_no_key() {
let tape = ReplayEffects::new(CapsuleEffects::default());
tape.cache_insert("token:sk-live-42", b"1", None);
let _ = tape.cache_get("token:sk-live-42");
let divergences = tape.divergences();
assert!(!divergences.is_empty());
for divergence in &divergences {
assert!(!divergence.actual.contains("sk-live-42"), "{divergence:?}");
assert!(!divergence.detail.contains("sk-live-42"), "{divergence:?}");
}
}
#[test]
fn both_placeholder_spellings_match_in_one_text() {
let recorded = "https://api.example/u/[FILTERED]/x?token=%5BFILTERED%5D";
let actual = "https://api.example/u/alice/x?token=sk%2Flive";
assert!(super::matches_redacted(recorded, actual));
assert_eq!(
super::redacted_spans(recorded, actual),
Some(vec![("alice", false), ("sk%2Flive", true)])
);
assert!(!super::matches_redacted(
recorded,
"https://api.example/u/alice/y?token=sk%2Flive"
));
}
#[test]
fn a_tenant_lookup_the_capsule_never_recorded_is_a_divergence() {
let tape = ReplayEffects::new(CapsuleEffects::default());
assert_eq!(tape.tenant(), super::TenantVerdict::Unrecorded);
let divergences = tape.divergences();
assert_eq!(divergences.len(), 1, "{divergences:?}");
assert_eq!(divergences[0].seam, EffectSeam::Tenant);
assert_eq!(divergences[0].kind, EffectDivergenceKind::Unrecorded);
let tape = ReplayEffects::new(CapsuleEffects {
tenant: Some(TenantEffect { id: None }),
..CapsuleEffects::default()
});
assert_eq!(tape.tenant(), super::TenantVerdict::RecordedFailure);
assert!(tape.finish().is_empty());
}
#[test]
fn a_recorded_miss_replays_as_a_miss_even_when_the_run_filled_the_key() {
let tape = ReplayEffects::new(CapsuleEffects {
cache: vec![
CacheEffect::Get {
key: "widgets".to_owned(),
value: None,
},
CacheEffect::Insert {
key: "widgets".to_owned(),
value: base64_of(b"41"),
ttl_secs: None,
},
],
..CapsuleEffects::default()
});
assert_eq!(
tape.cache_get("widgets"),
CachedValue::Miss,
"the recorded miss must replay as a miss"
);
tape.cache_insert("widgets", b"41", None);
assert_eq!(
tape.cache_get("widgets"),
CachedValue::Hit(b"41".to_vec()),
"the run's own fill is readable back, as it was in production"
);
assert!(tape.divergences().is_empty());
}
#[test]
fn an_ambiguous_masked_cache_key_is_treated_as_unrecorded() {
let tape = ReplayEffects::new(CapsuleEffects {
cache: vec![
CacheEffect::Get {
key: "user:[FILTERED]:a".to_owned(),
value: Some(base64_of(b"1")),
},
CacheEffect::Get {
key: "user:[FILTERED]:b".to_owned(),
value: Some(base64_of(b"2")),
},
],
..CapsuleEffects::default()
});
assert_eq!(
tape.cache_get("user:alice:a"),
CachedValue::Hit(b"1".to_vec())
);
let tape = ReplayEffects::new(CapsuleEffects {
cache: vec![
CacheEffect::Get {
key: "user:[FILTERED]".to_owned(),
value: Some(base64_of(b"1")),
},
CacheEffect::Get {
key: "user:[FILTERED]x".to_owned(),
value: Some(base64_of(b"2")),
},
],
..CapsuleEffects::default()
});
assert_eq!(tape.cache_get("user:bobx"), CachedValue::Unrecorded);
assert_eq!(tape.divergences().len(), 1);
}
#[test]
fn a_cache_read_is_served_by_key_and_an_unrecorded_key_diverges() {
let tape = ReplayEffects::new(CapsuleEffects {
cache: vec![
CacheEffect::Get {
key: "user:7".to_owned(),
value: Some(base64_of(b"{\"a\":1}")),
},
CacheEffect::Get {
key: "user:9".to_owned(),
value: None,
},
],
..CapsuleEffects::default()
});
assert_eq!(
tape.cache_get("user:7"),
CachedValue::Hit(b"{\"a\":1}".to_vec()),
"a recorded hit serves its bytes"
);
assert_eq!(
tape.cache_get("user:9"),
CachedValue::Miss,
"a recorded miss replays as a miss, not as a divergence"
);
assert_eq!(tape.cache_get("user:404"), CachedValue::Unrecorded);
assert_eq!(tape.divergences().len(), 1, "the unrecorded key diverges");
}
#[test]
fn a_mail_send_is_asserted_and_never_delivered() {
let tape = ReplayEffects::new(CapsuleEffects {
mail: vec![MailEffect {
to: vec!["a@example.com".to_owned()],
from: None,
subject: "Receipt".to_owned(),
body: CapsuleBody::Text("thanks".to_owned()),
error: None,
..Default::default()
}],
..CapsuleEffects::default()
});
let body = CapsuleBody::Text("thanks".to_owned());
assert_eq!(
tape.next_mail(&sent(&["a@example.com".to_owned()], "Receipt", &body)),
MailVerdict::Sent,
"a recorded success is served as one, and nothing is delivered"
);
assert!(tape.divergences().is_empty());
assert_eq!(
tape.next_mail(&sent(&["b@example.com".to_owned()], "Receipt", &body)),
MailVerdict::Diverged
);
assert_eq!(tape.divergences().len(), 1);
}
#[test]
fn a_changed_outbound_body_diverges_rather_than_being_served_the_old_response() {
let mut recorded = http("POST", "https://payments.example/charge", 502);
recorded.request_body = CapsuleBody::Text("{\"amount\":10}".to_owned());
let tape = ReplayEffects::new(CapsuleEffects {
http: vec![recorded],
..CapsuleEffects::default()
});
let body = CapsuleBody::Text("{\"amount\":9999}".to_owned());
assert!(
tape.next_http(&OutboundRequest {
method: "POST",
url: "https://payments.example/charge",
headers: &[],
body: &body,
})
.is_none(),
"a different amount to the same endpoint is not the recorded call"
);
let divergences = tape.divergences();
assert_eq!(divergences.len(), 1);
assert!(
divergences[0].detail.contains("its body"),
"the report must name what changed: {}",
divergences[0].detail
);
assert!(
!divergences[0].detail.contains("9999"),
"and must not quote the body itself: {}",
divergences[0].detail
);
}
#[test]
fn a_changed_outbound_header_diverges_but_a_masked_one_still_matches() {
let mut recorded = http("POST", "https://api.example/send", 200);
recorded.request_headers = vec![
("authorization".to_owned(), FILTERED.to_owned()),
("x-idempotency-key".to_owned(), "abc".to_owned()),
];
let tape = ReplayEffects::new(CapsuleEffects {
http: vec![recorded.clone(), recorded],
..CapsuleEffects::default()
});
let faithful = [
("authorization".to_owned(), "Bearer sk-live-42".to_owned()),
("x-idempotency-key".to_owned(), "abc".to_owned()),
];
assert!(
tape.next_http(&OutboundRequest {
method: "POST",
url: "https://api.example/send",
headers: &faithful,
body: &CapsuleBody::Absent,
})
.is_some(),
"a masked header value must match whatever really stood there"
);
assert!(tape.divergences().is_empty(), "{:?}", tape.divergences());
let changed = [
("authorization".to_owned(), "Bearer sk-live-42".to_owned()),
("x-idempotency-key".to_owned(), "zzz".to_owned()),
];
assert!(
tape.next_http(&OutboundRequest {
method: "POST",
url: "https://api.example/send",
headers: &changed,
body: &CapsuleBody::Absent,
})
.is_none()
);
let divergences = tape.divergences();
assert_eq!(divergences.len(), 1);
assert!(
divergences[0].detail.contains("x-idempotency-key")
&& !divergences[0].detail.contains("zzz"),
"the report names the header but never its value: {}",
divergences[0].detail
);
}
#[test]
fn a_changed_mail_body_or_sender_diverges_rather_than_replaying_clean() {
let recorded = MailEffect {
to: vec!["a@example.com".to_owned()],
from: Some("billing@shop.example".to_owned()),
subject: "Receipt".to_owned(),
body: CapsuleBody::Text("you paid $10".to_owned()),
error: None,
..Default::default()
};
let tape = ReplayEffects::new(CapsuleEffects {
mail: vec![recorded.clone(), recorded],
..CapsuleEffects::default()
});
let to = ["a@example.com".to_owned()];
let wrong_body = CapsuleBody::Text("you paid $10000".to_owned());
assert_eq!(
tape.next_mail(&SentMail {
to: &to,
from: Some("billing@shop.example"),
subject: "Receipt",
body: &wrong_body,
alternate_body: &NO_BODY,
reply_to: None,
list_unsubscribe: None,
extra_headers: &[],
attachments: &[],
}),
MailVerdict::Diverged,
"same envelope, different letter — the recorded success must not be served"
);
let right_body = CapsuleBody::Text("you paid $10".to_owned());
assert_eq!(
tape.next_mail(&SentMail {
to: &to,
from: Some("noreply@shop.example"),
subject: "Receipt",
body: &right_body,
alternate_body: &NO_BODY,
reply_to: None,
list_unsubscribe: None,
extra_headers: &[],
attachments: &[],
}),
MailVerdict::Diverged,
"and mail sent from a different address is a change too"
);
let divergences = tape.divergences();
assert_eq!(divergences.len(), 2);
assert!(
divergences[0].detail.contains("its body")
&& divergences[1].detail.contains("its sender"),
"each report names what changed: {divergences:?}"
);
}
#[test]
fn a_sender_the_replay_never_configured_is_not_a_changed_sender() {
let tape = ReplayEffects::new(CapsuleEffects {
mail: vec![MailEffect {
to: vec!["a@example.com".to_owned()],
from: Some("noreply@shop.example".to_owned()),
subject: "Receipt".to_owned(),
body: CapsuleBody::Text("thanks".to_owned()),
error: None,
..Default::default()
}],
..CapsuleEffects::default()
});
let to = ["a@example.com".to_owned()];
let body = CapsuleBody::Text("thanks".to_owned());
assert_eq!(
tape.next_mail(&SentMail {
to: &to,
from: None,
subject: "Receipt",
body: &body,
alternate_body: &NO_BODY,
reply_to: None,
list_unsubscribe: None,
extra_headers: &[],
attachments: &[],
}),
MailVerdict::Sent
);
assert!(tape.divergences().is_empty(), "{:?}", tape.divergences());
}
#[test]
fn a_changed_mail_attachment_or_alternate_body_diverges() {
let recorded = MailEffect {
to: vec!["a@example.com".to_owned()],
from: Some("billing@shop.example".to_owned()),
subject: "Receipt".to_owned(),
body: CapsuleBody::Text("you paid $10".to_owned()),
alternate_body: CapsuleBody::Text("<p>you paid $10</p>".to_owned()),
attachments: vec![crate::capsule::schema::MailAttachmentEffect {
filename: "invoice.pdf".to_owned(),
content_type: "application/pdf".to_owned(),
len: 12,
sha256: "cd".repeat(32),
}],
..Default::default()
};
let tape = ReplayEffects::new(CapsuleEffects {
mail: vec![recorded.clone(), recorded.clone(), recorded],
..CapsuleEffects::default()
});
let to = ["a@example.com".to_owned()];
let body = CapsuleBody::Text("you paid $10".to_owned());
let same_html = CapsuleBody::Text("<p>you paid $10</p>".to_owned());
let new_html = CapsuleBody::Text("<p>you paid $10000</p>".to_owned());
let invoice = [crate::capsule::schema::MailAttachmentEffect {
filename: "invoice.pdf".to_owned(),
content_type: "application/pdf".to_owned(),
len: 12,
sha256: "cd".repeat(32),
}];
assert_eq!(
tape.next_mail(&sent_full(&to, &body, &same_html, &[])),
MailVerdict::Diverged,
"an attachment the run stopped sending is a different email"
);
assert_eq!(
tape.next_mail(&sent_full(&to, &body, &new_html, &invoice)),
MailVerdict::Diverged,
"a rewritten HTML half must not ride along under an unchanged text fallback"
);
assert_eq!(
tape.next_mail(&sent_full(&to, &body, &same_html, &invoice)),
MailVerdict::Sent
);
let divergences = tape.divergences();
assert_eq!(divergences.len(), 2);
assert!(
divergences[0].detail.contains("its attachments")
&& divergences[1].detail.contains("multipart"),
"each report names what changed: {divergences:?}"
);
}
#[test]
fn a_recorded_mail_failure_replays_as_its_own_variant() {
use crate::capsule::schema::MailErrorKind;
let tape = ReplayEffects::new(CapsuleEffects {
mail: vec![MailEffect {
to: vec!["a@example.com".to_owned()],
subject: "Receipt".to_owned(),
body: CapsuleBody::Text("thanks".to_owned()),
error: Some(
"all recipients are on the mail suppression list; nothing was sent".to_owned(),
),
error_kind: Some(MailErrorKind::AllRecipientsSuppressed),
..Default::default()
}],
..CapsuleEffects::default()
});
let to = ["a@example.com".to_owned()];
let body = CapsuleBody::Text("thanks".to_owned());
assert_eq!(
tape.next_mail(&sent(&to, "Receipt", &body)),
MailVerdict::Failed(
"all recipients are on the mail suppression list; nothing was sent".to_owned(),
Some(MailErrorKind::AllRecipientsSuppressed)
)
);
}
#[test]
fn a_moved_absolute_deadline_is_a_divergence() {
use crate::job::EnqueueSchedule;
let deadline = chrono::DateTime::parse_from_rfc3339("2026-01-01T00:00:00Z")
.expect("parses")
.with_timezone(&chrono::Utc);
let recorded = JobEffect {
name: "send_receipt".to_owned(),
payload: serde_json::json!({}),
delay_secs: Some(3600),
due_at: Some(deadline),
..Default::default()
};
let tape = ReplayEffects::new(CapsuleEffects {
jobs: vec![recorded.clone(), recorded],
..CapsuleEffects::default()
});
assert_eq!(
tape.next_job(
"send_receipt",
&serde_json::json!({}),
EnqueueSchedule::At(deadline + chrono::Duration::hours(1))
),
EnqueueVerdict::Diverged,
"a job rescheduled to a different instant is a behaviour change"
);
assert_eq!(
tape.next_job(
"send_receipt",
&serde_json::json!({}),
EnqueueSchedule::At(deadline)
),
EnqueueVerdict::Queued,
"and the same instant still replays, whatever the clock now reads"
);
}
#[test]
fn a_changed_cache_expiry_is_a_divergence() {
let tape = ReplayEffects::new(CapsuleEffects {
cache: vec![CacheEffect::Insert {
key: "user:7".to_owned(),
value: "eyJhIjoxfQ==".to_owned(),
ttl_secs: Some(5),
}],
..CapsuleEffects::default()
});
tape.cache_insert("user:7", b"{\"a\":1}", Some(3600));
let divergences = tape.divergences();
assert_eq!(divergences.len(), 1, "{divergences:?}");
assert_eq!(divergences[0].seam, EffectSeam::Cache);
}
#[test]
fn a_masked_attachment_filename_still_matches_the_file_being_sent() {
let attachment = |filename: &str| crate::capsule::schema::MailAttachmentEffect {
filename: filename.to_owned(),
content_type: "application/pdf".to_owned(),
len: 12,
sha256: "cd".repeat(32),
};
let tape = ReplayEffects::new(CapsuleEffects {
mail: vec![MailEffect {
to: vec!["a@example.com".to_owned()],
from: Some("billing@shop.example".to_owned()),
subject: "Receipt".to_owned(),
body: CapsuleBody::Text("you paid $10".to_owned()),
attachments: vec![attachment(FILTERED)],
..Default::default()
}],
..CapsuleEffects::default()
});
let to = ["a@example.com".to_owned()];
let body = CapsuleBody::Text("you paid $10".to_owned());
let sent = [attachment("invoice-sk-live-42.pdf")];
assert_eq!(
tape.next_mail(&sent_full(&to, &body, &NO_BODY, &sent)),
MailVerdict::Sent,
"a masked filename stands for the real one"
);
assert!(tape.divergences().is_empty(), "{:?}", tape.divergences());
}
#[test]
fn a_recorded_enqueue_failure_is_reproduced_as_a_failure() {
let tape = ReplayEffects::new(CapsuleEffects {
jobs: vec![JobEffect {
name: "send_receipt".to_owned(),
payload: serde_json::json!({}),
delay_secs: None,
due_at: None,
requested_due_at: None,
error: Some("queue is down".to_owned()),
error_status: None,
}],
..CapsuleEffects::default()
});
assert_eq!(
tape.next_job(
"send_receipt",
&serde_json::json!({}),
crate::job::EnqueueSchedule::Immediate
),
EnqueueVerdict::Failed("queue is down".to_owned(), None)
);
assert!(tape.divergences().is_empty());
}
#[test]
fn a_cache_write_the_code_dropped_is_a_divergence() {
let tape = ReplayEffects::new(CapsuleEffects {
cache: vec![CacheEffect::Insert {
key: "widgets".to_owned(),
value: base64_of(b"41"),
ttl_secs: None,
}],
..CapsuleEffects::default()
});
let divergences = tape.finish();
assert_eq!(divergences.len(), 1, "{divergences:?}");
assert_eq!(divergences[0].kind, EffectDivergenceKind::Unconsumed);
}
#[test]
fn a_cache_write_with_a_changed_value_is_a_divergence() {
let tape = ReplayEffects::new(CapsuleEffects {
cache: vec![CacheEffect::Insert {
key: "widgets".to_owned(),
value: base64_of(b"41"),
ttl_secs: None,
}],
..CapsuleEffects::default()
});
tape.cache_insert("widgets", b"42", None);
let divergences = tape.divergences();
assert_eq!(divergences.len(), 1, "{divergences:?}");
assert_eq!(divergences[0].kind, EffectDivergenceKind::Mismatch);
}
#[test]
fn a_read_of_a_key_the_recording_only_wrote_diverges() {
let tape = ReplayEffects::new(CapsuleEffects {
cache: vec![CacheEffect::Insert {
key: "widgets".to_owned(),
value: base64_of(b"41"),
ttl_secs: None,
}],
..CapsuleEffects::default()
});
tape.cache_insert("widgets", b"41", None);
let _ = tape.cache_get("widgets");
let divergences = tape.divergences();
assert_eq!(divergences.len(), 1, "{divergences:?}");
assert_eq!(divergences[0].kind, EffectDivergenceKind::Unrecorded);
assert_eq!(divergences[0].seam, EffectSeam::Cache);
}
#[test]
fn a_masked_fill_then_read_back_is_not_a_divergence() {
let tape = ReplayEffects::new(CapsuleEffects {
cache: vec![
CacheEffect::Get {
key: "user:[FILTERED]".to_owned(),
value: None,
},
CacheEffect::Insert {
key: "user:[FILTERED]".to_owned(),
value: base64_of(b"41"),
ttl_secs: None,
},
CacheEffect::Get {
key: "user:[FILTERED]".to_owned(),
value: Some(base64_of(b"41")),
},
],
..CapsuleEffects::default()
});
assert_eq!(tape.cache_get("user:alice"), CachedValue::Miss);
tape.cache_insert("user:alice", b"41", None);
assert_eq!(
tape.cache_get("user:alice"),
CachedValue::Hit(b"41".to_vec())
);
let divergences = tape.finish();
assert!(divergences.is_empty(), "{divergences:?}");
}
#[test]
fn a_recorded_invalidate_is_consumed_and_the_next_read_misses() {
let tape = ReplayEffects::new(CapsuleEffects {
cache: vec![
CacheEffect::Get {
key: "widgets".to_owned(),
value: Some(base64_of(b"41")),
},
CacheEffect::Invalidate {
key: "widgets".to_owned(),
error: None,
},
CacheEffect::Clear,
],
..CapsuleEffects::default()
});
assert_eq!(tape.cache_get("widgets"), CachedValue::Hit(b"41".to_vec()));
assert_eq!(tape.cache_invalidate("widgets"), Ok(()));
assert_eq!(tape.cache_get("widgets"), CachedValue::Miss);
tape.cache_clear();
let divergences = tape.finish();
assert!(divergences.is_empty(), "{divergences:?}");
}
#[test]
fn a_dropped_or_added_invalidate_is_a_divergence() {
let tape = ReplayEffects::new(CapsuleEffects {
cache: vec![CacheEffect::Invalidate {
key: "widgets".to_owned(),
error: None,
}],
..CapsuleEffects::default()
});
let divergences = tape.finish();
assert_eq!(divergences.len(), 1, "{divergences:?}");
assert_eq!(divergences[0].kind, EffectDivergenceKind::Unconsumed);
let tape = ReplayEffects::new(CapsuleEffects::default());
let _ = tape.cache_invalidate("widgets");
tape.cache_clear();
let _ = tape.cache_invalidate_namespace("widgets");
tape.cache_untyped_insert("widgets");
let divergences = tape.divergences();
assert_eq!(divergences.len(), 4, "{divergences:?}");
assert!(
divergences
.iter()
.all(|d| d.kind == EffectDivergenceKind::Unrecorded)
);
}
#[test]
fn a_recorded_invalidate_failure_replays_as_a_failure() {
let tape = ReplayEffects::new(CapsuleEffects {
cache: vec![
CacheEffect::Invalidate {
key: "widgets".to_owned(),
error: Some(crate::capsule::schema::CacheInvalidationError {
attempts: 3,
reason: "READONLY".to_owned(),
}),
},
CacheEffect::InvalidateNamespace {
namespace: "widgets".to_owned(),
error: Some(crate::capsule::schema::CacheInvalidationError {
attempts: 1,
reason: "no sweep".to_owned(),
}),
},
],
..CapsuleEffects::default()
});
let err = tape
.cache_invalidate("widgets")
.expect_err("recorded failure");
assert_eq!(err.attempts(), 3);
assert_eq!(err.reason(), "READONLY");
assert!(tape.cache_invalidate_namespace("widgets").is_err());
assert!(tape.finish().is_empty());
}
#[test]
fn a_failed_removal_keeps_the_entry() {
let failed = Some(crate::capsule::schema::CacheInvalidationError {
attempts: 1,
reason: "READONLY".to_owned(),
});
let tape = ReplayEffects::new(CapsuleEffects {
cache: vec![
CacheEffect::Get {
key: "ns:k".to_owned(),
value: Some(base64_of(b"41")),
},
CacheEffect::Invalidate {
key: "ns:k".to_owned(),
error: failed.clone(),
},
CacheEffect::InvalidateNamespace {
namespace: "ns".to_owned(),
error: failed,
},
],
..CapsuleEffects::default()
});
assert_eq!(tape.cache_get("ns:k"), CachedValue::Hit(b"41".to_vec()));
assert!(tape.cache_invalidate("ns:k").is_err());
assert!(tape.cache_invalidate_namespace("ns").is_err());
assert_eq!(tape.cache_get("ns:k"), CachedValue::Hit(b"41".to_vec()));
assert!(tape.finish().is_empty(), "{:?}", tape.finish());
}
#[test]
fn a_matching_cache_write_is_consumed_and_readable_back() {
let tape = ReplayEffects::new(CapsuleEffects {
cache: vec![
CacheEffect::Insert {
key: "widgets".to_owned(),
value: base64_of(b"41"),
ttl_secs: None,
},
CacheEffect::Get {
key: "widgets".to_owned(),
value: Some(base64_of(b"41")),
},
],
..CapsuleEffects::default()
});
tape.cache_insert("widgets", b"41", None);
assert_eq!(
tape.cache_get("widgets"),
CachedValue::Hit(b"41".to_vec()),
"the run's own write is readable back, as it was in production"
);
assert!(tape.finish().is_empty(), "{:?}", tape.finish());
}
#[test]
fn a_recorded_tenant_the_replayed_run_never_read_is_a_divergence() {
let tape = ReplayEffects::new(CapsuleEffects {
tenant: Some(TenantEffect {
id: Some("acme".to_owned()),
}),
..CapsuleEffects::default()
});
let divergences = tape.finish();
assert_eq!(divergences.len(), 1, "{divergences:?}");
assert_eq!(divergences[0].seam, EffectSeam::Tenant);
}
#[test]
fn the_recorded_tenant_is_served_without_consulting_live_config() {
let tape = ReplayEffects::new(CapsuleEffects {
tenant: Some(TenantEffect {
id: Some("acme".to_owned()),
}),
..CapsuleEffects::default()
});
assert_eq!(
tape.tenant(),
super::TenantVerdict::Resolved("acme".to_owned())
);
}
fn base64_of(bytes: &[u8]) -> String {
use base64::Engine as _;
base64::engine::general_purpose::STANDARD.encode(bytes)
}
#[test]
fn a_divergence_report_never_prints_url_userinfo() {
let described = super::describe_http("GET", "https://svc:hunter2pw@api.example/x?k=v");
assert_eq!(described, "GET https://[FILTERED]@api.example/x?…");
}
}