#![cfg(test)]
use std::error::Error;
use std::fmt;
use std::panic::AssertUnwindSafe;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Barrier, Mutex, mpsc};
use crate::cancel::Token;
use super::controller::{ControllerError, ReservationResources, TranscriptController};
use crate::audit;
use crate::opaque::{Envelope, Payload};
use crate::runtime::hook::{
self, DispatchError, Emitter, HookError, HookResult, ReadScope, TranscriptHooks,
TranscriptMutation, TranscriptReservation,
};
use crate::runtime::session::{Lease, LeaseError, Session, install_transaction_arrival};
use crate::transcript::item::{Assistant, Item, ModelVisibility, System};
use crate::transcript::store::{self, Snapshot, Version};
use crate::value::content::{Segment, SegmentExtensions};
use crate::value::lifecycle::Coord;
#[derive(Debug)]
struct TestError(&'static str);
impl fmt::Display for TestError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str(self.0)
}
}
impl Error for TestError {}
#[derive(Default)]
struct SilentEmitter;
impl Emitter for SilentEmitter {
fn emit_audit(&self, _event: audit::Event) {}
fn emit_error(&self, _error: Arc<dyn Error + Send + Sync>) {}
}
type Journal = Arc<Mutex<Vec<String>>>;
fn journal() -> Journal {
Arc::new(Mutex::new(Vec::new()))
}
fn entries(journal: &Journal) -> Vec<String> {
journal.lock().expect("journal healthy").clone()
}
struct RecordingReservation {
journal: Journal,
name: &'static str,
panic_on_publish: Option<Arc<AtomicUsize>>,
panic_on_abort: bool,
block_on_publish: Option<Arc<PublishGate>>,
}
impl TranscriptReservation for RecordingReservation {
fn publish(self: Box<Self>) {
self.journal
.lock()
.expect("journal healthy")
.push(format!("publish:{}", self.name));
assert!(
!self.panic_on_abort,
"an abort-panicking reservation must not reach publish"
);
if let Some(gate) = &self.block_on_publish {
gate.hold();
}
if let Some(gate) = &self.panic_on_publish {
if gate.fetch_add(1, Ordering::SeqCst) == 0 {
panic!("publisher panic from {}", self.name);
}
}
}
fn abort(self: Box<Self>) {
self.journal
.lock()
.expect("journal healthy")
.push(format!("abort:{}", self.name));
if self.panic_on_abort {
panic!("abort panic from {}", self.name);
}
}
}
struct PublishGate {
entered: Mutex<Option<mpsc::Sender<()>>>,
release: Barrier,
}
impl PublishGate {
fn new() -> (Arc<Self>, mpsc::Receiver<()>) {
let (sender, entered) = mpsc::channel();
(
Arc::new(Self {
entered: Mutex::new(Some(sender)),
release: Barrier::new(2),
}),
entered,
)
}
fn hold(&self) {
let entered = self.entered.lock().expect("gate healthy").take();
if let Some(entered) = entered {
entered.send(()).expect("the test observes publication");
self.release.wait();
}
}
fn release(&self) {
self.release.wait();
}
}
enum Behavior {
Reserve,
ReservePanicOnPublish(Arc<AtomicUsize>),
ReservePanicOnAbort,
ReserveBlockOnPublish(Arc<PublishGate>),
Reject,
CancelThenReserve(Token),
CancelBeforeReturn(Token),
Panic,
PanicOnce(AtomicUsize),
}
struct Reserver {
journal: Journal,
name: &'static str,
behavior: Behavior,
observed: Arc<Mutex<Vec<(Version, Version, usize)>>>,
replacement: Mutex<Option<Snapshot>>,
}
impl Reserver {
fn new(journal: &Journal, name: &'static str, behavior: Behavior) -> Arc<Self> {
Arc::new(Self {
journal: Arc::clone(journal),
name,
behavior,
observed: Arc::new(Mutex::new(Vec::new())),
replacement: Mutex::new(None),
})
}
}
impl TranscriptHooks for Reserver {
fn reserve(
&self,
_scope: &ReadScope<'_>,
mutation: &TranscriptMutation,
) -> HookResult<Box<dyn TranscriptReservation>> {
self.journal
.lock()
.expect("journal healthy")
.push(format!("reserve:{}", self.name));
self.observed.lock().expect("observed healthy").push((
mutation.previous_version(),
mutation.version(),
mutation.items().len(),
));
if let Some(snapshot) = mutation.replacement() {
*self.replacement.lock().expect("replacement healthy") = Some(snapshot.clone());
}
let mut reservation = RecordingReservation {
journal: Arc::clone(&self.journal),
name: self.name,
panic_on_publish: None,
panic_on_abort: false,
block_on_publish: None,
};
match &self.behavior {
Behavior::Reserve => {}
Behavior::ReservePanicOnPublish(gate) => {
reservation.panic_on_publish = Some(Arc::clone(gate));
}
Behavior::ReservePanicOnAbort => reservation.panic_on_abort = true,
Behavior::ReserveBlockOnPublish(gate) => {
reservation.block_on_publish = Some(Arc::clone(gate));
}
Behavior::Reject => return Err(HookError::new(TestError("reservation rejected"))),
Behavior::CancelThenReserve(token) | Behavior::CancelBeforeReturn(token) => {
token.cancel();
}
Behavior::Panic => panic!("reservation callback panic from {}", self.name),
Behavior::PanicOnce(seen) => {
if seen.fetch_add(1, Ordering::SeqCst) == 0 {
panic!("first reservation callback panic from {}", self.name);
}
}
}
Ok(Box::new(reservation))
}
}
#[derive(Default)]
struct ItemCapture {
items: Mutex<Vec<Item>>,
}
struct CapturedReservation;
impl TranscriptReservation for CapturedReservation {
fn publish(self: Box<Self>) {}
}
impl TranscriptHooks for ItemCapture {
fn reserve(
&self,
_scope: &ReadScope<'_>,
mutation: &TranscriptMutation,
) -> HookResult<Box<dyn TranscriptReservation>> {
*self.items.lock().expect("capture healthy") = mutation.items().to_vec();
Ok(Box::new(CapturedReservation))
}
}
fn registry(reservers: Vec<(&str, Arc<Reserver>)>) -> hook::RuntimeRegistry {
let mut builder = hook::RegistryBuilder::new();
for (name, hooks) in reservers {
builder.register(hook::Registration::new(name.to_owned()).transcript(hooks));
}
builder.build_runtime().expect("distinct non-empty names")
}
fn inert(name: &str) -> hook::Registration {
hook::Registration::new(name.to_owned())
}
struct Fixture {
session: Session,
lease: Lease,
coord: Coord,
emitter: SilentEmitter,
}
impl Fixture {
fn new() -> Self {
Self::with_snapshot(None)
}
fn with_snapshot(snapshot: Option<store::Snapshot>) -> Self {
let session = Session::new("session".to_owned(), snapshot);
let lease = session.try_acquire().expect("fresh session is idle");
lease.admit().expect("first admission succeeds");
let coord = Coord::at_run("session".to_owned(), "run".to_owned()).expect("named run");
Self {
session,
lease,
coord,
emitter: SilentEmitter,
}
}
fn resources(&self, registrations: usize) -> ReservationResources {
ReservationResources::new(registrations, self.lease.state().reader())
}
fn controller<'a>(
&'a self,
registry: &'a hook::RuntimeRegistry,
resources: &'a ReservationResources,
) -> TranscriptController<'a> {
TranscriptController::new(&self.lease, registry, resources, &self.coord, &self.emitter)
}
}
fn item(text: &str) -> Item {
Item::System(System::new(
"note".to_owned(),
text.to_owned(),
ModelVisibility::Visible,
))
}
fn live(fixture: &Fixture) -> Vec<String> {
fixture
.lease
.transcript_snapshot()
.expect("healthy store")
.items()
.iter()
.map(|entry| match entry {
Item::System(system) => system.text().to_owned(),
_ => panic!("tests only append system items"),
})
.collect()
}
#[test]
fn append_without_a_retained_observation_keeps_the_live_backing() {
let fixture = Fixture::new();
let registry = hook::RegistryBuilder::new().build_runtime().unwrap();
let resources = fixture.resources(0);
let controller = fixture.controller(®istry, &resources);
let before = fixture.lease.transcript_snapshot().unwrap();
let backing = before.items_backing_ptr();
drop(before);
controller
.append_batch(vec![item("appended")], &Token::new())
.unwrap();
let after = fixture.lease.transcript_snapshot().unwrap();
assert!(std::ptr::eq(after.items_backing_ptr(), backing));
assert_eq!(live(&fixture), vec!["appended"]);
}
#[test]
fn append_reserves_in_registration_order_then_publishes_in_the_same_order() {
let fixture = Fixture::new();
let log = journal();
let first = Reserver::new(&log, "first", Behavior::Reserve);
let second = Reserver::new(&log, "second", Behavior::Reserve);
let registry = registry(vec![
("first", Arc::clone(&first)),
("second", Arc::clone(&second)),
]);
let resources = fixture.resources(registry.len());
let controller = fixture.controller(®istry, &resources);
let version = controller
.append_batch(vec![item("a"), item("b")], &Token::new())
.expect("no reservation rejects");
assert_eq!(version, Version::from_u64(2));
assert_eq!(
entries(&log),
vec![
"reserve:first",
"reserve:second",
"publish:first",
"publish:second",
]
);
assert_eq!(live(&fixture), vec!["a", "b"]);
assert_eq!(
*first.observed.lock().expect("observed healthy"),
vec![(Version::ZERO, Version::from_u64(2), 2)]
);
assert_eq!(
*second.observed.lock().expect("observed healthy"),
vec![(Version::ZERO, Version::from_u64(2), 2)]
);
}
#[test]
fn append_reports_the_commit_before_any_reservation_publishes() {
let fixture = Fixture::new();
let log = journal();
let registry = registry(vec![
("first", Reserver::new(&log, "first", Behavior::Reserve)),
("second", Reserver::new(&log, "second", Behavior::Reserve)),
]);
let resources = fixture.resources(registry.len());
let controller = fixture.controller(®istry, &resources);
let committed = Arc::clone(&log);
controller
.append_batch_settling(vec![item("a")], &Token::new(), &mut || {
committed
.lock()
.expect("journal healthy")
.push("commit".to_owned());
})
.expect("no reservation rejects");
assert_eq!(
entries(&log),
vec![
"reserve:first",
"reserve:second",
"commit",
"publish:first",
"publish:second",
]
);
}
#[test]
fn append_reports_no_commit_when_nothing_commits() {
let fixture = Fixture::new();
let log = journal();
let registry = registry(vec![(
"reject",
Reserver::new(&log, "reject", Behavior::Reject),
)]);
let resources = fixture.resources(registry.len());
let controller = fixture.controller(®istry, &resources);
let mut commits = 0;
let refused = controller.append_batch_settling(vec![item("a")], &Token::new(), &mut || {
commits += 1;
});
assert!(refused.is_err());
let empty = controller.append_batch_settling(Vec::new(), &Token::new(), &mut || {
commits += 1;
});
assert_eq!(empty.expect("an empty batch is a no-op"), Version::ZERO);
assert_eq!(commits, 0);
assert!(live(&fixture).is_empty());
}
#[test]
fn transcript_reservation_receives_provider_continuation_state_intact() {
let fixture = Fixture::new();
let capture = Arc::new(ItemCapture::default());
let mut builder = hook::RegistryBuilder::new();
builder.register(hook::Registration::new("capture".to_owned()).transcript(capture.clone()));
let registry = builder.build_runtime().expect("one named registration");
let resources = fixture.resources(registry.len());
let controller = fixture.controller(®istry, &resources);
let segment_provider_data = Envelope::try_new(
"provider".to_owned(),
"segment".to_owned(),
Payload::copy_from_slice(br#"{"segment":true}"#),
)
.expect("segment provider data is valid");
let record_provider_data = Envelope::try_new(
"provider".to_owned(),
"record".to_owned(),
Payload::copy_from_slice(br#"{"record":true}"#),
)
.expect("record provider data is valid");
let raw = br#"{"type":"redacted_thinking","data":"opaque"}"#;
let extensions = SegmentExtensions::new(
"provider-signature".to_owned(),
Some(Payload::copy_from_slice(raw)),
Some(vec![segment_provider_data]),
)
.with_item_id("thinking-1".to_owned());
controller
.append_batch(
vec![Item::Assistant(Assistant::new(
vec![Segment::Thinking {
text: String::new(),
extensions,
}],
Some(vec![record_provider_data]),
))],
&Token::new(),
)
.expect("the complete record is committed");
let captured = capture.items.lock().expect("capture healthy");
let Item::Assistant(assistant) = &captured[0] else {
panic!("the reservation must receive the assistant record");
};
let record_data = assistant
.provider_data()
.expect("record-level provider data is preserved");
assert_eq!(record_data[0].data().as_bytes(), br#"{"record":true}"#);
let Segment::Thinking { text, extensions } = &assistant.content()[0] else {
panic!("the reservation must receive the thinking segment");
};
assert!(text.is_empty(), "redacted thinking has no display text");
assert_eq!(extensions.signature(), "provider-signature");
assert_eq!(extensions.item_id(), Some("thinking-1"));
assert_eq!(
extensions
.raw_payload()
.expect("raw replay payload is preserved")
.as_bytes(),
raw
);
let segment_data = extensions
.provider_data()
.expect("segment provider data is preserved");
assert_eq!(segment_data[0].data().as_bytes(), br#"{"segment":true}"#);
}
#[test]
fn rejecting_reservation_aborts_earlier_holders_in_reverse_order_and_commits_nothing() {
let fixture = Fixture::new();
let log = journal();
let registry = registry(vec![
("first", Reserver::new(&log, "first", Behavior::Reserve)),
("second", Reserver::new(&log, "second", Behavior::Reserve)),
("third", Reserver::new(&log, "third", Behavior::Reject)),
("fourth", Reserver::new(&log, "fourth", Behavior::Reserve)),
]);
let resources = fixture.resources(registry.len());
let controller = fixture.controller(®istry, &resources);
let error = controller
.append_batch(vec![item("a")], &Token::new())
.expect_err("the third reservation rejects");
match error {
ControllerError::Dispatch(DispatchError::Callback(failure)) => {
assert_eq!(failure.name(), "third");
assert_eq!(failure.error().to_string(), "reservation rejected");
}
other => panic!("expected a callback failure, got {other:?}"),
}
assert_eq!(
entries(&log),
vec![
"reserve:first",
"reserve:second",
"reserve:third",
"abort:second",
"abort:first",
]
);
assert!(live(&fixture).is_empty());
assert_eq!(
controller.version().expect("healthy store"),
Version::ZERO,
"a rejected transaction must not advance the version"
);
}
#[test]
fn cancellation_observed_after_a_successful_reservation_aborts_before_commit() {
let fixture = Fixture::new();
let log = journal();
let token = Token::new();
let registry = registry(vec![
("first", Reserver::new(&log, "first", Behavior::Reserve)),
(
"second",
Reserver::new(&log, "second", Behavior::CancelBeforeReturn(token.clone())),
),
("third", Reserver::new(&log, "third", Behavior::Reserve)),
]);
let resources = fixture.resources(registry.len());
let controller = fixture.controller(®istry, &resources);
let error = controller
.append_batch(vec![item("a")], &token)
.expect_err("cancellation is observed pre-commit");
assert!(matches!(
error,
ControllerError::Dispatch(DispatchError::Cancelled)
));
assert_eq!(
entries(&log),
vec![
"reserve:first",
"reserve:second",
"abort:second",
"abort:first",
]
);
assert!(live(&fixture).is_empty());
}
#[test]
fn cancellation_before_the_first_reservation_runs_no_callback() {
let fixture = Fixture::new();
let log = journal();
let registry = registry(vec![(
"only",
Reserver::new(&log, "only", Behavior::Reserve),
)]);
let resources = fixture.resources(registry.len());
let controller = fixture.controller(®istry, &resources);
let token = Token::new();
token.cancel();
let error = controller
.append_batch(vec![item("a")], &token)
.expect_err("an already-cancelled transaction cannot commit");
assert!(matches!(
error,
ControllerError::Dispatch(DispatchError::Cancelled)
));
assert!(entries(&log).is_empty());
assert!(live(&fixture).is_empty());
}
#[test]
fn cancellation_between_the_last_reservation_and_the_commit_aborts_everything() {
let fixture = Fixture::new();
let log = journal();
let token = Token::new();
let registry = registry(vec![(
"only",
Reserver::new(&log, "only", Behavior::CancelThenReserve(token.clone())),
)]);
let resources = fixture.resources(registry.len());
let controller = fixture.controller(®istry, &resources);
let error = controller
.append_batch(vec![item("a")], &token)
.expect_err("cancellation precedes the commit");
assert!(matches!(
error,
ControllerError::Dispatch(DispatchError::Cancelled)
));
assert_eq!(entries(&log), vec!["reserve:only", "abort:only"]);
assert!(live(&fixture).is_empty());
}
#[test]
fn cancellation_with_no_transcript_hooks_still_blocks_the_commit() {
let fixture = Fixture::new();
let mut builder = hook::RegistryBuilder::new();
builder.register(inert("inert"));
let registry = builder.build_runtime().expect("named registration");
let resources = fixture.resources(registry.len());
let controller = fixture.controller(®istry, &resources);
let token = Token::new();
token.cancel();
let error = controller
.append_batch(vec![item("a")], &token)
.expect_err("cancellation fails closed without any reservation");
assert!(matches!(
error,
ControllerError::Dispatch(DispatchError::Cancelled)
));
assert!(live(&fixture).is_empty());
}
#[test]
fn a_reservation_callback_panic_aborts_held_reservations_and_resumes_the_panic() {
let fixture = Fixture::new();
let log = journal();
let registry = registry(vec![
("first", Reserver::new(&log, "first", Behavior::Reserve)),
("second", Reserver::new(&log, "second", Behavior::Panic)),
("third", Reserver::new(&log, "third", Behavior::Reserve)),
]);
let resources = fixture.resources(registry.len());
let controller = fixture.controller(®istry, &resources);
let token = Token::new();
let panicked = std::panic::catch_unwind(AssertUnwindSafe(|| {
controller.append_batch(vec![item("a")], &token)
}))
.expect_err("the callback panic is not downgraded to an error");
let message = panicked
.downcast_ref::<String>()
.expect("test panics carry a String payload");
assert_eq!(message, "reservation callback panic from second");
assert_eq!(
entries(&log),
vec!["reserve:first", "reserve:second", "abort:first"]
);
assert!(
live(&fixture).is_empty(),
"a panicking reservation must not commit"
);
}
#[test]
fn an_aborting_reservation_panic_still_releases_the_remaining_reservations() {
let fixture = Fixture::new();
let log = journal();
let registry = registry(vec![
("first", Reserver::new(&log, "first", Behavior::Reserve)),
(
"second",
Reserver::new(&log, "second", Behavior::ReservePanicOnAbort),
),
("third", Reserver::new(&log, "third", Behavior::Reject)),
]);
let resources = fixture.resources(registry.len());
let controller = fixture.controller(®istry, &resources);
let token = Token::new();
let panicked = std::panic::catch_unwind(AssertUnwindSafe(|| {
controller.append_batch(vec![item("a")], &token)
}))
.expect_err("the first abort panic is resumed after the whole set is released");
let message = panicked
.downcast_ref::<String>()
.expect("test panics carry a String payload");
assert_eq!(message, "abort panic from second");
assert_eq!(
entries(&log),
vec![
"reserve:first",
"reserve:second",
"reserve:third",
"abort:second",
"abort:first",
]
);
assert!(live(&fixture).is_empty());
}
#[test]
fn a_publisher_panic_does_not_stop_later_publishers_and_never_aborts() {
let fixture = Fixture::new();
let log = journal();
let registry = registry(vec![
(
"first",
Reserver::new(
&log,
"first",
Behavior::ReservePanicOnPublish(Arc::new(AtomicUsize::new(0))),
),
),
("second", Reserver::new(&log, "second", Behavior::Reserve)),
("third", Reserver::new(&log, "third", Behavior::Reserve)),
]);
let resources = fixture.resources(registry.len());
let controller = fixture.controller(®istry, &resources);
let token = Token::new();
let panicked = std::panic::catch_unwind(AssertUnwindSafe(|| {
controller.append_batch(vec![item("a")], &token)
}))
.expect_err("the publisher panic is resumed after the whole set has published");
let message = panicked
.downcast_ref::<String>()
.expect("test panics carry a String payload");
assert_eq!(message, "publisher panic from first");
let observed = entries(&log);
assert_eq!(
observed,
vec![
"reserve:first",
"reserve:second",
"reserve:third",
"publish:first",
"publish:second",
"publish:third",
]
);
assert!(
observed.iter().all(|entry| !entry.starts_with("abort:")),
"a committed reservation must never be aborted: {observed:?}"
);
assert_eq!(live(&fixture), vec!["a"]);
}
#[test]
fn a_stale_replace_is_rejected_before_any_reservation() {
let fixture = Fixture::new();
let log = journal();
let registry = registry(vec![
("first", Reserver::new(&log, "first", Behavior::Reserve)),
("second", Reserver::new(&log, "second", Behavior::Reserve)),
]);
let resources = fixture.resources(registry.len());
let controller = fixture.controller(®istry, &resources);
let token = Token::new();
controller
.append_batch(vec![item("a")], &token)
.expect("the first transaction commits");
log.lock().expect("journal healthy").clear();
let error = controller
.replace_snapshot(
Snapshot::new(Version::ZERO, vec![item("z")]),
Version::ZERO,
&token,
)
.expect_err("a stale expected version cannot commit");
match error {
ControllerError::Lease(LeaseError::Transcript(store::Error::VersionConflict {
expected,
actual,
})) => {
assert_eq!(expected, Version::ZERO);
assert_eq!(actual, Version::from_u64(1));
}
other => panic!("expected a version conflict, got {other:?}"),
}
assert!(entries(&log).is_empty());
assert_eq!(
live(&fixture),
vec!["a"],
"a failed compare-and-set must leave the committed history intact"
);
}
#[test]
fn a_current_replace_commits_and_publishes_the_replacement_fact() {
let fixture = Fixture::new();
let log = journal();
let reserver = Reserver::new(&log, "only", Behavior::Reserve);
let registry = registry(vec![("only", Arc::clone(&reserver))]);
let resources = fixture.resources(registry.len());
let controller = fixture.controller(®istry, &resources);
let token = Token::new();
controller
.append_batch(vec![item("a")], &token)
.expect("the first transaction commits");
let candidate = Snapshot::new(Version::ZERO, vec![item("x"), item("y")]);
let version = controller
.replace_snapshot(candidate, Version::from_u64(1), &token)
.expect("a current expected version commits");
assert_eq!(version, Version::from_u64(2));
assert_eq!(live(&fixture), vec!["x", "y"]);
let reserved = reserver
.replacement
.lock()
.expect("replacement healthy")
.clone()
.expect("replacement reservation captures the candidate");
let installed = fixture.lease.transcript_snapshot().unwrap();
assert_eq!(reserved.version(), Version::from_u64(2));
assert!(reserved.shares_items_with(&installed));
assert_eq!(
*reserver.observed.lock().expect("observed healthy"),
vec![
(Version::ZERO, Version::from_u64(1), 1),
(Version::from_u64(1), Version::from_u64(2), 2),
]
);
}
#[test]
fn an_overflowing_transition_is_rejected_before_any_reservation_runs() {
let fixture = Fixture::with_snapshot(Some(store::Snapshot::new(Version::MAX, Vec::new())));
let log = journal();
let registry = registry(vec![(
"only",
Reserver::new(&log, "only", Behavior::Reserve),
)]);
let resources = fixture.resources(registry.len());
let controller = fixture.controller(®istry, &resources);
let token = Token::new();
let error = controller
.replace_snapshot(
Snapshot::new(Version::MAX, vec![item("x")]),
Version::MAX,
&token,
)
.expect_err("MAX cannot advance");
match error {
ControllerError::InvalidTransition { previous, delta } => {
assert_eq!(previous, Version::MAX);
assert_eq!(delta, 1);
}
other => panic!("expected an invalid transition, got {other:?}"),
}
assert!(
entries(&log).is_empty(),
"no hook may observe a transition the store could not produce"
);
assert!(live(&fixture).is_empty());
}
#[test]
fn an_empty_batch_is_a_no_op_that_runs_no_reservation() {
let fixture = Fixture::new();
let log = journal();
let registry = registry(vec![(
"only",
Reserver::new(&log, "only", Behavior::Reserve),
)]);
let resources = fixture.resources(registry.len());
let controller = fixture.controller(®istry, &resources);
let version = controller
.append_batch(Vec::new(), &Token::new())
.expect("an empty batch is a successful no-op");
assert_eq!(version, Version::ZERO);
assert!(
entries(&log).is_empty(),
"there is no transition to reserve"
);
assert!(live(&fixture).is_empty());
}
#[test]
fn a_released_lease_rejects_every_controller_entry_point() {
let fixture = Fixture::new();
let log = journal();
let registry = registry(vec![(
"only",
Reserver::new(&log, "only", Behavior::Reserve),
)]);
let resources = fixture.resources(registry.len());
let controller = fixture.controller(®istry, &resources);
let token = Token::new();
fixture.lease.release();
assert!(matches!(
controller.version(),
Err(ControllerError::Lease(LeaseError::Released))
));
assert!(matches!(
controller.append_batch(vec![item("a")], &token),
Err(ControllerError::Lease(LeaseError::Released))
));
assert!(matches!(
controller.append_batch(Vec::new(), &token),
Err(ControllerError::Lease(LeaseError::Released))
));
assert!(matches!(
controller.replace_snapshot(
Snapshot::new(Version::ZERO, vec![item("a")]),
Version::ZERO,
&token,
),
Err(ControllerError::Lease(LeaseError::Released))
));
assert!(entries(&log).is_empty());
}
#[test]
fn registrations_without_the_transcript_family_keep_their_own_resource_slots() {
let fixture = Fixture::new();
let log = journal();
let mut builder = hook::RegistryBuilder::new();
builder
.register(inert("inert-first"))
.register(
hook::Registration::new("reserver".to_owned()).transcript(Reserver::new(
&log,
"reserver",
Behavior::Reserve,
)),
)
.register(inert("inert-last"));
let registry = builder.build_runtime().expect("distinct names");
let resources = fixture.resources(registry.len());
let controller = fixture.controller(®istry, &resources);
let version = controller
.append_batch(vec![item("a")], &Token::new())
.expect("the sole reservation succeeds");
assert_eq!(version, Version::from_u64(1));
assert_eq!(entries(&log), vec!["reserve:reserver", "publish:reserver"]);
}
#[test]
fn concurrent_transactions_never_interleave_commit_and_publication() {
let fixture = Arc::new(Fixture::new());
let log = journal();
let depth = Arc::new(AtomicUsize::new(0));
let observer = Arc::new(GapObserver {
journal: Arc::clone(&log),
depth: Arc::clone(&depth),
});
let mut builder = hook::RegistryBuilder::new();
builder.register(
hook::Registration::new("gap".to_owned()).transcript(Arc::clone(&observer) as Arc<_>),
);
let registry = builder.build_runtime().expect("named registration");
let resources = fixture.resources(registry.len());
let controller = Arc::new(fixture.controller(®istry, &resources));
std::thread::scope(|scope| {
for _ in 0..8 {
let controller = Arc::clone(&controller);
scope.spawn(move || {
let token = Token::new();
for _ in 0..16 {
controller
.append_batch(vec![item("a")], &token)
.expect("every transaction commits under the outer lock");
}
});
}
});
assert_eq!(
observer.depth.load(Ordering::SeqCst),
0,
"every transaction closed"
);
assert_eq!(
fixture
.lease
.transcript_snapshot()
.expect("healthy store")
.version(),
Version::from_u64(8 * 16)
);
assert!(
!entries(&log).iter().any(|entry| entry == "nested"),
"reserve, commit, and publish must share one outer lock domain"
);
}
struct GapObserver {
journal: Journal,
depth: Arc<AtomicUsize>,
}
impl TranscriptHooks for GapObserver {
fn reserve(
&self,
_scope: &ReadScope<'_>,
_mutation: &TranscriptMutation,
) -> HookResult<Box<dyn TranscriptReservation>> {
if self.depth.fetch_add(1, Ordering::SeqCst) != 0 {
self.journal
.lock()
.expect("journal healthy")
.push("nested".to_owned());
}
Ok(Box::new(GapReservation {
depth: Arc::clone(&self.depth),
}))
}
}
struct GapReservation {
depth: Arc<AtomicUsize>,
}
impl TranscriptReservation for GapReservation {
fn publish(self: Box<Self>) {
self.depth.fetch_sub(1, Ordering::SeqCst);
}
fn abort(self: Box<Self>) {
self.depth.fetch_sub(1, Ordering::SeqCst);
}
}
#[test]
fn controller_error_messages_name_the_failing_boundary() {
let fixture = Fixture::new();
let log = journal();
let registry = registry(vec![(
"rejecting",
Reserver::new(&log, "rejecting", Behavior::Reject),
)]);
let resources = fixture.resources(registry.len());
let controller = fixture.controller(®istry, &resources);
let callback = controller
.append_batch(vec![item("a")], &Token::new())
.expect_err("the reservation rejects");
assert_eq!(
callback.to_string(),
"transcript reservation rejected by hook rejecting: reservation rejected"
);
let dispatch = ControllerError::Dispatch(DispatchError::Cancelled);
assert_eq!(
dispatch.to_string(),
"transcript mutation cancelled before commit"
);
let lease = ControllerError::Lease(LeaseError::Released);
assert_eq!(
lease.to_string(),
"transcript mutation rejected: session lease released"
);
let transition = ControllerError::InvalidTransition {
previous: Version::MAX,
delta: 1,
};
assert_eq!(
transition.to_string(),
"invalid transcript transition: previous 18446744073709551615, increment 1"
);
}
#[test]
fn a_transaction_that_panicked_does_not_wedge_the_outer_lock() {
let fixture = Fixture::new();
let log = journal();
let registry = registry(vec![(
"panic-once",
Reserver::new(&log, "panic-once", Behavior::PanicOnce(AtomicUsize::new(0))),
)]);
let resources = fixture.resources(registry.len());
let controller = fixture.controller(®istry, &resources);
let token = Token::new();
let _ = std::panic::catch_unwind(AssertUnwindSafe(|| {
controller.append_batch(vec![item("a")], &token)
}))
.expect_err("the first reservation callback panics");
assert!(
live(&fixture).is_empty(),
"the panicking transaction committed nothing"
);
assert_eq!(
controller
.append_batch(vec![item("b")], &token)
.expect("a poisoned outer lock must be recovered, not propagated"),
Version::from_u64(1)
);
assert_eq!(live(&fixture), vec!["b"]);
assert_eq!(
entries(&log),
vec![
"reserve:panic-once",
"reserve:panic-once",
"publish:panic-once"
]
);
}
#[test]
fn two_publisher_panics_keep_the_first_and_leave_the_lock_reusable() {
let fixture = Fixture::new();
let log = journal();
let registry = registry(vec![
(
"first",
Reserver::new(
&log,
"first",
Behavior::ReservePanicOnPublish(Arc::new(AtomicUsize::new(0))),
),
),
(
"second",
Reserver::new(
&log,
"second",
Behavior::ReservePanicOnPublish(Arc::new(AtomicUsize::new(0))),
),
),
]);
let resources = fixture.resources(registry.len());
let controller = fixture.controller(®istry, &resources);
let token = Token::new();
let panicked = std::panic::catch_unwind(AssertUnwindSafe(|| {
controller.append_batch(vec![item("a")], &token)
}))
.expect_err("the publisher panics propagate");
let message = panicked
.downcast_ref::<String>()
.expect("test panics carry a String payload");
assert_eq!(message, "publisher panic from first");
assert_eq!(
entries(&log),
vec![
"reserve:first",
"reserve:second",
"publish:first",
"publish:second",
],
"a panicking publisher must not stop the later publisher"
);
assert_eq!(live(&fixture), vec!["a"]);
log.lock().expect("journal healthy").clear();
assert_eq!(
controller
.append_batch(vec![item("b")], &token)
.expect("a poisoned outer lock must be recovered, not propagated"),
Version::from_u64(2)
);
assert_eq!(live(&fixture), vec!["a", "b"]);
}
#[test]
fn a_reservation_callback_panic_outranks_an_abort_panic_but_still_aborts_everything() {
let fixture = Fixture::new();
let log = journal();
let registry = registry(vec![
("first", Reserver::new(&log, "first", Behavior::Reserve)),
(
"second",
Reserver::new(&log, "second", Behavior::ReservePanicOnAbort),
),
("third", Reserver::new(&log, "third", Behavior::Panic)),
]);
let resources = fixture.resources(registry.len());
let controller = fixture.controller(®istry, &resources);
let token = Token::new();
let panicked = std::panic::catch_unwind(AssertUnwindSafe(|| {
controller.append_batch(vec![item("a")], &token)
}))
.expect_err("the reservation callback panic is not downgraded");
let message = panicked
.downcast_ref::<String>()
.expect("test panics carry a String payload");
assert_eq!(
message, "reservation callback panic from third",
"the original callback panic must outrank the abort panic"
);
assert_eq!(
entries(&log),
vec![
"reserve:first",
"reserve:second",
"reserve:third",
"abort:second",
"abort:first",
]
);
assert!(live(&fixture).is_empty(), "nothing committed");
}
fn assert_blocked_by_an_unpublished_commit(
blocked: impl FnOnce(&Fixture) + Send,
expected_publications: usize,
) -> Vec<String> {
let fixture = Fixture::new();
let log = journal();
let (gate, entered) = PublishGate::new();
let registry = registry(vec![(
"blocking",
Reserver::new(
&log,
"blocking",
Behavior::ReserveBlockOnPublish(Arc::clone(&gate)),
),
)]);
let resources = fixture.resources(registry.len());
let controller = fixture.controller(®istry, &resources);
let arrival = install_transaction_arrival(&fixture.session);
std::thread::scope(|scope| {
let publishing = scope.spawn(|| {
controller
.append_batch(vec![item("committed")], &Token::new())
.expect("the transaction commits")
});
entered
.recv()
.expect("the publisher reports that it is publishing");
assert_eq!(
fixture
.session
.transcript()
.version()
.expect("healthy store"),
Version::from_u64(1)
);
arrival.arm();
let contender = scope.spawn(|| blocked(&fixture));
arrival.wait_until_blocked();
assert_eq!(
entries(&log)
.iter()
.filter(|entry| entry.starts_with("publish:"))
.count(),
expected_publications
);
gate.release();
assert_eq!(publishing.join().expect("no panic"), Version::from_u64(1));
contender.join().expect("no panic");
});
entries(&log)
}
#[test]
fn a_direct_lease_commit_cannot_cross_an_unpublished_transaction() {
let log = assert_blocked_by_an_unpublished_commit(
|fixture| {
fixture
.lease
.append_batch_if_version(vec![item("direct")], Version::from_u64(1))
.expect("the direct commit succeeds once the transaction resolves");
},
1,
);
assert_eq!(log, vec!["reserve:blocking", "publish:blocking"]);
}
#[test]
fn a_release_cannot_cross_an_unpublished_transaction() {
let log = assert_blocked_by_an_unpublished_commit(
|fixture| {
fixture.lease.release();
},
1,
);
assert_eq!(log, vec!["reserve:blocking", "publish:blocking"]);
}
#[test]
fn a_successor_acquisition_and_commit_cannot_cross_an_unpublished_transaction() {
let log = assert_blocked_by_an_unpublished_commit(
|fixture| {
fixture.lease.release();
let successor = fixture
.session
.try_acquire()
.expect("the released session is idle again");
successor.admit().expect("first admission");
assert_eq!(
successor
.replace_if_version(vec![item("successor")], Version::from_u64(1))
.expect("the successor commits after the predecessor published"),
Version::from_u64(2)
);
},
1,
);
assert_eq!(log, vec!["reserve:blocking", "publish:blocking"]);
}