#![cfg(test)]
use std::error::Error;
use std::fmt;
use std::panic::AssertUnwindSafe;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use crate::cancel::Token;
use super::hook::*;
use super::model::{
self, ClientTool, CompletionReason, Info, RequestPlan, SamplingParams, TokenUsage, ToolChoice,
};
use super::session::{self, InputTarget, Session};
use super::tool::{self, CallDisposition, ExposeAll};
use crate::audit;
use crate::opaque::{Envelope, Payload};
use crate::redact_tests::{SECRET, assert_redacted, assert_states};
use crate::runtime::agent::ClosePanickedError;
use crate::state::run_state::{self};
use crate::state::session_state::{self, SnapshotValue};
use crate::transcript::item::{Assistant, Item, ModelVisibility, System};
use crate::transcript::store::{self, Snapshot, Version};
use crate::value::content::Part;
use crate::value::lifecycle::Coord;
use crate::value::tool_call::{self, Id, ToolCall, ToolResult};
#[derive(Debug)]
struct TestError(&'static str);
impl fmt::Display for TestError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str(self.0)
}
}
impl Error for TestError {}
#[derive(Debug)]
struct ChainedError {
message: &'static str,
cause: TestError,
}
impl fmt::Display for ChainedError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str(self.message)
}
}
impl Error for ChainedError {
fn source(&self) -> Option<&(dyn Error + 'static)> {
Some(&self.cause)
}
}
#[derive(Default)]
struct RecordingEmitter {
audits: Mutex<Vec<audit::Event>>,
errors: Mutex<Vec<String>>,
}
impl Emitter for RecordingEmitter {
fn emit_audit(&self, event: audit::Event) {
self.audits.lock().unwrap().push(event);
}
fn emit_error(&self, error: Arc<dyn Error + Send + Sync>) {
self.errors.lock().unwrap().push(error.to_string());
}
}
fn dispatch_error<T>(result: Result<T, DispatchError>) -> DispatchError {
match result {
Ok(_) => panic!("expected dispatch failure"),
Err(error) => error,
}
}
struct ScopeCapabilities {
session: Session,
lease: session::Lease,
run_state: run_state::Store,
session_state: session_state::Reader,
transcript: store::SnapshotReader,
}
impl ScopeCapabilities {
fn new() -> Self {
let session = Session::new("session".into(), None);
let lease = session.try_acquire().unwrap();
let session_state = session.state();
let transcript = lease.transcript_reader();
Self {
session,
lease,
run_state: run_state::Store::new(),
session_state,
transcript,
}
}
fn factory<'a>(
&'a self,
coord: &'a Coord,
emitter: &'a dyn Emitter,
requires_session_state: bool,
) -> ScopeFactory<'a> {
ScopeFactory::new(
coord,
&self.run_state,
&self.session_state,
Some(self.lease.state()),
requires_session_state,
TranscriptSource::Reader(&self.transcript),
emitter,
)
}
}
fn capability_model(id: &str) -> Info {
Info::new(id.to_owned(), 1024)
.with_tools(true)
.with_thinking(true)
}
fn fixtures() -> (
ScopeCapabilities,
Coord,
RecordingEmitter,
Info,
super::tool::StepToolSnapshot,
ToolCall,
ToolResult,
Assistant,
) {
let capabilities = ScopeCapabilities::new();
let coord = Coord::at_run("session".into(), "run".into()).unwrap();
let model = capability_model("model");
let tools = tool::RegistryBuilder::new()
.build()
.unwrap()
.resolve_step_without_providers(
Snapshot::new(Version::ZERO, Vec::new()),
&Token::new(),
&ExposeAll,
)
.unwrap()
.into_snapshot();
let call = ToolCall::new(
Id::new("call".into()),
tool_call::Name::new("tool".into()),
Payload::from_vec(vec![]),
);
(
capabilities,
coord,
RecordingEmitter::default(),
model,
tools,
call,
ToolResult::no_output(),
Assistant::new(vec![], None),
)
}
fn lifecycle_observations(coord: &Coord, call: &ToolCall) -> Vec<LifecycleObservation> {
let plan = RequestPlan::new();
let request = plan.clone().try_into_request(vec![]).unwrap();
let tools = tool::RegistryBuilder::new()
.build()
.unwrap()
.resolve_step_without_providers(
Snapshot::new(Version::ZERO, Vec::new()),
&Token::new(),
&ExposeAll,
)
.unwrap()
.into_snapshot();
let snapshot = StepSnapshot::freeze(
Info::new("lifecycle-model".into(), 2048)
.with_tools(true)
.with_vision(true)
.with_thinking(true),
plan,
tools,
);
vec![
LifecycleObservation::RunStart {
coord: coord.clone(),
input: vec![Part::text("input".into())],
},
LifecycleObservation::RunEnd {
coord: coord.clone(),
end: LifecycleEnd::Completed,
turns: 1,
},
LifecycleObservation::TurnStart {
coord: coord.clone(),
trigger: vec![],
},
LifecycleObservation::TurnEnd {
coord: coord.clone(),
end: LifecycleEnd::Stopped { code: None },
steps: 1,
},
LifecycleObservation::StepStart {
coord: coord.clone(),
snapshot: snapshot.clone(),
},
LifecycleObservation::StepEnd {
coord: coord.clone(),
end: LifecycleEnd::Cancelled {
error: Arc::new(TestError("cancelled")),
},
snapshot,
},
LifecycleObservation::GenerationStart {
coord: coord.clone(),
request,
},
LifecycleObservation::GenerationEnd {
coord: coord.clone(),
end: LifecycleEnd::Failed {
error: Arc::new(TestError("failed")),
},
finish_reason: "length".into(),
completion: CompletionReason::Length,
},
LifecycleObservation::ToolCallStart {
coord: coord.clone(),
call: call.clone(),
},
LifecycleObservation::ToolCallEnd(
ToolCallEndObservation::try_new(
coord.clone(),
LifecycleEnd::Completed,
call.clone(),
call.clone(),
ToolCallProgress::completed_post_hooks(
CallDisposition::Executed,
ToolResult::no_output(),
),
)
.unwrap(),
),
]
}
#[test]
fn inventory_matches_all_28_callbacks_in_exact_order_and_adapter_shape() {
use DecisionKind as D;
use HookFamily as F;
use ScopeKind as S;
let expected = [
hook("OnPrepareRun", F::Run, Some(S::Prepare), Some(D::Prepare)),
hook(
"OnFinalizeRun",
F::Run,
Some(S::Finalize),
Some(D::Finalize),
),
hook("OnRunAdmitted", F::Observer, Some(S::Read), None),
hook(
"OnBeforeToolCall",
F::Tool,
Some(S::ToolCall),
Some(D::ToolCall),
),
hook(
"OnBeforeToolResult",
F::Tool,
Some(S::ToolResult),
Some(D::ToolResult),
),
hook(
"OnAfterToolExecute",
F::Tool,
Some(S::AfterToolCommit),
None,
),
hook(
"OnBeforeModel",
F::Model,
Some(S::BeforeModel),
Some(D::BeforeModel),
),
hook(
"OnAfterModel",
F::Model,
Some(S::AfterModel),
Some(D::AfterModel),
),
hook(
"OnBeforeInputAdmit",
F::Input,
Some(S::InputAdmit),
Some(D::InputAdmit),
),
hook("OnBeforeTurn", F::TurnStep, Some(S::Turn), None),
hook("OnAfterTurn", F::TurnStep, Some(S::Turn), None),
hook("OnBeforeStep", F::TurnStep, Some(S::BeforeStep), None),
hook(
"OnAfterStep",
F::TurnStep,
Some(S::AfterStep),
Some(D::AfterStep),
),
hook("OnClose", F::AgentTeardown, None, None),
hook(
"OnContinueRun",
F::Run,
Some(S::Continue),
Some(D::Continue),
),
hook("OnRunStart", F::Observer, Some(S::Observer), None),
hook("OnRunEnd", F::Observer, Some(S::Observer), None),
hook("OnTurnStart", F::Observer, Some(S::Observer), None),
hook("OnTurnEnd", F::Observer, Some(S::Observer), None),
hook("OnStepStart", F::Observer, Some(S::Observer), None),
hook("OnStepEnd", F::Observer, Some(S::Observer), None),
hook("OnGenerationStart", F::Observer, Some(S::Observer), None),
hook("OnGenerationEnd", F::Observer, Some(S::Observer), None),
hook("OnToolCallStart", F::Observer, Some(S::Observer), None),
hook("OnToolCallEnd", F::Observer, Some(S::Observer), None),
hook("OnModelEvent", F::Observer, Some(S::Observer), None),
hook("OnAuditEvent", F::Observer, Some(S::Read), None),
hook("OnTranscriptMutation", F::Transcript, Some(S::Read), None),
];
assert_eq!(HOOK_INVENTORY, expected.as_slice());
assert_eq!(
HOOK_INVENTORY
.iter()
.filter_map(|spec| spec.scope)
.collect::<std::collections::HashSet<_>>()
.len(),
14
);
assert_eq!(
HOOK_INVENTORY
.iter()
.filter_map(|spec| spec.decision)
.collect::<std::collections::HashSet<_>>()
.len(),
9
);
let lifecycle = [
("OnRunStart", LifecycleKind::RunStart),
("OnRunEnd", LifecycleKind::RunEnd),
("OnTurnStart", LifecycleKind::TurnStart),
("OnTurnEnd", LifecycleKind::TurnEnd),
("OnStepStart", LifecycleKind::StepStart),
("OnStepEnd", LifecycleKind::StepEnd),
("OnGenerationStart", LifecycleKind::GenerationStart),
("OnGenerationEnd", LifecycleKind::GenerationEnd),
("OnToolCallStart", LifecycleKind::ToolCallStart),
("OnToolCallEnd", LifecycleKind::ToolCallEnd),
];
for (name, kind) in lifecycle {
assert_eq!(lifecycle_hook_name(kind), name);
assert!(
expected
.iter()
.any(|spec| spec.name == name && spec.family == F::Observer)
);
}
}
#[test]
fn lifecycle_observations_are_one_to_one_and_preserve_terminal_facts() {
let (_, coord, _, _, _, call, _, _) = fixtures();
let observations = lifecycle_observations(&coord, &call);
let expected = [
LifecycleKind::RunStart,
LifecycleKind::RunEnd,
LifecycleKind::TurnStart,
LifecycleKind::TurnEnd,
LifecycleKind::StepStart,
LifecycleKind::StepEnd,
LifecycleKind::GenerationStart,
LifecycleKind::GenerationEnd,
LifecycleKind::ToolCallStart,
LifecycleKind::ToolCallEnd,
];
assert_eq!(
observations
.iter()
.map(LifecycleObservation::kind)
.collect::<Vec<_>>(),
expected
);
assert!(
observations
.iter()
.all(|observation| observation.coord() == &coord)
);
let completed = LifecycleEnd::Completed;
assert_eq!(
completed.outcome(),
crate::value::lifecycle::Outcome::Completed
);
assert!(completed.stop_code().is_none());
assert!(completed.error().is_none());
let stopped = LifecycleEnd::Stopped {
code: Some(super::event::StopCode::new("done".into()).unwrap()),
};
assert_eq!(stopped.outcome(), crate::value::lifecycle::Outcome::Stopped);
assert_eq!(stopped.stop_code().unwrap().as_str(), "done");
assert!(stopped.error().is_none());
let canceled = LifecycleEnd::Cancelled {
error: Arc::new(TestError("cancel")),
};
assert_eq!(
canceled.outcome(),
crate::value::lifecycle::Outcome::Cancelled
);
assert_eq!(canceled.error().unwrap().to_string(), "cancel");
assert!(canceled.stop_code().is_none());
let failed = LifecycleEnd::Failed {
error: Arc::new(TestError("fail")),
};
assert_eq!(failed.outcome(), crate::value::lifecycle::Outcome::Failed);
assert_eq!(failed.error().unwrap().to_string(), "fail");
assert!(failed.stop_code().is_none());
let dispositions = [
CallDisposition::Executed,
CallDisposition::ShortCircuited,
CallDisposition::Blocked,
CallDisposition::Rejected,
CallDisposition::Truncated,
CallDisposition::NotExecuted,
];
assert_eq!(dispositions.len(), 6);
let routed: crate::runtime::tool::CallDisposition = CallDisposition::Rejected;
assert_eq!(routed, CallDisposition::Rejected);
assert_eq!(
ToolCallProgress::result_available(routed, ToolResult::no_output()).disposition(),
crate::runtime::tool::CallDisposition::Rejected
);
}
#[test]
fn tool_call_end_accepts_only_the_complete_legal_terminal_matrix() {
use crate::value::lifecycle::Outcome;
let (_, coord, _, _, _, original, _, _) = fixtures();
let effective = ToolCall::new(
Id::new("effective".into()),
tool_call::Name::new("rewritten".into()),
Payload::from_vec(vec![1]),
);
let dispositions = [
CallDisposition::Executed,
CallDisposition::ShortCircuited,
CallDisposition::Blocked,
CallDisposition::Rejected,
CallDisposition::Truncated,
CallDisposition::NotExecuted,
];
let kinds = [
ToolCallProgressKind::Observed,
ToolCallProgressKind::ResultAvailable,
ToolCallProgressKind::ResultCommitted,
ToolCallProgressKind::PostHooksCompleted,
];
let outcomes = [
Outcome::Completed,
Outcome::Stopped,
Outcome::Cancelled,
Outcome::Failed,
];
let mut legal = 0;
let mut illegal = 0;
for disposition in dispositions {
for kind in kinds {
for outcome in outcomes {
let milestone = match kind {
ToolCallProgressKind::Observed => ToolCallProgress::observed(disposition),
ToolCallProgressKind::ResultAvailable => {
ToolCallProgress::result_available(disposition, ToolResult::no_output())
}
ToolCallProgressKind::ResultCommitted => {
ToolCallProgress::result_committed(disposition, ToolResult::no_output())
}
ToolCallProgressKind::PostHooksCompleted => {
ToolCallProgress::completed_post_hooks(disposition, ToolResult::no_output())
}
};
assert_eq!(milestone.kind(), kind);
assert_eq!(
milestone.result().is_some(),
kind != ToolCallProgressKind::Observed
);
assert_eq!(
milestone.is_result_committed(),
matches!(
kind,
ToolCallProgressKind::ResultCommitted
| ToolCallProgressKind::PostHooksCompleted
)
);
assert_eq!(
milestone.post_hooks_completed(),
kind == ToolCallProgressKind::PostHooksCompleted
);
let end = match outcome {
Outcome::Completed => LifecycleEnd::Completed,
Outcome::Stopped => LifecycleEnd::Stopped { code: None },
Outcome::Cancelled => LifecycleEnd::Cancelled {
error: Arc::new(TestError("cancelled")),
},
Outcome::Failed => LifecycleEnd::Failed {
error: Arc::new(TestError("failed")),
},
};
let result = ToolCallEndObservation::try_new(
coord.clone(),
end,
original.clone(),
effective.clone(),
milestone,
);
let result_disposition = !matches!(
disposition,
CallDisposition::Truncated | CallDisposition::NotExecuted
);
let disposition_valid =
(kind == ToolCallProgressKind::Observed) != result_disposition;
let terminal_valid = (outcome == Outcome::Completed)
== (kind == ToolCallProgressKind::PostHooksCompleted);
if disposition_valid && terminal_valid {
legal += 1;
let observation = result.unwrap();
assert_eq!(observation.coord(), &coord);
assert_eq!(observation.end().outcome(), outcome);
assert_eq!(observation.original_call().id().as_str(), "call");
assert_eq!(observation.effective_call().id().as_str(), "effective");
assert_eq!(observation.progress().kind(), kind);
assert_eq!(observation.progress().disposition(), disposition);
let wrapped = LifecycleObservation::ToolCallEnd(observation);
assert_eq!(wrapped.kind(), LifecycleKind::ToolCallEnd);
assert_eq!(wrapped.coord(), &coord);
} else {
illegal += 1;
let error = result.err().unwrap();
if !disposition_valid {
assert_eq!(
error,
ToolCallEndError::InvalidDisposition {
disposition,
progress: kind,
}
);
} else if outcome == Outcome::Completed {
assert_eq!(error, ToolCallEndError::CompletedBeforePostHooks(kind));
} else {
assert_eq!(error, ToolCallEndError::PostHooksWithoutCompletion(outcome));
}
}
}
}
}
assert_eq!((legal, illegal), (34, 62));
}
#[test]
fn all_scopes_expose_the_real_narrow_capability_matrix() {
let (capabilities, coord, emitter, model, tools, call, result, output) = fixtures();
assert_eq!(capabilities.session.id(), "session");
assert_eq!(
capabilities.session.transcript().version().unwrap(),
Version::ZERO
);
let factory = capabilities.factory(&coord, &emitter, true);
let staged_store = store::Store::new();
let staged = StagedTranscript::from_store(&staged_store);
let base = Snapshot::new(Version::ZERO, vec![]);
let parts = vec![Part::text("input".into())];
let items = vec![];
let prepare_state_stage = capabilities.lease.state().staged();
let prepare = factory.prepare(&staged, Some(&prepare_state_stage), &base, &parts);
assert_eq!(prepare.session().id(), "session");
assert_eq!(prepare.coord().run_id(), "run");
assert_eq!(prepare.input().len(), 1);
assert_eq!(prepare.base_snapshot().version(), Version::ZERO);
assert_eq!(prepare.transcript().unwrap().version(), Version::ZERO);
assert!(prepare.staged_session_state_writer().is_some());
let _ = prepare.run_state();
let _ = prepare.session_state();
let finalize = factory.compose_continuation(&staged, &parts, 2, 3);
assert_eq!(finalize.session().id(), "session");
assert_eq!(finalize.coord().run_id(), "run");
assert_eq!(finalize.transcript().unwrap().version(), Version::ZERO);
let _ = finalize.session_state();
assert_eq!(finalize.continuation().len(), 1);
assert_eq!((finalize.continuations(), finalize.turns()), (2, 3));
assert_eq!(
finalize.staged().append_batch(vec![]).unwrap(),
Version::ZERO
);
finalize.emit_audit(audit::Event::new("finalize".into(), None));
finalize.emit_error(Arc::new(TestError("finalize-error")));
let continuation = factory.decide_stop();
assert_eq!(continuation.transcript().unwrap().version(), Version::ZERO);
assert!(continuation.session_state_writer().is_some());
continuation.emit_audit(audit::Event::new("continue".into(), None));
continuation.emit_error(Arc::new(TestError("continue-error")));
let input = factory.input_admit(InputTarget::NextStep, &items);
assert_eq!(input.transcript().unwrap().version(), Version::ZERO);
assert_eq!(input.target(), InputTarget::NextStep);
assert!(input.items().is_empty());
input.emit_audit(audit::Event::new("input".into(), None));
input.emit_error(Arc::new(TestError("input-error")));
let turn = factory.turn(&staged, &items);
assert_eq!(turn.transcript().unwrap().version(), Version::ZERO);
assert!(turn.trigger().is_empty());
assert_eq!(turn.staged().replace(vec![]).unwrap(), Version::from_u64(1));
turn.emit_audit(audit::Event::new("turn".into(), None));
turn.emit_error(Arc::new(TestError("turn-error")));
let mut plan = RequestPlan::new();
{
let mut before = factory.before_step(&staged, &model, &mut plan);
assert_eq!(before.transcript().unwrap().version(), Version::ZERO);
assert_eq!(before.model().id(), "model");
before.plan().push_ephemeral(Part::text("ephemeral".into()));
before.emit_audit(audit::Event::new("before-step".into(), None));
before.emit_error(Arc::new(TestError("before-step-error")));
}
let snapshot = StepSnapshot::freeze(model.clone(), plan, tools);
let after = factory.after_step(&staged, &snapshot);
assert_eq!(after.transcript().unwrap().version(), Version::ZERO);
assert_eq!(after.snapshot().model().id(), "model");
let _ = after.snapshot().plan();
after.emit_audit(audit::Event::new("after-step".into(), None));
after.emit_error(Arc::new(TestError("after-step-error")));
let request = RequestPlan::new().try_into_request(vec![]).unwrap();
let before_model = factory.before_model(&snapshot, &request);
assert_eq!(before_model.transcript().unwrap().version(), Version::ZERO);
assert_eq!(before_model.snapshot().model().id(), "model");
assert!(before_model.request().messages().is_empty());
before_model.emit_audit(audit::Event::new("before-model".into(), None));
before_model.emit_error(Arc::new(TestError("before-model-error")));
let after_model = factory.after_model(&snapshot, &output, true, "stop", CompletionReason::Stop);
assert_eq!(after_model.transcript().unwrap().version(), Version::ZERO);
assert_eq!(after_model.snapshot().model().id(), "model");
assert!(after_model.output().content().is_empty());
assert!(after_model.generated());
assert_eq!(after_model.finish_reason(), "stop");
assert_eq!(after_model.completion(), CompletionReason::Stop);
after_model.emit_audit(audit::Event::new("after-model".into(), None));
after_model.emit_error(Arc::new(TestError("after-model-error")));
let tool_call = factory.tool_call(&snapshot, &call);
assert_eq!(tool_call.transcript().unwrap().version(), Version::ZERO);
assert_eq!(tool_call.snapshot().model().id(), "model");
assert_eq!(tool_call.snapshot().tools().exposure().len(), 0);
assert_eq!(tool_call.call().id().as_str(), "call");
tool_call.emit_audit(audit::Event::new("tool-call".into(), None));
tool_call.emit_error(Arc::new(TestError("tool-call-error")));
let tool_result = factory.tool_result(&snapshot, &call, CallDisposition::Executed, &result);
assert_eq!(tool_result.transcript().unwrap().version(), Version::ZERO);
assert_eq!(tool_result.snapshot().model().id(), "model");
assert_eq!(tool_result.snapshot().tools().exposure().len(), 0);
assert_eq!(tool_result.call().id().as_str(), "call");
assert_eq!(tool_result.disposition(), CallDisposition::Executed);
assert!(!tool_result.result().has_output());
tool_result.emit_audit(audit::Event::new("tool-result".into(), None));
tool_result.emit_error(Arc::new(TestError("tool-result-error")));
let executed = factory.after_tool_commit(&staged, &snapshot, &result);
assert_eq!(executed.transcript().unwrap().version(), Version::ZERO);
assert_eq!(executed.snapshot().model().id(), "model");
assert_eq!(executed.snapshot().tools().exposure().len(), 0);
assert!(!executed.result().has_output());
executed.emit_audit(audit::Event::new("executed".into(), None));
executed.emit_error(Arc::new(TestError("executed-error")));
let observer = factory.observer(Some(&snapshot));
assert_eq!(observer.transcript().unwrap().version(), Version::ZERO);
assert_eq!(observer.session().id(), "session");
assert_eq!(observer.snapshot().unwrap().model().id(), "model");
assert_eq!(observer.snapshot().unwrap().tools().exposure().len(), 0);
let _ = observer.run_state();
let _ = observer.session_state();
observer.emit_audit(audit::Event::new("observer".into(), None));
let read = factory.read();
assert_eq!(read.transcript().unwrap().version(), Version::ZERO);
assert_eq!(read.session().id(), "session");
assert_eq!(read.coord().run_id(), "run");
let _ = read.run_state();
let _ = read.session_state();
assert_eq!(emitter.errors.lock().unwrap().len(), 11);
assert_eq!(emitter.audits.lock().unwrap().len(), 12);
let denied = capabilities.factory(&coord, &emitter, false);
assert!(denied.decide_stop().session_state_writer().is_none());
assert!(
denied
.prepare(&staged, Some(&prepare_state_stage), &base, &parts,)
.staged_session_state_writer()
.is_none()
);
}
#[test]
fn prepare_base_stays_fixed_while_phase_transcript_advances() {
let (capabilities, coord, emitter, _, _, _, _, _) = fixtures();
let staged_store = store::Store::from_items(vec![Item::System(System::new(
"test".into(),
"base".into(),
ModelVisibility::Visible,
))]);
let staged = StagedTranscript::from_store(&staged_store);
let reader = staged_store.snapshot_reader();
let base = reader.snapshot().unwrap();
let factory = ScopeFactory::new(
&coord,
&capabilities.run_state,
&capabilities.session_state,
Some(capabilities.lease.state()),
true,
TranscriptSource::Reader(&reader),
&emitter,
);
let scope = factory.prepare(&staged, None, &base, &[]);
scope
.staged()
.append_batch(vec![Item::System(System::new(
"test".into(),
"later".into(),
ModelVisibility::Visible,
))])
.unwrap();
assert_eq!(scope.base_snapshot().version(), Version::ZERO);
assert_eq!(scope.base_snapshot().items().len(), 1);
assert_eq!(scope.transcript().unwrap().version(), Version::from_u64(1));
assert_eq!(scope.transcript().unwrap().items().len(), 2);
}
#[test]
fn staged_scope_reads_share_until_a_write_then_share_the_new_state() {
let (capabilities, coord, emitter, _, _, _, _, _) = fixtures();
let staged_store = store::Store::from_items(vec![Item::System(System::new(
"test".into(),
"seed".into(),
ModelVisibility::Visible,
))]);
let staged = StagedTranscript::from_store(&staged_store);
let reader = staged_store.snapshot_reader();
let factory = ScopeFactory::new(
&coord,
&capabilities.run_state,
&capabilities.session_state,
Some(capabilities.lease.state()),
true,
TranscriptSource::Reader(&reader),
&emitter,
);
let scope = factory.turn(&staged, &[]);
let first = scope.transcript().unwrap();
let second = scope.transcript().unwrap();
assert!(first.shares_items_with(&second));
scope
.staged()
.append_batch(vec![Item::System(System::new(
"test".into(),
"tail".into(),
ModelVisibility::Visible,
))])
.unwrap();
let after_write = scope.transcript().unwrap();
let repeated = scope.transcript().unwrap();
assert_eq!(first.items().len(), 1);
assert_eq!(after_write.items().len(), 2);
assert!(!first.shares_items_with(&after_write));
assert!(after_write.shares_items_with(&repeated));
}
#[test]
fn one_owned_step_snapshot_preserves_all_lanes_and_is_reused_everywhere() {
let (capabilities, coord, emitter, model, tools, call, result, output) = fixtures();
let mut plan = RequestPlan::new();
plan.push_system_instruction(Part::text("system".into()));
plan.push_tool(ClientTool::new(
tool_call::Name::new("local".into()),
"local tool".into(),
Payload::from_vec(br#"{"type":"object"}"#.to_vec()),
));
*plan.sampling_mut() = SamplingParams {
temperature: Some(0.25),
reasoning_effort: Some("high".into()),
max_output_tokens: Some(321),
parallel_tool_calls: true,
output_schema: Some(Payload::from_vec(br#"{"type":"string"}"#.to_vec())),
stop_sequences: vec!["stop".into()],
tool_choice: Some(ToolChoice::Specific(tool_call::Name::new("local".into()))),
};
plan.push_extension(
Envelope::try_new(
"provider".into(),
"hosted".into(),
Payload::from_vec(br#"{"enabled":true}"#.to_vec()),
)
.unwrap(),
)
.push_ephemeral(Part::text("ephemeral".into()));
let snapshot = StepSnapshot::freeze(model, plan, tools);
let request = snapshot.plan().clone().try_into_request(vec![]).unwrap();
assert_eq!(request.system_instruction().len(), 1);
assert_eq!(request.tools().len(), 1);
assert_eq!(request.tools()[0].name().as_str(), "local");
assert_eq!(request.sampling().temperature, Some(0.25));
assert_eq!(request.sampling().reasoning_effort.as_deref(), Some("high"));
assert_eq!(request.sampling().max_output_tokens, Some(321));
assert!(request.sampling().parallel_tool_calls);
assert_eq!(
request
.sampling()
.output_schema
.as_ref()
.unwrap()
.as_bytes(),
br#"{"type":"string"}"#
);
assert_eq!(request.sampling().stop_sequences, ["stop"]);
let Some(ToolChoice::Specific(name)) = &request.sampling().tool_choice else {
panic!("specific tool choice must survive the snapshot")
};
assert_eq!(name.as_str(), "local");
assert_eq!(request.extensions().len(), 1);
assert_eq!(request.extensions()[0].namespace(), "provider");
assert_eq!(request.extensions()[0].kind(), "hosted");
assert_eq!(request.ephemeral().len(), 1);
assert_eq!(snapshot.model().id(), "model");
assert_eq!(snapshot.model().context_window(), 1024);
assert!(snapshot.model().supports_tools());
assert!(!snapshot.model().supports_vision());
assert!(snapshot.model().supports_thinking());
let factory = capabilities.factory(&coord, &emitter, true);
let staged_store = store::Store::new();
let staged = StagedTranscript::from_store(&staged_store);
let after_model = factory.after_model(&snapshot, &output, true, "stop", CompletionReason::Stop);
let tool_call = factory.tool_call(&snapshot, &call);
let tool_result = factory.tool_result(&snapshot, &call, CallDisposition::Executed, &result);
let after_execute = factory.after_tool_commit(&staged, &snapshot, &result);
assert!(std::ptr::eq(after_model.snapshot(), Arc::as_ref(&snapshot)));
assert!(std::ptr::eq(tool_call.snapshot(), Arc::as_ref(&snapshot)));
assert!(std::ptr::eq(tool_result.snapshot(), Arc::as_ref(&snapshot)));
assert!(std::ptr::eq(
after_execute.snapshot(),
Arc::as_ref(&snapshot)
));
let start = LifecycleObservation::StepStart {
coord: coord.clone(),
snapshot: snapshot.clone(),
};
let end = LifecycleObservation::StepEnd {
coord,
end: LifecycleEnd::Completed,
snapshot: snapshot.clone(),
};
let LifecycleObservation::StepStart {
snapshot: start_snapshot,
..
} = start
else {
panic!("expected StepStart")
};
let LifecycleObservation::StepEnd {
snapshot: end_snapshot,
..
} = end
else {
panic!("expected StepEnd")
};
assert!(Arc::ptr_eq(&snapshot, &start_snapshot));
assert!(Arc::ptr_eq(&snapshot, &end_snapshot));
}
#[derive(Clone, Copy)]
enum StepBehavior {
Success,
Cancel,
CancelAndFail,
}
struct StepHook {
name: &'static str,
log: Arc<Mutex<Vec<&'static str>>>,
cancellation: Token,
behavior: StepBehavior,
}
impl TurnStepHooks for StepHook {
fn before_turn(&self, _scope: &TurnScope<'_>) -> HookResult<Vec<Item>> {
Ok(vec![])
}
fn after_turn(&self, _scope: &TurnScope<'_>) -> HookResult<Vec<Item>> {
Ok(vec![])
}
fn before_step(&self, scope: &mut BeforeStepScope<'_>) -> HookResult<Vec<Item>> {
self.log.lock().unwrap().push(self.name);
scope.plan().push_ephemeral(Part::text(self.name.into()));
match self.behavior {
StepBehavior::Success => Ok(vec![Item::System(System::new(
self.name.into(),
self.name.into(),
ModelVisibility::Hidden,
))]),
StepBehavior::Cancel => {
self.cancellation.cancel();
Ok(vec![])
}
StepBehavior::CancelAndFail => {
self.cancellation.cancel();
Err(HookError::new(TestError("callback won")))
}
}
}
fn after_step(&self, _scope: &AfterStepScope<'_>) -> HookResult<AfterStepDecision> {
Ok(AfterStepDecision::Continue(vec![]))
}
}
struct AllFamilies;
impl RunHooks for AllFamilies {
fn prepare(&self, _scope: &PrepareScope<'_>) -> HookResult<PrepareDecision> {
Ok(PrepareDecision::Continue)
}
fn compose_continuation(
&self,
scope: &ContinuationScope<'_>,
) -> HookResult<ContinuationDecision> {
Ok(ContinuationDecision::Replace(scope.continuation().to_vec()))
}
fn decide_stop(&self, _scope: &StopDecisionScope<'_>) -> HookResult<StopDecision> {
Ok(StopDecision::Continue)
}
}
impl InputHooks for AllFamilies {
fn before_input_admit(&self, _scope: &InputAdmitScope<'_>) -> InputAdmitDecision {
InputAdmitDecision::Admit
}
}
impl ModelHooks for AllFamilies {
fn before_model(&self, _scope: &BeforeModelScope<'_>) -> HookResult<BeforeModelDecision> {
Ok(BeforeModelDecision::Continue)
}
fn after_model(&self, _scope: &AfterModelScope<'_>) -> AfterModelDecision {
AfterModelDecision::Continue
}
}
impl ToolHooks for AllFamilies {
fn before_tool_call(&self, _scope: &ToolCallScope<'_>) -> HookResult<ToolCallDecision> {
Ok(ToolCallDecision::Continue)
}
fn before_tool_result(&self, _scope: &ToolResultScope<'_>) -> HookResult<ToolResultDecision> {
Ok(ToolResultDecision::Continue)
}
fn after_tool_commit(&self, _scope: &AfterToolCommitScope<'_>) -> Vec<Item> {
vec![]
}
}
struct Published(Arc<Mutex<usize>>);
impl TranscriptReservation for Published {
fn publish(self: Box<Self>) {
*self.0.lock().unwrap() += 1;
}
}
impl TranscriptHooks for AllFamilies {
fn reserve(
&self,
_scope: &ReadScope<'_>,
_mutation: &TranscriptMutation,
) -> HookResult<Box<dyn TranscriptReservation>> {
Ok(Box::new(Published(Arc::new(Mutex::new(0)))))
}
}
impl AgentTeardown for AllFamilies {
fn close(&self) -> HookResult<()> {
Ok(())
}
}
#[test]
fn registry_into_requests_metadata_and_rejects_bad_names() {
let all = Arc::new(AllFamilies);
let step = Arc::new(StepHook {
name: "all",
log: Arc::new(Mutex::new(vec![])),
cancellation: Token::new(),
behavior: StepBehavior::Success,
});
let registration = Registration::new("all".into())
.requires_session_state(true)
.run(all.clone())
.input(all.clone())
.turn_step(step)
.model(all.clone())
.tool(all.clone())
.observer(Arc::new(HealthyObserver::default()))
.transcript(all.clone())
.teardown(all);
let mut builder = RegistryBuilder::new();
builder.register(registration);
builder.register(Registration::new("metadata-only".into()));
let registry = builder.build_runtime().unwrap();
assert_eq!(registry.len(), 2);
assert_eq!(
registry.names().collect::<Vec<_>>(),
vec!["all", "metadata-only"]
);
assert!(registry.requires_session_state());
let run = registry.run_hooks().next().unwrap();
assert_eq!(run.index(), 0);
assert_eq!(run.name(), "all");
assert!(run.requires_session_state());
let _ = run.hook();
assert!(
registry
.input_hooks()
.next()
.unwrap()
.requires_session_state()
);
assert!(
registry
.turn_step_hooks()
.next()
.unwrap()
.requires_session_state()
);
assert!(
registry
.model_hooks()
.next()
.unwrap()
.requires_session_state()
);
assert!(
registry
.tool_hooks()
.next()
.unwrap()
.requires_session_state()
);
assert!(
registry
.observer_hooks()
.next()
.unwrap()
.requires_session_state()
);
assert!(
registry
.transcript_hooks()
.next()
.unwrap()
.requires_session_state()
);
assert_eq!(
registry
.run_hooks()
.map(|entry| entry.name())
.collect::<Vec<_>>(),
vec!["all"]
);
assert_eq!(registry.input_hooks().count(), 1);
assert_eq!(registry.turn_step_hooks().count(), 1);
assert_eq!(registry.model_hooks().count(), 1);
assert_eq!(registry.tool_hooks().count(), 1);
assert_eq!(registry.observer_hooks().count(), 1);
assert_eq!(registry.transcript_hooks().count(), 1);
assert!(registry.close().is_empty());
assert_eq!(registry.clone().names().next(), Some("all"));
let mut empty = RegistryBuilder::new();
empty.register(Registration::new(String::new()));
let error = empty.build_runtime().err().unwrap();
assert_eq!(error, RegistryError::EmptyName);
assert_eq!(error.to_string(), "hook name must be non-empty");
let mut duplicate = RegistryBuilder::new();
duplicate.register(Registration::new("same".into()));
duplicate.register(Registration::new("same".into()));
let error = duplicate.build_runtime().err().unwrap();
assert_eq!(error, RegistryError::DuplicateName("same".into()));
assert_eq!(error.to_string(), "duplicate hook name: same");
let registry = RegistryBuilder::new().build_runtime().unwrap();
assert_eq!(registry.len(), 0);
assert!(registry.is_empty());
assert!(!registry.requires_session_state());
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct PreparedState(&'static str);
impl SnapshotValue for PreparedState {}
#[derive(Clone, Copy)]
enum PrepareBehavior {
Success,
Fail,
Cancel,
CancelAndFail,
BlankModify,
}
struct IsolatedPrepareHook {
key: run_state::Key<String>,
session_key: session_state::Key<PreparedState>,
value: &'static str,
behavior: PrepareBehavior,
cancellation: Token,
saw_empty: Arc<Mutex<Vec<bool>>>,
}
impl RunHooks for IsolatedPrepareHook {
fn prepare(&self, scope: &PrepareScope<'_>) -> HookResult<PrepareDecision> {
self.saw_empty
.lock()
.unwrap()
.push(scope.run_state().load(&self.key).is_none());
scope
.run_state()
.set(&self.key, Arc::new(self.value.to_owned()));
scope
.staged_session_state_writer()
.expect("declared prepare writer")
.set(&self.session_key, &PreparedState(self.value))
.unwrap();
scope
.staged()
.append_batch(vec![Item::System(System::new(
self.value.into(),
self.value.into(),
ModelVisibility::Hidden,
))])
.unwrap();
match self.behavior {
PrepareBehavior::Success => Ok(PrepareDecision::Modify(scope.input().to_vec())),
PrepareBehavior::Fail => Err(HookError::new(TestError("prepare failed"))),
PrepareBehavior::Cancel => {
self.cancellation.cancel();
Ok(PrepareDecision::Continue)
}
PrepareBehavior::CancelAndFail => {
self.cancellation.cancel();
Err(HookError::new(TestError("prepare failure won")))
}
PrepareBehavior::BlankModify => Ok(PrepareDecision::Modify(Vec::new())),
}
}
fn compose_continuation(
&self,
scope: &ContinuationScope<'_>,
) -> HookResult<ContinuationDecision> {
Ok(ContinuationDecision::Replace(scope.continuation().to_vec()))
}
fn decide_stop(&self, _scope: &StopDecisionScope<'_>) -> HookResult<StopDecision> {
Ok(StopDecision::Continue)
}
}
#[test]
fn prepare_adapters_isolate_transcript_and_session_stages_without_auto_commit() {
let (session, coord, emitter, _, _, _, _, _) = fixtures();
let run_key = run_state::Key::new();
let session_key = session_state::Key::new();
let saw_empty = Arc::new(Mutex::new(Vec::new()));
let mut builder = RegistryBuilder::new();
builder.register(Registration::new("metadata-only".into()));
for (name, value, behavior) in [
("success", "success", PrepareBehavior::Success),
("failure", "failure", PrepareBehavior::Fail),
] {
builder.register(
Registration::new(name.into())
.requires_session_state(true)
.run(Arc::new(IsolatedPrepareHook {
key: run_key.clone(),
session_key: session_key.clone(),
value,
behavior,
cancellation: Token::new(),
saw_empty: saw_empty.clone(),
})),
);
}
let registry = builder.build_runtime().unwrap();
assert_eq!(
registry
.run_hooks()
.map(|adapter| (adapter.index(), adapter.name()))
.collect::<Vec<_>>(),
[(1, "success"), (2, "failure")]
);
let base = Snapshot::new(Version::ZERO, vec![]);
let input = vec![Part::text("input".into())];
let mut adapters = registry.run_hooks();
let success = adapters.next().unwrap();
let failure = adapters.next().unwrap();
let success_run_state = run_state::Store::new();
let success_transcript_store = store::Store::new();
let success_transcript = StagedTranscript::from_store(&success_transcript_store);
let success_session = session.lease.state().staged();
success
.invoke_prepare(
CallbackResources::new(
&coord,
&success_run_state,
&session.session_state,
Some(session.lease.state()),
&emitter,
),
&success_transcript,
Some(&success_session),
&base,
&input,
&Token::new(),
)
.unwrap();
let failure_run_state = run_state::Store::new();
let failure_transcript_store = store::Store::new();
let failure_transcript = StagedTranscript::from_store(&failure_transcript_store);
let failure_session = session.lease.state().staged();
let error = dispatch_error(failure.invoke_prepare(
CallbackResources::new(
&coord,
&failure_run_state,
&session.session_state,
Some(session.lease.state()),
&emitter,
),
&failure_transcript,
Some(&failure_session),
&base,
&input,
&Token::new(),
));
assert!(matches!(error, DispatchError::Callback(_)));
assert_eq!(*saw_empty.lock().unwrap(), [true, true]);
assert_eq!(
success_run_state.load(&run_key).unwrap().as_str(),
"success"
);
assert_eq!(
failure_run_state.load(&run_key).unwrap().as_str(),
"failure"
);
assert_eq!(
success_transcript_store
.snapshot_reader()
.snapshot()
.unwrap()
.items()
.len(),
1
);
assert_eq!(
failure_transcript_store
.snapshot_reader()
.snapshot()
.unwrap()
.items()
.len(),
1
);
assert!(session.session_state.load(&session_key).is_none());
assert_eq!(success_session.load(&session_key).unwrap().0, "success");
assert_eq!(failure_session.load(&session_key).unwrap().0, "failure");
assert_eq!(
session.session.transcript().version().unwrap(),
Version::ZERO
);
success_session.commit().unwrap();
failure_session.discard();
assert_eq!(
session.session_state.load(&session_key).unwrap().0,
"success"
);
}
#[test]
fn prepare_adapter_preserves_cancellation_and_callback_error_priority() {
let (session, coord, emitter, _, _, _, _, _) = fixtures();
let run_key = run_state::Key::new();
let session_key = session_state::Key::new();
let saw_empty = Arc::new(Mutex::new(Vec::new()));
for (behavior, expected_callback) in [
(PrepareBehavior::Cancel, false),
(PrepareBehavior::CancelAndFail, true),
] {
let cancellation = Token::new();
let hook = Arc::new(IsolatedPrepareHook {
key: run_key.clone(),
session_key: session_key.clone(),
value: "cancel",
behavior,
cancellation: cancellation.clone(),
saw_empty: saw_empty.clone(),
});
let mut builder = RegistryBuilder::new();
builder.register(
Registration::new("prepare".into())
.requires_session_state(true)
.run(hook),
);
let registry = builder.build_runtime().unwrap();
let run_state = run_state::Store::new();
let transcript_store = store::Store::new();
let transcript = StagedTranscript::from_store(&transcript_store);
let session_stage = session.lease.state().staged();
let result = registry.run_hooks().next().unwrap().invoke_prepare(
CallbackResources::new(
&coord,
&run_state,
&session.session_state,
Some(session.lease.state()),
&emitter,
),
&transcript,
Some(&session_stage),
&Snapshot::new(Version::ZERO, vec![]),
&[],
&cancellation,
);
assert_eq!(
matches!(dispatch_error(result), DispatchError::Callback(_)),
expected_callback
);
session_stage.discard();
}
let cancellation = Token::new();
cancellation.cancel();
let mut builder = RegistryBuilder::new();
builder.register(
Registration::new("pre-cancel".into())
.requires_session_state(true)
.run(Arc::new(IsolatedPrepareHook {
key: run_key,
session_key,
value: "never",
behavior: PrepareBehavior::Success,
cancellation: cancellation.clone(),
saw_empty: saw_empty.clone(),
})),
);
let registry = builder.build_runtime().unwrap();
let run_state = run_state::Store::new();
let transcript_store = store::Store::new();
let transcript = StagedTranscript::from_store(&transcript_store);
let session_stage = session.lease.state().staged();
let error = dispatch_error(registry.run_hooks().next().unwrap().invoke_prepare(
CallbackResources::new(
&coord,
&run_state,
&session.session_state,
Some(session.lease.state()),
&emitter,
),
&transcript,
Some(&session_stage),
&Snapshot::new(Version::ZERO, vec![]),
&[],
&cancellation,
));
assert!(matches!(error, DispatchError::Cancelled));
assert_eq!(saw_empty.lock().unwrap().len(), 2);
}
#[test]
fn a_prepare_rewrite_to_nothing_is_refused_and_named_at_the_dispatch_boundary() {
let (session, coord, emitter, _, _, _, _, _) = fixtures();
let mut builder = RegistryBuilder::new();
builder.register(
Registration::new("blanker".into())
.requires_session_state(true)
.run(Arc::new(IsolatedPrepareHook {
key: run_state::Key::new(),
session_key: session_state::Key::new(),
value: "blank",
behavior: PrepareBehavior::BlankModify,
cancellation: Token::new(),
saw_empty: Arc::new(Mutex::new(Vec::new())),
})),
);
let registry = builder.build_runtime().unwrap();
let run_state = run_state::Store::new();
let transcript_store = store::Store::new();
let transcript = StagedTranscript::from_store(&transcript_store);
let session_stage = session.lease.state().staged();
let error = dispatch_error(registry.run_hooks().next().unwrap().invoke_prepare(
CallbackResources::new(
&coord,
&run_state,
&session.session_state,
Some(session.lease.state()),
&emitter,
),
&transcript,
Some(&session_stage),
&Snapshot::new(Version::ZERO, vec![]),
&[Part::text("caller said this".into())],
&Token::new(),
));
let DispatchError::Callback(failure) = error else {
panic!("an empty rewrite is a callback failure, not cancellation");
};
assert_eq!(failure.name(), "blanker");
assert_eq!(
failure.to_string(),
"hook blanker failed: a registration rewrote the initial input to nothing"
);
assert_eq!(
failure
.error()
.source_ref()
.downcast_ref::<PrepareError>()
.copied(),
Some(PrepareError::EmptyModify)
);
session_stage.discard();
}
#[test]
fn before_step_adapters_preserve_order_index_and_error_priority_over_cancel() {
let (session, coord, emitter, model, _, _, _, _) = fixtures();
let cancellation = Token::new();
let log = Arc::new(Mutex::new(Vec::new()));
let first = Arc::new(StepHook {
name: "first",
log: log.clone(),
cancellation: cancellation.clone(),
behavior: StepBehavior::Success,
});
let second = Arc::new(StepHook {
name: "second",
log: log.clone(),
cancellation: cancellation.clone(),
behavior: StepBehavior::Success,
});
let mut builder = RegistryBuilder::new();
builder.register(Registration::new("metadata-only".into()));
builder.register(Registration::new("first".into()).turn_step(first));
builder.register(Registration::new("second".into()).turn_step(second));
let registry = builder.build_runtime().unwrap();
let mut plan = RequestPlan::new();
let mut items = Vec::new();
for adapter in registry.turn_step_hooks() {
let run_state = run_state::Store::new();
let staged_store = store::Store::new();
let staged = StagedTranscript::from_store(&staged_store);
let resources = CallbackResources::new(
&coord,
&run_state,
&session.session_state,
Some(session.lease.state()),
&emitter,
);
items.extend(
adapter
.invoke_before_step(resources, &staged, &model, &mut plan, &cancellation)
.unwrap(),
);
}
assert_eq!(
registry
.turn_step_hooks()
.map(|adapter| (adapter.index(), adapter.name()))
.collect::<Vec<_>>(),
vec![(1, "first"), (2, "second")]
);
assert_eq!(*log.lock().unwrap(), vec!["first", "second"]);
assert_eq!(items.len(), 2);
let cancellation = Token::new();
cancellation.cancel();
let run_state = run_state::Store::new();
let staged_store = store::Store::new();
let staged = StagedTranscript::from_store(&staged_store);
let resources = CallbackResources::new(
&coord,
&run_state,
&session.session_state,
Some(session.lease.state()),
&emitter,
);
let error = dispatch_error(
registry
.turn_step_hooks()
.next()
.unwrap()
.invoke_before_step(resources, &staged, &model, &mut plan, &cancellation),
);
assert!(matches!(error, DispatchError::Cancelled));
let cancellation = Token::new();
let cancelling = Arc::new(StepHook {
name: "cancel",
log: log.clone(),
cancellation: cancellation.clone(),
behavior: StepBehavior::Cancel,
});
let mut builder = RegistryBuilder::new();
builder.register(Registration::new("cancel".into()).turn_step(cancelling));
let cancel_registry = builder.build_runtime().unwrap();
let error = dispatch_error(
cancel_registry
.turn_step_hooks()
.next()
.unwrap()
.invoke_before_step(resources, &staged, &model, &mut plan, &cancellation),
);
assert!(matches!(error, DispatchError::Cancelled));
let cancellation = Token::new();
let failing = Arc::new(StepHook {
name: "failing",
log,
cancellation: cancellation.clone(),
behavior: StepBehavior::CancelAndFail,
});
let mut builder = RegistryBuilder::new();
builder.register(Registration::new("failing".into()).turn_step(failing));
let failing_registry = builder.build_runtime().unwrap();
let error = dispatch_error(
failing_registry
.turn_step_hooks()
.next()
.unwrap()
.invoke_before_step(resources, &staged, &model, &mut plan, &cancellation),
);
let DispatchError::Callback(failure) = error else {
panic!("callback error must outrank cancellation")
};
assert_eq!(failure.name(), "failing");
assert_eq!(failure.error().to_string(), "callback won");
assert!(
failure
.error()
.source_ref()
.downcast_ref::<TestError>()
.is_some()
);
assert_eq!(
format!("{:?}", failure.error()),
"HookError(\"callback won\")"
);
assert!(failure.error().source().is_some());
}
#[test]
fn hook_error_debug_states_every_link_of_the_retained_chain() {
let wrapped = HookError::new(ChainedError {
message: "writing the journal failed",
cause: TestError("disk full"),
});
let expected = "HookError(\"writing the journal failed\", \"disk full\")";
assert_eq!(format!("{wrapped:?}"), expected);
assert_eq!(
format!("{:?}", HookError::new(TestError("disk full"))),
"HookError(\"disk full\")",
"a leaf callback error still renders as one field"
);
}
#[test]
fn dispatch_error_keeps_the_registration_name_in_its_chain() {
let mut builder = RegistryBuilder::new();
builder.register(
Registration::new("teardown".into()).teardown(Arc::new(FailingTeardown("disk full"))),
);
let failure = builder.build_runtime().unwrap().close().pop().unwrap();
let dispatch = DispatchError::Callback(failure);
let attributed = dispatch
.source()
.and_then(|cause| cause.downcast_ref::<AttributedError>())
.expect("a callback dispatch failure retains its attribution");
assert_eq!(attributed.name(), "teardown");
assert_eq!(
attributed
.source()
.and_then(|cause| cause.downcast_ref::<HookError>())
.map(ToString::to_string),
Some("disk full".to_owned()),
"the callback error is still one link further along"
);
assert!(DispatchError::Cancelled.source().is_none());
}
struct WriterProbeHook {
seen: Arc<Mutex<Vec<bool>>>,
}
impl TurnStepHooks for WriterProbeHook {
fn before_turn(&self, _scope: &TurnScope<'_>) -> HookResult<Vec<Item>> {
Ok(vec![])
}
fn after_turn(&self, _scope: &TurnScope<'_>) -> HookResult<Vec<Item>> {
Ok(vec![])
}
fn before_step(&self, scope: &mut BeforeStepScope<'_>) -> HookResult<Vec<Item>> {
self.seen
.lock()
.unwrap()
.push(scope.session_state_writer().is_some());
Ok(vec![])
}
fn after_step(&self, _scope: &AfterStepScope<'_>) -> HookResult<AfterStepDecision> {
Ok(AfterStepDecision::Continue(vec![]))
}
}
#[test]
fn adapter_grants_writer_only_to_registration_that_declared_requirement() {
let (session, coord, emitter, model, _, _, _, _) = fixtures();
let seen = Arc::new(Mutex::new(Vec::new()));
let mut builder = RegistryBuilder::new();
builder.register(
Registration::new("reader".into())
.turn_step(Arc::new(WriterProbeHook { seen: seen.clone() })),
);
builder.register(
Registration::new("writer".into())
.requires_session_state(true)
.turn_step(Arc::new(WriterProbeHook { seen: seen.clone() })),
);
let registry = builder.build_runtime().unwrap();
let mut plan = RequestPlan::new();
for adapter in registry.turn_step_hooks() {
let run_state = run_state::Store::new();
let staged_store = store::Store::new();
let staged = StagedTranscript::from_store(&staged_store);
let resources = CallbackResources::new(
&coord,
&run_state,
&session.session_state,
Some(session.lease.state()),
&emitter,
);
adapter
.invoke_before_step(resources, &staged, &model, &mut plan, &Token::new())
.unwrap();
}
assert_eq!(*seen.lock().unwrap(), [false, true]);
}
struct IsolatedStepHook {
key: run_state::Key<String>,
value: &'static str,
fail: bool,
saw_empty: Arc<Mutex<Vec<bool>>>,
}
impl TurnStepHooks for IsolatedStepHook {
fn before_turn(&self, _scope: &TurnScope<'_>) -> HookResult<Vec<Item>> {
Ok(vec![])
}
fn after_turn(&self, _scope: &TurnScope<'_>) -> HookResult<Vec<Item>> {
Ok(vec![])
}
fn before_step(&self, scope: &mut BeforeStepScope<'_>) -> HookResult<Vec<Item>> {
self.saw_empty
.lock()
.unwrap()
.push(scope.run_state().load(&self.key).is_none());
scope
.run_state()
.set(&self.key, Arc::new(self.value.to_owned()));
assert!(scope.session_state_writer().is_some());
scope
.staged()
.append_batch(vec![Item::System(System::new(
self.value.into(),
self.value.into(),
ModelVisibility::Hidden,
))])
.unwrap();
if self.fail {
Err(HookError::new(TestError("isolated failure")))
} else {
Ok(vec![])
}
}
fn after_step(&self, _scope: &AfterStepScope<'_>) -> HookResult<AfterStepDecision> {
Ok(AfterStepDecision::Continue(vec![]))
}
}
#[test]
fn registration_callbacks_isolate_run_state_stage_and_successor() {
let (session, coord, emitter, model, _, _, _, _) = fixtures();
let key = run_state::Key::new();
let saw_empty = Arc::new(Mutex::new(Vec::new()));
let mut builder = RegistryBuilder::new();
builder.register(
Registration::new("success".into())
.requires_session_state(true)
.turn_step(Arc::new(IsolatedStepHook {
key: key.clone(),
value: "success",
fail: false,
saw_empty: saw_empty.clone(),
})),
);
builder.register(
Registration::new("failure".into())
.requires_session_state(true)
.turn_step(Arc::new(IsolatedStepHook {
key: key.clone(),
value: "failure",
fail: true,
saw_empty: saw_empty.clone(),
})),
);
let registry = builder.build_runtime().unwrap();
let mut adapters = registry.turn_step_hooks();
let success = adapters.next().unwrap();
let failure = adapters.next().unwrap();
let mut plan = RequestPlan::new();
let cancellation = Token::new();
let success_state = run_state::Store::new();
let success_store = store::Store::new();
let success_stage = StagedTranscript::from_store(&success_store);
success
.invoke_before_step(
CallbackResources::new(
&coord,
&success_state,
&session.session_state,
Some(session.lease.state()),
&emitter,
),
&success_stage,
&model,
&mut plan,
&cancellation,
)
.unwrap();
let failure_state = run_state::Store::new();
let failure_store = store::Store::new();
let failure_stage = StagedTranscript::from_store(&failure_store);
let error = dispatch_error(failure.invoke_before_step(
CallbackResources::new(
&coord,
&failure_state,
&session.session_state,
Some(session.lease.state()),
&emitter,
),
&failure_stage,
&model,
&mut plan,
&cancellation,
));
assert!(matches!(error, DispatchError::Callback(_)));
assert_eq!(*saw_empty.lock().unwrap(), [true, true]);
assert_eq!(success_state.load(&key).unwrap().as_str(), "success");
assert_eq!(failure_state.load(&key).unwrap().as_str(), "failure");
assert_eq!(
success_store
.snapshot_reader()
.snapshot()
.unwrap()
.items()
.len(),
1
);
assert_eq!(
failure_store
.snapshot_reader()
.snapshot()
.unwrap()
.items()
.len(),
1
);
assert_eq!(
session.session.transcript().version().unwrap(),
Version::ZERO
);
let successor_state = run_state::Store::new();
let successor_store = store::Store::new();
assert!(successor_state.load(&key).is_none());
assert_eq!(
successor_store
.snapshot_reader()
.snapshot()
.unwrap()
.version(),
Version::ZERO
);
}
struct IsolatedObserver {
key: run_state::Key<String>,
value: &'static str,
expected_model: &'static str,
saw_empty: Arc<Mutex<Vec<bool>>>,
saw_model: Arc<Mutex<Vec<String>>>,
}
impl ObserverHooks for IsolatedObserver {
fn run_admitted(&self, _scope: &ReadScope<'_>) {}
fn lifecycle(&self, _observation: &LifecycleObservation, scope: &ObserverScope<'_>) {
self.saw_empty
.lock()
.unwrap()
.push(scope.run_state().load(&self.key).is_none());
scope
.run_state()
.set(&self.key, Arc::new(self.value.to_owned()));
let model = scope.snapshot().unwrap().model().id().to_owned();
assert_eq!(model, self.expected_model);
self.saw_model.lock().unwrap().push(model);
}
fn model_event(&self, _scope: &ObserverScope<'_>, _event: &model::Event) {}
fn audit_event(&self, _scope: &ReadScope<'_>, _event: &audit::Event) {}
}
#[test]
fn observer_adapters_preserve_index_and_isolate_same_key_scope_and_step_arc() {
let (session, coord, emitter, _, tools, call, _, _) = fixtures();
let key = run_state::Key::new();
let saw_empty = Arc::new(Mutex::new(Vec::new()));
let saw_model = Arc::new(Mutex::new(Vec::new()));
let mut builder = RegistryBuilder::new();
builder.register(Registration::new("metadata-only".into()));
for (name, model_id) in [("first", "first-model"), ("second", "second-model")] {
builder.register(
Registration::new(name.into()).observer(Arc::new(IsolatedObserver {
key: key.clone(),
value: name,
expected_model: model_id,
saw_empty: saw_empty.clone(),
saw_model: saw_model.clone(),
})),
);
}
let registry = builder.build_runtime().unwrap();
assert_eq!(
registry
.observer_hooks()
.map(|adapter| (adapter.index(), adapter.name()))
.collect::<Vec<_>>(),
[(1, "first"), (2, "second")]
);
let observation = lifecycle_observations(&coord, &call).remove(0);
let first_state = run_state::Store::new();
let first_reader = session.session.state();
let first_tools = tool::RegistryBuilder::new()
.build()
.unwrap()
.resolve_step_without_providers(
Snapshot::new(Version::ZERO, Vec::new()),
&Token::new(),
&ExposeAll,
)
.unwrap()
.into_snapshot();
let first_snapshot = StepSnapshot::freeze(
capability_model("first-model"),
RequestPlan::new(),
first_tools,
);
registry.observer_hooks().next().unwrap().invoke_lifecycle(
CallbackResources::new(&coord, &first_state, &first_reader, None, &emitter),
TranscriptSource::Reader(&session.lease.transcript_reader()),
Some(&first_snapshot),
&observation,
);
let second_state = run_state::Store::new();
let second_reader = session.session.state();
let second_snapshot =
StepSnapshot::freeze(capability_model("second-model"), RequestPlan::new(), tools);
registry.observer_hooks().nth(1).unwrap().invoke_lifecycle(
CallbackResources::new(&coord, &second_state, &second_reader, None, &emitter),
TranscriptSource::Reader(&session.lease.transcript_reader()),
Some(&second_snapshot),
&observation,
);
assert_eq!(*saw_empty.lock().unwrap(), [true, true]);
assert_eq!(*saw_model.lock().unwrap(), ["first-model", "second-model"]);
assert_eq!(first_state.load(&key).unwrap().as_str(), "first");
assert_eq!(second_state.load(&key).unwrap().as_str(), "second");
assert!(!Arc::ptr_eq(&first_snapshot, &second_snapshot));
}
#[derive(Default)]
struct HealthyObserver {
calls: Mutex<Vec<&'static str>>,
}
impl ObserverHooks for HealthyObserver {
fn run_admitted(&self, _scope: &ReadScope<'_>) {
self.calls.lock().unwrap().push("admitted");
}
fn lifecycle(&self, _observation: &LifecycleObservation, _scope: &ObserverScope<'_>) {
self.calls.lock().unwrap().push("lifecycle");
}
fn model_event(&self, _scope: &ObserverScope<'_>, _event: &model::Event) {
self.calls.lock().unwrap().push("model");
}
fn audit_event(&self, _scope: &ReadScope<'_>, _event: &audit::Event) {
self.calls.lock().unwrap().push("audit");
}
}
struct InertObserver;
impl ObserverHooks for InertObserver {}
#[test]
fn an_observer_that_overrides_nothing_is_dispatched_and_observes_nothing() {
let (session, coord, emitter, model, tools, call, _, _) = fixtures();
let mut builder = RegistryBuilder::new();
builder.register(Registration::new("inert".into()).observer(Arc::new(InertObserver)));
let registry = builder.build_runtime().unwrap();
let snapshot = StepSnapshot::freeze(model, RequestPlan::new(), tools);
let run_state = run_state::Store::new();
let adapter = registry.observer_hooks().next().unwrap();
for observation in lifecycle_observations(&coord, &call) {
adapter.invoke_lifecycle(
CallbackResources::new(&coord, &run_state, &session.session_state, None, &emitter),
TranscriptSource::Reader(&session.lease.transcript_reader()),
Some(&snapshot),
&observation,
);
}
let resources =
CallbackResources::new(&coord, &run_state, &session.session_state, None, &emitter);
adapter.invoke_run_admitted(resources, &session.lease.transcript_reader());
adapter.invoke_model_event(
resources,
snapshot.tools().transcript(),
Some(&snapshot),
&model::Event::Usage(TokenUsage::default()),
);
adapter.invoke_audit_event(
resources,
&session.lease.transcript_reader(),
&audit::Event::new("source".into(), None),
);
assert!(
emitter.audits.lock().unwrap().is_empty(),
"an observer that observes nothing emits nothing"
);
assert!(emitter.errors.lock().unwrap().is_empty());
}
struct PanickingObserver;
impl ObserverHooks for PanickingObserver {
fn run_admitted(&self, _scope: &ReadScope<'_>) {
panic!("observer admitted panic");
}
fn lifecycle(&self, _observation: &LifecycleObservation, _scope: &ObserverScope<'_>) {
panic!("observer lifecycle panic");
}
fn model_event(&self, _scope: &ObserverScope<'_>, _event: &model::Event) {
panic!("observer model panic");
}
fn audit_event(&self, _scope: &ReadScope<'_>, _event: &audit::Event) {
panic!("observer audit panic");
}
}
#[test]
fn observer_panics_are_audited_at_each_exact_boundary_and_peers_continue() {
let (session, coord, emitter, model, tools, call, _, _) = fixtures();
let healthy = Arc::new(HealthyObserver::default());
let mut builder = RegistryBuilder::new();
builder.register(Registration::new("without-observer".into()));
builder.register(Registration::new("panic".into()).observer(Arc::new(PanickingObserver)));
builder.register(Registration::new("healthy".into()).observer(healthy.clone()));
let registry = builder.build_runtime().unwrap();
let snapshot = StepSnapshot::freeze(model, RequestPlan::new(), tools);
let panic_state = run_state::Store::new();
let healthy_state = run_state::Store::new();
for observation in lifecycle_observations(&coord, &call) {
for adapter in registry.observer_hooks() {
let run_state = if adapter.name() == "panic" {
&panic_state
} else {
&healthy_state
};
adapter.invoke_lifecycle(
CallbackResources::new(&coord, run_state, &session.session_state, None, &emitter),
TranscriptSource::Reader(&session.lease.transcript_reader()),
Some(&snapshot),
&observation,
);
}
}
for adapter in registry.observer_hooks() {
let run_state = if adapter.name() == "panic" {
&panic_state
} else {
&healthy_state
};
let resources =
CallbackResources::new(&coord, run_state, &session.session_state, None, &emitter);
adapter.invoke_run_admitted(resources, &session.lease.transcript_reader());
adapter.invoke_model_event(
resources,
snapshot.tools().transcript(),
Some(&snapshot),
&model::Event::Usage(TokenUsage::default()),
);
adapter.invoke_audit_event(
resources,
&session.lease.transcript_reader(),
&audit::Event::new("source".into(), None),
);
}
let calls = healthy.calls.lock().unwrap();
assert_eq!(
calls.iter().filter(|call| **call == "lifecycle").count(),
10
);
assert_eq!(calls.iter().filter(|call| **call == "admitted").count(), 1);
assert_eq!(calls.iter().filter(|call| **call == "model").count(), 1);
assert_eq!(calls.iter().filter(|call| **call == "audit").count(), 1);
drop(calls);
let audits = emitter.audits.lock().unwrap();
assert_eq!(audits.len(), 13);
assert!(
audits
.iter()
.all(|event| event.kind() == "hook.observer_panic")
);
assert!(
audits
.iter()
.all(|event| event.payload().unwrap().starts_with(b"panic:On"))
);
}
struct AuditThenPanicObserver;
impl ObserverHooks for AuditThenPanicObserver {
fn run_admitted(&self, _scope: &ReadScope<'_>) {}
fn lifecycle(&self, _observation: &LifecycleObservation, scope: &ObserverScope<'_>) {
scope.emit_audit(audit::Event::new("queued.one".into(), None));
scope.emit_audit(audit::Event::new("queued.two".into(), None));
panic!("after queueing");
}
fn model_event(&self, _scope: &ObserverScope<'_>, _event: &model::Event) {}
fn audit_event(&self, _scope: &ReadScope<'_>, _event: &audit::Event) {}
}
#[test]
fn callback_panic_flushes_prior_audits_fifo_then_audits_panic_and_continues() {
let (session, coord, emitter, model, tools, call, _, _) = fixtures();
let healthy = Arc::new(HealthyObserver::default());
let mut builder = RegistryBuilder::new();
builder.register(
Registration::new("queue-panic".into()).observer(Arc::new(AuditThenPanicObserver)),
);
builder.register(Registration::new("healthy".into()).observer(healthy.clone()));
let registry = builder.build_runtime().unwrap();
let snapshot = StepSnapshot::freeze(model, RequestPlan::new(), tools);
let observation = lifecycle_observations(&coord, &call).remove(0);
for adapter in registry.observer_hooks() {
let run_state = run_state::Store::new();
adapter.invoke_lifecycle(
CallbackResources::new(&coord, &run_state, &session.session_state, None, &emitter),
TranscriptSource::Reader(&session.lease.transcript_reader()),
Some(&snapshot),
&observation,
);
}
let kinds = emitter
.audits
.lock()
.unwrap()
.iter()
.map(|event| event.kind().to_owned())
.collect::<Vec<_>>();
assert_eq!(kinds, ["queued.one", "queued.two", "hook.observer_panic"]);
assert_eq!(*healthy.calls.lock().unwrap(), ["lifecycle"]);
}
struct CountingPanicOnDropPayload {
drops: Arc<AtomicUsize>,
}
impl Drop for CountingPanicOnDropPayload {
fn drop(&mut self) {
self.drops.fetch_add(1, Ordering::SeqCst);
panic!("panic-on-drop observer payload dropped");
}
}
struct PanicOnDropObserver {
drops: Arc<AtomicUsize>,
}
impl PanicOnDropObserver {
fn panic(&self) -> ! {
std::panic::panic_any(CountingPanicOnDropPayload {
drops: self.drops.clone(),
})
}
}
impl ObserverHooks for PanicOnDropObserver {
fn run_admitted(&self, _scope: &ReadScope<'_>) {
self.panic();
}
fn lifecycle(&self, _observation: &LifecycleObservation, scope: &ObserverScope<'_>) {
scope.emit_audit(audit::Event::new("queued.lifecycle".into(), None));
self.panic();
}
fn model_event(&self, scope: &ObserverScope<'_>, _event: &model::Event) {
scope.emit_audit(audit::Event::new("queued.model".into(), None));
self.panic();
}
fn audit_event(&self, _scope: &ReadScope<'_>, _event: &audit::Event) {
self.panic();
}
}
#[test]
fn observer_adapters_forget_panic_on_drop_payloads_and_continue_peers() {
let (session, coord, emitter, model, tools, call, _, _) = fixtures();
let drops = Arc::new(AtomicUsize::new(0));
let healthy = Arc::new(HealthyObserver::default());
let mut builder = RegistryBuilder::new();
builder.register(Registration::new("panic-on-drop".into()).observer(Arc::new(
PanicOnDropObserver {
drops: drops.clone(),
},
)));
builder.register(Registration::new("healthy".into()).observer(healthy.clone()));
let registry = builder.build_runtime().unwrap();
let snapshot = StepSnapshot::freeze(model, RequestPlan::new(), tools);
let observation = lifecycle_observations(&coord, &call).remove(0);
let lifecycle = std::panic::catch_unwind(AssertUnwindSafe(|| {
for adapter in registry.observer_hooks() {
let run_state = run_state::Store::new();
adapter.invoke_lifecycle(
CallbackResources::new(&coord, &run_state, &session.session_state, None, &emitter),
TranscriptSource::Reader(&session.lease.transcript_reader()),
Some(&snapshot),
&observation,
);
}
}));
let admitted = std::panic::catch_unwind(AssertUnwindSafe(|| {
for adapter in registry.observer_hooks() {
let run_state = run_state::Store::new();
adapter.invoke_run_admitted(
CallbackResources::new(&coord, &run_state, &session.session_state, None, &emitter),
&session.lease.transcript_reader(),
);
}
}));
let model = std::panic::catch_unwind(AssertUnwindSafe(|| {
for adapter in registry.observer_hooks() {
let run_state = run_state::Store::new();
adapter.invoke_model_event(
CallbackResources::new(&coord, &run_state, &session.session_state, None, &emitter),
snapshot.tools().transcript(),
Some(&snapshot),
&model::Event::Usage(TokenUsage::default()),
);
}
}));
let audit = std::panic::catch_unwind(AssertUnwindSafe(|| {
for adapter in registry.observer_hooks() {
let run_state = run_state::Store::new();
adapter.invoke_audit_event(
CallbackResources::new(&coord, &run_state, &session.session_state, None, &emitter),
&session.lease.transcript_reader(),
&audit::Event::new("source".into(), None),
);
}
}));
assert!(lifecycle.is_ok(), "lifecycle payload escaped isolation");
assert!(admitted.is_ok(), "run-admitted payload escaped isolation");
assert!(model.is_ok(), "model-event payload escaped isolation");
assert!(audit.is_ok(), "audit-event payload escaped isolation");
assert_eq!(drops.load(Ordering::SeqCst), 0);
assert_eq!(
*healthy.calls.lock().unwrap(),
["lifecycle", "admitted", "model", "audit"]
);
let audits = emitter.audits.lock().unwrap();
let kinds = audits.iter().map(|event| event.kind()).collect::<Vec<_>>();
assert_eq!(
kinds,
[
"queued.lifecycle",
"hook.observer_panic",
"hook.observer_panic",
"queued.model",
"hook.observer_panic",
"hook.observer_panic",
]
);
let panic_callbacks = audits
.iter()
.filter(|event| event.kind() == "hook.observer_panic")
.map(|event| std::str::from_utf8(event.payload().unwrap()).unwrap())
.collect::<Vec<_>>();
assert_eq!(
panic_callbacks,
[
"panic-on-drop:OnRunStart",
"panic-on-drop:OnRunAdmitted",
"panic-on-drop:OnModelEvent",
"panic-on-drop:OnAuditEvent",
]
);
}
struct EmitAuditObserver;
impl ObserverHooks for EmitAuditObserver {
fn run_admitted(&self, _scope: &ReadScope<'_>) {}
fn lifecycle(&self, _observation: &LifecycleObservation, scope: &ObserverScope<'_>) {
scope.emit_audit(audit::Event::new("core-boundary".into(), None));
}
fn model_event(&self, _scope: &ObserverScope<'_>, _event: &model::Event) {}
fn audit_event(&self, _scope: &ReadScope<'_>, _event: &audit::Event) {}
}
struct OncePanickingEmitter {
calls: AtomicUsize,
}
impl Emitter for OncePanickingEmitter {
fn emit_audit(&self, _event: audit::Event) {
if self.calls.fetch_add(1, Ordering::SeqCst) == 0 {
panic!("one-shot core emitter panic");
}
}
fn emit_error(&self, _error: Arc<dyn Error + Send + Sync>) {}
}
#[test]
fn one_shot_core_emitter_panic_propagates_and_prevents_peer_callback() {
let (session, coord, _, model, tools, call, _, _) = fixtures();
let healthy = Arc::new(HealthyObserver::default());
let mut builder = RegistryBuilder::new();
builder.register(Registration::new("emitting".into()).observer(Arc::new(EmitAuditObserver)));
builder.register(Registration::new("peer".into()).observer(healthy.clone()));
let registry = builder.build_runtime().unwrap();
let emitter = OncePanickingEmitter {
calls: AtomicUsize::new(0),
};
let snapshot = StepSnapshot::freeze(model, RequestPlan::new(), tools);
let observation = lifecycle_observations(&coord, &call).remove(0);
let unwind = std::panic::catch_unwind(AssertUnwindSafe(|| {
for adapter in registry.observer_hooks() {
let run_state = run_state::Store::new();
adapter.invoke_lifecycle(
CallbackResources::new(&coord, &run_state, &session.session_state, None, &emitter),
TranscriptSource::Reader(&session.lease.transcript_reader()),
Some(&snapshot),
&observation,
);
}
}));
assert!(unwind.is_err());
assert_eq!(emitter.calls.load(Ordering::SeqCst), 1);
assert!(healthy.calls.lock().unwrap().is_empty());
}
struct PanickingEmitter;
impl Emitter for PanickingEmitter {
fn emit_audit(&self, _event: audit::Event) {
panic!("core audit boundary panic");
}
fn emit_error(&self, _error: Arc<dyn Error + Send + Sync>) {}
}
#[test]
fn panic_boundary_does_not_swallow_core_or_payload_panics() {
let (session, coord, _, model, tools, call, _, _) = fixtures();
let mut builder = RegistryBuilder::new();
builder.register(Registration::new("panic".into()).observer(Arc::new(PanickingObserver)));
let registry = builder.build_runtime().unwrap();
let emitter = PanickingEmitter;
let snapshot = StepSnapshot::freeze(model, RequestPlan::new(), tools);
let observation = lifecycle_observations(&coord, &call).remove(0);
let run_state = run_state::Store::new();
let result = std::panic::catch_unwind(AssertUnwindSafe(|| {
registry.observer_hooks().next().unwrap().invoke_lifecycle(
CallbackResources::new(&coord, &run_state, &session.session_state, None, &emitter),
TranscriptSource::Reader(&session.lease.transcript_reader()),
Some(&snapshot),
&observation,
);
}));
assert!(result.is_err());
let core = std::panic::catch_unwind(|| panic!("core invariant"));
assert!(core.is_err());
}
struct FailingTeardown(&'static str);
impl AgentTeardown for FailingTeardown {
fn close(&self) -> HookResult<()> {
Err(HookError::new(TestError(self.0)))
}
}
struct PanicOnDropPayload;
impl Drop for PanicOnDropPayload {
fn drop(&mut self) {
panic!("panic payload dropped")
}
}
struct PanicOnDropTeardown;
impl AgentTeardown for PanicOnDropTeardown {
fn close(&self) -> HookResult<()> {
std::panic::panic_any(PanicOnDropPayload)
}
}
#[test]
fn an_attributed_failure_renders_its_registration_and_keeps_the_callback_error() {
let mut builder = RegistryBuilder::new();
builder.register(
Registration::new("teardown".into()).teardown(Arc::new(FailingTeardown("disk full"))),
);
let registry = builder.build_runtime().unwrap();
let failures = registry.close();
assert_eq!(failures.len(), 1);
let failure = &failures[0];
assert_eq!(failure.name(), "teardown");
assert_eq!(failure.to_string(), "hook teardown failed: disk full");
let hook = failure
.source()
.expect("an attributed failure retains the callback error");
assert_eq!(hook.to_string(), "disk full");
assert_eq!(
hook.source()
.and_then(|cause| cause.downcast_ref::<TestError>())
.map(|cause| cause.0),
Some("disk full")
);
}
#[test]
fn close_forgets_panic_on_drop_payload_and_continues_later_hooks() {
let mut builder = RegistryBuilder::new();
builder.register(Registration::new("no-close".into()));
builder.register(Registration::new("first".into()).teardown(Arc::new(FailingTeardown("one"))));
builder.register(
Registration::new("panic-on-drop".into()).teardown(Arc::new(PanicOnDropTeardown)),
);
builder.register(Registration::new("ok".into()).teardown(Arc::new(AllFamilies)));
builder.register(Registration::new("second".into()).teardown(Arc::new(FailingTeardown("two"))));
let failures = builder.build_runtime().unwrap().close();
assert_eq!(failures.len(), 3);
assert_eq!(failures[0].name(), "first");
assert_eq!(failures[0].error().to_string(), "one");
assert_eq!(failures[1].name(), "panic-on-drop");
assert!(
failures[1]
.error()
.source_ref()
.downcast_ref::<ClosePanickedError>()
.is_some()
);
assert_eq!(failures[2].name(), "second");
assert_eq!(failures[2].error().to_string(), "two");
}
#[test]
fn transcript_mutation_views_cover_append_and_replace() {
let append = TranscriptMutation::Append {
previous: Version::ZERO,
version: Version::from_u64(1),
items: vec![],
};
assert_eq!(append.previous_version(), Version::ZERO);
assert_eq!(append.version(), Version::from_u64(1));
assert!(append.items().is_empty());
assert!(append.replacement().is_none());
let replace = TranscriptMutation::Replace {
previous: Version::from_u64(1),
snapshot: Snapshot::new(Version::from_u64(2), vec![]),
};
assert_eq!(replace.previous_version(), Version::from_u64(1));
assert_eq!(replace.version(), Version::from_u64(2));
assert!(replace.items().is_empty());
assert_eq!(
replace.replacement().map(Snapshot::version),
Some(Version::from_u64(2))
);
let published = Arc::new(Mutex::new(0));
let reservation: Box<dyn TranscriptReservation> = Box::new(Published(published.clone()));
reservation.publish();
assert_eq!(*published.lock().unwrap(), 1);
}
struct FamilyProbe {
calls: Arc<Mutex<Vec<&'static str>>>,
fail: bool,
cancel_on_call: Mutex<Option<Token>>,
}
impl FamilyProbe {
fn new(calls: &Arc<Mutex<Vec<&'static str>>>) -> Arc<Self> {
Arc::new(Self {
calls: calls.clone(),
fail: false,
cancel_on_call: Mutex::new(None),
})
}
fn failing(calls: &Arc<Mutex<Vec<&'static str>>>) -> Arc<Self> {
Arc::new(Self {
calls: calls.clone(),
fail: true,
cancel_on_call: Mutex::new(None),
})
}
fn cancelling(calls: &Arc<Mutex<Vec<&'static str>>>) -> Arc<Self> {
Arc::new(Self {
calls: calls.clone(),
fail: false,
cancel_on_call: Mutex::new(None),
})
}
fn arm(&self) -> Token {
let token = Token::new();
*self.cancel_on_call.lock().unwrap() = Some(token.clone());
token
}
fn enter(&self, name: &'static str) {
self.calls.lock().unwrap().push(name);
if let Some(token) = self.cancel_on_call.lock().unwrap().as_ref() {
token.cancel();
}
}
}
impl InputHooks for FamilyProbe {
fn before_input_admit(&self, scope: &InputAdmitScope<'_>) -> InputAdmitDecision {
scope.transcript().expect("input scope has transcript");
self.enter("before_input_admit");
InputAdmitDecision::Reject("probe".into())
}
}
impl ModelHooks for FamilyProbe {
fn before_model(&self, scope: &BeforeModelScope<'_>) -> HookResult<BeforeModelDecision> {
scope
.transcript()
.expect("before-model scope has transcript");
self.enter("before_model");
if self.fail {
return Err(HookError::new(TestError("before-model failed")));
}
Ok(BeforeModelDecision::Abort("probe".into()))
}
fn after_model(&self, scope: &AfterModelScope<'_>) -> AfterModelDecision {
scope
.transcript()
.expect("after-model scope has transcript");
self.enter("after_model");
AfterModelDecision::Continue
}
}
impl ToolHooks for FamilyProbe {
fn before_tool_call(&self, scope: &ToolCallScope<'_>) -> HookResult<ToolCallDecision> {
scope.transcript().expect("tool-call scope has transcript");
self.enter("before_tool_call");
Ok(ToolCallDecision::Block("probe".into()))
}
fn before_tool_result(&self, scope: &ToolResultScope<'_>) -> HookResult<ToolResultDecision> {
scope
.transcript()
.expect("tool-result scope has transcript");
self.enter("before_tool_result");
Ok(ToolResultDecision::Continue)
}
fn after_tool_commit(&self, scope: &AfterToolCommitScope<'_>) -> Vec<Item> {
scope.transcript().expect("after-tool scope has transcript");
self.enter("after_tool_commit");
vec![Item::System(System::new(
"after".into(),
"durable".into(),
ModelVisibility::Visible,
))]
}
}
impl TranscriptHooks for FamilyProbe {
fn reserve(
&self,
scope: &ReadScope<'_>,
_mutation: &TranscriptMutation,
) -> HookResult<Box<dyn TranscriptReservation>> {
scope
.transcript()
.expect("reservation scope has transcript");
self.enter("reserve");
if self.fail {
return Err(HookError::new(TestError("reserve failed")));
}
Ok(Box::new(Published(Arc::new(Mutex::new(0)))))
}
}
fn probe_registry(build: impl FnOnce(&mut RegistryBuilder)) -> RuntimeRegistry {
let mut builder = RegistryBuilder::new();
build(&mut builder);
builder.build_runtime().unwrap()
}
#[test]
fn family_adapters_return_decisions_and_reservations_for_one_registration() {
let (session, coord, emitter, model, tools, call, result, output) = fixtures();
let calls = Arc::new(Mutex::new(Vec::new()));
let probe = FamilyProbe::new(&calls);
let registry = probe_registry(|builder| {
builder.register(
Registration::new("probe".into())
.input(probe.clone())
.model(probe.clone())
.tool(probe.clone())
.transcript(probe.clone()),
);
});
let snapshot = StepSnapshot::freeze(model, RequestPlan::new(), tools);
let request = RequestPlan::new().try_into_request(vec![]).unwrap();
let staged_store = store::Store::new();
let staged = StagedTranscript::from_store(&staged_store);
let run_state = run_state::Store::new();
let resources =
CallbackResources::new(&coord, &run_state, &session.session_state, None, &emitter);
let token = Token::new();
let input = registry.input_hooks().next().unwrap();
assert!(matches!(
input
.invoke_before_input_admit(
resources,
&session.lease.transcript_reader(),
InputTarget::NextStep,
&[],
&token,
)
.unwrap(),
InputAdmitDecision::Reject(reason) if reason == "probe"
));
let models = registry.model_hooks().next().unwrap();
assert!(matches!(
models
.invoke_before_model(resources, &snapshot, &request, &token)
.unwrap(),
BeforeModelDecision::Abort(reason) if reason == "probe"
));
assert!(matches!(
models
.invoke_after_model(
resources,
&snapshot,
&output,
true,
"stop",
CompletionReason::Stop,
&token,
)
.unwrap(),
AfterModelDecision::Continue
));
let tool = registry.tool_hooks().next().unwrap();
assert!(matches!(
tool.invoke_before_tool_call(resources, &snapshot, &call, &token)
.unwrap(),
ToolCallDecision::Block(reason) if reason == "probe"
));
assert!(matches!(
tool.invoke_before_tool_result(
resources,
&snapshot,
&call,
CallDisposition::Executed,
&result,
&token,
)
.unwrap(),
ToolResultDecision::Continue
));
assert_eq!(
tool.invoke_after_tool_commit(resources, &staged, &snapshot, &result)
.len(),
1
);
let transcripts = registry.transcript_hooks().next().unwrap();
let reservation = transcripts
.invoke_reserve(
resources,
snapshot.tools().transcript(),
&TranscriptMutation::Append {
previous: Version::ZERO,
version: Version::from_u64(1),
items: vec![],
},
)
.unwrap();
reservation.publish();
assert_eq!(
*calls.lock().unwrap(),
vec![
"before_input_admit",
"before_model",
"after_model",
"before_tool_call",
"before_tool_result",
"after_tool_commit",
"reserve",
]
);
}
#[test]
fn family_adapters_report_callback_errors_with_the_registration_name() {
let (session, coord, emitter, model, tools, _, _, _) = fixtures();
let calls = Arc::new(Mutex::new(Vec::new()));
let probe = FamilyProbe::failing(&calls);
let registry = probe_registry(|builder| {
builder.register(
Registration::new("failing".into())
.model(probe.clone())
.transcript(probe.clone()),
);
});
let snapshot = StepSnapshot::freeze(model, RequestPlan::new(), tools);
let request = RequestPlan::new().try_into_request(vec![]).unwrap();
let run_state = run_state::Store::new();
let resources =
CallbackResources::new(&coord, &run_state, &session.session_state, None, &emitter);
let token = Token::new();
let model_error = dispatch_error(
registry
.model_hooks()
.next()
.unwrap()
.invoke_before_model(resources, &snapshot, &request, &token),
);
match model_error {
DispatchError::Callback(failure) => {
assert_eq!(failure.name(), "failing");
assert_eq!(failure.error().to_string(), "before-model failed");
}
DispatchError::Cancelled => panic!("a real callback error outranks cancellation"),
}
let reserve_error = dispatch_error(registry.transcript_hooks().next().unwrap().invoke_reserve(
resources,
snapshot.tools().transcript(),
&TranscriptMutation::Append {
previous: Version::ZERO,
version: Version::from_u64(1),
items: vec![],
},
));
match reserve_error {
DispatchError::Callback(failure) => {
assert_eq!(failure.name(), "failing");
assert_eq!(failure.error().to_string(), "reserve failed");
}
DispatchError::Cancelled => panic!("a real callback error outranks cancellation"),
}
}
#[test]
fn family_adapters_fail_closed_before_and_after_each_cancellable_callback() {
let (session, coord, emitter, model, tools, call, result, output) = fixtures();
let calls = Arc::new(Mutex::new(Vec::new()));
let probe = FamilyProbe::cancelling(&calls);
let registry = probe_registry(|builder| {
builder.register(
Registration::new("probe".into())
.input(probe.clone())
.model(probe.clone())
.tool(probe.clone()),
);
});
let snapshot = StepSnapshot::freeze(model, RequestPlan::new(), tools);
let request = RequestPlan::new().try_into_request(vec![]).unwrap();
let run_state = run_state::Store::new();
let resources =
CallbackResources::new(&coord, &run_state, &session.session_state, None, &emitter);
let input = registry.input_hooks().next().unwrap();
let models = registry.model_hooks().next().unwrap();
let tool = registry.tool_hooks().next().unwrap();
let token = probe.arm();
assert!(matches!(
dispatch_error(input.invoke_before_input_admit(
resources,
&session.lease.transcript_reader(),
InputTarget::NextStep,
&[],
&token
)),
DispatchError::Cancelled
));
let token = probe.arm();
assert!(matches!(
dispatch_error(models.invoke_before_model(resources, &snapshot, &request, &token)),
DispatchError::Cancelled
));
let token = probe.arm();
assert!(matches!(
dispatch_error(models.invoke_after_model(
resources,
&snapshot,
&output,
false,
"stop",
CompletionReason::Stop,
&token,
)),
DispatchError::Cancelled
));
let token = probe.arm();
assert!(matches!(
dispatch_error(tool.invoke_before_tool_call(resources, &snapshot, &call, &token)),
DispatchError::Cancelled
));
let token = probe.arm();
assert!(matches!(
dispatch_error(tool.invoke_before_tool_result(
resources,
&snapshot,
&call,
CallDisposition::Executed,
&result,
&token,
)),
DispatchError::Cancelled
));
assert_eq!(
*calls.lock().unwrap(),
vec![
"before_input_admit",
"before_model",
"after_model",
"before_tool_call",
"before_tool_result",
],
"every cancellable callback ran exactly once before its cancellation"
);
let cancelled = Token::new();
cancelled.cancel();
*probe.cancel_on_call.lock().unwrap() = None;
let callbacks_before = calls.lock().unwrap().len();
assert!(matches!(
dispatch_error(input.invoke_before_input_admit(
resources,
&session.lease.transcript_reader(),
InputTarget::NextStep,
&[],
&cancelled
)),
DispatchError::Cancelled
));
assert!(matches!(
dispatch_error(models.invoke_before_model(resources, &snapshot, &request, &cancelled)),
DispatchError::Cancelled
));
assert!(matches!(
dispatch_error(models.invoke_after_model(
resources,
&snapshot,
&output,
false,
"stop",
CompletionReason::Stop,
&cancelled,
)),
DispatchError::Cancelled
));
assert!(matches!(
dispatch_error(tool.invoke_before_tool_call(resources, &snapshot, &call, &cancelled)),
DispatchError::Cancelled
));
assert!(matches!(
dispatch_error(tool.invoke_before_tool_result(
resources,
&snapshot,
&call,
CallDisposition::Executed,
&result,
&cancelled,
)),
DispatchError::Cancelled
));
assert_eq!(
calls.lock().unwrap().len(),
callbacks_before,
"an already-cancelled adapter must invoke no callback"
);
}
struct DefaultAbortReservation(Arc<Mutex<usize>>);
impl TranscriptReservation for DefaultAbortReservation {
fn publish(self: Box<Self>) {
*self.0.lock().unwrap() += 1;
}
}
#[test]
fn the_default_reservation_abort_publishes_nothing() {
let published = Arc::new(Mutex::new(0));
let reservation: Box<dyn TranscriptReservation> =
Box::new(DefaultAbortReservation(published.clone()));
reservation.abort();
assert_eq!(
*published.lock().unwrap(),
0,
"the default abort must not publish"
);
}
#[test]
fn every_scope_debug_locates_the_callback_and_elides_its_capabilities() {
let (capabilities, coord, emitter, model, tools, call, result, output) = fixtures();
let factory = capabilities.factory(&coord, &emitter, true);
let staged_store = store::Store::new();
let staged = StagedTranscript::from_store(&staged_store);
let base = Snapshot::new(Version::ZERO, vec![]);
let parts = vec![Part::text(SECRET.into())];
let items = vec![Item::System(System::new(
"instruction".into(),
SECRET.into(),
ModelVisibility::Visible,
))];
let stage = capabilities.lease.state().staged();
let mut plan = RequestPlan::new();
plan.push_ephemeral(Part::text(SECRET.into()));
let snapshot = StepSnapshot::freeze(model.clone(), plan.clone(), tools);
let request = RequestPlan::new().try_into_request(vec![]).unwrap();
assert_states(
&factory.prepare(&staged, Some(&stage), &base, &parts),
&["PrepareScope", r#"run_id: "run""#, "text: <21 bytes>"],
);
assert_states(
&factory.compose_continuation(&staged, &parts, 2, 3),
&["ContinuationScope", "continuations: 2", "turns: 3", ".."],
);
assert_states(&factory.decide_stop(), &["StopDecisionScope", ".."]);
assert_states(
&factory.input_admit(InputTarget::NextStep, &items),
&["InputAdmitScope", "target: NextStep", "text: <21 bytes>"],
);
assert_states(
&factory.turn(&staged, &items),
&["TurnScope", "text: <21 bytes>"],
);
{
let mut before = plan.clone();
assert_states(
&factory.before_step(&staged, &model, &mut before),
&["BeforeStepScope", r#"id: "model""#],
);
}
assert_states(
&factory.after_step(&staged, &snapshot),
&["AfterStepScope", r#"id: "model""#],
);
assert_states(
&factory.before_model(&snapshot, &request),
&["BeforeModelScope", "Request"],
);
assert_states(
&factory.after_model(&snapshot, &output, true, "stop", CompletionReason::Stop),
&[
"AfterModelScope",
"generated: true",
r#"finish_reason: "stop""#,
"completion: Stop",
],
);
assert_states(
&factory.tool_call(&snapshot, &call),
&["ToolCallScope", r#"Id("call")"#],
);
assert_states(
&factory.tool_result(&snapshot, &call, CallDisposition::Executed, &result),
&["ToolResultScope", "disposition: Executed", "NoOutput"],
);
assert_states(
&factory.after_tool_commit(&staged, &snapshot, &result),
&["AfterToolCommitScope", "NoOutput"],
);
assert_states(
&factory.observer(Some(&snapshot)),
&["ObserverScope", "snapshot: <present>", ".."],
);
assert_states(
&factory.observer(None),
&["ObserverScope", "snapshot: <absent>"],
);
assert_states(
&factory.read(),
&["ReadScope", r#"run_id: "run""#, "Reader { slots: 0 }"],
);
for rendering in [
format!(
"{:?}",
factory.prepare(&staged, Some(&stage), &base, &parts)
),
format!(
"{:#?}",
factory.prepare(&staged, Some(&stage), &base, &parts)
),
format!("{:?}", factory.after_step(&staged, &snapshot)),
format!("{:#?}", factory.after_step(&staged, &snapshot)),
format!("{:?}", factory.turn(&staged, &items)),
format!("{:#?}", factory.turn(&staged, &items)),
] {
assert!(!rendering.contains(SECRET), "a scope leaked: {rendering}");
}
assert_states(&staged, &["StagedTranscript", "Appender", "Replacer"]);
assert_states(&snapshot, &["StepSnapshot", "ephemeral"]);
}
#[test]
fn lifecycle_end_debug_renders_its_cause_through_display() {
assert_eq!(format!("{:?}", LifecycleEnd::Completed), "Completed");
assert_eq!(
format!("{:?}", LifecycleEnd::Stopped { code: None }),
"Stopped { code: None }"
);
assert_eq!(
format!(
"{:?}",
LifecycleEnd::Stopped {
code: Some(crate::runtime::event::StopCode::new("done".into()).unwrap()),
}
),
r#"Stopped { code: Some(StopCode("done")) }"#
);
assert_eq!(
format!(
"{:?}",
LifecycleEnd::Cancelled {
error: Arc::new(TestError("cancelled")),
}
),
r#"Cancelled { error: "cancelled" }"#
);
assert_eq!(
format!(
"{:?}",
LifecycleEnd::Failed {
error: Arc::new(TestError("failed")),
}
),
r#"Failed { error: "failed" }"#
);
}
#[test]
fn decision_debug_keeps_operator_reasons_and_redacts_model_payloads() {
let coord = Coord::at_run("session".into(), "run".into()).unwrap();
let call = ToolCall::new(
Id::new("call".into()),
tool_call::Name::new("tool".into()),
Payload::copy_from_slice(SECRET.as_bytes()),
);
let assistant = Assistant::new(
vec![crate::value::content::Segment::Text {
text: SECRET.into(),
extensions: crate::value::content::SegmentExtensions::new(SECRET.into(), None, None),
}],
None,
);
let parts = vec![Part::text(SECRET.into())];
for decision in [
PrepareDecision::Continue,
PrepareDecision::Modify(parts.clone()),
] {
assert_redacted(&decision);
}
assert_states(
&PrepareDecision::Modify(parts.clone()),
&["Modify", "text: <21 bytes>"],
);
assert_redacted(&ContinuationDecision::Replace(parts.clone()));
assert_redacted(&BeforeModelDecision::ShortCircuit(assistant.clone()));
assert_redacted(&AfterModelDecision::Modify(assistant));
assert_redacted(&ToolCallDecision::Modify(call));
assert_redacted(&ToolResultDecision::Modify(ToolResult::success_text(
SECRET.into(),
)));
assert_redacted(&InputAdmitDecision::Modify(vec![Item::System(
System::new(
"instruction".into(),
SECRET.into(),
ModelVisibility::Visible,
),
)]));
assert_redacted(&AfterStepDecision::Continue(vec![]));
assert_eq!(
format!("{:?}", BeforeModelDecision::Abort("budget spent".into())),
r#"Abort("budget spent")"#
);
assert_eq!(
format!("{:?}", ToolCallDecision::Block("not approved".into())),
r#"Block("not approved")"#
);
assert_eq!(
format!("{:?}", InputAdmitDecision::Reject("closed".into())),
r#"Reject("closed")"#
);
assert_eq!(
format!(
"{:?}",
StopDecision::Stop(crate::runtime::event::StopCode::new("done".into()).unwrap())
),
r#"Stop(StopCode("done"))"#
);
assert_states(
&LifecycleObservation::RunStart {
coord,
input: parts,
},
&["RunStart", "text: <21 bytes>"],
);
}
#[test]
fn registration_registry_and_builder_debug_state_the_roster() {
let registration = Registration::new("recorder".to_owned())
.requires_session_state(true)
.observer(Arc::new(HealthyObserver {
calls: Mutex::new(Vec::new()),
}));
assert_states(
®istration,
&[
r#"name: "recorder""#,
"requires_session_state: true",
"observer: <present>",
"run: <absent>",
"input: <absent>",
"turn_step: <absent>",
"model: <absent>",
"tool: <absent>",
"transcript: <absent>",
"teardown: <absent>",
],
);
let mut builder = RegistryBuilder::new();
assert_eq!(format!("{builder:?}"), "RegistryBuilder { hooks: [] }");
builder.register(registration);
builder.register(Registration::new("second".to_owned()));
assert_eq!(
format!("{builder:?}"),
r#"RegistryBuilder { hooks: ["recorder", "second"] }"#
);
let registry = builder.build_runtime().expect("distinct names build");
assert_eq!(
format!("{registry:?}"),
r#"RuntimeRegistry { hooks: ["recorder", "second"] }"#
);
}