use std::fmt;
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum Observation {
ParseStarted,
ParseSucceeded,
ParseFailed,
RunStarted,
RunSucceeded,
RunFailed,
SectionStarted,
SectionFinished,
ModelTurnCompleted,
ModelTurnFailed,
ModelTurnTruncated,
ToolCallSucceeded,
ToolCallFailed,
LuaCompilationStarted,
LuaCompilationSucceeded,
LuaCompilationFailed,
LuaSharedLoadStarted,
LuaSharedLoadSucceeded,
LuaSharedLoadFailed,
LuaPrologueStarted,
LuaPrologueSucceeded,
LuaPrologueFailed,
LuaReplyBindingStarted,
LuaReplyBindingSucceeded,
LuaReplyBindingFailed,
LuaEpilogStarted,
LuaEpilogSucceeded,
LuaEpilogFailed,
LuaTeardownStarted,
LuaTeardownSucceeded,
ToolRegistryValidationStarted,
ToolRegistryValidationSucceeded,
ToolRegistryValidationFailed,
ToolScopeClosing,
ToolScopeClosed,
ToolScopeFailed,
ToolScopeValidationStarted,
ToolScopeValidationSucceeded,
ToolScopeValidationFailed,
ModelCatalogValidationStarted,
ModelCatalogValidationSucceeded,
ModelCatalogValidationFailed,
ModelScopeClosing,
ModelScopeClosed,
ModelScopeFailed,
StoreWriteSucceeded,
StoreWriteFailed,
StoreAppendSucceeded,
StoreAppendFailed,
StoreReadLinesSucceeded,
StoreReadLinesFailed,
StoreReadSucceeded,
StoreReadFailed,
StoreInjectSucceeded,
StoreInjectFailed,
StoreReplaceSucceeded,
StoreReplaceFailed,
StoreDeleteSucceeded,
StoreDeleteFailed,
StoreGlobSucceeded,
StoreGlobFailed,
FanoutArmStarted,
FanoutArmFinished,
FanoutArmSucceeded,
FanoutArmExhausted,
FanoutArmFailed,
FanoutArmCancelled,
Lua(String),
Other(String),
}
impl Observation {
#[must_use]
pub fn label(&self) -> Option<&'static str> {
let label = match self {
Observation::ParseStarted => "Parse started",
Observation::ParseSucceeded => "Parse succeeded",
Observation::ParseFailed => "Parse failed",
Observation::RunStarted => "Run started",
Observation::RunSucceeded => "Run succeeded",
Observation::RunFailed => "Run failed",
Observation::SectionStarted => "Section started",
Observation::SectionFinished => "Section finished",
Observation::ModelTurnCompleted => "Model turn completed",
Observation::ModelTurnFailed => "Model turn failed",
Observation::ModelTurnTruncated => "Model turn truncated",
Observation::ToolCallSucceeded => "Tool call succeeded",
Observation::ToolCallFailed => "Tool call failed",
Observation::LuaCompilationStarted => "Lua compilation started",
Observation::LuaCompilationSucceeded => "Lua compilation succeeded",
Observation::LuaCompilationFailed => "Lua compilation failed",
Observation::LuaSharedLoadStarted => "Lua shared load started",
Observation::LuaSharedLoadSucceeded => "Lua shared load succeeded",
Observation::LuaSharedLoadFailed => "Lua shared load failed",
Observation::LuaPrologueStarted => "Lua prologue started",
Observation::LuaPrologueSucceeded => "Lua prologue succeeded",
Observation::LuaPrologueFailed => "Lua prologue failed",
Observation::LuaReplyBindingStarted => "Lua reply binding started",
Observation::LuaReplyBindingSucceeded => "Lua reply binding succeeded",
Observation::LuaReplyBindingFailed => "Lua reply binding failed",
Observation::LuaEpilogStarted => "Lua epilog started",
Observation::LuaEpilogSucceeded => "Lua epilog succeeded",
Observation::LuaEpilogFailed => "Lua epilog failed",
Observation::LuaTeardownStarted => "Lua teardown started",
Observation::LuaTeardownSucceeded => "Lua teardown succeeded",
Observation::ToolRegistryValidationStarted => "Tool registry validation started",
Observation::ToolRegistryValidationSucceeded => "Tool registry validation succeeded",
Observation::ToolRegistryValidationFailed => "Tool registry validation failed",
Observation::ToolScopeClosing => "Tool scope closing",
Observation::ToolScopeClosed => "Tool scope closed",
Observation::ToolScopeFailed => "Tool scope failed",
Observation::ToolScopeValidationStarted => "Tool scope validation started",
Observation::ToolScopeValidationSucceeded => "Tool scope validation succeeded",
Observation::ToolScopeValidationFailed => "Tool scope validation failed",
Observation::ModelCatalogValidationStarted => "Model catalog validation started",
Observation::ModelCatalogValidationSucceeded => "Model catalog validation succeeded",
Observation::ModelCatalogValidationFailed => "Model catalog validation failed",
Observation::ModelScopeClosing => "Model scope closing",
Observation::ModelScopeClosed => "Model scope closed",
Observation::ModelScopeFailed => "Model scope failed",
Observation::StoreWriteSucceeded => "Store write succeeded",
Observation::StoreWriteFailed => "Store write failed",
Observation::StoreAppendSucceeded => "Store append succeeded",
Observation::StoreAppendFailed => "Store append failed",
Observation::StoreReadLinesSucceeded => "Store read_lines succeeded",
Observation::StoreReadLinesFailed => "Store read_lines failed",
Observation::StoreReadSucceeded => "Store read succeeded",
Observation::StoreReadFailed => "Store read failed",
Observation::StoreInjectSucceeded => "Store inject succeeded",
Observation::StoreInjectFailed => "Store inject failed",
Observation::StoreReplaceSucceeded => "Store replace succeeded",
Observation::StoreReplaceFailed => "Store replace failed",
Observation::StoreDeleteSucceeded => "Store delete succeeded",
Observation::StoreDeleteFailed => "Store delete failed",
Observation::StoreGlobSucceeded => "Store glob succeeded",
Observation::StoreGlobFailed => "Store glob failed",
Observation::FanoutArmStarted => "Fanout arm started",
Observation::FanoutArmFinished => "Fanout arm finished",
Observation::FanoutArmSucceeded => "Fanout arm succeeded",
Observation::FanoutArmExhausted => "Fanout arm exhausted",
Observation::FanoutArmFailed => "Fanout arm failed",
Observation::FanoutArmCancelled => "Fanout arm cancelled",
Observation::Lua(_) | Observation::Other(_) => return None,
};
Some(label)
}
}
impl fmt::Display for Observation {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Observation::Lua(message) => write!(f, "Lua: {message}"),
Observation::Other(message) => f.write_str(message),
fixed => f.write_str(fixed.label().unwrap_or_default()),
}
}
}
pub(crate) mod detail {
use super::Observation;
pub(crate) const PARSE_STARTED: Observation = Observation::ParseStarted;
pub(crate) const PARSE_SUCCEEDED: Observation = Observation::ParseSucceeded;
pub(crate) const PARSE_FAILED: Observation = Observation::ParseFailed;
pub(crate) const RUN_STARTED: Observation = Observation::RunStarted;
pub(crate) const RUN_SUCCEEDED: Observation = Observation::RunSucceeded;
pub(crate) const RUN_FAILED: Observation = Observation::RunFailed;
pub(crate) const SECTION_STARTED: Observation = Observation::SectionStarted;
pub(crate) const SECTION_FINISHED: Observation = Observation::SectionFinished;
pub(crate) const MODEL_TURN_COMPLETED: Observation = Observation::ModelTurnCompleted;
pub(crate) const MODEL_TURN_FAILED: Observation = Observation::ModelTurnFailed;
pub(crate) const MODEL_TURN_TRUNCATED: Observation = Observation::ModelTurnTruncated;
pub(crate) const TOOL_CALL_SUCCEEDED: Observation = Observation::ToolCallSucceeded;
pub(crate) const TOOL_CALL_FAILED: Observation = Observation::ToolCallFailed;
pub(crate) const LUA_COMPILATION_STARTED: Observation = Observation::LuaCompilationStarted;
pub(crate) const LUA_COMPILATION_SUCCEEDED: Observation = Observation::LuaCompilationSucceeded;
pub(crate) const LUA_COMPILATION_FAILED: Observation = Observation::LuaCompilationFailed;
pub(crate) const LUA_SHARED_LOAD_STARTED: Observation = Observation::LuaSharedLoadStarted;
pub(crate) const LUA_SHARED_LOAD_SUCCEEDED: Observation = Observation::LuaSharedLoadSucceeded;
pub(crate) const LUA_SHARED_LOAD_FAILED: Observation = Observation::LuaSharedLoadFailed;
pub(crate) const LUA_PROLOGUE_STARTED: Observation = Observation::LuaPrologueStarted;
pub(crate) const LUA_PROLOGUE_SUCCEEDED: Observation = Observation::LuaPrologueSucceeded;
pub(crate) const LUA_PROLOGUE_FAILED: Observation = Observation::LuaPrologueFailed;
pub(crate) const LUA_REPLY_BINDING_STARTED: Observation = Observation::LuaReplyBindingStarted;
pub(crate) const LUA_REPLY_BINDING_SUCCEEDED: Observation =
Observation::LuaReplyBindingSucceeded;
pub(crate) const LUA_REPLY_BINDING_FAILED: Observation = Observation::LuaReplyBindingFailed;
pub(crate) const LUA_EPILOG_STARTED: Observation = Observation::LuaEpilogStarted;
pub(crate) const LUA_EPILOG_SUCCEEDED: Observation = Observation::LuaEpilogSucceeded;
pub(crate) const LUA_EPILOG_FAILED: Observation = Observation::LuaEpilogFailed;
pub(crate) const LUA_TEARDOWN_STARTED: Observation = Observation::LuaTeardownStarted;
pub(crate) const LUA_TEARDOWN_SUCCEEDED: Observation = Observation::LuaTeardownSucceeded;
pub(crate) const TOOL_SCOPE_CLOSING: Observation = Observation::ToolScopeClosing;
pub(crate) const TOOL_SCOPE_CLOSED: Observation = Observation::ToolScopeClosed;
pub(crate) const TOOL_SCOPE_FAILED: Observation = Observation::ToolScopeFailed;
pub(crate) const TOOL_SCOPE_VALIDATION_STARTED: Observation =
Observation::ToolScopeValidationStarted;
pub(crate) const TOOL_SCOPE_VALIDATION_SUCCEEDED: Observation =
Observation::ToolScopeValidationSucceeded;
pub(crate) const TOOL_SCOPE_VALIDATION_FAILED: Observation =
Observation::ToolScopeValidationFailed;
pub(crate) const MODEL_SCOPE_CLOSING: Observation = Observation::ModelScopeClosing;
pub(crate) const MODEL_SCOPE_CLOSED: Observation = Observation::ModelScopeClosed;
pub(crate) const MODEL_SCOPE_FAILED: Observation = Observation::ModelScopeFailed;
pub(crate) const STORE_WRITE_SUCCEEDED: Observation = Observation::StoreWriteSucceeded;
pub(crate) const STORE_WRITE_FAILED: Observation = Observation::StoreWriteFailed;
pub(crate) const STORE_APPEND_SUCCEEDED: Observation = Observation::StoreAppendSucceeded;
pub(crate) const STORE_APPEND_FAILED: Observation = Observation::StoreAppendFailed;
pub(crate) const STORE_READ_LINES_SUCCEEDED: Observation = Observation::StoreReadLinesSucceeded;
pub(crate) const STORE_READ_LINES_FAILED: Observation = Observation::StoreReadLinesFailed;
pub(crate) const STORE_READ_SUCCEEDED: Observation = Observation::StoreReadSucceeded;
pub(crate) const STORE_READ_FAILED: Observation = Observation::StoreReadFailed;
pub(crate) const STORE_INJECT_SUCCEEDED: Observation = Observation::StoreInjectSucceeded;
pub(crate) const STORE_INJECT_FAILED: Observation = Observation::StoreInjectFailed;
pub(crate) const STORE_REPLACE_SUCCEEDED: Observation = Observation::StoreReplaceSucceeded;
pub(crate) const STORE_REPLACE_FAILED: Observation = Observation::StoreReplaceFailed;
pub(crate) const STORE_DELETE_SUCCEEDED: Observation = Observation::StoreDeleteSucceeded;
pub(crate) const STORE_DELETE_FAILED: Observation = Observation::StoreDeleteFailed;
pub(crate) const STORE_GLOB_SUCCEEDED: Observation = Observation::StoreGlobSucceeded;
pub(crate) const STORE_GLOB_FAILED: Observation = Observation::StoreGlobFailed;
pub(crate) const FANOUT_ARM_STARTED: Observation = Observation::FanoutArmStarted;
pub(crate) const FANOUT_ARM_SUCCEEDED: Observation = Observation::FanoutArmSucceeded;
pub(crate) const FANOUT_ARM_EXHAUSTED: Observation = Observation::FanoutArmExhausted;
pub(crate) const FANOUT_ARM_FAILED: Observation = Observation::FanoutArmFailed;
pub(crate) const FANOUT_ARM_CANCELLED: Observation = Observation::FanoutArmCancelled;
}
pub trait Observer: Send + Sync {
fn observe(&self, execution: &str, section: &str, event: Observation);
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
#[non_exhaustive]
pub struct NullObserver;
impl Observer for NullObserver {
fn observe(&self, _execution: &str, _section: &str, _event: Observation) {}
}
#[cfg(test)]
mod tests {
use std::sync::{Arc, Barrier, Mutex};
use super::*;
#[test]
fn null_observer_accepts_reports() {
let observer = NullObserver;
observer.observe("example-run", "Prompt", Observation::RunStarted);
observer.observe("example-run", "Gather", Observation::SectionStarted);
observer.observe("example-run", "Gather", Observation::SectionFinished);
observer.observe("example-run", "Prompt", Observation::RunSucceeded);
}
#[test]
fn display_renders_stable_strings() {
assert_eq!(Observation::RunStarted.to_string(), "Run started");
assert_eq!(
Observation::StoreReadLinesSucceeded.to_string(),
"Store read_lines succeeded"
);
assert_eq!(Observation::Lua("hi".to_owned()).to_string(), "Lua: hi");
assert_eq!(Observation::Other("x".to_owned()).to_string(), "x");
assert_eq!(Observation::RunStarted.label(), Some("Run started"));
assert_eq!(Observation::Lua("hi".to_owned()).label(), None);
}
#[test]
fn observer_is_dyn_compatible_and_shareable() {
fn assert_send_sync<T: Send + Sync + ?Sized>() {}
assert_send_sync::<dyn Observer>();
let observer: &dyn Observer = &NullObserver;
observer.observe("example-run", "Gather", Observation::SectionFinished);
}
#[derive(Default)]
struct Recorder(Mutex<Vec<(String, String, Observation)>>);
impl Observer for Recorder {
fn observe(&self, execution: &str, section: &str, event: Observation) {
self.0
.lock()
.expect("recorder mutex must remain usable")
.push((execution.to_owned(), section.to_owned(), event));
}
}
impl Recorder {
fn records(&self) -> Vec<(String, String, Observation)> {
self.0
.lock()
.expect("recorder mutex must remain usable")
.clone()
}
}
#[test]
fn unknown_and_message_variants_are_tolerated_by_a_wildcard_consumer() {
fn classify(event: &Observation) -> &'static str {
match event {
Observation::RunStarted => "known-fixed",
Observation::Lua(_) => "lua-checkpoint",
_ => "unknown-or-other",
}
}
assert_eq!(classify(&Observation::RunStarted), "known-fixed");
assert_eq!(
classify(&Observation::Lua("hi".to_owned())),
"lua-checkpoint"
);
assert_eq!(
classify(&Observation::Other("future".to_owned())),
"unknown-or-other"
);
assert_eq!(classify(&Observation::SectionFinished), "unknown-or-other");
assert_eq!(
Observation::Lua("secret note".to_owned()).to_string(),
"Lua: secret note"
);
assert_eq!(
Observation::Other("verbatim".to_owned()).to_string(),
"verbatim"
);
}
#[test]
fn parse_failure_pairs_started_with_failed_and_carries_author_labels() {
use crate::parser::Prompt;
let recorder = Recorder::default();
let execution = "author/controlled:run id";
let _ = Prompt::parse("no frontmatter here", execution, &recorder)
.expect_err("a source without frontmatter must fail to parse");
let records = recorder.records();
assert_eq!(
records.first().map(|(_, _, event)| event),
Some(&Observation::ParseStarted),
"the lifecycle must open with ParseStarted: {records:?}"
);
assert_eq!(
records.last().map(|(_, _, event)| event),
Some(&Observation::ParseFailed),
"a failed parse must close with ParseFailed: {records:?}"
);
assert!(
records
.iter()
.all(|(seen_execution, _, _)| seen_execution == execution),
"the author-controlled execution id must be carried verbatim: {records:?}"
);
let recorder = Recorder::default();
let source =
"---\nname: greeter\ndescription: d\npromptforge: 1\n---\n\n# T\n\n## S\n\nhi\n";
Prompt::parse(source, execution, &recorder).expect("a well-formed source must parse");
let events: Vec<Observation> = recorder
.records()
.into_iter()
.map(|(_, _, event)| event)
.collect();
assert_eq!(events.first(), Some(&Observation::ParseStarted));
assert_eq!(events.last(), Some(&Observation::ParseSucceeded));
}
#[test]
fn interleaved_reports_stay_correlated_by_execution_and_section() {
#[derive(Default)]
struct Recorder(Mutex<Vec<(String, String, Observation)>>);
impl Observer for Recorder {
fn observe(&self, execution: &str, section: &str, event: Observation) {
self.0
.lock()
.expect("recorder mutex must remain usable")
.push((execution.to_owned(), section.to_owned(), event));
}
}
let recorder = Arc::new(Recorder::default());
let barrier = Arc::new(Barrier::new(2));
let first_recorder = Arc::clone(&recorder);
let first_barrier = Arc::clone(&barrier);
let first = std::thread::spawn(move || {
first_recorder.observe("execution-a", "First", detail::SECTION_STARTED);
first_barrier.wait();
first_barrier.wait();
first_recorder.observe("execution-a", "First", detail::SECTION_FINISHED);
first_barrier.wait();
first_barrier.wait();
});
let second_recorder = Arc::clone(&recorder);
let second = std::thread::spawn(move || {
barrier.wait();
second_recorder.observe("execution-b", "Second", detail::SECTION_STARTED);
barrier.wait();
barrier.wait();
second_recorder.observe("execution-b", "Second", detail::SECTION_FINISHED);
barrier.wait();
});
first.join().expect("first recording thread must finish");
second.join().expect("second recording thread must finish");
assert_eq!(
*recorder
.0
.lock()
.expect("recorder mutex must remain usable"),
[
(
"execution-a".to_owned(),
"First".to_owned(),
Observation::SectionStarted,
),
(
"execution-b".to_owned(),
"Second".to_owned(),
Observation::SectionStarted,
),
(
"execution-a".to_owned(),
"First".to_owned(),
Observation::SectionFinished,
),
(
"execution-b".to_owned(),
"Second".to_owned(),
Observation::SectionFinished,
),
]
);
}
}