use std::fmt;
use std::str::FromStr;
fn normalized_episode_token(input: &str) -> String {
let trimmed = input.trim();
let mut normalized = String::with_capacity(trimmed.len());
let mut previous_was_lowercase = false;
let mut previous_was_separator = false;
for character in trimmed.chars() {
match character {
'-' | '_' => {
if !normalized.is_empty() && !previous_was_separator {
normalized.push('_');
}
previous_was_lowercase = false;
previous_was_separator = true;
}
character if character.is_ascii_uppercase() => {
if previous_was_lowercase && !previous_was_separator {
normalized.push('_');
}
normalized.push(character.to_ascii_lowercase());
previous_was_lowercase = false;
previous_was_separator = false;
}
character => {
normalized.push(character.to_ascii_lowercase());
previous_was_lowercase = character.is_ascii_lowercase();
previous_was_separator = false;
}
}
}
normalized
}
pub const TASK_EPISODE_SCHEMA_V1: &str = "ee.task_episode.v1";
pub const INTERVENTION_SCHEMA_V1: &str = "ee.intervention.v1";
pub const COUNTERFACTUAL_RUN_SCHEMA_V1: &str = "ee.counterfactual_run.v1";
pub const REGRET_LEDGER_SCHEMA_V1: &str = "ee.regret_ledger.v1";
pub const REGRET_ENTRY_SCHEMA_V1: &str = "ee.regret_entry.v1";
pub const EPISODE_ID_PREFIX: &str = "ep_";
pub const INTERVENTION_ID_PREFIX: &str = "int_";
pub const COUNTERFACTUAL_RUN_ID_PREFIX: &str = "cfr_";
pub const REGRET_ENTRY_ID_PREFIX: &str = "reg_";
#[derive(Clone, Debug, Default, Eq, PartialEq)]
pub struct TaskEpisode {
pub schema: &'static str,
pub id: String,
pub workspace_id: Option<String>,
pub session_id: Option<String>,
pub task_input: String,
pub retrieved_memory_ids: Vec<String>,
pub context_pack_id: Option<String>,
pub actions: Vec<EpisodeAction>,
pub outcome: EpisodeOutcome,
pub started_at: String,
pub ended_at: Option<String>,
pub duration_ms: Option<u64>,
pub agent: Option<String>,
pub episode_hash: Option<String>,
}
impl TaskEpisode {
#[must_use]
pub fn new(
id: impl Into<String>,
task_input: impl Into<String>,
started_at: impl Into<String>,
) -> Self {
Self {
schema: TASK_EPISODE_SCHEMA_V1,
id: id.into(),
task_input: task_input.into(),
started_at: started_at.into(),
outcome: EpisodeOutcome::Unknown,
..Default::default()
}
}
#[must_use]
pub fn with_workspace_id(mut self, id: impl Into<String>) -> Self {
self.workspace_id = Some(id.into());
self
}
#[must_use]
pub fn with_session_id(mut self, id: impl Into<String>) -> Self {
self.session_id = Some(id.into());
self
}
#[must_use]
pub fn with_context_pack_id(mut self, id: impl Into<String>) -> Self {
self.context_pack_id = Some(id.into());
self
}
pub fn add_retrieved_memory(&mut self, id: impl Into<String>) {
self.retrieved_memory_ids.push(id.into());
}
pub fn add_action(&mut self, action: EpisodeAction) {
self.actions.push(action);
}
#[must_use]
pub fn with_outcome(mut self, outcome: EpisodeOutcome) -> Self {
self.outcome = outcome;
self
}
#[must_use]
pub fn with_ended_at(mut self, ts: impl Into<String>) -> Self {
self.ended_at = Some(ts.into());
self
}
#[must_use]
pub fn with_duration_ms(mut self, ms: u64) -> Self {
self.duration_ms = Some(ms);
self
}
#[must_use]
pub fn with_agent(mut self, agent: impl Into<String>) -> Self {
self.agent = Some(agent.into());
self
}
#[must_use]
pub fn with_episode_hash(mut self, hash: impl Into<String>) -> Self {
self.episode_hash = Some(hash.into());
self
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct EpisodeAction {
pub sequence: u32,
pub action_type: ActionType,
pub description: String,
pub timestamp: String,
pub succeeded: bool,
pub error: Option<String>,
}
impl EpisodeAction {
#[must_use]
pub fn new(
sequence: u32,
action_type: ActionType,
description: impl Into<String>,
timestamp: impl Into<String>,
) -> Self {
Self {
sequence,
action_type,
description: description.into(),
timestamp: timestamp.into(),
succeeded: true,
error: None,
}
}
#[must_use]
pub fn with_error(mut self, err: impl Into<String>) -> Self {
self.succeeded = false;
self.error = Some(err.into());
self
}
}
#[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq)]
pub enum ActionType {
ToolCall,
Edit,
Command,
Search,
Retrieval,
Output,
#[default]
Other,
}
impl ActionType {
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::ToolCall => "tool_call",
Self::Edit => "edit",
Self::Command => "command",
Self::Search => "search",
Self::Retrieval => "retrieval",
Self::Output => "output",
Self::Other => "other",
}
}
}
impl fmt::Display for ActionType {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(self.as_str())
}
}
impl FromStr for ActionType {
type Err = ParseActionTypeError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match normalized_episode_token(s).as_str() {
"tool_call" => Ok(Self::ToolCall),
"edit" => Ok(Self::Edit),
"command" => Ok(Self::Command),
"search" => Ok(Self::Search),
"retrieval" => Ok(Self::Retrieval),
"output" => Ok(Self::Output),
"other" => Ok(Self::Other),
_ => Err(ParseActionTypeError {
input: s.to_owned(),
}),
}
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ParseActionTypeError {
input: String,
}
impl fmt::Display for ParseActionTypeError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"unknown action type `{}`; expected tool_call, edit, command, search, retrieval, output, or other",
self.input
)
}
}
impl std::error::Error for ParseActionTypeError {}
#[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq)]
pub enum EpisodeOutcome {
Success,
Failure,
Cancelled,
Timeout,
#[default]
Unknown,
}
impl EpisodeOutcome {
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::Success => "success",
Self::Failure => "failure",
Self::Cancelled => "cancelled",
Self::Timeout => "timeout",
Self::Unknown => "unknown",
}
}
#[must_use]
pub const fn is_negative(self) -> bool {
matches!(self, Self::Failure | Self::Cancelled | Self::Timeout)
}
}
impl fmt::Display for EpisodeOutcome {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(self.as_str())
}
}
impl FromStr for EpisodeOutcome {
type Err = ParseEpisodeOutcomeError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match normalized_episode_token(s).as_str() {
"success" => Ok(Self::Success),
"failure" => Ok(Self::Failure),
"cancelled" => Ok(Self::Cancelled),
"timeout" => Ok(Self::Timeout),
"unknown" => Ok(Self::Unknown),
_ => Err(ParseEpisodeOutcomeError {
input: s.to_owned(),
}),
}
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ParseEpisodeOutcomeError {
input: String,
}
impl fmt::Display for ParseEpisodeOutcomeError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"unknown episode outcome `{}`; expected success, failure, cancelled, timeout, or unknown",
self.input
)
}
}
impl std::error::Error for ParseEpisodeOutcomeError {}
#[derive(Clone, Debug, PartialEq)]
pub struct Intervention {
pub schema: &'static str,
pub id: String,
pub intervention_type: InterventionType,
pub target_memory_id: Option<String>,
pub hypothetical_content: Option<String>,
pub score_delta: Option<f64>,
pub description: String,
pub rationale: Option<String>,
pub created_at: String,
pub created_by: Option<String>,
}
impl Intervention {
#[must_use]
pub fn new(
id: impl Into<String>,
intervention_type: InterventionType,
description: impl Into<String>,
created_at: impl Into<String>,
) -> Self {
Self {
schema: INTERVENTION_SCHEMA_V1,
id: id.into(),
intervention_type,
description: description.into(),
created_at: created_at.into(),
target_memory_id: None,
hypothetical_content: None,
score_delta: None,
rationale: None,
created_by: None,
}
}
#[must_use]
pub fn with_target_memory(mut self, id: impl Into<String>) -> Self {
self.target_memory_id = Some(id.into());
self
}
#[must_use]
pub fn with_hypothetical_content(mut self, content: impl Into<String>) -> Self {
self.hypothetical_content = Some(content.into());
self
}
#[must_use]
pub fn with_score_delta(mut self, delta: f64) -> Self {
self.score_delta = Some(delta);
self
}
#[must_use]
pub fn with_rationale(mut self, rationale: impl Into<String>) -> Self {
self.rationale = Some(rationale.into());
self
}
#[must_use]
pub fn with_created_by(mut self, by: impl Into<String>) -> Self {
self.created_by = Some(by.into());
self
}
}
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub enum InterventionType {
AddMemory,
RemoveMemory,
ReplaceContent,
Strengthen,
Weaken,
Rerank,
}
impl InterventionType {
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::AddMemory => "add_memory",
Self::RemoveMemory => "remove_memory",
Self::ReplaceContent => "replace_content",
Self::Strengthen => "strengthen",
Self::Weaken => "weaken",
Self::Rerank => "rerank",
}
}
}
impl fmt::Display for InterventionType {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(self.as_str())
}
}
impl FromStr for InterventionType {
type Err = ParseInterventionTypeError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match normalized_episode_token(s).as_str() {
"add_memory" => Ok(Self::AddMemory),
"remove_memory" => Ok(Self::RemoveMemory),
"replace_content" => Ok(Self::ReplaceContent),
"strengthen" => Ok(Self::Strengthen),
"weaken" => Ok(Self::Weaken),
"rerank" => Ok(Self::Rerank),
_ => Err(ParseInterventionTypeError {
input: s.to_owned(),
}),
}
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ParseInterventionTypeError {
input: String,
}
impl fmt::Display for ParseInterventionTypeError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"unknown intervention type `{}`; expected add_memory, remove_memory, replace_content, strengthen, weaken, or rerank",
self.input
)
}
}
impl std::error::Error for ParseInterventionTypeError {}
#[derive(Clone, Debug, PartialEq)]
pub struct CounterfactualRun {
pub schema: &'static str,
pub id: String,
pub episode_id: String,
pub intervention_ids: Vec<String>,
pub hypothetical_outcome: EpisodeOutcome,
pub confidence: f64,
pub method: CounterfactualMethod,
pub analysis: Option<String>,
pub executed_at: String,
pub analysis_duration_ms: Option<u64>,
}
impl CounterfactualRun {
#[must_use]
pub fn new(
id: impl Into<String>,
episode_id: impl Into<String>,
hypothetical_outcome: EpisodeOutcome,
confidence: f64,
method: CounterfactualMethod,
executed_at: impl Into<String>,
) -> Self {
Self {
schema: COUNTERFACTUAL_RUN_SCHEMA_V1,
id: id.into(),
episode_id: episode_id.into(),
intervention_ids: Vec::new(),
hypothetical_outcome,
confidence,
method,
analysis: None,
executed_at: executed_at.into(),
analysis_duration_ms: None,
}
}
pub fn add_intervention(&mut self, id: impl Into<String>) {
self.intervention_ids.push(id.into());
}
#[must_use]
pub fn with_analysis(mut self, analysis: impl Into<String>) -> Self {
self.analysis = Some(analysis.into());
self
}
#[must_use]
pub fn with_analysis_duration_ms(mut self, ms: u64) -> Self {
self.analysis_duration_ms = Some(ms);
self
}
}
#[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq)]
pub enum CounterfactualMethod {
DeterministicReplay,
HeuristicEstimate,
LlmReasoning,
HumanJudgment,
#[default]
Unknown,
}
impl CounterfactualMethod {
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::DeterministicReplay => "deterministic_replay",
Self::HeuristicEstimate => "heuristic_estimate",
Self::LlmReasoning => "llm_reasoning",
Self::HumanJudgment => "human_judgment",
Self::Unknown => "unknown",
}
}
}
impl fmt::Display for CounterfactualMethod {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(self.as_str())
}
}
impl FromStr for CounterfactualMethod {
type Err = ParseCounterfactualMethodError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match normalized_episode_token(s).as_str() {
"deterministic_replay" => Ok(Self::DeterministicReplay),
"heuristic_estimate" => Ok(Self::HeuristicEstimate),
"llm_reasoning" => Ok(Self::LlmReasoning),
"human_judgment" => Ok(Self::HumanJudgment),
"unknown" => Ok(Self::Unknown),
_ => Err(ParseCounterfactualMethodError {
input: s.to_owned(),
}),
}
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ParseCounterfactualMethodError {
input: String,
}
impl fmt::Display for ParseCounterfactualMethodError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"unknown counterfactual method `{}`; expected deterministic_replay, heuristic_estimate, llm_reasoning, human_judgment, or unknown",
self.input
)
}
}
impl std::error::Error for ParseCounterfactualMethodError {}
#[derive(Clone, Debug, Default, PartialEq)]
pub struct RegretLedger {
pub schema: &'static str,
pub workspace_id: Option<String>,
pub entries: Vec<RegretEntry>,
pub summary: Option<RegretSummary>,
pub updated_at: String,
}
impl RegretLedger {
#[must_use]
pub fn new(updated_at: impl Into<String>) -> Self {
Self {
schema: REGRET_LEDGER_SCHEMA_V1,
updated_at: updated_at.into(),
..Default::default()
}
}
#[must_use]
pub fn with_workspace_id(mut self, id: impl Into<String>) -> Self {
self.workspace_id = Some(id.into());
self
}
pub fn add_entry(&mut self, entry: RegretEntry) {
self.entries.push(entry);
}
#[must_use]
pub fn with_summary(mut self, summary: RegretSummary) -> Self {
self.summary = Some(summary);
self
}
}
#[derive(Clone, Debug, PartialEq)]
pub struct RegretEntry {
pub schema: &'static str,
pub id: String,
pub episode_id: String,
pub counterfactual_run_id: String,
pub intervention_id: String,
pub regret_score: f64,
pub confidence: f64,
pub category: RegretCategory,
pub promoted: bool,
pub promoted_memory_id: Option<String>,
pub created_at: String,
}
impl RegretEntry {
#[must_use]
#[allow(clippy::too_many_arguments)]
pub fn new(
id: impl Into<String>,
episode_id: impl Into<String>,
counterfactual_run_id: impl Into<String>,
intervention_id: impl Into<String>,
regret_score: f64,
confidence: f64,
category: RegretCategory,
created_at: impl Into<String>,
) -> Self {
Self {
schema: REGRET_ENTRY_SCHEMA_V1,
id: id.into(),
episode_id: episode_id.into(),
counterfactual_run_id: counterfactual_run_id.into(),
intervention_id: intervention_id.into(),
regret_score,
confidence,
category,
promoted: false,
promoted_memory_id: None,
created_at: created_at.into(),
}
}
#[must_use]
pub fn with_promotion(mut self, memory_id: impl Into<String>) -> Self {
self.promoted = true;
self.promoted_memory_id = Some(memory_id.into());
self
}
}
#[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq)]
pub enum RegretCategory {
MissingKnowledge,
StaleInformation,
RetrievalFailure,
UnderutilizedMemory,
Misinformation,
#[default]
Other,
}
impl RegretCategory {
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::MissingKnowledge => "missing_knowledge",
Self::StaleInformation => "stale_information",
Self::RetrievalFailure => "retrieval_failure",
Self::UnderutilizedMemory => "underutilized_memory",
Self::Misinformation => "misinformation",
Self::Other => "other",
}
}
}
impl fmt::Display for RegretCategory {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(self.as_str())
}
}
impl FromStr for RegretCategory {
type Err = ParseRegretCategoryError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match normalized_episode_token(s).as_str() {
"missing_knowledge" => Ok(Self::MissingKnowledge),
"stale_information" => Ok(Self::StaleInformation),
"retrieval_failure" => Ok(Self::RetrievalFailure),
"underutilized_memory" => Ok(Self::UnderutilizedMemory),
"misinformation" => Ok(Self::Misinformation),
"other" => Ok(Self::Other),
_ => Err(ParseRegretCategoryError {
input: s.to_owned(),
}),
}
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ParseRegretCategoryError {
input: String,
}
impl fmt::Display for ParseRegretCategoryError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"unknown regret category `{}`; expected missing_knowledge, stale_information, retrieval_failure, underutilized_memory, misinformation, or other",
self.input
)
}
}
impl std::error::Error for ParseRegretCategoryError {}
#[derive(Clone, Debug, Default, Eq, PartialEq)]
pub struct RegretSummary {
pub total_entries: u32,
pub promoted_count: u32,
pub by_category: Vec<(RegretCategory, u32)>,
pub average_regret: Option<String>,
pub average_confidence: Option<String>,
}
impl RegretSummary {
#[must_use]
pub fn new(total_entries: u32, promoted_count: u32) -> Self {
Self {
total_entries,
promoted_count,
..Default::default()
}
}
pub fn add_category_count(&mut self, category: RegretCategory, count: u32) {
self.by_category.push((category, count));
}
#[must_use]
pub fn with_average_regret(mut self, avg: impl Into<String>) -> Self {
self.average_regret = Some(avg.into());
self
}
#[must_use]
pub fn with_average_confidence(mut self, avg: impl Into<String>) -> Self {
self.average_confidence = Some(avg.into());
self
}
}
pub const COUNTERFACTUAL_CLAIM_SCHEMA_V1: &str = "ee.counterfactual_claim.v1";
pub const REGRET_DELTA_SCHEMA_V1: &str = "ee.regret_delta.v1";
pub const COUNTERFACTUAL_CLAIM_ID_PREFIX: &str = "cfc_";
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub enum CounterfactualClaimType {
WouldHaveSurfaced,
RegretDelta,
MissedRetrieval,
InsufficientRank,
}
impl CounterfactualClaimType {
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::WouldHaveSurfaced => "would_have_surfaced",
Self::RegretDelta => "regret_delta",
Self::MissedRetrieval => "missed_retrieval",
Self::InsufficientRank => "insufficient_rank",
}
}
#[must_use]
pub const fn all() -> [Self; 4] {
[
Self::WouldHaveSurfaced,
Self::RegretDelta,
Self::MissedRetrieval,
Self::InsufficientRank,
]
}
}
impl fmt::Display for CounterfactualClaimType {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(self.as_str())
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ParseCounterfactualClaimTypeError {
input: String,
}
impl ParseCounterfactualClaimTypeError {
pub fn input(&self) -> &str {
&self.input
}
}
impl fmt::Display for ParseCounterfactualClaimTypeError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"unknown counterfactual claim type `{}`; expected would_have_surfaced, regret_delta, missed_retrieval, or insufficient_rank",
self.input
)
}
}
impl std::error::Error for ParseCounterfactualClaimTypeError {}
impl FromStr for CounterfactualClaimType {
type Err = ParseCounterfactualClaimTypeError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match normalized_episode_token(s).as_str() {
"would_have_surfaced" => Ok(Self::WouldHaveSurfaced),
"regret_delta" => Ok(Self::RegretDelta),
"missed_retrieval" => Ok(Self::MissedRetrieval),
"insufficient_rank" => Ok(Self::InsufficientRank),
other => Err(ParseCounterfactualClaimTypeError {
input: other.to_owned(),
}),
}
}
}
#[derive(Clone, Debug, PartialEq)]
pub struct CounterfactualClaim {
pub schema: &'static str,
pub id: String,
pub claim_type: CounterfactualClaimType,
pub episode_id: String,
pub counterfactual_run_id: String,
pub memory_id: Option<String>,
pub description: String,
pub confidence: f64,
pub evidence: Vec<String>,
pub suggested_action: Option<String>,
pub created_at: String,
}
impl CounterfactualClaim {
#[must_use]
pub fn new(
id: impl Into<String>,
claim_type: CounterfactualClaimType,
episode_id: impl Into<String>,
counterfactual_run_id: impl Into<String>,
description: impl Into<String>,
confidence: f64,
created_at: impl Into<String>,
) -> Self {
Self {
schema: COUNTERFACTUAL_CLAIM_SCHEMA_V1,
id: id.into(),
claim_type,
episode_id: episode_id.into(),
counterfactual_run_id: counterfactual_run_id.into(),
memory_id: None,
description: description.into(),
confidence,
evidence: Vec::new(),
suggested_action: None,
created_at: created_at.into(),
}
}
#[must_use]
pub fn would_have_surfaced(
id: impl Into<String>,
episode_id: impl Into<String>,
counterfactual_run_id: impl Into<String>,
memory_id: impl Into<String>,
confidence: f64,
created_at: impl Into<String>,
) -> Self {
let memory_id_str = memory_id.into();
Self {
schema: COUNTERFACTUAL_CLAIM_SCHEMA_V1,
id: id.into(),
claim_type: CounterfactualClaimType::WouldHaveSurfaced,
episode_id: episode_id.into(),
counterfactual_run_id: counterfactual_run_id.into(),
memory_id: Some(memory_id_str.clone()),
description: format!(
"Memory {} would have been surfaced under alternate retrieval parameters",
memory_id_str
),
confidence,
evidence: Vec::new(),
suggested_action: Some(format!(
"Consider adjusting retrieval parameters to surface memory {}",
memory_id_str
)),
created_at: created_at.into(),
}
}
#[must_use]
pub fn regret_delta(
id: impl Into<String>,
episode_id: impl Into<String>,
counterfactual_run_id: impl Into<String>,
delta: &RegretDelta,
created_at: impl Into<String>,
) -> Self {
Self {
schema: COUNTERFACTUAL_CLAIM_SCHEMA_V1,
id: id.into(),
claim_type: CounterfactualClaimType::RegretDelta,
episode_id: episode_id.into(),
counterfactual_run_id: counterfactual_run_id.into(),
memory_id: None,
description: format!(
"Outcome would have changed from {} to {} with regret delta {:.3}",
delta.actual_outcome, delta.hypothetical_outcome, delta.delta_score
),
confidence: delta.confidence,
evidence: Vec::new(),
suggested_action: if delta.delta_score > 0.5 {
Some("High-impact improvement opportunity identified".to_owned())
} else {
None
},
created_at: created_at.into(),
}
}
#[must_use]
pub fn with_memory_id(mut self, memory_id: impl Into<String>) -> Self {
self.memory_id = Some(memory_id.into());
self
}
pub fn add_evidence(&mut self, evidence: impl Into<String>) {
self.evidence.push(evidence.into());
}
#[must_use]
pub fn with_suggested_action(mut self, action: impl Into<String>) -> Self {
self.suggested_action = Some(action.into());
self
}
}
#[derive(Clone, Debug, PartialEq)]
pub struct RegretDelta {
pub schema: &'static str,
pub id: String,
pub episode_id: String,
pub counterfactual_run_id: String,
pub actual_outcome: EpisodeOutcome,
pub hypothetical_outcome: EpisodeOutcome,
pub delta_score: f64,
pub confidence: f64,
pub contributing_factors: Vec<String>,
pub computed_at: String,
}
impl RegretDelta {
#[must_use]
#[allow(clippy::too_many_arguments)]
pub fn new(
id: impl Into<String>,
episode_id: impl Into<String>,
counterfactual_run_id: impl Into<String>,
actual_outcome: EpisodeOutcome,
hypothetical_outcome: EpisodeOutcome,
delta_score: f64,
confidence: f64,
computed_at: impl Into<String>,
) -> Self {
Self {
schema: REGRET_DELTA_SCHEMA_V1,
id: id.into(),
episode_id: episode_id.into(),
counterfactual_run_id: counterfactual_run_id.into(),
actual_outcome,
hypothetical_outcome,
delta_score,
confidence,
contributing_factors: Vec::new(),
computed_at: computed_at.into(),
}
}
#[must_use]
pub fn compute_score(actual: EpisodeOutcome, hypothetical: EpisodeOutcome) -> f64 {
let actual_value = Self::outcome_value(actual);
let hypothetical_value = Self::outcome_value(hypothetical);
hypothetical_value - actual_value
}
#[must_use]
pub const fn outcome_value(outcome: EpisodeOutcome) -> f64 {
match outcome {
EpisodeOutcome::Success => 1.0,
EpisodeOutcome::Cancelled => 0.3,
EpisodeOutcome::Timeout => 0.2,
EpisodeOutcome::Unknown => 0.0,
EpisodeOutcome::Failure => -0.5,
}
}
pub fn add_contributing_factor(&mut self, factor: impl Into<String>) {
self.contributing_factors.push(factor.into());
}
#[must_use]
pub fn is_improvement(&self) -> bool {
self.delta_score > 0.0
}
#[must_use]
pub fn is_significant(&self, threshold: f64) -> bool {
self.delta_score.abs() >= threshold && self.confidence >= 0.5
}
}
#[cfg(test)]
mod tests {
use super::*;
type TestResult = Result<(), String>;
fn ensure<T: std::fmt::Debug + PartialEq>(actual: T, expected: T, ctx: &str) -> TestResult {
if actual == expected {
Ok(())
} else {
Err(format!("{ctx}: expected {expected:?}, got {actual:?}"))
}
}
#[test]
fn episode_schema_versions_are_stable() -> TestResult {
ensure(TASK_EPISODE_SCHEMA_V1, "ee.task_episode.v1", "episode")?;
ensure(INTERVENTION_SCHEMA_V1, "ee.intervention.v1", "intervention")?;
ensure(
COUNTERFACTUAL_RUN_SCHEMA_V1,
"ee.counterfactual_run.v1",
"cfr",
)?;
ensure(REGRET_LEDGER_SCHEMA_V1, "ee.regret_ledger.v1", "ledger")?;
ensure(REGRET_ENTRY_SCHEMA_V1, "ee.regret_entry.v1", "entry")
}
#[test]
fn task_episode_builder() -> TestResult {
let mut ep = TaskEpisode::new("ep_001", "Fix the bug", "2026-04-30T12:00:00Z")
.with_workspace_id("ws_001")
.with_session_id("sess_001")
.with_context_pack_id("pack_001")
.with_outcome(EpisodeOutcome::Success)
.with_ended_at("2026-04-30T12:05:00Z")
.with_duration_ms(300000)
.with_agent("claude-code");
ep.add_retrieved_memory("mem_001");
ep.add_action(EpisodeAction::new(
1,
ActionType::Edit,
"Edit file",
"2026-04-30T12:01:00Z",
));
ensure(ep.schema, TASK_EPISODE_SCHEMA_V1, "schema")?;
ensure(ep.task_input, "Fix the bug".to_string(), "task")?;
ensure(ep.outcome, EpisodeOutcome::Success, "outcome")?;
ensure(ep.retrieved_memory_ids.len(), 1, "memories")?;
ensure(ep.actions.len(), 1, "actions")
}
#[test]
fn action_type_strings_are_stable() -> TestResult {
ensure(ActionType::ToolCall.as_str(), "tool_call", "tool_call")?;
ensure(ActionType::Edit.as_str(), "edit", "edit")?;
ensure(ActionType::Command.as_str(), "command", "command")?;
ensure(ActionType::Search.as_str(), "search", "search")?;
ensure(ActionType::Retrieval.as_str(), "retrieval", "retrieval")?;
ensure(ActionType::Output.as_str(), "output", "output")?;
ensure(ActionType::Other.as_str(), "other", "other")
}
#[test]
fn action_type_round_trip() -> TestResult {
for at in [
ActionType::ToolCall,
ActionType::Edit,
ActionType::Command,
ActionType::Search,
ActionType::Retrieval,
ActionType::Output,
ActionType::Other,
] {
let parsed = ActionType::from_str(at.as_str());
ensure(parsed, Ok(at), at.as_str())?;
}
Ok(())
}
#[test]
fn action_type_rejects_invalid() {
assert!(ActionType::from_str("invalid").is_err());
}
#[test]
fn episode_outcome_strings_are_stable() -> TestResult {
ensure(EpisodeOutcome::Success.as_str(), "success", "success")?;
ensure(EpisodeOutcome::Failure.as_str(), "failure", "failure")?;
ensure(EpisodeOutcome::Cancelled.as_str(), "cancelled", "cancelled")?;
ensure(EpisodeOutcome::Timeout.as_str(), "timeout", "timeout")?;
ensure(EpisodeOutcome::Unknown.as_str(), "unknown", "unknown")
}
#[test]
fn episode_outcome_round_trip() -> TestResult {
for eo in [
EpisodeOutcome::Success,
EpisodeOutcome::Failure,
EpisodeOutcome::Cancelled,
EpisodeOutcome::Timeout,
EpisodeOutcome::Unknown,
] {
let parsed = EpisodeOutcome::from_str(eo.as_str());
ensure(parsed, Ok(eo), eo.as_str())?;
}
Ok(())
}
#[test]
fn episode_outcome_is_negative() -> TestResult {
ensure(EpisodeOutcome::Success.is_negative(), false, "success")?;
ensure(EpisodeOutcome::Failure.is_negative(), true, "failure")?;
ensure(EpisodeOutcome::Cancelled.is_negative(), true, "cancelled")?;
ensure(EpisodeOutcome::Timeout.is_negative(), true, "timeout")?;
ensure(EpisodeOutcome::Unknown.is_negative(), false, "unknown")
}
#[test]
fn intervention_builder() -> TestResult {
let int = Intervention::new(
"int_001",
InterventionType::AddMemory,
"Add missing rule",
"2026-04-30T12:00:00Z",
)
.with_target_memory("mem_001")
.with_hypothetical_content("Always run tests")
.with_rationale("Would have prevented test failure")
.with_created_by("analyst");
ensure(int.schema, INTERVENTION_SCHEMA_V1, "schema")?;
ensure(int.intervention_type, InterventionType::AddMemory, "type")?;
ensure(int.target_memory_id, Some("mem_001".to_string()), "target")
}
#[test]
fn intervention_type_strings_are_stable() -> TestResult {
ensure(InterventionType::AddMemory.as_str(), "add_memory", "add")?;
ensure(
InterventionType::RemoveMemory.as_str(),
"remove_memory",
"remove",
)?;
ensure(
InterventionType::ReplaceContent.as_str(),
"replace_content",
"replace",
)?;
ensure(
InterventionType::Strengthen.as_str(),
"strengthen",
"strengthen",
)?;
ensure(InterventionType::Weaken.as_str(), "weaken", "weaken")?;
ensure(InterventionType::Rerank.as_str(), "rerank", "rerank")
}
#[test]
fn intervention_type_round_trip() -> TestResult {
for it in [
InterventionType::AddMemory,
InterventionType::RemoveMemory,
InterventionType::ReplaceContent,
InterventionType::Strengthen,
InterventionType::Weaken,
InterventionType::Rerank,
] {
let parsed = InterventionType::from_str(it.as_str());
ensure(parsed, Ok(it), it.as_str())?;
}
Ok(())
}
#[test]
fn counterfactual_run_builder() -> TestResult {
let mut cfr = CounterfactualRun::new(
"cfr_001",
"ep_001",
EpisodeOutcome::Success,
0.85,
CounterfactualMethod::DeterministicReplay,
"2026-04-30T12:00:00Z",
)
.with_analysis("Adding the rule would have prevented failure")
.with_analysis_duration_ms(500);
cfr.add_intervention("int_001");
ensure(cfr.schema, COUNTERFACTUAL_RUN_SCHEMA_V1, "schema")?;
ensure(cfr.hypothetical_outcome, EpisodeOutcome::Success, "outcome")?;
ensure(cfr.intervention_ids.len(), 1, "interventions")
}
#[test]
fn counterfactual_method_strings_are_stable() -> TestResult {
ensure(
CounterfactualMethod::DeterministicReplay.as_str(),
"deterministic_replay",
"replay",
)?;
ensure(
CounterfactualMethod::HeuristicEstimate.as_str(),
"heuristic_estimate",
"heuristic",
)?;
ensure(
CounterfactualMethod::LlmReasoning.as_str(),
"llm_reasoning",
"llm",
)?;
ensure(
CounterfactualMethod::HumanJudgment.as_str(),
"human_judgment",
"human",
)?;
ensure(CounterfactualMethod::Unknown.as_str(), "unknown", "unknown")
}
#[test]
fn counterfactual_method_round_trip() -> TestResult {
for cm in [
CounterfactualMethod::DeterministicReplay,
CounterfactualMethod::HeuristicEstimate,
CounterfactualMethod::LlmReasoning,
CounterfactualMethod::HumanJudgment,
CounterfactualMethod::Unknown,
] {
let parsed = CounterfactualMethod::from_str(cm.as_str());
ensure(parsed, Ok(cm), cm.as_str())?;
}
Ok(())
}
#[test]
fn regret_ledger_builder() -> TestResult {
let mut ledger = RegretLedger::new("2026-04-30T12:00:00Z")
.with_workspace_id("ws_001")
.with_summary(RegretSummary::new(10, 3));
ledger.add_entry(RegretEntry::new(
"reg_001",
"ep_001",
"cfr_001",
"int_001",
0.7,
0.85,
RegretCategory::MissingKnowledge,
"2026-04-30T12:00:00Z",
));
ensure(ledger.schema, REGRET_LEDGER_SCHEMA_V1, "schema")?;
ensure(ledger.entries.len(), 1, "entries")
}
#[test]
fn regret_entry_builder() -> TestResult {
let entry = RegretEntry::new(
"reg_001",
"ep_001",
"cfr_001",
"int_001",
0.7,
0.85,
RegretCategory::RetrievalFailure,
"2026-04-30T12:00:00Z",
)
.with_promotion("mem_new_001");
ensure(entry.schema, REGRET_ENTRY_SCHEMA_V1, "schema")?;
ensure(entry.promoted, true, "promoted")?;
ensure(entry.category, RegretCategory::RetrievalFailure, "category")
}
#[test]
fn regret_category_strings_are_stable() -> TestResult {
ensure(
RegretCategory::MissingKnowledge.as_str(),
"missing_knowledge",
"missing",
)?;
ensure(
RegretCategory::StaleInformation.as_str(),
"stale_information",
"stale",
)?;
ensure(
RegretCategory::RetrievalFailure.as_str(),
"retrieval_failure",
"retrieval",
)?;
ensure(
RegretCategory::UnderutilizedMemory.as_str(),
"underutilized_memory",
"underutilized",
)?;
ensure(
RegretCategory::Misinformation.as_str(),
"misinformation",
"misinformation",
)?;
ensure(RegretCategory::Other.as_str(), "other", "other")
}
#[test]
fn regret_category_round_trip() -> TestResult {
for rc in [
RegretCategory::MissingKnowledge,
RegretCategory::StaleInformation,
RegretCategory::RetrievalFailure,
RegretCategory::UnderutilizedMemory,
RegretCategory::Misinformation,
RegretCategory::Other,
] {
let parsed = RegretCategory::from_str(rc.as_str());
ensure(parsed, Ok(rc), rc.as_str())?;
}
Ok(())
}
#[test]
fn regret_summary_builder() -> TestResult {
let mut summary = RegretSummary::new(100, 25)
.with_average_regret("0.65")
.with_average_confidence("0.80");
summary.add_category_count(RegretCategory::MissingKnowledge, 40);
summary.add_category_count(RegretCategory::RetrievalFailure, 30);
ensure(summary.total_entries, 100, "total")?;
ensure(summary.promoted_count, 25, "promoted")?;
ensure(summary.by_category.len(), 2, "categories")
}
#[test]
fn counterfactual_claim_schema_versions_are_stable() -> TestResult {
ensure(
COUNTERFACTUAL_CLAIM_SCHEMA_V1,
"ee.counterfactual_claim.v1",
"claim",
)?;
ensure(REGRET_DELTA_SCHEMA_V1, "ee.regret_delta.v1", "delta")
}
#[test]
fn counterfactual_claim_type_strings_are_stable() -> TestResult {
ensure(
CounterfactualClaimType::WouldHaveSurfaced.as_str(),
"would_have_surfaced",
"surfaced",
)?;
ensure(
CounterfactualClaimType::RegretDelta.as_str(),
"regret_delta",
"delta",
)?;
ensure(
CounterfactualClaimType::MissedRetrieval.as_str(),
"missed_retrieval",
"missed",
)?;
ensure(
CounterfactualClaimType::InsufficientRank.as_str(),
"insufficient_rank",
"rank",
)
}
#[test]
fn counterfactual_claim_type_round_trip() -> TestResult {
for ct in CounterfactualClaimType::all() {
let parsed = CounterfactualClaimType::from_str(ct.as_str());
ensure(parsed, Ok(ct), ct.as_str())?;
}
Ok(())
}
#[test]
fn episode_enums_accept_operator_spelling_variants() -> TestResult {
ensure(
ActionType::from_str(" Tool-Call "),
Ok(ActionType::ToolCall),
"action type alias",
)?;
ensure(
ActionType::from_str("toolCall"),
Ok(ActionType::ToolCall),
"camel action type alias",
)?;
ensure(
EpisodeOutcome::from_str("FAILURE"),
Ok(EpisodeOutcome::Failure),
"outcome alias",
)?;
ensure(
InterventionType::from_str("replace-content"),
Ok(InterventionType::ReplaceContent),
"intervention alias",
)?;
ensure(
InterventionType::from_str("removeMemory"),
Ok(InterventionType::RemoveMemory),
"camel intervention alias",
)?;
ensure(
CounterfactualMethod::from_str(" Human-Judgment "),
Ok(CounterfactualMethod::HumanJudgment),
"method alias",
)?;
ensure(
CounterfactualMethod::from_str("humanJudgment"),
Ok(CounterfactualMethod::HumanJudgment),
"camel method alias",
)?;
ensure(
RegretCategory::from_str("retrieval-failure"),
Ok(RegretCategory::RetrievalFailure),
"regret category alias",
)?;
ensure(
RegretCategory::from_str("staleInformation"),
Ok(RegretCategory::StaleInformation),
"camel regret category alias",
)?;
ensure(
CounterfactualClaimType::from_str("INSUFFICIENT_RANK"),
Ok(CounterfactualClaimType::InsufficientRank),
"claim type alias",
)?;
ensure(
CounterfactualClaimType::from_str("wouldHaveSurfaced"),
Ok(CounterfactualClaimType::WouldHaveSurfaced),
"camel claim type alias",
)
}
#[test]
fn counterfactual_claim_would_have_surfaced() -> TestResult {
let claim = CounterfactualClaim::would_have_surfaced(
"cfc_001",
"ep_001",
"cfr_001",
"mem_001",
0.85,
"2026-04-30T12:00:00Z",
);
ensure(claim.schema, COUNTERFACTUAL_CLAIM_SCHEMA_V1, "schema")?;
ensure(
claim.claim_type,
CounterfactualClaimType::WouldHaveSurfaced,
"type",
)?;
ensure(claim.memory_id, Some("mem_001".to_string()), "memory")?;
ensure(claim.confidence, 0.85, "confidence")?;
ensure(claim.suggested_action.is_some(), true, "has action")
}
#[test]
fn counterfactual_claim_regret_delta() -> TestResult {
let delta = RegretDelta::new(
"rd_001",
"ep_001",
"cfr_001",
EpisodeOutcome::Failure,
EpisodeOutcome::Success,
1.5,
0.9,
"2026-04-30T12:00:00Z",
);
let claim = CounterfactualClaim::regret_delta(
"cfc_002",
"ep_001",
"cfr_001",
&delta,
"2026-04-30T12:00:00Z",
);
ensure(claim.schema, COUNTERFACTUAL_CLAIM_SCHEMA_V1, "schema")?;
ensure(
claim.claim_type,
CounterfactualClaimType::RegretDelta,
"type",
)?;
ensure(claim.confidence, 0.9, "confidence")?;
ensure(
claim.suggested_action.is_some(),
true,
"has action for high delta",
)
}
#[test]
fn regret_delta_compute_score() -> TestResult {
let failure_to_success =
RegretDelta::compute_score(EpisodeOutcome::Failure, EpisodeOutcome::Success);
ensure(failure_to_success, 1.5, "failure->success")?;
let success_to_failure =
RegretDelta::compute_score(EpisodeOutcome::Success, EpisodeOutcome::Failure);
ensure(success_to_failure, -1.5, "success->failure")?;
let no_change =
RegretDelta::compute_score(EpisodeOutcome::Success, EpisodeOutcome::Success);
ensure(no_change, 0.0, "no change")
}
#[test]
fn regret_delta_is_improvement() -> TestResult {
let mut delta = RegretDelta::new(
"rd_001",
"ep_001",
"cfr_001",
EpisodeOutcome::Failure,
EpisodeOutcome::Success,
1.5,
0.9,
"2026-04-30T12:00:00Z",
);
ensure(delta.is_improvement(), true, "positive is improvement")?;
delta.delta_score = -0.5;
ensure(delta.is_improvement(), false, "negative is not improvement")?;
delta.delta_score = 0.0;
ensure(delta.is_improvement(), false, "zero is not improvement")
}
#[test]
fn regret_delta_is_significant() -> TestResult {
let mut delta = RegretDelta::new(
"rd_001",
"ep_001",
"cfr_001",
EpisodeOutcome::Failure,
EpisodeOutcome::Success,
0.8,
0.7,
"2026-04-30T12:00:00Z",
);
ensure(delta.is_significant(0.5), true, "high delta high conf")?;
delta.confidence = 0.3;
ensure(delta.is_significant(0.5), false, "low confidence")?;
delta.confidence = 0.7;
delta.delta_score = 0.2;
ensure(delta.is_significant(0.5), false, "low delta")
}
#[test]
fn regret_delta_builder() -> TestResult {
let mut delta = RegretDelta::new(
"rd_001",
"ep_001",
"cfr_001",
EpisodeOutcome::Failure,
EpisodeOutcome::Success,
1.0,
0.85,
"2026-04-30T12:00:00Z",
);
delta.add_contributing_factor("Memory boost");
delta.add_contributing_factor("Better context");
ensure(delta.schema, REGRET_DELTA_SCHEMA_V1, "schema")?;
ensure(delta.contributing_factors.len(), 2, "factors")
}
#[test]
fn counterfactual_claim_builder() -> TestResult {
let mut claim = CounterfactualClaim::new(
"cfc_001",
CounterfactualClaimType::MissedRetrieval,
"ep_001",
"cfr_001",
"Memory was not retrieved",
0.75,
"2026-04-30T12:00:00Z",
)
.with_memory_id("mem_001")
.with_suggested_action("Adjust retrieval threshold");
claim.add_evidence("Memory existed in index");
claim.add_evidence("Query matched memory");
ensure(claim.schema, COUNTERFACTUAL_CLAIM_SCHEMA_V1, "schema")?;
ensure(claim.evidence.len(), 2, "evidence count")
}
}