use std::future::Future;
use std::sync::{Arc, Mutex};
use clawft_cow_memory::BranchableMemory;
use clawft_types::error::ClawftError;
use tracing::error;
use super::turn_ledger::TurnLedger;
#[derive(Debug)]
pub struct HeldTurn {
pub label: String,
pre_turn_head: Option<clawft_cow_memory::CheckpointId>,
pub created_at: std::time::Instant,
}
struct BracketGuard {
mem: Arc<Mutex<BranchableMemory>>,
ledger: Option<Arc<dyn TurnLedger>>,
label: String,
pre_turn_head: Option<clawft_cow_memory::CheckpointId>,
armed: bool,
}
impl Drop for BracketGuard {
fn drop(&mut self) {
if !self.armed {
return;
}
let rolled_back = {
let mut guard = self.mem.lock().unwrap_or_else(|p| p.into_inner());
match guard.rollback(self.pre_turn_head) {
Ok(()) => true,
Err(e) => {
error!(
error = %e,
label = %self.label,
"cancelled-turn rollback failed; checkpoint chain may dangle"
);
false
}
}
};
if let Some(ledger) = &self.ledger {
ledger.on_revert(
&self.label,
"turn cancelled (dispatch future dropped)",
rolled_back,
);
}
}
}
pub fn accept_held(
mem: &Arc<Mutex<BranchableMemory>>,
ledger: Option<&Arc<dyn TurnLedger>>,
held: HeldTurn,
) -> clawft_types::Result<()> {
let mut guard = mem.lock().unwrap_or_else(|p| p.into_inner());
let report = guard.promote().map_err(cow_err)?;
if let Some(ledger) = ledger {
ledger.on_promote(
&held.label,
true,
Some(super::turn_ledger::PromotedLineage {
child_id: report.child_id,
parent_id: report.parent_id,
parent_hash: report.parent_hash,
mutation_count: (report.ingested + report.deleted) as u32,
}),
);
}
Ok(())
}
pub fn discard_held(
mem: &Arc<Mutex<BranchableMemory>>,
ledger: Option<&Arc<dyn TurnLedger>>,
held: HeldTurn,
reason: &str,
) -> clawft_types::Result<()> {
let rolled_back = {
let mut guard = mem.lock().unwrap_or_else(|p| p.into_inner());
match guard.rollback(held.pre_turn_head) {
Ok(()) => true,
Err(e) => {
error!(error = %e, label = %held.label, "discard rollback failed");
false
}
}
};
if let Some(ledger) = ledger {
ledger.on_revert(&held.label, reason, rolled_back);
}
if rolled_back {
Ok(())
} else {
Err(ClawftError::CowMemory {
reason: "discard rollback failed; lineage may hold the undecided chain".into(),
})
}
}
pub(crate) async fn with_turn_checkpoint<F, Fut, T, I>(
mem: &Arc<Mutex<BranchableMemory>>,
ledger: Option<&Arc<dyn TurnLedger>>,
label: impl Into<String>,
turn: F,
items_on_ok: I,
review: bool,
held_out: &mut Option<HeldTurn>,
) -> clawft_types::Result<T>
where
F: FnOnce() -> Fut,
Fut: Future<Output = clawft_types::Result<T>>,
I: FnOnce(&T) -> Vec<clawft_cow_memory::VectorItem>,
{
let label: String = label.into();
let pre_turn_head = {
let guard = mem.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
guard
.lineage()
.into_iter()
.find(|entry| matches!(entry.role, clawft_cow_memory::NodeRole::Checkpoint))
.map(|entry| entry.id)
};
{
let mut guard = mem.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
if guard.is_paused() {
tracing::info!(label, "cow memory: auto-resuming parked lineage for incoming turn");
guard.resume().map_err(cow_err)?;
}
guard.checkpoint(label.clone()).map_err(cow_err)?;
}
if let Some(ledger) = ledger {
ledger.on_checkpoint(&label);
}
let mut cancel_guard = BracketGuard {
mem: Arc::clone(mem),
ledger: ledger.cloned(),
label: label.clone(),
pre_turn_head,
armed: true,
};
let result = turn().await;
cancel_guard.armed = false;
match &result {
Ok(val) => {
let items = items_on_ok(val);
if review {
let mut guard = mem.lock().unwrap_or_else(|p| p.into_inner());
if !items.is_empty()
&& let Err(e) = guard.ingest(&items)
{
error!(error = %e, label, "review-mode ingest failed; holding without the exchange");
}
*held_out = Some(HeldTurn {
label,
pre_turn_head,
created_at: std::time::Instant::now(),
});
return result;
}
let (promoted, lineage) = {
let mut guard = mem.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
if !items.is_empty()
&& let Err(e) = guard.ingest(&items)
{
error!(
error = %e,
label,
"clawft-cow-memory: turn-exchange ingest failed; \
promoting without the exchange"
);
}
match guard.promote() {
Ok(report) => (
true,
Some(super::turn_ledger::PromotedLineage {
child_id: report.child_id,
parent_id: report.parent_id,
parent_hash: report.parent_hash,
mutation_count: (report.ingested + report.deleted) as u32,
}),
),
Err(e) => {
error!(
error = %e,
"clawft-cow-memory: promote failed after successful turn; \
checkpoint chain left in place for the next turn to build on"
);
(false, None)
}
}
};
if let Some(ledger) = ledger {
ledger.on_promote(&label, promoted, lineage);
}
}
Err(turn_err) => {
let rolled_back = {
let mut guard = mem.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
match guard.rollback(pre_turn_head) {
Ok(()) => true,
Err(e) => {
error!(
error = %e,
"clawft-cow-memory: rollback failed after turn error; \
original turn error still returned to the caller"
);
false
}
}
};
if let Some(ledger) = ledger {
ledger.on_revert(&label, &turn_err.to_string(), rolled_back);
}
}
}
result
}
pub(crate) fn exchange_items(
dim: u16,
channel: &str,
chat_id: &str,
user_text: &str,
reply_text: &str,
) -> Vec<clawft_cow_memory::VectorItem> {
use clawft_cow_memory::VectorItem;
let embedder = crate::embeddings::hash_embedder::HashEmbedder::new(dim as usize);
let mut tags = clawft_cow_memory::VectorTags::new();
tags.insert("channel".into(), channel.to_string());
tags.insert("chat_id".into(), chat_id.to_string());
let mut items = Vec::with_capacity(2);
for (role, text) in [("user", user_text), ("assistant", reply_text)] {
if text.trim().is_empty() {
continue;
}
let mut t = tags.clone();
t.insert("role".into(), role.to_string());
items.push(
VectorItem::new(embedder.compute_embedding(text))
.with_text(text)
.with_tags(t),
);
}
items
}
fn cow_err(e: clawft_cow_memory::CowMemoryError) -> ClawftError {
ClawftError::CowMemory {
reason: e.to_string(),
}
}
#[cfg(test)]
mod tests {
use super::*;
use tempfile::TempDir;
#[derive(Default)]
struct RecordingLedger {
log: Mutex<Vec<String>>,
}
impl RecordingLedger {
fn events(&self) -> Vec<String> {
self.log.lock().unwrap().clone()
}
}
impl TurnLedger for RecordingLedger {
fn on_checkpoint(&self, label: &str) {
self.log.lock().unwrap().push(format!("checkpoint:{label}"));
}
fn on_revert(&self, label: &str, _err: &str, rolled_back: bool) {
self.log
.lock()
.unwrap()
.push(format!("revert:{label}:rolled_back={rolled_back}"));
}
fn on_promote(&self, label: &str, promoted: bool, lineage: Option<super::super::turn_ledger::PromotedLineage>) {
self.log.lock().unwrap().push(format!(
"promote:{label}:promoted={promoted}:lineage={}",
lineage.is_some()
));
}
}
#[tokio::test]
async fn ledger_witness_sequence_ok_and_err() {
let dir = TempDir::new().unwrap();
let mem = Arc::new(Mutex::new(BranchableMemory::create(dir.path(), 4).unwrap()));
let concrete = Arc::new(RecordingLedger::default());
let ledger: Arc<dyn TurnLedger> = concrete.clone();
let ok: clawft_types::Result<u8> =
with_turn_checkpoint(&mem, Some(&ledger), "t-ok", || async { Ok(1u8) }, no_items, false, &mut None).await;
assert!(ok.is_ok());
let err: clawft_types::Result<u8> = with_turn_checkpoint(&mem, Some(&ledger), "t-err", || async {
Err(ClawftError::CowMemory { reason: "boom".into() })
}, no_items, false, &mut None)
.await;
assert!(err.is_err());
let events = concrete.events();
assert_eq!(
events,
vec![
"checkpoint:t-ok".to_string(),
"promote:t-ok:promoted=true:lineage=true".to_string(),
"checkpoint:t-err".to_string(),
"revert:t-err:rolled_back=true".to_string(),
],
"append-only witness order: checkpoint precedes its outcome, \
revert witnesses the discard"
);
}
#[tokio::test]
async fn exchange_ingest_promotes_on_ok_and_adds_nothing_on_err() {
let dir = TempDir::new().unwrap();
let mem = test_mem(&dir); let items = |reply: &'static str| {
move |_: &String| exchange_items(4, "cli", "c1", "hello loom", reply)
};
let ok: clawft_types::Result<String> = with_turn_checkpoint(
&mem,
None,
"t-ok",
|| async { Ok("threads woven".to_string()) },
items("threads woven"),
false,
&mut None,
)
.await;
assert!(ok.is_ok());
{
let guard = mem.lock().unwrap();
let probe = crate::embeddings::hash_embedder::HashEmbedder::new(4)
.compute_embedding("threads woven");
let hits = guard.query(&probe, 4).expect("query");
assert!(!hits.is_empty(), "promoted exchange must be queryable");
}
let before = {
let guard = mem.lock().unwrap();
let s = guard.status();
s.base.total_vectors + s.working.total_vectors
};
let err: clawft_types::Result<String> = with_turn_checkpoint(
&mem,
None,
"t-err",
|| async {
Err(ClawftError::Timeout {
operation: "boom".into(),
})
},
items("never spoken"),
false,
&mut None,
)
.await;
assert!(err.is_err());
let after = {
let guard = mem.lock().unwrap();
let s = guard.status();
s.base.total_vectors + s.working.total_vectors
};
assert_eq!(before, after, "a failed turn must add nothing to memory");
}
#[test]
fn exchange_items_shape() {
let items = exchange_items(8, "cli", "c1", "ask", "answer");
assert_eq!(items.len(), 2);
assert_eq!(items[0].tags.get("role").map(String::as_str), Some("user"));
assert_eq!(
items[1].tags.get("role").map(String::as_str),
Some("assistant")
);
assert!(items.iter().all(|i| i.vector.len() == 8));
assert_eq!(exchange_items(8, "cli", "c1", "ask", " ").len(), 1);
}
#[tokio::test]
async fn bracket_auto_resumes_a_parked_lineage() {
let dir = TempDir::new().unwrap();
let mem = test_mem(&dir);
{
let mut guard = mem.lock().unwrap();
guard.pause("test-park").expect("pause");
assert!(guard.is_paused());
}
let items = |_: &String| exchange_items(4, "cli", "c1", "wake up", "awake");
let out: clawft_types::Result<String> =
with_turn_checkpoint(&mem, None, "t-wake", || async { Ok("awake".to_string()) }, items, false, &mut None)
.await;
assert!(out.is_ok(), "turn on a parked lineage must auto-resume, not fail");
let guard = mem.lock().unwrap();
assert!(!guard.is_paused(), "lineage resumed");
let probe =
crate::embeddings::hash_embedder::HashEmbedder::new(4).compute_embedding("awake");
assert!(
!guard.query(&probe, 4).expect("query").is_empty(),
"post-resume exchange promoted and queryable"
);
}
#[tokio::test]
async fn review_mode_holds_then_accept_promotes_witnessed() {
let dir = TempDir::new().unwrap();
let mem = test_mem(&dir);
let concrete = Arc::new(RecordingLedger::default());
let ledger: Arc<dyn TurnLedger> = concrete.clone();
let items = |_: &String| exchange_items(4, "cli", "c1", "hold me", "held reply");
let mut held = None;
let out: clawft_types::Result<String> = with_turn_checkpoint(
&mem, Some(&ledger), "t-hold", || async { Ok("held reply".to_string()) },
items, true, &mut held,
)
.await;
assert!(out.is_ok());
let held = held.expect("review mode must yield a HeldTurn");
assert_eq!(held.label, "t-hold");
{
let events = concrete.events();
assert!(events.iter().any(|e| e.starts_with("checkpoint:t-hold")));
assert!(
!events.iter().any(|e| e.starts_with("promote:")),
"no promote witness while held: {events:?}"
);
}
let probe = crate::embeddings::hash_embedder::HashEmbedder::new(4)
.compute_embedding("held reply");
assert!(!mem.lock().unwrap().query(&probe, 4).unwrap().is_empty());
accept_held(&mem, Some(&ledger), held).expect("accept");
let events = concrete.events();
assert!(
events.iter().any(|e| e.starts_with("promote:t-hold:promoted=true:lineage=true")),
"accept promotes with lineage witnessed: {events:?}"
);
assert!(!mem.lock().unwrap().query(&probe, 4).unwrap().is_empty());
}
#[tokio::test]
async fn review_mode_discard_rolls_back_witnessed() {
let dir = TempDir::new().unwrap();
let mem = test_mem(&dir);
let concrete = Arc::new(RecordingLedger::default());
let ledger: Arc<dyn TurnLedger> = concrete.clone();
let items = |_: &String| exchange_items(4, "cli", "c1", "hold me", "unwanted");
let mut held = None;
let _: clawft_types::Result<String> = with_turn_checkpoint(
&mem, Some(&ledger), "t-no", || async { Ok("unwanted".to_string()) },
items, true, &mut held,
)
.await;
let held = held.expect("held");
discard_held(&mem, Some(&ledger), held, "reviewer said no").expect("discard");
let events = concrete.events();
assert!(
events.iter().any(|e| e.starts_with("revert:t-no:rolled_back=true")),
"discard witnesses the revert: {events:?}"
);
let probe = crate::embeddings::hash_embedder::HashEmbedder::new(4)
.compute_embedding("unwanted");
assert!(
mem.lock().unwrap().query(&probe, 4).unwrap().is_empty(),
"discarded exchange must vanish"
);
}
#[tokio::test]
async fn cancelled_drop_rolls_back_and_witnesses_like_a_discard() {
let dir = TempDir::new().unwrap();
let mem = test_mem(&dir);
let concrete = Arc::new(RecordingLedger::default());
let ledger: Arc<dyn TurnLedger> = concrete.clone();
{
let mut g = mem.lock().unwrap();
g.ingest(&[clawft_cow_memory::VectorItem::new(vec![0.5, 0.5, 0.0, 0.0])])
.unwrap();
g.promote().unwrap();
}
let count = |mem: &Arc<Mutex<BranchableMemory>>| {
let g = mem.lock().unwrap();
let s = g.status();
s.base.total_vectors + s.working.total_vectors
};
let before = count(&mem);
let never = tokio::sync::Notify::new();
let mut held_slot = None;
let bracket = with_turn_checkpoint(
&mem,
Some(&ledger),
"t-cancelled",
|| async {
{
let mut g = mem.lock().unwrap();
g.ingest(&[clawft_cow_memory::VectorItem::new(vec![
0.0, 0.0, 1.0, 0.0,
])])
.unwrap();
}
never.notified().await; Ok(String::from("unreachable"))
},
no_items,
false,
&mut held_slot,
);
tokio::select! {
biased;
_ = tokio::time::sleep(std::time::Duration::from_millis(50)) => {}
_ = bracket => panic!("bracket must not resolve"),
}
assert_eq!(count(&mem), before, "cancelled turn leaves no memory residue");
let events = concrete.events();
assert!(
events
.iter()
.any(|e| e.starts_with("revert:t-cancelled:rolled_back=true")),
"cancel witnesses the compensating revert like a discard: {events:?}"
);
let _: clawft_types::Result<u8> = with_turn_checkpoint(
&mem, Some(&ledger), "t-err", || async {
Err(ClawftError::Timeout { operation: "boom".into() })
},
no_items, false, &mut None,
)
.await;
let mut held = None;
let _: clawft_types::Result<String> = with_turn_checkpoint(
&mem, Some(&ledger), "t-disc", || async { Ok("x".to_string()) },
|_: &String| exchange_items(4, "cli", "c1", "q", "x"), true, &mut held,
)
.await;
discard_held(&mem, Some(&ledger), held.unwrap(), "reviewer").unwrap();
assert_eq!(count(&mem), before, "all three paths leave identical memory state");
let events = concrete.events();
for lbl in ["t-cancelled", "t-err", "t-disc"] {
assert!(
events.iter().any(|e| e.starts_with(&format!("revert:{lbl}:rolled_back=true"))),
"{lbl} must witness its revert: {events:?}"
);
}
}
fn no_items<T>(_: &T) -> Vec<clawft_cow_memory::VectorItem> {
Vec::new()
}
fn test_mem(dir: &TempDir) -> Arc<Mutex<BranchableMemory>> {
let mem = BranchableMemory::create(dir.path(), 4).expect("create BranchableMemory");
Arc::new(Mutex::new(mem))
}
#[tokio::test]
async fn successful_turn_promotes_writes() {
let dir = TempDir::new().unwrap();
let mem = test_mem(&dir);
let result: clawft_types::Result<()> = with_turn_checkpoint(&mem, None, "t1", || async {
let mut guard = mem.lock().unwrap();
guard
.ingest(&[clawft_cow_memory::VectorItem::new(vec![1.0, 0.0, 0.0, 0.0])])
.map_err(|e| ClawftError::CowMemory {
reason: e.to_string(),
})?;
Ok(())
}, no_items, false, &mut None)
.await;
assert!(result.is_ok());
let guard = mem.lock().unwrap();
let hits = guard.query(&[1.0, 0.0, 0.0, 0.0], 5).expect("query");
assert_eq!(hits.len(), 1, "promoted vector should be queryable");
}
#[tokio::test]
async fn failed_turn_rolls_back_writes() {
let dir = TempDir::new().unwrap();
let mem = test_mem(&dir);
let result: clawft_types::Result<()> = with_turn_checkpoint(&mem, None, "t1", || async {
let mut guard = mem.lock().unwrap();
guard
.ingest(&[clawft_cow_memory::VectorItem::new(vec![1.0, 0.0, 0.0, 0.0])])
.map_err(|e| ClawftError::CowMemory {
reason: e.to_string(),
})?;
drop(guard);
Err(ClawftError::Timeout {
operation: "simulated turn failure".into(),
})
}, no_items, false, &mut None)
.await;
assert!(result.is_err());
let guard = mem.lock().unwrap();
let hits = guard.query(&[1.0, 0.0, 0.0, 0.0], 5).expect("query");
assert!(
hits.is_empty(),
"rolled-back turn's writes must not be queryable"
);
}
#[tokio::test]
async fn disabled_memory_untouched_by_a_turn_that_never_wraps_it() {
let dir = TempDir::new().unwrap();
let mem = test_mem(&dir);
let guard = mem.lock().unwrap();
let status = guard.status();
assert_eq!(status.dimension, 4);
assert_eq!(status.own_checkpoint_depth, 0);
}
}