use std::sync::RwLock;
use second_brain_core::index::raw::{RawHit, RawIndex, RawOrigin};
use second_brain_core::redact::redact_for_store;
use second_brain_core::retrieval::RerankScorer;
use second_brain_core::retrieval::raw_fallback::{RawAnswer, judge, raw_candidates};
use second_brain_core::sqlite_store::{RawMemoryState, SqliteStore};
use second_brain_core::transcript_archive::TranscriptArchive;
use serde::Serialize;
use uuid::Uuid;
#[derive(Serialize)]
pub(super) struct RawPassageResponse {
text: String,
session_at: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
project_path: Option<String>,
relevance: f32,
origin: &'static str,
#[serde(rename = "ref")]
reference: String,
#[serde(skip_serializing_if = "Option::is_none")]
memory: Option<String>,
}
impl RawPassageResponse {
pub(super) fn memory_id(&self) -> Option<Uuid> {
self.memory
.as_deref()
.and_then(|id| Uuid::parse_str(id).ok())
}
}
#[derive(Clone, Copy)]
pub(super) struct HotOutcome {
pub results: usize,
pub abstention_on: bool,
pub narrowed: bool,
pub budgeted: bool,
pub excludes_session: bool,
}
#[derive(Clone, Copy)]
pub(super) struct RawQuery<'a> {
pub text: &'a str,
pub embedding: &'a [f32],
pub project: Option<&'a str>,
pub limit: usize,
pub archive: Option<&'a TranscriptArchive>,
}
pub(super) fn fallback(
index: Option<&RwLock<RawIndex>>,
scorer: Option<&dyn RerankScorer>,
hot: HotOutcome,
sqlite: &SqliteStore,
query: RawQuery<'_>,
) -> Vec<RawPassageResponse> {
let (Some(index), Some(scorer)) = (index, scorer) else {
return Vec::new();
};
if hot.results > 0 || !hot.abstention_on || hot.narrowed || hot.budgeted || hot.excludes_session
{
return Vec::new();
}
match answer(index, scorer, sqlite, query) {
Ok(passages) => passages,
Err(e) => {
tracing::warn!("raw transcript fallback failed: {e:#}");
Vec::new()
}
}
}
fn answer(
index: &RwLock<RawIndex>,
scorer: &dyn RerankScorer,
sqlite: &SqliteStore,
query: RawQuery<'_>,
) -> anyhow::Result<Vec<RawPassageResponse>> {
let hits = {
let guard = index.read().unwrap_or_else(|p| p.into_inner());
raw_candidates(&guard, query.text, query.embedding, query.project)?
};
Ok(judge(query.text, with_memory_texts(sqlite, hits), scorer)?
.into_iter()
.filter_map(|a| {
let memory = standing_memory(sqlite, query.archive, &a)?;
Some(to_response(a, memory))
})
.take(query.limit)
.collect())
}
fn with_memory_texts(sqlite: &SqliteStore, hits: Vec<RawHit>) -> Vec<RawHit> {
hits.into_iter()
.filter_map(|mut hit| {
let RawOrigin::Memory { id } = hit.origin else {
return Some(hit);
};
match sqlite.get_memory_full(id) {
Ok(Some(memory)) => {
hit.text = memory.content;
Some(hit)
}
Ok(None) => None,
Err(e) => {
tracing::warn!("raw passage for memory {id} left out: {e:#}");
None
}
}
})
.collect()
}
fn standing_memory(
sqlite: &SqliteStore,
archive: Option<&TranscriptArchive>,
answer: &RawAnswer,
) -> Option<Option<Uuid>> {
if let RawOrigin::Transcript { sha256 } = &answer.hit.origin
&& let Some(entry) = archive.and_then(|a| a.holder(sha256))
{
match sqlite.session_deleted(&entry.source) {
Ok(false) => {}
Ok(true) => return None,
Err(e) => {
tracing::warn!("raw passage withheld, its session could not be checked: {e:#}");
return None;
}
}
}
let standing = match (&answer.hit.origin, answer.hit.memory) {
(RawOrigin::Memory { id }, _) => sqlite
.is_archived_raw(*id)
.map(|ok| ok.then_some(Some(*id))),
(RawOrigin::Transcript { .. }, Some(id)) => sqlite.raw_memory_state(id).map(|s| match s {
RawMemoryState::Withdrawn => None,
RawMemoryState::InForce { id } => Some(Some(id)),
RawMemoryState::Unknown => Some(None),
}),
(RawOrigin::Transcript { .. }, None) => Ok(Some(None)),
};
standing.unwrap_or_else(|e| {
tracing::warn!("raw passage withheld, its memory could not be checked: {e:#}");
None
})
}
fn to_response(answer: RawAnswer, memory: Option<Uuid>) -> RawPassageResponse {
let (origin, reference) = match answer.hit.origin {
RawOrigin::Transcript { sha256 } => ("transcript", sha256),
RawOrigin::Memory { id } => ("archived-memory", id.to_string()),
};
RawPassageResponse {
text: redact_for_store(&answer.hit.text),
session_at: answer.hit.at.map(|t| t.to_rfc3339()),
project_path: answer.hit.project,
relevance: answer.relevance,
origin,
reference,
memory: memory.map(|id| id.to_string()),
}
}
#[cfg(test)]
mod tests {
use std::sync::atomic::{AtomicUsize, Ordering};
use super::*;
use anyhow::Result;
use second_brain_core::cold::{ArchiveKeys, ColdTier, Kek, KekLabel, KeyId, Zeroizing};
use second_brain_core::index::EMBEDDING_DIM;
use second_brain_core::index::raw::{RawPassage, text_hash};
use second_brain_core::schema::{Memory, MemoryType};
use second_brain_core::store::Store;
use uuid::Uuid;
#[derive(Default)]
struct Mirror(AtomicUsize);
impl RerankScorer for Mirror {
fn rerank(&self, _: &str, docs: &[&str]) -> Result<Vec<f32>> {
self.0.fetch_add(1, Ordering::SeqCst);
Ok(docs
.iter()
.map(|d| if d.contains("mirror") { 0.9 } else { 0.0 })
.collect())
}
}
const ABSTAINED: HotOutcome = HotOutcome {
results: 0,
abstention_on: true,
narrowed: false,
budgeted: false,
excludes_session: false,
};
fn unit(i: usize) -> Vec<f32> {
let mut v = vec![0.0; EMBEDDING_DIM];
v[i] = 1.0;
v
}
fn passage(text: &str, memory: Option<Uuid>) -> RawPassage {
RawPassage {
text: text.to_string(),
at: None,
project: None,
memory,
}
}
fn transcript(byte: &str) -> RawOrigin {
RawOrigin::Transcript {
sha256: byte.repeat(32),
}
}
struct Fixture {
_dir: tempfile::TempDir,
index: RwLock<RawIndex>,
store: SqliteStore,
embedding: Vec<f32>,
}
impl Fixture {
fn new(store: SqliteStore, sources: &[(RawOrigin, RawPassage)]) -> Self {
let dir = tempfile::tempdir().unwrap();
let mut index = RawIndex::open(dir.path()).unwrap();
for (origin, passage) in sources {
index
.add_source(origin, &[(passage.clone(), unit(1))])
.unwrap();
}
index.commit().unwrap();
Self {
_dir: dir,
index: RwLock::new(index),
store,
embedding: unit(1),
}
}
fn query(&self, limit: usize) -> RawQuery<'_> {
RawQuery {
text: "mirror",
embedding: &self.embedding,
project: None,
limit,
archive: None,
}
}
fn ask(&self, scorer: &dyn RerankScorer, hot: HotOutcome) -> Vec<(&'static str, String)> {
let mut out: Vec<_> = fallback(
Some(&self.index),
Some(scorer),
hot,
&self.store,
self.query(5),
)
.into_iter()
.map(|p| (p.origin, p.text))
.collect();
out.sort();
out
}
}
fn store() -> SqliteStore {
SqliteStore::in_memory("machine-a".to_string()).unwrap()
}
fn raw_memory(store: &SqliteStore, content: &str) -> Uuid {
let mut m = Memory::new(
content.to_string(),
MemoryType::Semantic,
"claude-code".to_string(),
String::new(),
);
m.embedding = unit(1);
store.store_memory(&m).unwrap();
m.id
}
#[test]
fn raw_passages_answer_only_after_the_hot_path_abstained_with_nothing() {
let f = Fixture::new(
store(),
&[(transcript("ab"), passage("the mirror runs nightly", None))],
);
let scorer = Mirror::default();
assert_eq!(
f.ask(&scorer, ABSTAINED),
vec![("transcript", "the mirror runs nightly".to_string())]
);
for hot in [
HotOutcome {
results: 1,
..ABSTAINED
},
HotOutcome {
abstention_on: false,
..ABSTAINED
},
HotOutcome {
narrowed: true,
..ABSTAINED
},
] {
assert!(f.ask(&scorer, hot).is_empty());
}
assert_eq!(scorer.0.load(Ordering::SeqCst), 1);
assert!(fallback(Some(&f.index), None, ABSTAINED, &f.store, f.query(5)).is_empty());
assert!(fallback(None, Some(&scorer), ABSTAINED, &f.store, f.query(5)).is_empty());
}
#[test]
fn a_recall_with_a_deadline_or_an_excluded_session_gets_no_raw_passages() {
let f = Fixture::new(
store(),
&[(transcript("ab"), passage("the mirror runs nightly", None))],
);
let scorer = Mirror::default();
for hot in [
HotOutcome {
budgeted: true,
..ABSTAINED
},
HotOutcome {
excludes_session: true,
..ABSTAINED
},
] {
assert!(f.ask(&scorer, hot).is_empty());
}
assert_eq!(scorer.0.load(Ordering::SeqCst), 0);
}
#[test]
fn an_archived_memory_stops_answering_once_it_is_retired() {
let s = store();
let id = raw_memory(&s, "the mirror token rotates monthly");
s.archive_memory(id).unwrap();
let f = Fixture::new(
s,
&[
(transcript("ab"), passage("the mirror runs nightly", None)),
(
RawOrigin::Memory { id },
passage("the mirror token rotates monthly", Some(id)),
),
],
);
let scorer = Mirror::default();
assert_eq!(f.ask(&scorer, ABSTAINED).len(), 2);
f.store.retire_memory(id, "wrong").unwrap();
assert_eq!(
f.ask(&scorer, ABSTAINED),
vec![("transcript", "the mirror runs nightly".to_string())]
);
}
#[test]
fn a_transcript_passage_stops_answering_once_its_raw_memory_is_forgotten() {
let s = store();
let id = raw_memory(&s, "the mirror key lives in the vault");
let f = Fixture::new(
s,
&[(
transcript("ab"),
passage("the mirror key lives in the vault", Some(id)),
)],
);
let scorer = Mirror::default();
let served = fallback(
Some(&f.index),
Some(&scorer),
ABSTAINED,
&f.store,
f.query(5),
);
assert_eq!(served[0].memory, Some(id.to_string()));
f.store.retire_memory(id, "should not exist").unwrap();
assert!(f.ask(&scorer, ABSTAINED).is_empty());
}
#[test]
fn served_text_is_redacted_with_the_current_rules() {
let token = ["gh", "p_"].concat() + &"a1B2".repeat(9);
let f = Fixture::new(
store(),
&[(
transcript("ab"),
passage(&format!("the mirror reads {token}"), None),
)],
);
let served = f.ask(&Mirror::default(), ABSTAINED);
assert_eq!(served.len(), 1);
assert!(!served[0].1.contains(&token), "{}", served[0].1);
assert!(served[0].1.contains("[REDACTED:"), "{}", served[0].1);
}
#[test]
fn withdrawn_passages_do_not_shorten_the_answer_below_the_limit() {
let s = store();
let gone = raw_memory(&s, "the mirror moved to the old host");
s.retire_memory(gone, "wrong").unwrap();
let f = Fixture::new(
s,
&[
(
transcript("ab"),
passage("the mirror moved to the old host", Some(gone)),
),
(
transcript("cd"),
passage("the mirror moved to the new host", None),
),
],
);
let served = fallback(
Some(&f.index),
Some(&Mirror::default()),
ABSTAINED,
&f.store,
f.query(1),
);
assert_eq!(served.len(), 1);
assert_eq!(served[0].text, "the mirror moved to the new host");
}
fn answer(origin: RawOrigin) -> RawAnswer {
RawAnswer {
hit: RawHit {
key: 1,
text: "the mirror moved".to_string(),
text_hash: text_hash("the mirror moved"),
at: None,
project: None,
memory: None,
origin,
},
relevance: 0.9,
}
}
#[test]
fn memory_id_reads_back_the_served_memory() {
let id = Uuid::new_v4();
assert_eq!(
to_response(answer(RawOrigin::Memory { id }), Some(id)).memory_id(),
Some(id)
);
assert_eq!(
to_response(answer(transcript("ab")), None).memory_id(),
None
);
let mut malformed = to_response(answer(transcript("ab")), Some(id));
malformed.memory = Some("not-a-uuid".to_string());
assert_eq!(malformed.memory_id(), None);
}
struct OneKey;
impl ArchiveKeys for OneKey {
fn current(&self) -> Result<Option<Kek>> {
Ok(Some(Kek {
label: KekLabel {
epoch: 1,
key_id: KeyId([1; 8]),
},
key: Zeroizing::new([1; 32]),
}))
}
fn by_label(&self, label: &KekLabel) -> Result<Option<Zeroizing<[u8; 32]>>> {
Ok((label.epoch == 1).then(|| Zeroizing::new([1; 32])))
}
}
fn cold_store(dir: &std::path::Path, content: &str) -> (SqliteStore, Uuid, std::path::PathBuf) {
let s = store();
let tier = std::sync::Arc::new(
ColdTier::open(dir.join("brain.cold"), std::sync::Arc::new(OneKey)).unwrap(),
);
s.attach_cold_tier(tier.clone());
let mut m = Memory::new(
content.to_string(),
MemoryType::Semantic,
"claude-code".to_string(),
String::new(),
);
m.embedding = unit(1);
m.session_at = Some(chrono::Utc::now() - chrono::Duration::days(400));
s.store_memory(&m).unwrap();
s.archive_memory(m.id).unwrap();
s.run_stored_redaction(None).unwrap();
let report = s
.run_cold_pass(chrono::Utc::now(), std::time::Duration::ZERO)
.unwrap();
assert_eq!(report.sealed_rows, 1);
let segment = s
.get_memory(m.id)
.unwrap()
.unwrap()
.archived_segment
.unwrap();
let path = tier.dir().join(format!("{segment}.seg"));
(s, m.id, path)
}
#[test]
fn a_cold_archived_memory_answers_with_the_text_in_its_segment() {
let dir = tempfile::tempdir().unwrap();
let full = "the mirror runs nightly and writes to the second disk";
let (s, id, _) = cold_store(dir.path(), full);
let f = Fixture::new(
s,
&[(
RawOrigin::Memory { id },
passage("mirror index copy", Some(id)),
)],
);
let served = fallback(
Some(&f.index),
Some(&Mirror::default()),
ABSTAINED,
&f.store,
f.query(5),
);
assert_eq!(served.len(), 1);
assert_eq!(served[0].text, full);
assert_eq!(served[0].memory_id(), Some(id));
}
#[test]
fn an_archived_memory_answers_with_the_stores_text_and_the_index_keeps_none() {
let s = store();
let full = "the mirror token rotates monthly at the second disk";
let id = raw_memory(&s, full);
s.archive_memory(id).unwrap();
let f = Fixture::new(
s,
&[(
RawOrigin::Memory { id },
passage("mirror index copy", Some(id)),
)],
);
let hit = {
let index = f.index.read().unwrap();
let key = index.bm25_search("mirror", 1, None).unwrap()[0].0;
index.passage(key).unwrap().unwrap()
};
assert!(hit.text.is_empty());
let served = fallback(
Some(&f.index),
Some(&Mirror::default()),
ABSTAINED,
&f.store,
f.query(5),
);
assert_eq!(served.len(), 1);
assert_eq!(served[0].origin, "archived-memory");
assert_eq!(served[0].text, full);
}
#[test]
fn a_cold_memory_whose_segment_fails_to_open_is_left_out_and_recorded() {
let dir = tempfile::tempdir().unwrap();
let (s, id, path) = cold_store(dir.path(), "the mirror runs nightly");
std::fs::write(&path, b"not a segment").unwrap();
let f = Fixture::new(
s,
&[
(
RawOrigin::Memory { id },
passage("mirror index copy", Some(id)),
),
(transcript("ab"), passage("the mirror moved", None)),
],
);
let served = fallback(
Some(&f.index),
Some(&Mirror::default()),
ABSTAINED,
&f.store,
f.query(5),
);
assert_eq!(served.len(), 1);
assert_eq!(served[0].text, "the mirror moved");
let segment = path.file_stem().unwrap().to_string_lossy().to_string();
assert_eq!(f.store.cold_stats().unwrap().unreadable[0].segment, segment);
}
}