use std::collections::{BTreeMap, HashMap, HashSet};
use anyhow::Result;
use chrono::{DateTime, Utc};
use rusqlite::params;
use serde::Serialize;
use uuid::Uuid;
use crate::schema::{EXPLICIT_SOURCES, Memory, Tier};
use crate::sqlite_store::{MEMORY_COLS, SqliteStore, parse_dt, parse_uuid, row_to_memory};
pub fn latest_session<'a>(members: impl IntoIterator<Item = &'a Memory>) -> Option<DateTime<Utc>> {
members.into_iter().filter_map(|m| m.session_at).max()
}
pub trait SessionEvidence {
fn line_time(&self, transcript: &str, text: &str) -> Option<DateTime<Utc>>;
fn first_time(&self, transcript: &str) -> Option<DateTime<Utc>>;
fn note_modified_at(&self, path: &str) -> Option<DateTime<Utc>>;
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum SessionTier {
TranscriptLine,
ConversationStart,
TranscriptStart,
FileMtime,
RememberedAt,
Unknown,
}
#[derive(Debug, Default, Serialize)]
#[serde(transparent)]
pub struct TierCounts(BTreeMap<SessionTier, u64>);
impl TierCounts {
pub fn get(&self, tier: SessionTier) -> u64 {
self.0.get(&tier).copied().unwrap_or(0)
}
fn add(&mut self, tier: SessionTier) {
*self.0.entry(tier).or_insert(0) += 1;
}
}
#[derive(Debug, Serialize)]
pub struct SessionSample {
pub id: Uuid,
pub tier: SessionTier,
pub session_at: DateTime<Utc>,
pub created_at: DateTime<Utc>,
pub source: String,
pub source_id: String,
pub content: String,
}
#[derive(Debug, Default, Serialize)]
pub struct SessionBackfillReport {
pub dry_run: bool,
pub candidates: u64,
pub all: TierCounts,
pub live: TierCounts,
pub fallback_conversations_skipped: u64,
pub written: u64,
pub reemitted: u64,
pub sample: Vec<SessionSample>,
}
const SAMPLE_SIZE: usize = 20;
const WRITE_BATCH: usize = 500;
struct ConversationLink {
id: Uuid,
source: String,
started_at: DateTime<Utc>,
}
struct StoreFacts {
memories: Vec<Memory>,
raws_of: HashMap<Uuid, Vec<Uuid>>,
conversations_of: HashMap<Uuid, Vec<ConversationLink>>,
}
impl SqliteStore {
pub fn backfill_session_at(
&self,
evidence: &dyn SessionEvidence,
dry_run: bool,
reemit: bool,
) -> Result<SessionBackfillReport> {
let facts = self.session_facts()?;
let by_id: HashMap<Uuid, &Memory> = facts.memories.iter().map(|m| (m.id, m)).collect();
let candidates: Vec<&Memory> = facts
.memories
.iter()
.filter(|m| m.session_at.is_none())
.collect();
let line_times = transcript_line_times(&candidates, &facts.raws_of, &by_id, evidence);
let mut report = SessionBackfillReport {
dry_run,
candidates: candidates.len() as u64,
..Default::default()
};
let mut fallbacks: HashSet<Uuid> = HashSet::new();
let mut dated: Vec<(&Memory, SessionTier, DateTime<Utc>)> = Vec::new();
for m in &candidates {
let raws = raws_of(m, &facts.raws_of, &by_id);
let (tier, at) = date_of(
m,
&raws,
&line_times,
&facts.conversations_of,
evidence,
&mut fallbacks,
);
report.all.add(tier);
if m.tier == Tier::Hot && m.superseded_by.is_none() {
report.live.add(tier);
}
if let Some(at) = at {
dated.push((m, tier, at));
}
}
report.fallback_conversations_skipped = fallbacks.len() as u64;
report.sample = sample(&dated);
if dry_run {
return Ok(report);
}
for batch in dated.chunks(WRITE_BATCH) {
report.written += self.with_txn(|tx| {
let mut written = 0;
for (m, _, at) in batch {
let changed = tx.execute(
"UPDATE memories SET session_at = ?2 WHERE id = ?1 AND session_at IS NULL",
params![m.id.to_string(), at.to_rfc3339()],
)?;
if changed == 1
&& let Some(row) = Self::memory_on_tx(tx, m.id)?
{
self.log_memory_update(tx, &row)?;
written += 1;
}
}
Ok(written)
})?;
}
if reemit {
let predated: Vec<Uuid> = facts
.memories
.iter()
.filter(|m| m.session_at.is_some())
.map(|m| m.id)
.collect();
for batch in predated.chunks(WRITE_BATCH) {
report.reemitted += self.with_txn(|tx| {
let mut logged = 0;
for &id in batch {
if let Some(row) = Self::memory_on_tx(tx, id)? {
self.log_memory_update(tx, &row)?;
logged += 1;
}
}
Ok(logged)
})?;
}
}
Ok(report)
}
fn session_facts(&self) -> Result<StoreFacts> {
let conn = self.reader();
let memories = {
let mut stmt =
conn.prepare(&format!("SELECT {MEMORY_COLS} FROM memories ORDER BY id"))?;
stmt.query_map([], |r| row_to_memory(r, false))?
.collect::<rusqlite::Result<Vec<_>>>()?
};
let mut raws_of: HashMap<Uuid, Vec<Uuid>> = HashMap::new();
{
let mut stmt = conn.prepare(
"SELECT distilled_id, memory_id FROM distillation_log \
WHERE memory_id <> distilled_id",
)?;
let rows = stmt.query_map([], |r| {
Ok((
parse_uuid(&r.get::<_, String>(0)?)?,
parse_uuid(&r.get::<_, String>(1)?)?,
))
})?;
for row in rows {
let (distilled, raw) = row?;
raws_of.entry(distilled).or_default().push(raw);
}
}
let mut conversations_of: HashMap<Uuid, Vec<ConversationLink>> = HashMap::new();
{
let mut stmt = conn.prepare(
"SELECT r.from_id, c.id, c.source, c.started_at FROM relations r \
JOIN conversations c ON c.id = r.to_id \
WHERE r.relation_type = 'derived_from'",
)?;
let rows = stmt.query_map([], |r| {
Ok((
parse_uuid(&r.get::<_, String>(0)?)?,
ConversationLink {
id: parse_uuid(&r.get::<_, String>(1)?)?,
source: r.get(2)?,
started_at: parse_dt(&r.get::<_, String>(3)?)?,
},
))
})?;
for row in rows {
let (memory, link) = row?;
conversations_of.entry(memory).or_default().push(link);
}
}
Ok(StoreFacts {
memories,
raws_of,
conversations_of,
})
}
}
fn is_distillate(m: &Memory) -> bool {
m.source == "consolidation"
}
fn is_explicit(m: &Memory) -> bool {
EXPLICIT_SOURCES.contains(&m.source.as_str())
}
fn raws_of<'a>(
m: &'a Memory,
raws_of: &HashMap<Uuid, Vec<Uuid>>,
by_id: &HashMap<Uuid, &'a Memory>,
) -> Vec<&'a Memory> {
if is_explicit(m) {
Vec::new()
} else if is_distillate(m) {
raws_of
.get(&m.id)
.into_iter()
.flatten()
.filter_map(|id| by_id.get(id).copied())
.filter(|raw| !is_distillate(raw) && !is_explicit(raw))
.collect()
} else {
vec![m]
}
}
fn transcript_line_times(
candidates: &[&Memory],
raws: &HashMap<Uuid, Vec<Uuid>>,
by_id: &HashMap<Uuid, &Memory>,
evidence: &dyn SessionEvidence,
) -> HashMap<Uuid, DateTime<Utc>> {
let mut needed: Vec<&Memory> = candidates
.iter()
.flat_map(|m| raws_of(m, raws, by_id))
.collect();
needed.sort_by(|a, b| (&a.source_id, a.id).cmp(&(&b.source_id, b.id)));
needed.dedup_by_key(|m| m.id);
needed
.into_iter()
.filter_map(|raw| {
evidence
.line_time(&raw.source_id, &raw.content)
.map(|at| (raw.id, at))
})
.collect()
}
fn date_of(
m: &Memory,
raws: &[&Memory],
line_times: &HashMap<Uuid, DateTime<Utc>>,
conversations_of: &HashMap<Uuid, Vec<ConversationLink>>,
evidence: &dyn SessionEvidence,
fallbacks: &mut HashSet<Uuid>,
) -> (SessionTier, Option<DateTime<Utc>>) {
if is_explicit(m) {
return if m.source == "claude-memory" {
match note_time(m, evidence) {
Some(at) => (SessionTier::FileMtime, Some(at)),
None => (SessionTier::Unknown, None),
}
} else {
(SessionTier::RememberedAt, Some(m.created_at))
};
}
if let Some(at) = raws.iter().filter_map(|r| line_times.get(&r.id)).max() {
return (SessionTier::TranscriptLine, Some(*at));
}
let started = raws
.iter()
.flat_map(|raw| {
conversations_of
.get(&raw.id)
.into_iter()
.flatten()
.map(move |c| (raw, c))
})
.filter(|(raw, c)| {
if c.source == "claude-memory" {
return false;
}
let parsed = c.started_at < raw.created_at;
if !parsed {
fallbacks.insert(c.id);
}
parsed
})
.map(|(_, c)| c.started_at)
.max();
if let Some(at) = started {
return (SessionTier::ConversationStart, Some(at));
}
if let Some(at) = evidence.first_time(&m.source_id) {
return (SessionTier::TranscriptStart, Some(at));
}
match note_time(m, evidence) {
Some(at) => (SessionTier::FileMtime, Some(at)),
None => (SessionTier::Unknown, None),
}
}
fn note_time(m: &Memory, evidence: &dyn SessionEvidence) -> Option<DateTime<Utc>> {
evidence
.note_modified_at(&m.source_id)
.filter(|at| *at <= m.created_at)
}
fn sample(dated: &[(&Memory, SessionTier, DateTime<Utc>)]) -> Vec<SessionSample> {
let step = (dated.len() / SAMPLE_SIZE).max(1);
dated
.iter()
.step_by(step)
.take(SAMPLE_SIZE)
.map(|(m, tier, at)| SessionSample {
id: m.id,
tier: *tier,
session_at: *at,
created_at: m.created_at,
source: m.source.clone(),
source_id: m.source_id.clone(),
content: m.content.chars().take(160).collect(),
})
.collect()
}