1use std::collections::{BTreeMap, HashMap, HashSet};
7
8use anyhow::Result;
9use chrono::{DateTime, Utc};
10use rusqlite::params;
11use serde::Serialize;
12use uuid::Uuid;
13
14use crate::schema::{EXPLICIT_SOURCES, Memory, Tier};
15use crate::sqlite_store::{MEMORY_COLS, SqliteStore, parse_dt, parse_uuid, row_to_memory};
16
17pub fn latest_session<'a>(members: impl IntoIterator<Item = &'a Memory>) -> Option<DateTime<Utc>> {
19 members.into_iter().filter_map(|m| m.session_at).max()
20}
21
22pub trait SessionEvidence {
25 fn line_time(&self, transcript: &str, text: &str) -> Option<DateTime<Utc>>;
27 fn first_time(&self, transcript: &str) -> Option<DateTime<Utc>>;
29 fn note_modified_at(&self, path: &str) -> Option<DateTime<Utc>>;
32}
33
34#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
39#[serde(rename_all = "snake_case")]
40pub enum SessionTier {
41 TranscriptLine,
42 ConversationStart,
43 TranscriptStart,
44 FileMtime,
45 RememberedAt,
46 Unknown,
47}
48
49#[derive(Debug, Default, Serialize)]
50#[serde(transparent)]
51pub struct TierCounts(BTreeMap<SessionTier, u64>);
52
53impl TierCounts {
54 pub fn get(&self, tier: SessionTier) -> u64 {
55 self.0.get(&tier).copied().unwrap_or(0)
56 }
57
58 fn add(&mut self, tier: SessionTier) {
59 *self.0.entry(tier).or_insert(0) += 1;
60 }
61}
62
63#[derive(Debug, Serialize)]
64pub struct SessionSample {
65 pub id: Uuid,
66 pub tier: SessionTier,
67 pub session_at: DateTime<Utc>,
68 pub created_at: DateTime<Utc>,
69 pub source: String,
70 pub source_id: String,
71 pub content: String,
72}
73
74#[derive(Debug, Default, Serialize)]
75pub struct SessionBackfillReport {
76 pub dry_run: bool,
77 pub candidates: u64,
79 pub all: TierCounts,
80 pub live: TierCounts,
82 pub fallback_conversations_skipped: u64,
85 pub written: u64,
86 pub reemitted: u64,
87 pub sample: Vec<SessionSample>,
88}
89
90const SAMPLE_SIZE: usize = 20;
91const WRITE_BATCH: usize = 500;
92
93struct ConversationLink {
94 id: Uuid,
95 source: String,
96 started_at: DateTime<Utc>,
97}
98
99struct StoreFacts {
102 memories: Vec<Memory>,
103 raws_of: HashMap<Uuid, Vec<Uuid>>,
104 conversations_of: HashMap<Uuid, Vec<ConversationLink>>,
105}
106
107impl SqliteStore {
108 pub fn backfill_session_at(
115 &self,
116 evidence: &dyn SessionEvidence,
117 dry_run: bool,
118 reemit: bool,
119 ) -> Result<SessionBackfillReport> {
120 let facts = self.session_facts()?;
121 let by_id: HashMap<Uuid, &Memory> = facts.memories.iter().map(|m| (m.id, m)).collect();
122
123 let candidates: Vec<&Memory> = facts
124 .memories
125 .iter()
126 .filter(|m| m.session_at.is_none())
127 .collect();
128 let line_times = transcript_line_times(&candidates, &facts.raws_of, &by_id, evidence);
129
130 let mut report = SessionBackfillReport {
131 dry_run,
132 candidates: candidates.len() as u64,
133 ..Default::default()
134 };
135 let mut fallbacks: HashSet<Uuid> = HashSet::new();
136 let mut dated: Vec<(&Memory, SessionTier, DateTime<Utc>)> = Vec::new();
137 for m in &candidates {
138 let raws = raws_of(m, &facts.raws_of, &by_id);
139 let (tier, at) = date_of(
140 m,
141 &raws,
142 &line_times,
143 &facts.conversations_of,
144 evidence,
145 &mut fallbacks,
146 );
147 report.all.add(tier);
148 if m.tier == Tier::Hot && m.superseded_by.is_none() {
149 report.live.add(tier);
150 }
151 if let Some(at) = at {
152 dated.push((m, tier, at));
153 }
154 }
155 report.fallback_conversations_skipped = fallbacks.len() as u64;
156 report.sample = sample(&dated);
157
158 if dry_run {
159 return Ok(report);
160 }
161 for batch in dated.chunks(WRITE_BATCH) {
162 report.written += self.with_txn(|tx| {
163 let mut written = 0;
164 for (m, _, at) in batch {
165 let changed = tx.execute(
166 "UPDATE memories SET session_at = ?2 WHERE id = ?1 AND session_at IS NULL",
167 params![m.id.to_string(), at.to_rfc3339()],
168 )?;
169 if changed == 1
170 && let Some(row) = Self::memory_on_tx(tx, m.id)?
171 {
172 self.log_memory_update(tx, &row)?;
173 written += 1;
174 }
175 }
176 Ok(written)
177 })?;
178 }
179 if reemit {
180 let predated: Vec<Uuid> = facts
181 .memories
182 .iter()
183 .filter(|m| m.session_at.is_some())
184 .map(|m| m.id)
185 .collect();
186 for batch in predated.chunks(WRITE_BATCH) {
187 report.reemitted += self.with_txn(|tx| {
188 let mut logged = 0;
189 for &id in batch {
190 if let Some(row) = Self::memory_on_tx(tx, id)? {
191 self.log_memory_update(tx, &row)?;
192 logged += 1;
193 }
194 }
195 Ok(logged)
196 })?;
197 }
198 }
199 Ok(report)
200 }
201
202 fn session_facts(&self) -> Result<StoreFacts> {
203 let conn = self.reader();
204 let memories = {
205 let mut stmt =
206 conn.prepare(&format!("SELECT {MEMORY_COLS} FROM memories ORDER BY id"))?;
207 stmt.query_map([], |r| row_to_memory(r, false))?
208 .collect::<rusqlite::Result<Vec<_>>>()?
209 };
210
211 let mut raws_of: HashMap<Uuid, Vec<Uuid>> = HashMap::new();
212 {
213 let mut stmt = conn.prepare(
214 "SELECT distilled_id, memory_id FROM distillation_log \
215 WHERE memory_id <> distilled_id",
216 )?;
217 let rows = stmt.query_map([], |r| {
218 Ok((
219 parse_uuid(&r.get::<_, String>(0)?)?,
220 parse_uuid(&r.get::<_, String>(1)?)?,
221 ))
222 })?;
223 for row in rows {
224 let (distilled, raw) = row?;
225 raws_of.entry(distilled).or_default().push(raw);
226 }
227 }
228
229 let mut conversations_of: HashMap<Uuid, Vec<ConversationLink>> = HashMap::new();
230 {
231 let mut stmt = conn.prepare(
232 "SELECT r.from_id, c.id, c.source, c.started_at FROM relations r \
233 JOIN conversations c ON c.id = r.to_id \
234 WHERE r.relation_type = 'derived_from'",
235 )?;
236 let rows = stmt.query_map([], |r| {
237 Ok((
238 parse_uuid(&r.get::<_, String>(0)?)?,
239 ConversationLink {
240 id: parse_uuid(&r.get::<_, String>(1)?)?,
241 source: r.get(2)?,
242 started_at: parse_dt(&r.get::<_, String>(3)?)?,
243 },
244 ))
245 })?;
246 for row in rows {
247 let (memory, link) = row?;
248 conversations_of.entry(memory).or_default().push(link);
249 }
250 }
251
252 Ok(StoreFacts {
253 memories,
254 raws_of,
255 conversations_of,
256 })
257 }
258}
259
260fn is_distillate(m: &Memory) -> bool {
261 m.source == "consolidation"
262}
263
264fn is_explicit(m: &Memory) -> bool {
265 EXPLICIT_SOURCES.contains(&m.source.as_str())
266}
267
268fn raws_of<'a>(
271 m: &'a Memory,
272 raws_of: &HashMap<Uuid, Vec<Uuid>>,
273 by_id: &HashMap<Uuid, &'a Memory>,
274) -> Vec<&'a Memory> {
275 if is_explicit(m) {
276 Vec::new()
277 } else if is_distillate(m) {
278 raws_of
279 .get(&m.id)
280 .into_iter()
281 .flatten()
282 .filter_map(|id| by_id.get(id).copied())
283 .filter(|raw| !is_distillate(raw) && !is_explicit(raw))
284 .collect()
285 } else {
286 vec![m]
287 }
288}
289
290fn transcript_line_times(
294 candidates: &[&Memory],
295 raws: &HashMap<Uuid, Vec<Uuid>>,
296 by_id: &HashMap<Uuid, &Memory>,
297 evidence: &dyn SessionEvidence,
298) -> HashMap<Uuid, DateTime<Utc>> {
299 let mut needed: Vec<&Memory> = candidates
300 .iter()
301 .flat_map(|m| raws_of(m, raws, by_id))
302 .collect();
303 needed.sort_by(|a, b| (&a.source_id, a.id).cmp(&(&b.source_id, b.id)));
304 needed.dedup_by_key(|m| m.id);
305 needed
306 .into_iter()
307 .filter_map(|raw| {
308 evidence
309 .line_time(&raw.source_id, &raw.content)
310 .map(|at| (raw.id, at))
311 })
312 .collect()
313}
314
315fn date_of(
316 m: &Memory,
317 raws: &[&Memory],
318 line_times: &HashMap<Uuid, DateTime<Utc>>,
319 conversations_of: &HashMap<Uuid, Vec<ConversationLink>>,
320 evidence: &dyn SessionEvidence,
321 fallbacks: &mut HashSet<Uuid>,
322) -> (SessionTier, Option<DateTime<Utc>>) {
323 if is_explicit(m) {
324 return if m.source == "claude-memory" {
328 match note_time(m, evidence) {
329 Some(at) => (SessionTier::FileMtime, Some(at)),
330 None => (SessionTier::Unknown, None),
331 }
332 } else {
333 (SessionTier::RememberedAt, Some(m.created_at))
334 };
335 }
336
337 if let Some(at) = raws.iter().filter_map(|r| line_times.get(&r.id)).max() {
338 return (SessionTier::TranscriptLine, Some(*at));
339 }
340
341 let started = raws
342 .iter()
343 .flat_map(|raw| {
344 conversations_of
345 .get(&raw.id)
346 .into_iter()
347 .flatten()
348 .map(move |c| (raw, c))
349 })
350 .filter(|(raw, c)| {
351 if c.source == "claude-memory" {
352 return false;
353 }
354 let parsed = c.started_at < raw.created_at;
358 if !parsed {
359 fallbacks.insert(c.id);
360 }
361 parsed
362 })
363 .map(|(_, c)| c.started_at)
364 .max();
365 if let Some(at) = started {
366 return (SessionTier::ConversationStart, Some(at));
367 }
368
369 if let Some(at) = evidence.first_time(&m.source_id) {
370 return (SessionTier::TranscriptStart, Some(at));
371 }
372 match note_time(m, evidence) {
375 Some(at) => (SessionTier::FileMtime, Some(at)),
376 None => (SessionTier::Unknown, None),
377 }
378}
379
380fn note_time(m: &Memory, evidence: &dyn SessionEvidence) -> Option<DateTime<Utc>> {
383 evidence
384 .note_modified_at(&m.source_id)
385 .filter(|at| *at <= m.created_at)
386}
387
388fn sample(dated: &[(&Memory, SessionTier, DateTime<Utc>)]) -> Vec<SessionSample> {
391 let step = (dated.len() / SAMPLE_SIZE).max(1);
392 dated
393 .iter()
394 .step_by(step)
395 .take(SAMPLE_SIZE)
396 .map(|(m, tier, at)| SessionSample {
397 id: m.id,
398 tier: *tier,
399 session_at: *at,
400 created_at: m.created_at,
401 source: m.source.clone(),
402 source_id: m.source_id.clone(),
403 content: m.content.chars().take(160).collect(),
404 })
405 .collect()
406}