Skip to main content

second_brain_core/
session_dates.rs

1//! When the session behind each memory happened. `created_at` is the time a
2//! row was ingested or distilled, which bulk re-ingests and re-distillation
3//! moved by weeks, so recall needs a separate date that only ever comes from
4//! evidence of the session itself and is left unknown otherwise.
5
6use std::collections::{BTreeMap, HashMap, HashSet};
7
8use anyhow::Result;
9use chrono::{DateTime, Utc};
10use rusqlite::params;
11use serde::Serialize;
12use uuid::Uuid;
13
14use crate::schema::{EXPLICIT_SOURCES, Memory, Tier};
15use crate::sqlite_store::{MEMORY_COLS, SqliteStore, parse_dt, parse_uuid, row_to_memory};
16
17/// A distillate or merge is as recent as the latest session it drew on.
18pub fn latest_session<'a>(members: impl IntoIterator<Item = &'a Memory>) -> Option<DateTime<Utc>> {
19    members.into_iter().filter_map(|m| m.session_at).max()
20}
21
22/// Session evidence that lives outside the store: transcripts and note
23/// files, which only the machine that captured the sessions has.
24pub trait SessionEvidence {
25    /// Time of the line in the transcript at `transcript` that holds `text`.
26    fn line_time(&self, transcript: &str, text: &str) -> Option<DateTime<Utc>>;
27    /// Time of the first timestamped line in the transcript.
28    fn first_time(&self, transcript: &str) -> Option<DateTime<Utc>>;
29    /// Modification time of the note file at `path`. None for a transcript,
30    /// whose mtime is its last line rather than when its content was said.
31    fn note_modified_at(&self, path: &str) -> Option<DateTime<Utc>>;
32}
33
34/// Where a backfilled date came from, strongest first. Everything after
35/// `TranscriptLine` except `RememberedAt` is a lower bound: a session start
36/// dates knowledge written later in that session as older than it was, the
37/// safe direction when judging staleness.
38#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
39#[serde(rename_all = "snake_case")]
40pub enum SessionTier {
41    TranscriptLine,
42    ConversationStart,
43    TranscriptStart,
44    FileMtime,
45    RememberedAt,
46    Unknown,
47}
48
49#[derive(Debug, Default, Serialize)]
50#[serde(transparent)]
51pub struct TierCounts(BTreeMap<SessionTier, u64>);
52
53impl TierCounts {
54    pub fn get(&self, tier: SessionTier) -> u64 {
55        self.0.get(&tier).copied().unwrap_or(0)
56    }
57
58    fn add(&mut self, tier: SessionTier) {
59        *self.0.entry(tier).or_insert(0) += 1;
60    }
61}
62
63#[derive(Debug, Serialize)]
64pub struct SessionSample {
65    pub id: Uuid,
66    pub tier: SessionTier,
67    pub session_at: DateTime<Utc>,
68    pub created_at: DateTime<Utc>,
69    pub source: String,
70    pub source_id: String,
71    pub content: String,
72}
73
74#[derive(Debug, Default, Serialize)]
75pub struct SessionBackfillReport {
76    pub dry_run: bool,
77    /// Rows that had no session date when the run started.
78    pub candidates: u64,
79    pub all: TierCounts,
80    /// The subset recall can return: hot and not superseded.
81    pub live: TierCounts,
82    /// Conversations whose `started_at` was stamped at ingest because the
83    /// transcript carried no timestamp, so it was not used as evidence.
84    pub fallback_conversations_skipped: u64,
85    pub written: u64,
86    pub reemitted: u64,
87    pub sample: Vec<SessionSample>,
88}
89
90const SAMPLE_SIZE: usize = 20;
91const WRITE_BATCH: usize = 500;
92
93struct ConversationLink {
94    id: Uuid,
95    source: String,
96    started_at: DateTime<Utc>,
97}
98
99/// Everything the tiers read from the store, loaded once so the evidence
100/// lookups run outside any transaction.
101struct StoreFacts {
102    memories: Vec<Memory>,
103    raws_of: HashMap<Uuid, Vec<Uuid>>,
104    conversations_of: HashMap<Uuid, Vec<ConversationLink>>,
105}
106
107impl SqliteStore {
108    /// Fills a NULL `session_at` from the strongest evidence each memory has
109    /// (see `SessionTier`) and logs a sync Update per row, without touching
110    /// `updated_at`: peers merge the date apart from last-write-wins, so the
111    /// Update never beats a newer edit made elsewhere. A date already set is
112    /// never replaced, so a second run writes nothing. `reemit` also relogs
113    /// every date set before this run, for a peer that upgraded late.
114    pub fn backfill_session_at(
115        &self,
116        evidence: &dyn SessionEvidence,
117        dry_run: bool,
118        reemit: bool,
119    ) -> Result<SessionBackfillReport> {
120        let facts = self.session_facts()?;
121        let by_id: HashMap<Uuid, &Memory> = facts.memories.iter().map(|m| (m.id, m)).collect();
122
123        let candidates: Vec<&Memory> = facts
124            .memories
125            .iter()
126            .filter(|m| m.session_at.is_none())
127            .collect();
128        let line_times = transcript_line_times(&candidates, &facts.raws_of, &by_id, evidence);
129
130        let mut report = SessionBackfillReport {
131            dry_run,
132            candidates: candidates.len() as u64,
133            ..Default::default()
134        };
135        let mut fallbacks: HashSet<Uuid> = HashSet::new();
136        let mut dated: Vec<(&Memory, SessionTier, DateTime<Utc>)> = Vec::new();
137        for m in &candidates {
138            let raws = raws_of(m, &facts.raws_of, &by_id);
139            let (tier, at) = date_of(
140                m,
141                &raws,
142                &line_times,
143                &facts.conversations_of,
144                evidence,
145                &mut fallbacks,
146            );
147            report.all.add(tier);
148            if m.tier == Tier::Hot && m.superseded_by.is_none() {
149                report.live.add(tier);
150            }
151            if let Some(at) = at {
152                dated.push((m, tier, at));
153            }
154        }
155        report.fallback_conversations_skipped = fallbacks.len() as u64;
156        report.sample = sample(&dated);
157
158        if dry_run {
159            return Ok(report);
160        }
161        for batch in dated.chunks(WRITE_BATCH) {
162            report.written += self.with_txn(|tx| {
163                let mut written = 0;
164                for (m, _, at) in batch {
165                    let changed = tx.execute(
166                        "UPDATE memories SET session_at = ?2 WHERE id = ?1 AND session_at IS NULL",
167                        params![m.id.to_string(), at.to_rfc3339()],
168                    )?;
169                    if changed == 1
170                        && let Some(row) = Self::memory_on_tx(tx, m.id)?
171                    {
172                        self.log_memory_update(tx, &row)?;
173                        written += 1;
174                    }
175                }
176                Ok(written)
177            })?;
178        }
179        if reemit {
180            let predated: Vec<Uuid> = facts
181                .memories
182                .iter()
183                .filter(|m| m.session_at.is_some())
184                .map(|m| m.id)
185                .collect();
186            for batch in predated.chunks(WRITE_BATCH) {
187                report.reemitted += self.with_txn(|tx| {
188                    let mut logged = 0;
189                    for &id in batch {
190                        if let Some(row) = Self::memory_on_tx(tx, id)? {
191                            self.log_memory_update(tx, &row)?;
192                            logged += 1;
193                        }
194                    }
195                    Ok(logged)
196                })?;
197            }
198        }
199        Ok(report)
200    }
201
202    fn session_facts(&self) -> Result<StoreFacts> {
203        let conn = self.reader();
204        let memories = {
205            let mut stmt =
206                conn.prepare(&format!("SELECT {MEMORY_COLS} FROM memories ORDER BY id"))?;
207            stmt.query_map([], |r| row_to_memory(r, false))?
208                .collect::<rusqlite::Result<Vec<_>>>()?
209        };
210
211        let mut raws_of: HashMap<Uuid, Vec<Uuid>> = HashMap::new();
212        {
213            let mut stmt = conn.prepare(
214                "SELECT distilled_id, memory_id FROM distillation_log \
215                 WHERE memory_id <> distilled_id",
216            )?;
217            let rows = stmt.query_map([], |r| {
218                Ok((
219                    parse_uuid(&r.get::<_, String>(0)?)?,
220                    parse_uuid(&r.get::<_, String>(1)?)?,
221                ))
222            })?;
223            for row in rows {
224                let (distilled, raw) = row?;
225                raws_of.entry(distilled).or_default().push(raw);
226            }
227        }
228
229        let mut conversations_of: HashMap<Uuid, Vec<ConversationLink>> = HashMap::new();
230        {
231            let mut stmt = conn.prepare(
232                "SELECT r.from_id, c.id, c.source, c.started_at FROM relations r \
233                 JOIN conversations c ON c.id = r.to_id \
234                 WHERE r.relation_type = 'derived_from'",
235            )?;
236            let rows = stmt.query_map([], |r| {
237                Ok((
238                    parse_uuid(&r.get::<_, String>(0)?)?,
239                    ConversationLink {
240                        id: parse_uuid(&r.get::<_, String>(1)?)?,
241                        source: r.get(2)?,
242                        started_at: parse_dt(&r.get::<_, String>(3)?)?,
243                    },
244                ))
245            })?;
246            for row in rows {
247                let (memory, link) = row?;
248                conversations_of.entry(memory).or_default().push(link);
249            }
250        }
251
252        Ok(StoreFacts {
253            memories,
254            raws_of,
255            conversations_of,
256        })
257    }
258}
259
260fn is_distillate(m: &Memory) -> bool {
261    m.source == "consolidation"
262}
263
264fn is_explicit(m: &Memory) -> bool {
265    EXPLICIT_SOURCES.contains(&m.source.as_str())
266}
267
268/// The transcript rows a memory's session is read from: its own row for a
269/// raw, the raws that still exist for a distillate, none for a note.
270fn raws_of<'a>(
271    m: &'a Memory,
272    raws_of: &HashMap<Uuid, Vec<Uuid>>,
273    by_id: &HashMap<Uuid, &'a Memory>,
274) -> Vec<&'a Memory> {
275    if is_explicit(m) {
276        Vec::new()
277    } else if is_distillate(m) {
278        raws_of
279            .get(&m.id)
280            .into_iter()
281            .flatten()
282            .filter_map(|id| by_id.get(id).copied())
283            .filter(|raw| !is_distillate(raw) && !is_explicit(raw))
284            .collect()
285    } else {
286        vec![m]
287    }
288}
289
290/// Line times for every raw some candidate needs, looked up in source_id
291/// order so an evidence source that keeps only its last transcript parsed
292/// reads each transcript once.
293fn transcript_line_times(
294    candidates: &[&Memory],
295    raws: &HashMap<Uuid, Vec<Uuid>>,
296    by_id: &HashMap<Uuid, &Memory>,
297    evidence: &dyn SessionEvidence,
298) -> HashMap<Uuid, DateTime<Utc>> {
299    let mut needed: Vec<&Memory> = candidates
300        .iter()
301        .flat_map(|m| raws_of(m, raws, by_id))
302        .collect();
303    needed.sort_by(|a, b| (&a.source_id, a.id).cmp(&(&b.source_id, b.id)));
304    needed.dedup_by_key(|m| m.id);
305    needed
306        .into_iter()
307        .filter_map(|raw| {
308            evidence
309                .line_time(&raw.source_id, &raw.content)
310                .map(|at| (raw.id, at))
311        })
312        .collect()
313}
314
315fn date_of(
316    m: &Memory,
317    raws: &[&Memory],
318    line_times: &HashMap<Uuid, DateTime<Utc>>,
319    conversations_of: &HashMap<Uuid, Vec<ConversationLink>>,
320    evidence: &dyn SessionEvidence,
321    fallbacks: &mut HashSet<Uuid>,
322) -> (SessionTier, Option<DateTime<Utc>>) {
323    if is_explicit(m) {
324        // Every explicit source except claude-memory is written by remember
325        // during the session it describes; a claude-memory note is a file
326        // whose conversation is always stamped at ingest.
327        return if m.source == "claude-memory" {
328            match note_time(m, evidence) {
329                Some(at) => (SessionTier::FileMtime, Some(at)),
330                None => (SessionTier::Unknown, None),
331            }
332        } else {
333            (SessionTier::RememberedAt, Some(m.created_at))
334        };
335    }
336
337    if let Some(at) = raws.iter().filter_map(|r| line_times.get(&r.id)).max() {
338        return (SessionTier::TranscriptLine, Some(*at));
339    }
340
341    let started = raws
342        .iter()
343        .flat_map(|raw| {
344            conversations_of
345                .get(&raw.id)
346                .into_iter()
347                .flatten()
348                .map(move |c| (raw, c))
349        })
350        .filter(|(raw, c)| {
351            if c.source == "claude-memory" {
352                return false;
353            }
354            // An adapter stamps `started_at` with the clock only after it
355            // has built the conversation's memories, so a start that is not
356            // before the memory's creation was never read from a transcript.
357            let parsed = c.started_at < raw.created_at;
358            if !parsed {
359                fallbacks.insert(c.id);
360            }
361            parsed
362        })
363        .map(|(_, c)| c.started_at)
364        .max();
365    if let Some(at) = started {
366        return (SessionTier::ConversationStart, Some(at));
367    }
368
369    if let Some(at) = evidence.first_time(&m.source_id) {
370        return (SessionTier::TranscriptStart, Some(at));
371    }
372    // Consolidation once distilled claude-memory notes, and such a
373    // distillate's source_id is the note file rather than a transcript.
374    match note_time(m, evidence) {
375        Some(at) => (SessionTier::FileMtime, Some(at)),
376        None => (SessionTier::Unknown, None),
377    }
378}
379
380/// A note's mtime dates a memory only while the file is unchanged since the
381/// memory was made from it, because a later edit would date it too new.
382fn note_time(m: &Memory, evidence: &dyn SessionEvidence) -> Option<DateTime<Utc>> {
383    evidence
384        .note_modified_at(&m.source_id)
385        .filter(|at| *at <= m.created_at)
386}
387
388/// Evenly spaced over the dated rows, which are in id order, so the sample
389/// spreads across sources and ages instead of clustering on one transcript.
390fn sample(dated: &[(&Memory, SessionTier, DateTime<Utc>)]) -> Vec<SessionSample> {
391    let step = (dated.len() / SAMPLE_SIZE).max(1);
392    dated
393        .iter()
394        .step_by(step)
395        .take(SAMPLE_SIZE)
396        .map(|(m, tier, at)| SessionSample {
397            id: m.id,
398            tier: *tier,
399            session_at: *at,
400            created_at: m.created_at,
401            source: m.source.clone(),
402            source_id: m.source_id.clone(),
403            content: m.content.chars().take(160).collect(),
404        })
405        .collect()
406}