microscope_memory/
narrative_memory.rs

1//! Narrative Memory Module for Microscope Memory.
2//!
3//! Automatically links sequences of recalls into coherent narratives -
4//! story arcs that emerge from thought patterns and can be replayed
5//! as structured episodes.
6//!
7//! Binary format: NVM1
8//!   magic: "NVM1" (4 bytes)
9//!   timestamp_ms: u64 (8 bytes)
10//!   episode_id: u32 (4 bytes)
11//!   title_len: u16 (2 bytes) + title bytes
12//!   summary_len: u16 (2 bytes) + summary bytes
13//!   block_count: u16 (2 bytes) + block_indices (u32 each)
14//!   emotional_arc: [f32; 21] (84 bytes)
15
16use std::fs;
17use std::path::Path;
18use std::time::{SystemTime, UNIX_EPOCH};
19
20use crate::config::Config;
21use crate::reader::MicroscopeReader;
22
23use colored::Colorize;
24
25const MAX_STR_LEN: usize = 512;
26const MAX_EPISODES: usize = 50;
27
28#[derive(Clone, Debug)]
29pub struct NarrativeEpisode {
30    pub timestamp_ms: u64,
31    pub episode_id: u32,
32    pub title: String,
33    pub summary: String,
34    pub block_indices: Vec<u32>,
35    pub emotional_arc: [f32; 21],
36}
37
38impl NarrativeEpisode {
39    pub fn to_bytes(&self) -> Vec<u8> {
40        let title_b = self.title.as_bytes();
41        let summary_b = self.summary.as_bytes();
42        let tlen = title_b.len().min(MAX_STR_LEN) as u16;
43        let slen = summary_b.len().min(MAX_STR_LEN) as u16;
44        let bcount = self.block_indices.len().min(50) as u16;
45        let mut buf = Vec::with_capacity(120 + tlen as usize + slen as usize + bcount as usize * 4);
46        buf.extend_from_slice(b"NVM1");
47        buf.extend_from_slice(&self.timestamp_ms.to_le_bytes());
48        buf.extend_from_slice(&self.episode_id.to_le_bytes());
49        buf.extend_from_slice(&tlen.to_le_bytes());
50        buf.extend_from_slice(&title_b[..tlen as usize]);
51        buf.extend_from_slice(&slen.to_le_bytes());
52        buf.extend_from_slice(&summary_b[..slen as usize]);
53        buf.extend_from_slice(&bcount.to_le_bytes());
54        for &idx in self.block_indices.iter().take(50) {
55            buf.extend_from_slice(&idx.to_le_bytes());
56        }
57        for v in self.emotional_arc {
58            buf.extend_from_slice(&v.to_le_bytes());
59        }
60        buf
61    }
62
63    pub fn from_bytes(data: &[u8]) -> Option<Self> {
64        if data.len() < 120 || &data[0..4] != b"NVM1" {
65            return None;
66        }
67        let mut pos = 4;
68        let ts = u64::from_le_bytes(data[pos..pos + 8].try_into().ok()?);
69        pos += 8;
70        let eid = u32::from_le_bytes(data[pos..pos + 4].try_into().ok()?);
71        pos += 4;
72        let tlen = u16::from_le_bytes(data[pos..pos + 2].try_into().unwrap_or([0; 2])) as usize;
73        pos += 2;
74        let title = if tlen > 0 {
75            String::from_utf8_lossy(&data[pos..pos + tlen]).to_string()
76        } else {
77            String::new()
78        };
79        pos += tlen;
80        let slen = u16::from_le_bytes(data[pos..pos + 2].try_into().unwrap_or([0; 2])) as usize;
81        pos += 2;
82        let summary = if slen > 0 {
83            String::from_utf8_lossy(&data[pos..pos + slen]).to_string()
84        } else {
85            String::new()
86        };
87        pos += slen;
88        let bcount = u16::from_le_bytes(data[pos..pos + 2].try_into().unwrap_or([0; 2])) as usize;
89        pos += 2;
90        let mut indices = Vec::new();
91        for _ in 0..bcount {
92            let idx = u32::from_le_bytes(data[pos..pos + 4].try_into().ok()?);
93            pos += 4;
94            indices.push(idx);
95        }
96        let mut emo = [0.0f32; 21];
97        for e in emo.iter_mut() {
98            *e = f32::from_le_bytes(data[pos..pos + 4].try_into().ok()?);
99            pos += 4;
100        }
101        Some(Self {
102            timestamp_ms: ts,
103            episode_id: eid,
104            title,
105            summary,
106            block_indices: indices,
107            emotional_arc: emo,
108        })
109    }
110}
111
112pub struct NarrativeMemory {
113    pub episodes: Vec<NarrativeEpisode>,
114    pub next_id: u32,
115}
116
117/// Byte-at-a-time resynchronisation steps taken by `NarrativeMemory::load_or_init`
118/// across the process, and the count from the most recent load that took any.
119///
120/// Exposed because the cost is otherwise invisible: reading
121/// `narrative_memory.bin` is 0.22 ms, and a layout disagreement between
122/// `NarrativeEpisode::to_bytes` and `from_bytes` turns the load into a scan
123/// over the whole file, a byte at a time. A count in the low hundreds means
124/// the two layouts agree and the cost really is the read; a count near the
125/// file size means they do not, and the parser walks the file every recall.
126pub static RESYNC_STEPS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
127/// Byte-at-a-time steps in the last load that took any. Zero means that load
128/// parsed cleanly; it does not mean the counter was never armed.
129pub static LAST_RESCANS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
130
131impl NarrativeMemory {
132    pub fn load_or_init(output_dir: &Path) -> Self {
133        let path = output_dir.join("narrative_memory.bin");
134        let mut episodes = Vec::new();
135        let mut next_id = 1u32;
136        if let Ok(data) = fs::read(&path) {
137            let mut pos = 0;
138            let mut rescans = 0u64;
139            while pos + 120 <= data.len() {
140                if let Some(ep) = NarrativeEpisode::from_bytes(&data[pos..]) {
141                    let size = 4
142                        + 8
143                        + 4
144                        + 2
145                        + ep.title.len().min(MAX_STR_LEN)
146                        + 2
147                        + ep.summary.len().min(MAX_STR_LEN)
148                        + 2
149                        + ep.block_indices.len().min(50) * 4
150                        + 84;
151                    pos += size;
152                    if ep.episode_id >= next_id {
153                        next_id = ep.episode_id + 1;
154                    }
155                    episodes.push(ep);
156                } else {
157                    // One byte at a time, re-parsing a whole episode each time.
158                    // On a file whose reader and writer layouts disagree this is
159                    // a linear scan over the entire file on every recall. It is
160                    // counted because the cost is invisible otherwise: the file
161                    // read itself is 0.22 ms and this can be an order of
162                    // magnitude more.
163                    RESYNC_STEPS.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
164                    rescans += 1;
165                    pos += 1;
166                }
167            }
168            if rescans > 0 {
169                LAST_RESCANS.store(rescans, std::sync::atomic::Ordering::Relaxed);
170            }
171        }
172        Self { episodes, next_id }
173    }
174
175    pub fn save(&self, output_dir: &Path) -> Result<(), String> {
176        let path = output_dir.join("narrative_memory.bin");
177        let mut buf = Vec::new();
178        for ep in self.episodes.iter().rev().take(MAX_EPISODES).rev() {
179            buf.extend_from_slice(&ep.to_bytes());
180        }
181        let tmp_path = output_dir.join("narrative_memory.bin.tmp");
182        fs::write(&tmp_path, &buf).map_err(|e| format!("write narrative_memory.bin: {}", e))?;
183        fs::rename(&tmp_path, &path).map_err(|e| format!("rename narrative_memory.bin: {}", e))
184    }
185
186    pub fn build_episode(
187        &mut self,
188        _config: &Config,
189        reader: &MicroscopeReader,
190        output_dir: &Path,
191        query: &str,
192        results: &[(f32, usize, bool)],
193    ) -> Option<NarrativeEpisode> {
194        if results.is_empty() {
195            return None;
196        }
197        let now_ms = SystemTime::now()
198            .duration_since(UNIX_EPOCH)
199            .unwrap_or_default()
200            .as_millis() as u64;
201
202        // Collect main index results
203        let indices: Vec<u32> = results
204            .iter()
205            .filter(|(_, _, is_main)| *is_main)
206            .take(10)
207            .map(|(_, idx, _)| *idx as u32)
208            .collect();
209        if indices.is_empty() {
210            return None;
211        }
212
213        // Build title from query
214        let title = format!("Episode: {}", crate::safe_truncate(query, 60));
215
216        // Build summary from result texts
217        let texts: Vec<String> = indices
218            .iter()
219            .take(5)
220            .map(|&idx| {
221                let text = reader.text(idx as usize);
222                crate::safe_truncate(text, 80)
223            })
224            .collect();
225        let summary = if texts.len() == 1 {
226            format!("Recalled: {}", texts[0])
227        } else {
228            format!("Connected {} memories: {}", texts.len(), texts.join(" -> "))
229        };
230
231        // Get emotional arc from emotional state
232        let esr = crate::emotional_state::EmotionalStateRing::load_or_init(output_dir);
233        let emotional_arc = if esr.is_active() {
234            esr.current
235        } else {
236            [0.0f32; 21]
237        };
238
239        let episode = NarrativeEpisode {
240            timestamp_ms: now_ms,
241            episode_id: self.next_id,
242            title,
243            summary,
244            block_indices: indices,
245            emotional_arc,
246        };
247        self.next_id += 1;
248        self.episodes.push(episode.clone());
249        let _ = self.save(output_dir);
250        Some(episode)
251    }
252
253    pub fn recent_episodes(&self, n: usize) -> Vec<&NarrativeEpisode> {
254        self.episodes.iter().rev().take(n).collect()
255    }
256}
257
258pub fn format_episode(ep: &NarrativeEpisode) -> String {
259    let labels = crate::reader::EMOTION_DIMS;
260    let mut emos: Vec<(usize, f32)> = ep
261        .emotional_arc
262        .iter()
263        .enumerate()
264        .map(|(i, &v)| (i, v))
265        .collect();
266    emos.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
267    let top_emo: Vec<String> = emos
268        .iter()
269        .take(2)
270        .filter(|(_, v)| *v > 0.1)
271        .map(|(i, v)| format!("{}={:.2}", labels[*i], v))
272        .collect();
273
274    format!(
275        "  {} [#{}] {}\n\
276         \x20       {}\n\
277         \x20       {} blocks, emotion: {}\n\
278         \x20       {}",
279        "STORY:".cyan().bold(),
280        ep.episode_id,
281        ep.title,
282        ep.summary,
283        ep.block_indices.len(),
284        if top_emo.is_empty() {
285            "neutral".to_string()
286        } else {
287            top_emo.join(", ")
288        },
289        chrono_str(ep.timestamp_ms),
290    )
291}
292
293/// Render an age string relative to now.
294///
295/// `ts_ms` is an absolute epoch timestamp, so it must be subtracted from the
296/// current time; otherwise a memory written moments ago reports an age equal
297/// to the whole Unix epoch in days. A zero timestamp means "no timestamp
298/// recorded", which is unknown age, not zero age.
299fn chrono_str(ts_ms: u64) -> String {
300    if ts_ms == 0 {
301        return "unknown date".to_string();
302    }
303    let now_ms = std::time::SystemTime::now()
304        .duration_since(std::time::UNIX_EPOCH)
305        .map(|d| d.as_millis() as u64)
306        .unwrap_or(ts_ms);
307    if ts_ms > now_ms {
308        // Clock skew or a timestamp from the future: report it plainly instead
309        // of wrapping around into a huge unsigned age.
310        return "in the future".to_string();
311    }
312    let secs = (now_ms - ts_ms) / 1000;
313    let days = secs / 86400;
314    let hours = (secs % 86400) / 3600;
315    let mins = (secs % 3600) / 60;
316    if days > 0 {
317        format!("{}d {}h {}m ago", days, hours, mins)
318    } else if hours > 0 {
319        format!("{}h {}m ago", hours, mins)
320    } else {
321        format!("{}m ago", mins)
322    }
323}
324
325#[cfg(test)]
326mod tests {
327    use super::chrono_str;
328
329    #[test]
330    fn zero_timestamp_is_unknown_not_zero_days() {
331        assert_eq!(chrono_str(0), "unknown date");
332    }
333
334    #[test]
335    fn fresh_memory_does_not_report_tens_of_thousands_of_days() {
336        let now_ms = std::time::SystemTime::now()
337            .duration_since(std::time::UNIX_EPOCH)
338            .unwrap()
339            .as_millis() as u64;
340        let s = chrono_str(now_ms);
341        // A just-written memory must be minutes old, never ~20722 days.
342        assert!(!s.contains('d'), "fresh memory reported days: {s}");
343    }
344
345    #[test]
346    fn one_day_old_reports_one_day() {
347        let now_ms = std::time::SystemTime::now()
348            .duration_since(std::time::UNIX_EPOCH)
349            .unwrap()
350            .as_millis() as u64;
351        let s = chrono_str(now_ms - 86_400_000);
352        assert!(s.starts_with('1'), "expected 1d, got {s}");
353    }
354
355    #[test]
356    fn future_timestamp_does_not_wrap() {
357        let now_ms = std::time::SystemTime::now()
358            .duration_since(std::time::UNIX_EPOCH)
359            .unwrap()
360            .as_millis() as u64;
361        assert_eq!(chrono_str(now_ms + 86_400_000), "in the future");
362    }
363}
microscope_memory/narrative_memory.rs

microscope_memory/
narrative_memory.rs
