tycode_core/modules/memory/
compaction.rs

1//! Compaction storage for memory summarization.
2//!
3//! Compactions are AI-generated summaries of memories stored as separate JSON files.
4//! Each compaction covers all memories through a specific sequence number.
5//! The raw memory log is never truncated - compactions provide a compressed view.
6//!
7//! Files are named `compaction_<through_seq>.json` (e.g., `compaction_42.json`).
8
9use std::collections::BTreeMap;
10use std::fs;
11use std::path::{Path, PathBuf};
12use std::sync::Arc;
13
14use anyhow::{Context, Result};
15use chrono::{DateTime, Utc};
16use serde::{Deserialize, Serialize};
17
18use crate::agents::agent::ActiveAgent;
19use crate::agents::memory_summarizer::MemorySummarizerAgent;
20use crate::agents::runner::AgentRunner;
21use crate::ai::provider::AiProvider;
22use crate::ai::Message;
23use crate::module::{ContextBuilder, Module, PromptBuilder};
24use crate::settings::manager::SettingsManager;
25use crate::spawn::complete_task::CompleteTask;
26use crate::steering::SteeringDocuments;
27use crate::tools::r#trait::ToolExecutor;
28
29use super::log::MemoryLog;
30
31/// A compaction record representing a summary of memories through a specific sequence number.
32#[derive(Debug, Clone, Serialize, Deserialize)]
33pub struct Compaction {
34    /// Last memory sequence number included in this compaction
35    pub through_seq: u64,
36    /// AI-generated summary of all memories through this sequence
37    pub summary: String,
38    /// When this compaction was created
39    pub created_at: DateTime<Utc>,
40    /// Number of memories that were compacted
41    pub memories_count: usize,
42    /// Sequence number of the previous compaction (for auditability)
43    pub previous_compaction_seq: Option<u64>,
44}
45
46/// Manages compaction files in a directory.
47///
48/// Files are named `compaction_<through_seq>.json` where through_seq is the
49/// last memory sequence number included in that compaction.
50#[derive(Debug)]
51pub struct CompactionStore {
52    directory: PathBuf,
53}
54
55impl CompactionStore {
56    pub fn new(directory: PathBuf) -> Self {
57        Self { directory }
58    }
59
60    /// Find the latest compaction by scanning for the highest sequence number.
61    pub fn find_latest(&self) -> Result<Option<Compaction>> {
62        if !self.directory.exists() {
63            return Ok(None);
64        }
65
66        let mut highest_seq: Option<u64> = None;
67        let entries = fs::read_dir(&self.directory)
68            .with_context(|| format!("Failed to read directory: {}", self.directory.display()))?;
69
70        for entry in entries {
71            let entry = entry.context("Failed to read directory entry")?;
72            let file_name = entry.file_name();
73            let file_name_str = file_name.to_string_lossy();
74
75            if let Some(seq) = parse_compaction_seq(&file_name_str) {
76                highest_seq = Some(highest_seq.map_or(seq, |h| h.max(seq)));
77            }
78        }
79
80        match highest_seq {
81            Some(seq) => self.read(seq).map(Some),
82            None => Ok(None),
83        }
84    }
85
86    /// Save a compaction to disk.
87    pub fn save(&self, compaction: &Compaction) -> Result<()> {
88        fs::create_dir_all(&self.directory).with_context(|| {
89            format!(
90                "Failed to create compaction directory: {}",
91                self.directory.display()
92            )
93        })?;
94
95        let path = self.compaction_path(compaction.through_seq);
96        let content =
97            serde_json::to_string_pretty(compaction).context("Failed to serialize compaction")?;
98
99        fs::write(&path, content)
100            .with_context(|| format!("Failed to write compaction file: {}", path.display()))?;
101
102        Ok(())
103    }
104
105    /// Read a specific compaction by its through_seq.
106    pub fn read(&self, through_seq: u64) -> Result<Compaction> {
107        let path = self.compaction_path(through_seq);
108        let content = fs::read_to_string(&path)
109            .with_context(|| format!("Failed to read compaction file: {}", path.display()))?;
110
111        serde_json::from_str(&content)
112            .with_context(|| format!("Failed to parse compaction file: {}", path.display()))
113    }
114
115    fn compaction_path(&self, through_seq: u64) -> PathBuf {
116        self.directory
117            .join(format!("compaction_{}.json", through_seq))
118    }
119
120    pub fn directory(&self) -> &Path {
121        &self.directory
122    }
123}
124
125/// Run compaction: summarize new memories since last compaction using an AI agent.
126/// Returns None if there are no new memories to compact.
127pub async fn run_compaction(
128    memory_log: &MemoryLog,
129    provider: Arc<dyn AiProvider>,
130    settings: SettingsManager,
131    modules: Vec<Arc<dyn Module>>,
132    steering: SteeringDocuments,
133    prompt_builder: PromptBuilder,
134    context_builder: ContextBuilder,
135) -> Result<Option<Compaction>> {
136    let memory_dir = memory_log
137        .path()
138        .parent()
139        .context("Failed to get memory directory")?;
140    let compaction_store = CompactionStore::new(memory_dir.to_path_buf());
141
142    let latest_compaction = compaction_store.find_latest()?;
143    let through_seq = latest_compaction
144        .as_ref()
145        .map(|c| c.through_seq)
146        .unwrap_or(0);
147    let previous_summary = latest_compaction.as_ref().map(|c| c.summary.clone());
148
149    let all_memories = memory_log.read_all()?;
150    let new_memories: Vec<_> = all_memories
151        .into_iter()
152        .filter(|m| m.seq > through_seq)
153        .collect();
154
155    if new_memories.is_empty() {
156        return Ok(None);
157    }
158
159    let memory_count = new_memories.len();
160    let max_seq = new_memories.iter().map(|m| m.seq).max().unwrap_or(0);
161
162    let mut formatted = String::new();
163    if let Some(prev_summary) = &previous_summary {
164        formatted.push_str("# Previous Compaction Summary\n\n");
165        formatted.push_str(prev_summary);
166        formatted.push_str("\n\n---\n\n");
167    }
168
169    formatted.push_str("# New Memories Since Last Compaction\n\n");
170    for memory in &new_memories {
171        formatted.push_str(&format!(
172            "## Memory #{} ({})\n",
173            memory.seq,
174            memory.source.as_deref().unwrap_or("global")
175        ));
176        formatted.push_str(&memory.content);
177        formatted.push_str("\n\n");
178    }
179
180    formatted.push_str(
181        "\n---\n\n\
182        Please consolidate the previous summary (if any) with the new memories \
183        into a single comprehensive summary.",
184    );
185
186    let mut tools: BTreeMap<String, Arc<dyn ToolExecutor + Send + Sync>> = BTreeMap::new();
187    tools.insert(
188        CompleteTask::tool_name().to_string(),
189        Arc::new(CompleteTask::standalone()),
190    );
191
192    let runner = AgentRunner::new(
193        provider,
194        settings,
195        tools,
196        modules,
197        steering,
198        prompt_builder,
199        context_builder,
200    );
201    let agent = MemorySummarizerAgent::new();
202    let mut active_agent = ActiveAgent::new(Arc::new(agent));
203    active_agent.conversation.push(Message::user(formatted));
204
205    let summary = runner.run(active_agent, 10).await?;
206
207    let compaction = Compaction {
208        through_seq: max_seq,
209        summary,
210        created_at: Utc::now(),
211        memories_count: memory_count,
212        previous_compaction_seq: latest_compaction.map(|c| c.through_seq),
213    };
214
215    compaction_store.save(&compaction)?;
216    Ok(Some(compaction))
217}
218
219/// Count memories since the last compaction.
220pub fn memories_since_last_compaction(memory_log: &MemoryLog) -> Result<usize> {
221    let memory_dir = memory_log
222        .path()
223        .parent()
224        .context("Failed to get memory directory")?;
225    let compaction_store = CompactionStore::new(memory_dir.to_path_buf());
226    let through_seq = compaction_store
227        .find_latest()?
228        .map(|c| c.through_seq)
229        .unwrap_or(0);
230    let all = memory_log.read_all()?;
231    Ok(all.into_iter().filter(|m| m.seq > through_seq).count())
232}
233
234/// Parse sequence number from a compaction filename.
235/// Returns None if the filename doesn't match the expected pattern.
236fn parse_compaction_seq(filename: &str) -> Option<u64> {
237    let stripped = filename.strip_prefix("compaction_")?;
238    let seq_str = stripped.strip_suffix(".json")?;
239    seq_str.parse().ok()
240}
241
242#[cfg(test)]
243mod tests {
244    use super::*;
245
246    #[test]
247    fn test_parse_compaction_seq_valid() {
248        assert_eq!(parse_compaction_seq("compaction_42.json"), Some(42));
249        assert_eq!(parse_compaction_seq("compaction_0.json"), Some(0));
250        assert_eq!(parse_compaction_seq("compaction_12345.json"), Some(12345));
251    }
252
253    #[test]
254    fn test_parse_compaction_seq_invalid() {
255        assert_eq!(parse_compaction_seq("compaction_.json"), None);
256        assert_eq!(parse_compaction_seq("compaction_abc.json"), None);
257        assert_eq!(parse_compaction_seq("other_42.json"), None);
258        assert_eq!(parse_compaction_seq("compaction_42.txt"), None);
259        assert_eq!(parse_compaction_seq("compaction_42"), None);
260    }
261}
tycode_core/modules/memory/compaction.rs

tycode_core/modules/memory/
compaction.rs
