Skip to main content

rig_memory/
compaction.rs

1//! Compacting a long conversation: [`prepare_compaction`] says what a compaction summarizes and
2//! builds its summary requests, and [`apply_summary`] puts the summary in place of the messages
3//! it read, when they are still as it read them.
4//!
5//! ```
6//! use rig_core::conversation::Conversation;
7//! use rig_memory::{CompactionSettings, SummaryState, prepare_compaction};
8//!
9//! let settings = CompactionSettings::default();
10//! assert!(settings.should_compact(120_000, 128_000));
11//! let conversation = Conversation::default();
12//! assert_eq!(prepare_compaction(&conversation, settings.keep_recent_tokens, false), None);
13//! assert_eq!(SummaryState::default().message(), None);
14//! ```
15
16use std::collections::BTreeMap;
17use std::sync::Arc;
18
19use rig_core::catalog::ModelSpec;
20use rig_core::completion::Message;
21use rig_core::conversation::{Author, Change, Conversation, Entry, MessageId, Spliced};
22
23use crate::session::replay;
24use crate::{SessionError, Store};
25
26mod pi;
27
28pub use pi::{
29    CompactionPreparation, CompactionSettings, FileLists, SUMMARIZATION_SYSTEM_PROMPT,
30    SummaryFailure, clamp_max_tokens, estimate_message_tokens, estimate_tokens, find_cut_point,
31    merge_split_turn, no_history, prepare_compaction, prepare_compaction_at, request_tokens,
32    serialize_conversation, summarization_request, summary_request, summary_text,
33    turn_prefix_request,
34};
35
36/// What the summary's text opens with in the conversation.
37const OPENS: &str = "The conversation history before this point was compacted into the following summary:\n\n<summary>\n";
38
39/// What the summary's text closes with in the conversation.
40const CLOSES: &str = "\n</summary>";
41
42/// A compaction's summary and the files listed after it, which the next compaction carries on.
43/// The default is the state before the first compaction. The summary's entry in the
44/// conversation is its only record: [`apply_summary`] writes it and [`read`](Self::read) reads
45/// it back.
46#[derive(Clone, Debug, Default, PartialEq, Eq)]
47pub struct SummaryState {
48    /// The summary; empty before the first compaction. File lists a model wrote at its end are
49    /// not kept: the lists below are the only ones.
50    pub summary: String,
51    /// The files read and not changed by the summarized messages, sorted.
52    pub read_files: Vec<String>,
53    /// The files changed by the summarized messages, sorted.
54    pub modified_files: Vec<String>,
55}
56
57impl SummaryState {
58    /// The summary followed by its file lists, as the next compaction's update prompt reads it,
59    /// as pi stores it: a summary with no text still carries its lists. `None` when both are
60    /// empty, as before the first compaction.
61    pub fn previous_summary(&self) -> Option<String> {
62        Some(self.with_files()).filter(|text| !text.is_empty())
63    }
64
65    /// The text that stands for the summarized messages in requests, `None` before the first
66    /// compaction.
67    pub fn message(&self) -> Option<String> {
68        self.previous_summary().map(|_| self.wrapped())
69    }
70
71    /// The state a summary entry's `text` holds: the summary, and the file lists rig put after
72    /// it. Text that is no summary of rig's reads as a summary with no lists.
73    pub fn read(text: &str) -> Self {
74        let framed = text.strip_prefix(OPENS);
75        let Some(body) = framed.and_then(|text| text.strip_suffix(CLOSES)) else {
76            return Self {
77                summary: text.to_owned(),
78                ..Self::default()
79            };
80        };
81        let (body, modified_files) = pi::take_file_list(body, "modified-files");
82        let (summary, read_files) = pi::take_file_list(body, "read-files");
83        Self {
84            summary: summary.to_owned(),
85            read_files,
86            modified_files,
87        }
88    }
89
90    /// The summary, without file lists a model wrote at its end, followed by its file lists.
91    fn with_files(&self) -> String {
92        let files = pi::format_file_operations(&self.read_files, &self.modified_files);
93        format!("{}{files}", pi::without_file_lists(&self.summary))
94    }
95
96    /// [`with_files`](Self::with_files) in the frame requests send it in.
97    fn wrapped(&self) -> String {
98        format!("{OPENS}{}{CLOSES}", self.with_files())
99    }
100}
101
102/// The end of `text`, a conversation another plugin summarizes for a model `spec` in at most
103/// `summary_tokens`, cut to fit that model as rig's summaries were fitted before compaction
104/// followed pi; the cut start says how many characters were left out. Returns the text and the
105/// output limit to ask for, `None` when the model's limits are unknown. The room below a reserve
106/// of 16384 tokens, or half the input limit when that is more, is split: the answer gets at most
107/// half, and the text the rest at three characters a token, but never less than 16384 tokens or an
108/// eighth of the input limit, whichever is smaller; with no limit known, 400000 characters.
109pub fn fit_to_model<'t>(
110    text: &'t str,
111    spec: &ModelSpec,
112    summary_tokens: u64,
113) -> (std::borrow::Cow<'t, str>, Option<u64>) {
114    let chars = |tokens: u64| usize::try_from(tokens.saturating_mul(3)).unwrap_or(usize::MAX);
115    let most = spec
116        .output_limit()
117        .map_or(summary_tokens, |limit| summary_tokens.min(u64::from(limit)));
118    let (most, budget) = match spec.input_limit().map(u64::from) {
119        None => (most, 400_000),
120        Some(limit) => {
121            let room = limit.saturating_sub(16_384).max(limit / 2);
122            let most = most.min(room / 2);
123            let least = chars(16_384u64.min(limit / 8));
124            (most, chars(room.saturating_sub(most)).max(least))
125        }
126    };
127    let known = spec.max_output_tokens.is_some() || spec.context_window.is_some();
128    let most = known.then_some(most);
129    let left_out = text.chars().count().saturating_sub(budget);
130    if left_out == 0 {
131        return (std::borrow::Cow::Borrowed(text), most);
132    }
133    let start = text
134        .char_indices()
135        .nth(left_out)
136        .map_or(text.len(), |(start, _)| start);
137    let rest = text.get(start..).unwrap_or_default();
138    let cut = format!("[\u{2026} {left_out} earlier characters left out]\n{rest}");
139    (std::borrow::Cow::Owned(cut), most)
140}
141
142/// Who writes a compaction's summary: the author of its entry.
143pub fn summary_author() -> Author {
144    Author::plugin("compaction")
145}
146
147/// The summary a compaction put in, when `change` is one: the id of the
148/// entry by [`summary_author`] in a splice that took messages out.
149/// A splice by the same author that keeps every message, as an earlier
150/// version's clearing of old tool outputs did, is none.
151pub fn summary_in(change: &Change) -> Option<MessageId> {
152    let Change::Splice { removed, with, .. } = change else {
153        return None;
154    };
155    let kept = |id: &MessageId| with.iter().any(|entry| entry.id == Some(*id));
156    if removed.iter().all(kept) {
157        return None;
158    }
159    let summary = summary_author();
160    with.iter()
161        .find(|entry| entry.author.as_ref() == Some(&summary))
162        .and_then(|entry| entry.id)
163}
164
165/// How many of the first entries of `conversation` are summaries (by
166/// [`summary_author`]): what a new compaction builds on from its
167/// [`SummaryState`] instead of reading it again, so [`prepare_compaction`]
168/// never cuts them again.
169pub fn summarized_prefix(conversation: &Conversation) -> usize {
170    let author = summary_author();
171    let summaries = conversation.entries();
172    summaries
173        .take_while(|entry| entry.author == Some(&author))
174        .count()
175}
176
177/// Whether `conversation` ends in its last compaction, as pi's session does when nothing was
178/// added after it: a summary is in, and every other message is older than it.
179pub fn already_compacted(conversation: &Conversation) -> bool {
180    let author = summary_author();
181    let mut summaries = conversation.entries();
182    let summary = summaries
183        .find(|entry| entry.author == Some(&author))
184        .map(|entry| entry.id);
185    summary.is_some_and(|summary| conversation.entries().all(|entry| entry.id <= summary))
186}
187
188/// Puts the summary of `state` in place of `replaces`, the first
189/// messages of `conversation` as a compaction read them
190/// ([`CompactionPreparation::replaces`]), earlier summaries included: one
191/// user entry by [`summary_author`], in one splice, which is the
192/// compaction's whole record. Messages added after them stay. `None`,
193/// changing nothing, when `replaces` is empty, when the conversation no
194/// longer starts with those very messages, such as after an edit of one, so
195/// a summary never replaces what it did not read, or when what stays would
196/// start with a tool result whose call the summary replaced.
197pub fn apply_summary(
198    conversation: &mut Conversation,
199    replaces: &[Arc<Message>],
200    state: &SummaryState,
201) -> Option<Spliced> {
202    let end = replaces.len();
203    let now = conversation.messages();
204    let read = now.get(..end).filter(|now| !now.is_empty())?;
205    let same = read
206        .iter()
207        .zip(replaces)
208        .all(|(now, read)| Arc::ptr_eq(now, read));
209    if !same || !pi::may_keep_from(now, end) {
210        return None;
211    }
212    let summary = Entry::new(Message::user(state.wrapped())).by(summary_author());
213    Some(conversation.splice(..end, [summary], summary_author()))
214}
215
216/// The summary state the next compaction of `conversation` builds on: the
217/// one its newest leading summary entry holds ([`summarized_prefix`],
218/// [`SummaryState::read`]), or the state before any.
219pub fn summary_state(conversation: &Conversation) -> SummaryState {
220    let newest = summarized_prefix(conversation).checked_sub(1);
221    let newest = newest.and_then(|at| conversation.messages().get(at)?.rag_text());
222    newest
223        .map(|text| SummaryState::read(&text))
224        .unwrap_or_default()
225}
226
227/// The entries each compaction in the log `log` of `store` replaced
228/// ([`summary_in`]), by the summary's id, as they were just before it, read
229/// in one pass over the log. A summary named again keeps what it first
230/// replaced. A reset, the log's value replaced by a whole other one, drops
231/// them all: nothing says the new value is the same conversation, so its
232/// summaries show nothing of the old one's messages.
233///
234/// # Errors
235///
236/// As [`load`](crate::load).
237pub fn summarized_in_log(
238    store: &dyn Store,
239    log: &str,
240) -> Result<BTreeMap<MessageId, Vec<Entry>>, SessionError> {
241    let mut replaced = BTreeMap::new();
242    replay(store, log, |before, recorded| {
243        let change = &recorded.change;
244        let (Some(summary), Change::Splice { removed, with, .. }) = (summary_in(change), change)
245        else {
246            if matches!(change, Change::Reset { .. }) {
247                replaced.clear();
248            }
249            return;
250        };
251        let kept = |id: &&MessageId| with.iter().any(|entry| entry.id == Some(**id));
252        let taken = removed.iter().filter(|id| !kept(id));
253        let taken = taken.filter_map(|id| before.entry(before.index_of(*id)?));
254        replaced.entry(summary).or_insert_with(|| taken.collect());
255    })?;
256    Ok(replaced)
257}
258
259#[cfg(test)]
260mod tests;