Skip to main content

rig_memory/compaction/
pi.rs

1// The design, prompts and logic of this module are ported from the pi coding agent's compaction
2// (https://github.com/badlogic/pi-mono, packages/coding-agent/src/core/compaction), under its
3// license:
4//
5// MIT License
6//
7// Copyright (c) 2025 Mario Zechner
8//
9// Permission is hereby granted, free of charge, to any person obtaining a copy
10// of this software and associated documentation files (the "Software"), to deal
11// in the Software without restriction, including without limitation the rights
12// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
13// copies of the Software, and to permit persons to whom the Software is
14// furnished to do so, subject to the following conditions:
15//
16// The above copyright notice and this permission notice shall be included in all
17// copies or substantial portions of the Software.
18//
19// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
22// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
23// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
24// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
25// SOFTWARE.
26
27//! When to compact, where to cut, and the summary requests: pure functions over a conversation's
28//! messages. The caller sends the requests, reads the replies with [`summary_text`] and puts the
29//! summary in with [`apply_summary`](crate::apply_summary).
30
31use std::borrow::Borrow;
32use std::collections::BTreeSet;
33use std::sync::Arc;
34
35use rig_core::catalog::ModelSpec;
36use rig_core::completion::{
37    AssistantContent, CompletionRequest, CompletionResponse, Message, Reasoning, UnsupportedOption,
38};
39use rig_core::conversation::Conversation;
40use rig_core::message::{StopReason, ToolResultContent, UserContent};
41use serde::{Deserialize, Serialize};
42
43use super::{SummaryState, summarized_prefix};
44
45/// When compaction runs and how much it keeps: `[compaction]` in settings.
46#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
47#[serde(default, deny_unknown_fields)]
48pub struct CompactionSettings {
49    /// Whether a conversation is compacted on its own: near the window, and after a request
50    /// refused as too long or a reply the window cut short. `/compact` works either way.
51    pub enabled: bool,
52    /// Tokens of the model's input limit kept free for the reply.
53    pub reserve_tokens: u64,
54    /// About how many tokens of the newest messages stay as they are.
55    pub keep_recent_tokens: u64,
56}
57
58impl Default for CompactionSettings {
59    /// Enabled, 16384 tokens reserved, 20000 kept.
60    fn default() -> Self {
61        Self {
62            enabled: true,
63            reserve_tokens: 16_384,
64            keep_recent_tokens: 20_000,
65        }
66    }
67}
68
69impl CompactionSettings {
70    /// Whether a context of `context_tokens` in a window of `context_window` tokens is compacted
71    /// on its own: when enabled and it leaves less than the reserve free.
72    pub fn should_compact(&self, context_tokens: u64, context_window: u64) -> bool {
73        self.enabled && context_tokens > context_window.saturating_sub(self.reserve_tokens)
74    }
75}
76
77/// Characters an image counts for.
78const ESTIMATED_IMAGE_CHARS: u64 = 4_800;
79
80/// The length of `text` in UTF-16 code units, as JavaScript counts it.
81fn chars(text: &str) -> u64 {
82    text.encode_utf16().count() as u64
83}
84
85/// Characters of tool-result content, an image as [`ESTIMATED_IMAGE_CHARS`].
86fn result_chars(content: &[ToolResultContent]) -> u64 {
87    content
88        .iter()
89        .map(|part| match part {
90            ToolResultContent::Text(text) => chars(&text.text),
91            ToolResultContent::Image(_) => ESTIMATED_IMAGE_CHARS,
92            ToolResultContent::Json { value } => chars(&value.to_string()),
93        })
94        .fold(0, u64::saturating_add)
95}
96
97/// The characters of each message pi would make of `message`: each tool result is a message of
98/// its own, beside the rest of a user message.
99fn message_chars(message: &Message) -> Vec<u64> {
100    match message {
101        Message::System { content } => vec![chars(content)],
102        Message::User { content } => {
103            let mut parts = Vec::new();
104            let mut rest = None::<u64>;
105            for item in content {
106                match item {
107                    UserContent::ToolResult(result) => parts.push(result_chars(&result.content)),
108                    UserContent::Text(text) => {
109                        rest = Some(rest.unwrap_or(0).saturating_add(chars(&text.text)));
110                    }
111                    UserContent::Image(_) => {
112                        rest = Some(rest.unwrap_or(0).saturating_add(ESTIMATED_IMAGE_CHARS));
113                    }
114                    _ => rest = Some(rest.unwrap_or(0)),
115                }
116            }
117            parts.extend(rest);
118            parts
119        }
120        Message::Assistant(reply) => vec![
121            reply
122                .content
123                .iter()
124                .map(|item| match item {
125                    AssistantContent::Text(text) => chars(&text.text),
126                    AssistantContent::Reasoning(reasoning) => chars(&reasoning.text),
127                    AssistantContent::ToolCall(call) => chars(call.function.name.as_str())
128                        .saturating_add(chars(
129                            &serde_json::to_string(&call.function.arguments).unwrap_or_default(),
130                        )),
131                    _ => 0,
132                })
133                .fold(0, u64::saturating_add),
134        ],
135    }
136}
137
138/// The estimated tokens of `message`, at four characters a token, rounded up: on the high side,
139/// as pi's compaction counts. Each tool result counts as a message of its own, rounded apart,
140/// beside the rest of a user message.
141pub fn estimate_message_tokens(message: &Message) -> u64 {
142    message_chars(message)
143        .into_iter()
144        .map(|chars| chars.div_ceil(4))
145        .fold(0, u64::saturating_add)
146}
147
148/// The estimated tokens of `messages` ([`estimate_message_tokens`]).
149pub fn estimate_tokens(messages: &[impl Borrow<Message>]) -> u64 {
150    messages
151        .iter()
152        .map(|message| estimate_message_tokens(message.borrow()))
153        .fold(0, u64::saturating_add)
154}
155
156/// Tokens of `chars` characters at pi-ai's 3.5 characters a token, rounded up.
157fn provider_tokens(chars: u64) -> u64 {
158    chars.saturating_mul(2).div_ceil(7)
159}
160
161/// The tokens of a request as pi-ai's provider layer estimates them before it fits the output
162/// limit: the `system` prompt and the `tools` (their definitions as JSON) and each message at 3.5
163/// characters a token, or, with an `anchor` (the tokens reported for the reply at that index), that
164/// count plus an estimate of the messages after it.
165pub fn request_tokens(
166    (system, tools): (Option<&str>, Option<&str>),
167    messages: &[impl Borrow<Message>],
168    anchor: Option<(u64, usize)>,
169) -> u64 {
170    let estimate = |messages: &[_]| {
171        messages
172            .iter()
173            .flat_map(|message: &_| message_chars(Borrow::<Message>::borrow(message)))
174            .map(provider_tokens)
175            .fold(0, u64::saturating_add)
176    };
177    if let Some((tokens, at)) = anchor {
178        let after = messages.get(at.saturating_add(1)..).unwrap_or_default();
179        return tokens.saturating_add(estimate(after));
180    }
181    let system = system.map_or(0, |text| provider_tokens(chars(text)));
182    let tools = tools.map_or(0, |json| provider_tokens(chars(json)));
183    system
184        .saturating_add(tools)
185        .saturating_add(estimate(messages))
186}
187
188/// Whether `message` answers tool calls: a user message holding a tool result.
189fn is_tool_result(message: &Message) -> bool {
190    matches!(message, Message::User { content } if content.iter().any(|item| matches!(item, UserContent::ToolResult(_))))
191}
192
193/// Whether `message` starts a turn: a user or system message that is no tool result.
194fn is_turn_start(message: &Message) -> bool {
195    !matches!(message, Message::Assistant(_)) && !is_tool_result(message)
196}
197
198/// For each of `messages`, whether a compaction may keep from it: no tool result, and nothing
199/// between a call and its result, so a result is never kept without its call. Keeping from the
200/// reply that made a call keeps its results too.
201fn cut_points(messages: &[Arc<Message>]) -> Vec<bool> {
202    let mut allowed: Vec<bool> = messages
203        .iter()
204        .map(|message| !is_tool_result(message))
205        .collect();
206    let mut calls: Vec<(usize, &rig_core::message::CallId)> = Vec::new();
207    for (at, message) in messages.iter().enumerate() {
208        match &**message {
209            Message::Assistant(reply) => {
210                calls.extend(reply.tool_calls().map(|call| (at, &call.id)))
211            }
212            Message::User { content } => {
213                for result in content.iter().filter_map(UserContent::as_tool_result) {
214                    let made = calls.iter().rev().find(|(_, id)| **id == result.call);
215                    if let Some(&(made, _)) = made {
216                        for inside in allowed.iter_mut().take(at.saturating_add(1)).skip(made + 1) {
217                            *inside = false;
218                        }
219                    }
220                }
221            }
222            Message::System { .. } => {}
223        }
224    }
225    allowed
226}
227
228/// Whether a compaction of `messages` may keep from index `at`: from a message outside every
229/// tool exchange that is no tool result, or from the end, keeping none.
230pub(crate) fn may_keep_from(messages: &[Arc<Message>], at: usize) -> bool {
231    at == messages.len() || cut_points(messages).get(at) == Some(&true)
232}
233
234/// The index of the first message kept by the cut of `messages[start..end]` that keeps about
235/// `keep_recent_tokens`: walking back from the newest message until that many are counted, it
236/// cuts at the nearest message at or after that point that is no tool result and lies outside
237/// every tool exchange. Cut at a reply, its results stay after it, and the turn it is part of is
238/// split. With nothing to cut at, it keeps from `start`. After an overflow (`overflowed`), a last
239/// message over the budget alone is summarized too, as the turn's earlier part, since sending it
240/// again would overflow again.
241pub fn find_cut_point(
242    messages: &[Arc<Message>],
243    (start, end): (usize, usize),
244    keep_recent_tokens: u64,
245    overflowed: bool,
246) -> usize {
247    let end = end.min(messages.len());
248    let allowed = cut_points(messages);
249    let cut_points: Vec<usize> = (start..end)
250        .filter(|&at| allowed.get(at).copied().unwrap_or(false))
251        .collect();
252    let Some(&first) = cut_points.first() else {
253        return start;
254    };
255    let mut cut = first;
256    let mut accumulated = 0u64;
257    for at in (start..end).rev() {
258        let tokens = messages
259            .get(at)
260            .map_or(0, |message| estimate_message_tokens(message));
261        if tokens == 0 {
262            continue;
263        }
264        accumulated = accumulated.saturating_add(tokens);
265        if accumulated >= keep_recent_tokens {
266            // Trailing tool results over the budget keep the reply that called them.
267            cut = cut_points
268                .iter()
269                .copied()
270                .find(|&candidate| candidate >= at)
271                .or(cut_points.last().copied())
272                .unwrap_or(first);
273            if overflowed && cut.saturating_add(1) == end {
274                cut = end;
275            }
276            break;
277        }
278    }
279    cut
280}
281
282/// When keeping from `first_kept` splits a turn (it keeps from a reply, or nothing of the last
283/// turn), the index of the message that started that turn, at or after `start`.
284fn split_turn_start(messages: &[Arc<Message>], start: usize, first_kept: usize) -> Option<usize> {
285    let starts = |at: usize| messages.get(at).is_some_and(|it| is_turn_start(it));
286    let last = messages.len().checked_sub(1)?;
287    (!starts(first_kept)).then(|| (start..=first_kept.min(last)).rev().find(|&at| starts(at)))?
288}
289
290/// A compaction ready to summarize ([`prepare_compaction`]).
291#[derive(Clone, Debug, PartialEq)]
292pub struct CompactionPreparation {
293    /// Every message the summary replaces, as the conversation holds them now: all before the
294    /// first one kept, earlier summaries included.
295    pub replaces: Vec<Arc<Message>>,
296    /// The messages summarized into the history summary, after the earlier summaries.
297    pub to_summarize: Vec<Arc<Message>>,
298    /// When the cut splits a turn, that turn's messages before the cut, summarized apart.
299    pub turn_prefix: Vec<Arc<Message>>,
300    /// The newest compaction's state, an empty one included, as pi keeps it; `None` before the
301    /// first compaction.
302    pub previous: Option<SummaryState>,
303    /// The files read and not changed, and the files changed, by every summarized message and
304    /// earlier compaction.
305    pub files: FileLists,
306}
307
308/// The files a compaction lists after its summary.
309#[derive(Clone, Debug, Default, PartialEq, Eq)]
310pub struct FileLists {
311    /// Files only read, sorted.
312    pub read_files: Vec<String>,
313    /// Files edited or written, sorted.
314    pub modified_files: Vec<String>,
315}
316
317/// What a compaction of `conversation`, keeping about `keep_recent_tokens`, summarizes, or `None`
318/// when nothing would be, as when nothing was added since the last one
319/// ([`already_compacted`](super::already_compacted)); `overflowed` after the provider refused a
320/// request as too long ([`find_cut_point`]), which is compacted for even then, as the summary
321/// before it may have gone in after its prompt. The earlier summaries ([`summarized_prefix`])
322/// are never cut again: the new summary updates the newest one.
323pub fn prepare_compaction(
324    conversation: &Conversation,
325    keep_recent_tokens: u64,
326    overflowed: bool,
327) -> Option<CompactionPreparation> {
328    if !overflowed && super::already_compacted(conversation) {
329        return None;
330    }
331    let messages = conversation.messages();
332    let range = (summarized_prefix(conversation), messages.len());
333    let first_kept = find_cut_point(messages, range, keep_recent_tokens, overflowed);
334    prepare_compaction_at(conversation, first_kept)
335}
336
337/// What a compaction of `conversation` that keeps from message `first_kept` summarizes: the
338/// messages after the earlier summaries and before it (or before the start of the turn it
339/// splits), and a split turn's earlier part. `None` when `first_kept` replaces no message after
340/// the earlier summaries, lies past the end, or would keep a tool result without its call.
341pub fn prepare_compaction_at(
342    conversation: &Conversation,
343    first_kept: usize,
344) -> Option<CompactionPreparation> {
345    let messages = conversation.messages();
346    let start = summarized_prefix(conversation);
347    if first_kept <= start || !may_keep_from(messages, first_kept) {
348        return None;
349    }
350    let turn_start = split_turn_start(messages, start, first_kept);
351    let to_summarize = messages.get(start..turn_start.unwrap_or(first_kept))?;
352    let turn_prefix = turn_start.and_then(|turn_start| messages.get(turn_start..first_kept));
353    let turn_prefix = turn_prefix.unwrap_or_default();
354    let previous = (start > 0).then(|| super::summary_state(conversation));
355    let mut ops = FileOperations::default();
356    if let Some(state) = &previous {
357        ops.read.extend(state.read_files.iter().cloned());
358        ops.edited.extend(state.modified_files.iter().cloned());
359    }
360    for message in to_summarize.iter().chain(turn_prefix) {
361        ops.extract(message);
362    }
363    Some(CompactionPreparation {
364        replaces: messages.get(..first_kept)?.to_vec(),
365        to_summarize: to_summarize.to_vec(),
366        turn_prefix: turn_prefix.to_vec(),
367        previous,
368        files: ops.lists(),
369    })
370}
371
372/// The files the built-in file tools were called with.
373#[derive(Default)]
374struct FileOperations {
375    read: BTreeSet<String>,
376    written: BTreeSet<String>,
377    edited: BTreeSet<String>,
378}
379
380impl FileOperations {
381    /// Adds the `path` of each `read`, `write` and `edit` call of `message`.
382    fn extract(&mut self, message: &Message) {
383        let Message::Assistant(reply) = message else {
384            return;
385        };
386        for call in reply.tool_calls() {
387            let Some(path) = call
388                .function
389                .arguments
390                .get("path")
391                .and_then(|path| path.as_str())
392                .filter(|path| is_listed(path))
393            else {
394                continue;
395            };
396            let set = match call.function.name.as_str() {
397                "read" => &mut self.read,
398                "write" => &mut self.written,
399                "edit" => &mut self.edited,
400                _ => continue,
401            };
402            set.insert(path.to_owned());
403        }
404    }
405
406    /// The files only read, and the files changed.
407    fn lists(&self) -> FileLists {
408        let modified: BTreeSet<&String> = self.edited.iter().chain(&self.written).collect();
409        FileLists {
410            read_files: self
411                .read
412                .iter()
413                .filter(|path| !modified.contains(path))
414                .cloned()
415                .collect(),
416            modified_files: modified.into_iter().cloned().collect(),
417        }
418    }
419}
420
421/// Whether a file list holds `path`: not an empty one, and not one over several lines, which
422/// would read back as several.
423fn is_listed(path: &str) -> bool {
424    !path.is_empty() && !path.contains('\n')
425}
426
427/// The file lists as tags after a summary, or nothing when both are empty.
428pub(crate) fn format_file_operations(read_files: &[String], modified_files: &[String]) -> String {
429    let lists = [
430        ("read-files", read_files),
431        ("modified-files", modified_files),
432    ];
433    let mut text = String::new();
434    for (tag, files) in lists {
435        let files: Vec<&str> = files.iter().map(String::as_str).collect();
436        let files: Vec<&str> = files.into_iter().filter(|path| is_listed(path)).collect();
437        if !files.is_empty() {
438            text.push_str(&format!("\n\n<{tag}>\n{}\n</{tag}>", files.join("\n")));
439        }
440    }
441    text
442}
443
444/// `body` without the file list tagged `tag` that [`format_file_operations`] put at its end, and
445/// that list; all of `body` and no files when it ends in none.
446pub(crate) fn take_file_list<'t>(body: &'t str, tag: &str) -> (&'t str, Vec<String>) {
447    let (open, close) = (format!("\n\n<{tag}>\n"), format!("\n</{tag}>"));
448    let listed = body.strip_suffix(close.as_str());
449    match listed.and_then(|listed| listed.rsplit_once(open.as_str())) {
450        Some((rest, files)) => (rest, files.split('\n').map(str::to_owned).collect()),
451        None => (body, Vec::new()),
452    }
453}
454
455/// `summary` without the file lists a model wrote at its end, as one that copies the previous
456/// summary's does: rig lists the files itself, from the tool calls.
457pub(crate) fn without_file_lists(summary: &str) -> &str {
458    let mut text = summary;
459    loop {
460        let end = text.trim_end();
461        let tagged = ["read-files", "modified-files"]
462            .into_iter()
463            .find_map(|tag| {
464                let listed = end.strip_suffix(&format!("</{tag}>"))?;
465                listed.rfind(&format!("<{tag}>"))
466            });
467        match tagged.and_then(|at| end.get(..at)) {
468            Some(before) => text = before.trim_end(),
469            None => return text,
470        }
471    }
472}
473
474/// The most characters of a tool result a summary request shows.
475const TOOL_RESULT_MAX_CHARS: usize = 2_000;
476
477/// `text` cut to `max_chars` UTF-16 code units, saying how many more there were. A character
478/// that would straddle the cut is left out whole.
479fn truncate_for_summary(text: &str, max_chars: usize) -> String {
480    let total = text.encode_utf16().count();
481    if total <= max_chars {
482        return text.to_owned();
483    }
484    let mut units = 0;
485    let mut end = 0;
486    for (at, character) in text.char_indices() {
487        units += character.len_utf16();
488        if units > max_chars {
489            break;
490        }
491        end = at + character.len_utf8();
492    }
493    format!(
494        "{}\n\n[... {} more characters truncated]",
495        text.get(..end).unwrap_or_default(),
496        total - max_chars
497    )
498}
499
500/// `messages` as labeled text, so the model reads them rather than continues them. Thinking is
501/// left out, and tool results are cut to 2000 characters.
502pub fn serialize_conversation(messages: &[impl Borrow<Message>]) -> String {
503    let mut parts = Vec::new();
504    for message in messages {
505        match message.borrow() {
506            Message::System { content } if !content.is_empty() => {
507                parts.push(format!("[User]: {content}"));
508            }
509            Message::System { .. } => {}
510            Message::User { content } => {
511                for result in content.iter().filter_map(UserContent::as_tool_result) {
512                    let text: String = result.content.iter().map(|part| part.to_text("")).collect();
513                    if !text.is_empty() {
514                        parts.push(format!(
515                            "[Tool result]: {}",
516                            truncate_for_summary(&text, TOOL_RESULT_MAX_CHARS)
517                        ));
518                    }
519                }
520                let text: String = content
521                    .iter()
522                    .filter_map(|item| match item {
523                        UserContent::Text(text) => Some(text.text.as_str()),
524                        _ => None,
525                    })
526                    .collect();
527                if !text.is_empty() {
528                    parts.push(format!("[User]: {text}"));
529                }
530            }
531            Message::Assistant(reply) => {
532                let calls: Vec<String> = reply
533                    .tool_calls()
534                    .map(|call| {
535                        let arguments: Vec<String> = call
536                            .function
537                            .arguments
538                            .iter()
539                            .map(|(key, value)| format!("{key}={value}"))
540                            .collect();
541                        format!("{}({})", call.function.name.as_str(), arguments.join(", "))
542                    })
543                    .collect();
544                let texts: Vec<&str> = reply
545                    .content
546                    .iter()
547                    .filter_map(AssistantContent::as_text)
548                    .collect();
549                if !texts.is_empty() {
550                    parts.push(format!("[Assistant]: {}", texts.join("\n")));
551                }
552                if !calls.is_empty() {
553                    parts.push(format!("[Assistant tool calls]: {}", calls.join("; ")));
554                }
555            }
556        }
557    }
558    parts.join("\n\n")
559}
560
561/// The system prompt of every summary request.
562pub const SUMMARIZATION_SYSTEM_PROMPT: &str = "You are a context summarization assistant. Your task is to read a conversation between a user and an AI assistant, then produce a structured summary following the exact format specified.
563
564Do NOT continue the conversation. Do NOT respond to any questions in the conversation. ONLY output the structured summary.";
565
566const SUMMARIZATION_PROMPT: &str = "The messages above are a conversation to summarize. Create a structured context checkpoint summary that another LLM will use to continue the work.
567
568Use this EXACT format:
569
570## Goal
571[What is the user trying to accomplish? Can be multiple items if the session covers different tasks.]
572
573## Constraints & Preferences
574- [Any constraints, preferences, or requirements mentioned by user]
575- [Or \"(none)\" if none were mentioned]
576
577## Progress
578### Done
579- [x] [Completed tasks/changes]
580
581### In Progress
582- [ ] [Current work]
583
584### Blocked
585- [Issues preventing progress, if any]
586
587## Key Decisions
588- **[Decision]**: [Brief rationale]
589
590## Next Steps
5911. [Ordered list of what should happen next]
592
593## Critical Context
594- [Any data, examples, or references needed to continue]
595- [Or \"(none)\" if not applicable]
596
597Keep each section concise. Preserve exact file paths, function names, and error messages.";
598
599const UPDATE_SUMMARIZATION_PROMPT: &str = "The messages above are NEW conversation messages to incorporate into the existing summary provided in <previous-summary> tags.
600
601Update the existing structured summary with new information. RULES:
602- PRESERVE all existing information from the previous summary
603- ADD new progress, decisions, and context from the new messages
604- UPDATE the Progress section: move items from \"In Progress\" to \"Done\" when completed
605- UPDATE \"Next Steps\" based on what was accomplished
606- PRESERVE exact file paths, function names, and error messages
607- If something is no longer relevant, you may remove it
608
609Use this EXACT format:
610
611## Goal
612[Preserve existing goals, add new ones if the task expanded]
613
614## Constraints & Preferences
615- [Preserve existing, add new ones discovered]
616
617## Progress
618### Done
619- [x] [Include previously done items AND newly completed items]
620
621### In Progress
622- [ ] [Current work - update based on progress]
623
624### Blocked
625- [Current blockers - remove if resolved]
626
627## Key Decisions
628- **[Decision]**: [Brief rationale] (preserve all previous, add new)
629
630## Next Steps
6311. [Update based on current state]
632
633## Critical Context
634- [Preserve important context, add new if needed]
635
636Keep each section concise. Preserve exact file paths, function names, and error messages.";
637
638const TURN_PREFIX_SUMMARIZATION_PROMPT: &str = "The messages above are earlier context from an ongoing conversation. Later messages are stored separately and do not need to be reconstructed.
639
640Create a concise checkpoint of the user's request and the progress shown above. This checkpoint will be placed before the later messages so the conversation can continue with the necessary context.
641
642## Original Request
643[What did the user ask for?]
644
645## Progress So Far
646- [Key decisions and work completed in these messages]
647
648## Context Needed to Continue
649- [Information from these messages needed to understand the later work]
650
651Only summarize information explicitly present above. Do not infer or recreate later messages.";
652
653/// Tokens a request leaves free below the window besides its output, for an input counted higher
654/// than estimated.
655const CONTEXT_SAFETY_TOKENS: u64 = 4_096;
656
657/// The share of the input also left free while that leaves [`MIN_OUTPUT_WITH_RATIO_MARGIN`].
658const CONTEXT_SAFETY_PERCENT: u64 = 15;
659
660/// The least output the ratio margin is kept for; below it only the fixed margin is.
661const MIN_OUTPUT_WITH_RATIO_MARGIN: u64 = 32_768;
662
663/// `max_tokens` fitted into `spec`'s context window after an input of `input_tokens`, leaving a
664/// margin for an input counted higher than estimated: unchanged when the window is not known, and
665/// never below 1.
666pub fn clamp_max_tokens(spec: &ModelSpec, input_tokens: u64, max_tokens: u64) -> u64 {
667    let Some(window) = spec
668        .context_window
669        .map(u64::from)
670        .filter(|&window| window > 0)
671    else {
672        return max_tokens.max(1);
673    };
674    let ratio = input_tokens
675        .saturating_mul(CONTEXT_SAFETY_PERCENT)
676        .div_ceil(100);
677    let margin = CONTEXT_SAFETY_TOKENS.max(ratio);
678    let mut room = window.saturating_sub(input_tokens).saturating_sub(margin);
679    if room < MIN_OUTPUT_WITH_RATIO_MARGIN {
680        let fixed = window
681            .saturating_sub(input_tokens)
682            .saturating_sub(CONTEXT_SAFETY_TOKENS);
683        room = MIN_OUTPUT_WITH_RATIO_MARGIN.min(fixed);
684    }
685    max_tokens.min(room.max(1))
686}
687
688/// A one-off summary request to `spec`: `system`, then `prompt` as the user's message, with the
689/// `reasoning` effort when the model takes it and no prompt-cache writes. Its output limit is
690/// `max_output` exactly when given, as a caller that fitted its own sets it, else the model's
691/// output limit fitted by pi's clamp to the room the input leaves ([`clamp_max_tokens`]).
692///
693/// # Errors
694///
695/// When `spec` does not take its default options.
696pub fn summarization_request(
697    (system, prompt): (&str, String),
698    spec: &ModelSpec,
699    reasoning: Option<Reasoning>,
700    max_output: Option<u64>,
701) -> Result<CompletionRequest, UnsupportedOption> {
702    let mut options = spec.default_options(reasoning);
703    options.cache = None;
704    if reasoning.is_some() && spec.validate(&options).is_err() {
705        options.reasoning = None;
706    }
707    spec.validate(&options)?;
708    // On a model that caches by default, explicitly none, which survives the defaults filled in
709    // when the call starts; a model with no prompt cache is sent no cache option at all, as pi
710    // passes its retention only to providers that take one.
711    if spec.default_options(None).cache.is_some() {
712        options.cache = Some(rig_core::completion::CacheRetention::None);
713    }
714    let input = request_tokens(
715        (Some(system), None),
716        &[Message::user(prompt.as_str())],
717        None,
718    );
719    let request = CompletionRequest::new(Message::user(prompt))
720        .preamble(system.to_owned())
721        .options(options);
722    let most = match max_output {
723        Some(exact) => Some(exact),
724        None => spec
725            .output_limit()
726            .map(|most| clamp_max_tokens(spec, input, u64::from(most))),
727    };
728    Ok(match most {
729        Some(most) => request.max_tokens(most),
730        None => request,
731    })
732}
733
734/// `prompt` followed by the user's `instructions`, when there are any.
735fn with_focus(prompt: &str, instructions: &str) -> String {
736    let instructions = instructions.trim();
737    if instructions.is_empty() {
738        prompt.to_owned()
739    } else {
740        format!("{prompt}\n\nAdditional focus: {instructions}")
741    }
742}
743
744/// The request for the history summary of `messages` to `spec` with the `reasoning` effort,
745/// updating `previous` (the newest summary, sent with its file lists) when it holds anything,
746/// focused on `instructions`.
747///
748/// # Errors
749///
750/// As [`summarization_request`].
751pub fn summary_request(
752    messages: &[impl Borrow<Message>],
753    (previous, instructions): (Option<&SummaryState>, &str),
754    spec: &ModelSpec,
755    reasoning: Option<Reasoning>,
756) -> Result<CompletionRequest, UnsupportedOption> {
757    // An empty previous summary reads as none, as pi's `if (previousSummary)` reads it.
758    let previous = previous.and_then(SummaryState::previous_summary);
759    let base = match previous {
760        Some(_) => UPDATE_SUMMARIZATION_PROMPT,
761        None => SUMMARIZATION_PROMPT,
762    };
763    let mut prompt = format!(
764        "<conversation>\n{}\n</conversation>\n\n",
765        serialize_conversation(messages)
766    );
767    if let Some(previous) = previous {
768        prompt.push_str(&format!(
769            "<previous-summary>\n{previous}\n</previous-summary>\n\n"
770        ));
771    }
772    prompt.push_str(&with_focus(base, instructions));
773    summarization_request((SUMMARIZATION_SYSTEM_PROMPT, prompt), spec, reasoning, None)
774}
775
776/// The request for the summary of a split turn's earlier part, `messages`, to `spec` with the
777/// `reasoning` effort, focused on `instructions`.
778///
779/// # Errors
780///
781/// As [`summarization_request`].
782pub fn turn_prefix_request(
783    messages: &[impl Borrow<Message>],
784    instructions: &str,
785    spec: &ModelSpec,
786    reasoning: Option<Reasoning>,
787) -> Result<CompletionRequest, UnsupportedOption> {
788    let prompt = format!(
789        "# Conversation\n{}\n\n# Instructions\n{TURN_PREFIX_SUMMARIZATION_PROMPT}",
790        serialize_conversation(messages)
791    );
792    let prompt = with_focus(&prompt, instructions);
793    summarization_request((SUMMARIZATION_SYSTEM_PROMPT, prompt), spec, reasoning, None)
794}
795
796/// The history summary when a split turn's earlier part has no messages of its own to summarize:
797/// the previous summary without its file lists, or a note that there is none.
798pub fn no_history(previous: Option<&SummaryState>) -> String {
799    previous
800        .map_or("No prior history.", |state| &state.summary)
801        .to_owned()
802}
803
804/// One summary of a split turn: the `history` summary, without file lists a model wrote at its
805/// end, then its earlier part's `prefix`.
806pub fn merge_split_turn(history: &str, prefix: &str) -> String {
807    let history = without_file_lists(history);
808    format!("{history}\n\n---\n\n**Turn Context (split turn):**\n\n{prefix}")
809}
810
811/// Why a summary reply cannot be used.
812#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
813pub enum SummaryFailure {
814    /// The reply failed, for this reason.
815    #[error("{0} failed: {1}")]
816    Failed(&'static str, String),
817    /// The reply hit the output limit, so the summary is incomplete.
818    #[error("{0} failed: generation hit the token cap and the summary is incomplete")]
819    Truncated(&'static str),
820    /// The model called a tool.
821    #[error("{0} attempted to call a tool")]
822    CalledTool(&'static str),
823    /// The summary has no text.
824    #[error("{0} failed: the summary is empty")]
825    Empty(&'static str),
826}
827
828/// The summary in `response`, a reply to the request `label` names, its text parts joined by
829/// newlines, or why it cannot become a session's checkpoint: an errored reply, one cut at the
830/// output limit, one that calls a tool, or one with no text.
831///
832/// # Errors
833///
834/// [`SummaryFailure`].
835pub fn summary_text(
836    response: &CompletionResponse,
837    label: &'static str,
838) -> Result<String, SummaryFailure> {
839    match response.stop() {
840        StopReason::Error(why) | StopReason::Aborted(why) => {
841            let why = if why.is_empty() {
842                "Unknown error".to_owned()
843            } else {
844                why
845            };
846            Err(SummaryFailure::Failed(label, why))
847        }
848        StopReason::Length => Err(SummaryFailure::Truncated(label)),
849        _ if response.tool_calls().next().is_some() => Err(SummaryFailure::CalledTool(label)),
850        _ => {
851            let texts: Vec<&str> = response
852                .choice
853                .iter()
854                .filter_map(AssistantContent::as_text)
855                .collect();
856            let text = texts.join("\n");
857            match text.trim().is_empty() {
858                true => Err(SummaryFailure::Empty(label)),
859                false => Ok(text),
860            }
861        }
862    }
863}
864
865#[cfg(test)]
866mod tests;