use crate::context::{
self, compact_messages, message_timestamp, safe_head_end, safe_turn_start, total_tokens,
CompactionStrategy, ContextConfig,
};
use crate::provider::{ModelConfig, StreamConfig, StreamProvider};
use crate::retry::RetryConfig;
use crate::types::CacheConfig;
use crate::types::*;
use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};
use std::sync::{Arc, Mutex, MutexGuard};
use std::time::Duration;
use tokio::sync::mpsc::UnboundedSender;
use tokio_util::sync::CancellationToken;
pub const DEFAULT_TRIGGER_RATIO: f32 = 0.6;
pub const DEFAULT_RETAIN_TAIL_TOKENS: usize = 20_000;
pub const DEFAULT_REQUEST_TIMEOUT: Duration = Duration::from_secs(120);
pub const SUMMARY_MARKER: &str = "[Context compacted — summary of earlier conversation]";
const DEFAULT_SYSTEM_PROMPT: &str = "You are a context summarization assistant. You produce \
structured handoff briefings of agent conversations so that work can \
continue seamlessly with the summary in place of the original messages.";
const DEFAULT_INSTRUCTION: &str = "Summarize the conversation above as a handoff briefing for \
an agent that will continue this work without access to the original \
messages. Use exactly these sections:\n\
## Goal\nWhat the user is trying to accomplish, verbatim where possible.\n\
## State & progress\nWhat has been done, what is currently in flight.\n\
## Key decisions & constraints\nDecisions made and why. Record the \
constraints you were *given* as well as the choices made in response to \
them — deployment shape, scale, hard dependencies, things ruled out, \
stated preferences. A reader who keeps the decisions but loses the \
conditions that forced them cannot tell which are still binding.\n\
## Open items\nUnresolved questions and concrete next steps.\n\
Be dense and factual. Include exact identifiers (paths, names, versions, \
numbers) — those are the details the next agent cannot reconstruct.";
const TRANSCRIPT_PER_BLOCK_BYTES: usize = 2_000;
const TRANSCRIPT_TOTAL_BYTES: usize = 480_000;
const MIN_SUMMARIZED_SPAN: usize = 4;
const MIN_SUMMARY_TOKENS: u32 = 256;
#[derive(Debug, Clone, PartialEq, Eq)]
struct Fingerprint {
cut: usize,
hash: u64,
}
fn fingerprint(messages: &[AgentMessage], cut: usize) -> Fingerprint {
let mut hasher = DefaultHasher::new();
for (i, msg) in messages[..cut].iter().enumerate() {
i.hash(&mut hasher);
match serde_json::to_vec(msg) {
Ok(bytes) => bytes.hash(&mut hasher),
Err(e) => {
debug_assert!(false, "AgentMessage failed to serialize: {e}");
tracing::error!("llm compaction: message {i} failed to serialize: {e}");
u8::MAX.hash(&mut hasher);
}
}
}
Fingerprint {
cut,
hash: hasher.finish(),
}
}
fn safe_head_boundary(messages: &[AgentMessage], end: usize) -> usize {
let mut end = end;
for _ in 0..=messages.len() {
let pulled = safe_head_end(messages, safe_turn_start(messages, end));
if pulled == end {
break;
}
end = pulled;
}
end
}
struct Summary {
fingerprint: Fingerprint,
head_end: usize,
text: String,
usage: Usage,
}
enum Phase {
Idle,
Inflight,
Ready(Box<Summary>),
}
impl Phase {
fn is_idle(&self) -> bool {
matches!(self, Phase::Idle)
}
}
#[derive(Default)]
struct Warned {
inert: bool,
no_runtime: bool,
}
struct State {
phase: Phase,
warned: Warned,
}
impl Default for State {
fn default() -> Self {
Self {
phase: Phase::Idle,
warned: Warned::default(),
}
}
}
struct InflightGuard {
state: Arc<Mutex<State>>,
disarmed: bool,
}
impl InflightGuard {
fn disarm(&mut self) {
self.disarmed = true;
}
}
impl Drop for InflightGuard {
fn drop(&mut self) {
if self.disarmed {
return;
}
let mut state = lock(&self.state);
if matches!(state.phase, Phase::Inflight) {
state.phase = Phase::Idle;
}
}
}
fn lock(state: &Arc<Mutex<State>>) -> MutexGuard<'_, State> {
state
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
}
pub struct LlmCompaction {
provider: Arc<dyn StreamProvider>,
config: ModelConfig,
api_key: String,
trigger_ratio: f32,
retain_tail_tokens: Option<usize>,
system_prompt: String,
instruction: String,
max_summary_tokens: u32,
timeout: Duration,
retry: RetryConfig,
events: Option<UnboundedSender<AgentEvent>>,
cancel: CancellationToken,
state: Arc<Mutex<State>>,
}
impl std::fmt::Debug for LlmCompaction {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("LlmCompaction")
.field("config", &self.config)
.field("trigger_ratio", &self.trigger_ratio)
.field("retain_tail_tokens", &self.retain_tail_tokens)
.field("max_summary_tokens", &self.max_summary_tokens)
.field("timeout", &self.timeout)
.finish_non_exhaustive()
}
}
impl Drop for LlmCompaction {
fn drop(&mut self) {
self.cancel.cancel();
}
}
impl LlmCompaction {
pub fn from_config(config: ModelConfig) -> Self {
Self::from_config_with(&crate::provider::ProviderRegistry::default(), config)
.expect("default registry covers all built-in protocols")
}
pub fn from_config_with(
registry: &crate::provider::ProviderRegistry,
config: ModelConfig,
) -> Result<Self, crate::AgentBuildError> {
let provider = registry
.resolve(&config.api)
.ok_or(crate::AgentBuildError::NoProviderForProtocol(config.api))?;
let api_key = crate::provider::resolve_api_key_or_warn(&config.provider);
Ok(Self::build(provider, config, api_key))
}
pub fn from_provider(provider: Arc<dyn StreamProvider>, config: ModelConfig) -> Self {
let api_key = crate::provider::resolve_api_key_or_warn(&config.provider);
Self::build(provider, config, api_key)
}
fn build(provider: Arc<dyn StreamProvider>, config: ModelConfig, api_key: String) -> Self {
Self {
provider,
config,
api_key,
trigger_ratio: DEFAULT_TRIGGER_RATIO,
retain_tail_tokens: None,
system_prompt: DEFAULT_SYSTEM_PROMPT.into(),
instruction: DEFAULT_INSTRUCTION.into(),
max_summary_tokens: 2_000,
timeout: DEFAULT_REQUEST_TIMEOUT,
retry: RetryConfig::default(),
events: None,
cancel: CancellationToken::new(),
state: Arc::new(Mutex::new(State::default())),
}
}
pub fn with_api_key(mut self, key: impl Into<String>) -> Self {
self.api_key = key.into();
self
}
pub fn with_trigger_ratio(mut self, ratio: f32) -> Self {
let clamped = if ratio.is_finite() {
ratio.clamp(0.1, 0.95)
} else {
DEFAULT_TRIGGER_RATIO
};
if clamped != ratio {
tracing::warn!(
"llm compaction: trigger_ratio {ratio} is out of range, using {clamped}"
);
}
self.trigger_ratio = clamped;
self
}
pub fn with_retain_tail_tokens(mut self, tokens: usize) -> Self {
self.retain_tail_tokens = Some(tokens);
self
}
pub fn with_system_prompt(mut self, prompt: impl Into<String>) -> Self {
self.system_prompt = prompt.into();
self
}
pub fn with_instruction(mut self, instruction: impl Into<String>) -> Self {
self.instruction = instruction.into();
self
}
pub fn with_max_summary_tokens(mut self, tokens: u32) -> Self {
if tokens < MIN_SUMMARY_TOKENS {
tracing::warn!(
"llm compaction: max_summary_tokens {tokens} is below the {MIN_SUMMARY_TOKENS} \
floor; using the floor"
);
}
self.max_summary_tokens = tokens.max(MIN_SUMMARY_TOKENS);
self
}
pub fn with_timeout(mut self, timeout: Duration) -> Self {
self.timeout = timeout;
self
}
pub fn with_retry_config(mut self, retry: RetryConfig) -> Self {
self.retry = retry;
self
}
pub fn with_event_sender(mut self, events: UnboundedSender<AgentEvent>) -> Self {
self.events = Some(events);
self
}
fn emit(&self, event: AgentEvent) {
if let Some(tx) = &self.events {
if tx.send(event).is_err() {
tracing::debug!("llm compaction: event receiver dropped");
}
}
}
fn summary_cost(&self, usage: &Usage) -> Option<f64> {
let cost = &self.config.cost;
cost.is_configured().then(|| cost.cost_usd(usage))
}
fn effective_retain_tail(&self, budget: usize) -> usize {
self.retain_tail_tokens
.unwrap_or_else(|| DEFAULT_RETAIN_TAIL_TOKENS.min(budget / 4))
}
fn choose_cut(
&self,
messages: &[AgentMessage],
config: &ContextConfig,
budget: usize,
) -> Result<(usize, usize), NoCut> {
let len = messages.len();
let head_end = safe_head_boundary(messages, config.keep_first.min(len));
let retain = self.effective_retain_tail(budget);
let mut tail_tokens = 0usize;
let mut cut = len;
while cut > head_end && tail_tokens < retain {
cut -= 1;
tail_tokens += context::message_tokens(&messages[cut]);
}
cut = cut.min(len.saturating_sub(config.keep_recent));
let cut = safe_turn_start(messages, cut);
if cut > head_end && cut - head_end >= MIN_SUMMARIZED_SPAN {
return Ok((head_end, cut));
}
if len.saturating_sub(head_end) < MIN_SUMMARIZED_SPAN + config.keep_recent {
Err(NoCut::HistoryTooShort)
} else {
Err(NoCut::TailTooLarge)
}
}
fn warn_inert_once(&self, used: usize, budget: usize, config: &ContextConfig) {
{
let mut state = lock(&self.state);
if state.warned.inert {
return;
}
state.warned.inert = true;
}
let retain = self.effective_retain_tail(budget);
tracing::warn!(
"llm compaction is inert: past the trigger ({used} of {budget} budget tokens) there \
is still no split leaving {retain} tail tokens, {} recent messages and {} messages \
to summarize, so every compaction will fall back to the deterministic tiers. Lower \
retain_tail_tokens or keep_recent, or raise trigger_ratio (currently {}).",
config.keep_recent,
MIN_SUMMARIZED_SPAN,
self.trigger_ratio,
);
}
fn spawn_summarize(&self, messages: &[AgentMessage], head_end: usize, cut: usize) {
let Ok(handle) = tokio::runtime::Handle::try_current() else {
let mut state = lock(&self.state);
if !state.warned.no_runtime {
state.warned.no_runtime = true;
tracing::warn!(
"llm compaction: no tokio runtime, so background summarization is impossible \
and every compaction will use the deterministic tiers"
);
}
return;
};
let fp = fingerprint(messages, cut);
let transcript = serialize_transcript(&messages[head_end..cut]);
let provider = Arc::clone(&self.provider);
let state = Arc::clone(&self.state);
let cancel = self.cancel.child_token();
let (timeout, retry) = (self.timeout, self.retry.clone());
let mut stream_config = StreamConfig::new(self.config.id.clone(), self.api_key.clone());
stream_config.system_prompt = self.system_prompt.clone();
stream_config.messages = vec![Message::user(format!(
"<conversation>\n{transcript}\n</conversation>\n\n{}",
self.instruction
))];
stream_config.max_tokens = Some(self.max_summary_tokens);
stream_config.model_config = Some(self.config.clone());
debug_assert!(stream_config.temperature.is_none());
stream_config.cache_config = CacheConfig {
enabled: false,
..Default::default()
};
lock(&state).phase = Phase::Inflight;
tracing::debug!(
"llm compaction: summarizing messages[{}..{}) in background",
head_end,
cut
);
handle.spawn(async move {
let mut guard = InflightGuard {
state: Arc::clone(&state),
disarmed: false,
};
let outcome = summarize(&provider, stream_config, timeout, &retry, &cancel).await;
let Some((text, usage)) = outcome else { return };
lock(&state).phase = Phase::Ready(Box::new(Summary {
fingerprint: fp,
head_end,
usage,
text,
}));
guard.disarm();
});
}
fn splice(&self, messages: &[AgentMessage], summary: &Summary) -> Vec<AgentMessage> {
let cut = summary.fingerprint.cut;
let head_end = summary.head_end;
debug_assert!(head_end < cut, "summary span must be non-empty");
let ts = message_timestamp(&messages[cut.saturating_sub(1)]);
let summary_msg = AgentMessage::Llm(Message::User {
content: vec![Content::Text {
text: format!("{SUMMARY_MARKER}\n\n{}", summary.text),
}],
timestamp: ts,
});
let mut result = Vec::with_capacity(messages.len() - cut + head_end + 1);
result.extend_from_slice(&messages[..head_end]);
result.push(summary_msg);
result.extend_from_slice(&messages[cut..]);
result
}
fn shrink_tail(
&self,
mut result: Vec<AgentMessage>,
config: &ContextConfig,
budget: usize,
head_end: usize,
) -> Vec<AgentMessage> {
let keep = head_end + 1; if keep >= result.len() {
return result;
}
let tail = result.split_off(keep);
let fixed = total_tokens(&result);
tracing::debug!(
"llm compaction: spliced result exceeds the budget; compacting the tail against \
{} of {budget} tokens",
budget.saturating_sub(fixed)
);
let tail_config = ContextConfig {
max_context_tokens: budget.saturating_sub(fixed),
system_prompt_tokens: 0,
..config.clone()
};
result.extend(compact_messages(tail, &tail_config));
result
}
fn arm(&self, messages: &[AgentMessage], config: &ContextConfig, budget: usize) {
let used = total_tokens(messages);
if used <= (budget as f32 * self.trigger_ratio) as usize {
return;
}
if !lock(&self.state).phase.is_idle() {
return;
}
match self.choose_cut(messages, config, budget) {
Ok((head_end, cut)) => self.spawn_summarize(messages, head_end, cut),
Err(NoCut::HistoryTooShort) => tracing::debug!(
"llm compaction: history too short to summarize yet ({} messages)",
messages.len()
),
Err(NoCut::TailTooLarge) => self.warn_inert_once(used, budget, config),
}
}
}
#[derive(Debug)]
enum NoCut {
HistoryTooShort,
TailTooLarge,
}
async fn summarize(
provider: &Arc<dyn StreamProvider>,
stream_config: StreamConfig,
timeout: Duration,
retry: &RetryConfig,
cancel: &CancellationToken,
) -> Option<(String, Usage)> {
for attempt in 0..=retry.max_retries {
if cancel.is_cancelled() {
tracing::debug!("llm compaction: cancelled");
return None;
}
let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
let drain = tokio::spawn(async move { while rx.recv().await.is_some() {} });
let result = tokio::time::timeout(
timeout,
provider.stream(stream_config.clone(), tx, cancel.clone()),
)
.await;
drain.abort();
match result {
Err(_elapsed) => {
tracing::warn!(
"llm compaction: summarization timed out after {timeout:?} (attempt {})",
attempt + 1
);
}
Ok(Err(e)) => {
if e.is_retryable() && attempt < retry.max_retries {
let delay = e
.retry_after()
.unwrap_or_else(|| retry.delay_for_attempt(attempt));
tracing::debug!("llm compaction: {e}, retrying in {delay:?}");
tokio::time::sleep(delay).await;
continue;
}
tracing::warn!("llm compaction: summarization failed: {e}");
return None;
}
Ok(Ok(message)) => return accept_summary(message),
}
if attempt < retry.max_retries {
tokio::time::sleep(retry.delay_for_attempt(attempt)).await;
}
}
None
}
fn accept_summary(message: Message) -> Option<(String, Usage)> {
if let Message::Assistant {
stop_reason,
error_message,
..
} = &message
{
if !matches!(stop_reason, StopReason::Stop) {
tracing::warn!(
"llm compaction: rejecting summary (stop_reason={stop_reason:?}, overflow={}): \
{}; falling back to deterministic compaction",
message.is_context_overflow(),
error_message.as_deref().unwrap_or("no detail")
);
return None;
}
}
let text = assistant_text(&message);
if text.trim().is_empty() {
tracing::warn!("llm compaction: empty summary, discarding");
return None;
}
tracing::debug!("llm compaction: summary ready ({} chars)", text.len());
Some((text, assistant_usage(&message)))
}
impl CompactionStrategy for LlmCompaction {
fn compact(&self, messages: Vec<AgentMessage>, config: &ContextConfig) -> Vec<AgentMessage> {
let budget = config
.max_context_tokens
.saturating_sub(config.system_prompt_tokens);
let used = total_tokens(&messages);
let messages_before = messages.len();
if used > budget {
let ready = {
let mut state = lock(&self.state);
match std::mem::replace(&mut state.phase, Phase::Idle) {
Phase::Ready(summary) => Some(summary),
other => {
state.phase = other;
None
}
}
};
if let Some(summary) = ready {
let fp = &summary.fingerprint;
if fp.cut <= messages.len() && fingerprint(&messages, fp.cut) == *fp {
let mut summarized = fp.cut - summary.head_end;
let mut method = CompactionMethod::Summarized;
let mut result = self.splice(&messages, &summary);
if total_tokens(&result) > budget {
result = self.shrink_tail(result, config, budget, summary.head_end);
if total_tokens(&result) > budget {
tracing::warn!(
"llm compaction: head + summary exceed the budget on their own; \
discarding the summary and compacting deterministically"
);
result = compact_messages(result, config);
method = CompactionMethod::Deterministic;
summarized = 0;
}
}
let after = total_tokens(&result);
let cost = self.summary_cost(&summary.usage);
tracing::info!(
"llm compaction: {method:?} — summarized {summarized} messages, \
{messages_before} -> {} messages ({used} -> {after} tokens); \
request used {} in / {} out{}",
result.len(),
summary.usage.input,
summary.usage.output,
cost.map(|c| format!(", ${c:.4}")).unwrap_or_default(),
);
self.emit(AgentEvent::ContextCompacted {
method,
messages_before,
messages_after: result.len(),
tokens_before: used,
tokens_after: after,
summary: Some(SummaryStats::new(summarized, summary.usage, cost)),
});
self.arm(&result, config, budget);
return result;
}
tracing::warn!("llm compaction: history changed under summary, discarding");
}
}
if used > budget {
tracing::debug!("llm compaction: summary not ready, deterministic fallback");
let result = compact_messages(messages, config);
let after = total_tokens(&result);
self.emit(AgentEvent::ContextCompacted {
method: CompactionMethod::Deterministic,
messages_before,
messages_after: result.len(),
tokens_before: used,
tokens_after: after,
summary: None,
});
self.arm(&result, config, budget);
return result;
}
self.arm(&messages, config, budget);
messages
}
}
fn clip(text: &str, max: usize) -> &str {
if text.len() <= max {
return text;
}
let mut end = max;
while end > 0 && !text.is_char_boundary(end) {
end -= 1;
}
&text[..end]
}
fn serialize_transcript(messages: &[AgentMessage]) -> String {
let mut out = String::new();
for msg in messages {
if out.len() >= TRANSCRIPT_TOTAL_BYTES {
tracing::warn!(
"llm compaction: transcript hit the {TRANSCRIPT_TOTAL_BYTES}-byte cap; summarizing only the most recent part of the span"
);
out.push_str("[... earlier messages omitted: transcript size cap ...]\n");
break;
}
let AgentMessage::Llm(message) = msg else {
continue; };
match message {
Message::User { content, .. } => {
for c in content {
match c {
Content::Text { text } => {
out.push_str("User: ");
out.push_str(clip(text, TRANSCRIPT_PER_BLOCK_BYTES));
out.push('\n');
}
Content::Image { .. } => out.push_str("[image omitted]\n"),
_ => {}
}
}
}
Message::Assistant { content, .. } => {
for c in content {
match c {
Content::Text { text } => {
out.push_str("Assistant: ");
out.push_str(clip(text, TRANSCRIPT_PER_BLOCK_BYTES));
out.push('\n');
}
Content::ToolCall {
name, arguments, ..
} => {
let args = arguments.to_string();
out.push_str(&format!("[tool call] {name}({})\n", clip(&args, 300)));
}
Content::Thinking { .. } => out.push_str("[thinking omitted]\n"),
_ => {}
}
}
}
Message::ToolResult {
tool_name, content, ..
} => {
for c in content {
if let Content::Text { text } = c {
out.push_str(&format!(
"[tool result: {tool_name}] {}\n",
clip(text, TRANSCRIPT_PER_BLOCK_BYTES)
));
}
}
}
}
}
out
}
fn assistant_text(message: &Message) -> String {
let Message::Assistant { content, .. } = message else {
return String::new();
};
content
.iter()
.filter_map(|c| match c {
Content::Text { text } => Some(text.as_str()),
_ => None,
})
.collect::<Vec<_>>()
.join("\n")
}
fn assistant_usage(message: &Message) -> Usage {
match message {
Message::Assistant { usage, .. } => usage.clone(),
_ => Usage::default(),
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::provider::MockProvider;
fn turn(i: usize, bulk: usize) -> Vec<AgentMessage> {
vec![
AgentMessage::Llm(Message::User {
content: vec![Content::Text {
text: format!("user message {i}: {}", "x".repeat(bulk)),
}],
timestamp: i as u64,
}),
AgentMessage::Llm(
Message::assistant(
vec![Content::Text {
text: format!("assistant reply {i}: {}", "y".repeat(bulk)),
}],
StopReason::Stop,
"mock",
"mock",
Usage::default(),
)
.with_timestamp(i as u64),
),
]
}
fn history(turns: usize, bulk: usize) -> Vec<AgentMessage> {
(0..turns).flat_map(|i| turn(i, bulk)).collect()
}
fn config(budget: usize) -> ContextConfig {
ContextConfig {
max_context_tokens: budget,
system_prompt_tokens: 0,
keep_first: 1,
keep_recent: 2,
..Default::default()
}
}
fn mock(text: &str) -> LlmCompaction {
LlmCompaction::from_provider(Arc::new(MockProvider::text(text)), ModelConfig::mock())
}
async fn await_summary(strategy: &LlmCompaction) -> bool {
for _ in 0..50 {
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
if matches!(lock(&strategy.state).phase, Phase::Ready(_)) {
return true;
}
}
false
}
fn has_summary(messages: &[AgentMessage]) -> bool {
messages.iter().any(|m| {
matches!(m, AgentMessage::Llm(Message::User { content, .. })
if content.iter().any(|c| matches!(c, Content::Text { text }
if text.starts_with(SUMMARY_MARKER))))
})
}
async fn settle() {
for _ in 0..25 {
tokio::time::sleep(std::time::Duration::from_millis(1)).await;
}
}
struct Run {
messages: Vec<AgentMessage>,
rounds_with_summary: usize,
peak_tokens: usize,
}
async fn drive<F>(
strategy: &LlmCompaction,
cfg: &ContextConfig,
rounds: usize,
mut next_turn: F,
) -> Run
where
F: FnMut(usize) -> Vec<AgentMessage>,
{
let mut messages: Vec<AgentMessage> = Vec::new();
let mut run = Run {
messages: Vec::new(),
rounds_with_summary: 0,
peak_tokens: 0,
};
for i in 0..rounds {
messages.extend(next_turn(i));
messages = strategy.compact(std::mem::take(&mut messages), cfg);
if has_summary(&messages) {
run.rounds_with_summary += 1;
}
run.peak_tokens = run.peak_tokens.max(total_tokens(&messages));
settle().await;
}
run.messages = messages;
run
}
#[tokio::test(flavor = "multi_thread")]
async fn splices_summary_when_over_budget() {
let (provider, _) = ScriptedProvider::new("## Goal\nShip the parser.");
let strategy = LlmCompaction::from_provider(provider, ModelConfig::mock())
.with_trigger_ratio(0.1)
.with_retain_tail_tokens(200);
let cfg = config(2_000);
let run = drive(&strategy, &cfg, 30, |i| turn(i, 400)).await;
assert!(
run.rounds_with_summary > 0,
"a summary must be spliced at some point"
);
assert!(
has_summary(&run.messages),
"summary must be present at the end"
);
assert!(run.messages.iter().any(|m| {
matches!(m, AgentMessage::Llm(Message::User { content, .. })
if content.iter().any(|c| matches!(c, Content::Text { text }
if text.contains("Ship the parser"))))
}));
assert!(run.peak_tokens <= 2_000, "the budget must hold throughout");
}
#[tokio::test(flavor = "multi_thread")]
async fn under_trigger_is_a_no_op() {
let strategy = mock("unused");
let messages = history(3, 50);
let out = strategy.compact(messages.clone(), &config(1_000_000));
assert_eq!(out, messages, "below trigger nothing may change");
assert!(lock(&strategy.state).phase.is_idle());
}
#[tokio::test(flavor = "multi_thread")]
async fn stale_summary_is_discarded_not_spliced() {
let strategy = mock("## Goal\nStale.")
.with_trigger_ratio(0.1)
.with_retain_tail_tokens(200);
let messages = history(30, 400);
let cfg = config(2_000);
strategy.compact(messages.clone(), &cfg);
assert!(await_summary(&strategy).await);
let mut mutated = messages.clone();
mutated[0] = AgentMessage::Llm(Message::User {
content: vec![Content::Text {
text: "rewritten".into(),
}],
timestamp: 999,
});
let out = strategy.compact(mutated, &cfg);
assert!(!has_summary(&out), "stale summary must be discarded");
assert!(total_tokens(&out) <= 2_000, "fallback still fits budget");
}
#[tokio::test(flavor = "multi_thread")]
async fn same_shape_rewrite_is_still_detected() {
let strategy = mock("## Goal\nStale.")
.with_trigger_ratio(0.1)
.with_retain_tail_tokens(200);
let messages = history(30, 400);
let cfg = config(2_000);
strategy.compact(messages.clone(), &cfg);
assert!(await_summary(&strategy).await);
let mut mutated = messages.clone();
mutated[0] = AgentMessage::Llm(Message::User {
content: vec![Content::Text {
text: format!("user message 0: {}", "z".repeat(400)),
}],
timestamp: 0,
});
let out = strategy.compact(mutated, &cfg);
assert!(
!has_summary(&out),
"a same-length, same-timestamp rewrite must still invalidate the summary"
);
}
#[tokio::test(flavor = "multi_thread")]
async fn default_retain_tail_scales_to_a_small_budget() {
let strategy = mock("## Goal\nSmall budget."); let cfg = config(2_000);
assert_eq!(
strategy.effective_retain_tail(2_000),
500,
"the tail must derive from the budget, not the 20k ceiling"
);
let run = drive(&strategy, &cfg, 40, |i| turn(i, 400)).await;
assert!(
run.rounds_with_summary > 0,
"defaults must still splice on a small budget"
);
assert!(run.peak_tokens <= 2_000);
}
#[tokio::test(flavor = "multi_thread")]
async fn impossible_settings_warn_once_instead_of_silently_no_oping() {
let strategy = mock("unused")
.with_trigger_ratio(0.1)
.with_retain_tail_tokens(10_000_000);
let messages = history(30, 400);
let cfg = config(2_000);
for _ in 0..3 {
let out = strategy.compact(messages.clone(), &cfg);
assert!(!has_summary(&out));
}
let state = lock(&strategy.state);
assert!(
state.warned.inert,
"the inert case must be reported, not silent"
);
assert!(state.phase.is_idle(), "nothing should have been spawned");
}
#[tokio::test(flavor = "multi_thread")]
async fn emits_an_event_on_both_compaction_paths() {
let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
let strategy = mock("## Goal\nObserve me.")
.with_trigger_ratio(0.1)
.with_retain_tail_tokens(200)
.with_event_sender(tx);
let cfg = config(2_000);
let mut messages = history(30, 400);
for i in 100..140 {
messages.extend(turn(i, 400));
messages = strategy.compact(std::mem::take(&mut messages), &cfg);
settle().await;
}
let mut events = Vec::new();
while let Ok(e) = rx.try_recv() {
events.push(e);
}
assert!(!events.is_empty(), "compaction must emit events");
let mut saw_deterministic = false;
let mut saw_summarized = false;
for event in &events {
match event {
AgentEvent::ContextCompacted {
method,
tokens_before,
tokens_after,
summary,
..
} => {
assert!(tokens_after <= tokens_before);
match method {
CompactionMethod::Deterministic => saw_deterministic = true,
CompactionMethod::Summarized => {
saw_summarized = true;
let stats = summary
.as_ref()
.expect("a Summarized event must carry its request's cost");
assert!(stats.messages_summarized >= MIN_SUMMARIZED_SPAN);
assert!(stats.cost_usd.is_none());
}
}
}
other => panic!("unexpected event: {other:?}"),
}
}
assert!(saw_deterministic, "the fallback path must emit");
assert!(saw_summarized, "the splice path must emit");
}
#[tokio::test(flavor = "multi_thread")]
async fn head_is_kept_verbatim_or_summarized_but_not_both() {
let strategy = mock("summary")
.with_trigger_ratio(0.1)
.with_retain_tail_tokens(200);
let messages = history(30, 400);
let cfg = config(2_000);
let (head_end, cut) = strategy
.choose_cut(&messages, &cfg, 2_000)
.expect("a split exists");
assert_eq!(head_end, 1, "keep_first = 1");
let transcript = serialize_transcript(&messages[head_end..cut]);
assert!(
!transcript.contains("user message 0"),
"the verbatim head must not also appear inside the summarized span"
);
assert!(transcript.contains("user message 1"));
}
fn tool_turn(i: usize, lines: usize) -> Vec<AgentMessage> {
let call_id = format!("call-{i}");
vec![
AgentMessage::Llm(Message::User {
content: vec![Content::Text {
text: format!("run command {i}"),
}],
timestamp: i as u64,
}),
AgentMessage::Llm(
Message::assistant(
vec![Content::ToolCall {
id: call_id.clone(),
name: "bash".into(),
arguments: serde_json::json!({"command": format!("cmd {i}")}),
provider_metadata: None,
}],
StopReason::ToolUse,
"mock",
"mock",
Usage::default(),
)
.with_timestamp(i as u64),
),
AgentMessage::Llm(Message::ToolResult {
tool_call_id: call_id,
tool_name: "bash".into(),
content: vec![Content::Text {
text: (0..lines)
.map(|l| format!("output line {l} of turn {i}"))
.collect::<Vec<_>>()
.join("\n"),
}],
is_error: false,
timestamp: i as u64,
}),
]
}
#[tokio::test(flavor = "multi_thread")]
async fn spliced_history_is_level_1_stable() {
let (provider, _) = ScriptedProvider::new(format!("## Goal\n{}", "verbose ".repeat(500)));
let strategy = LlmCompaction::from_provider(provider, ModelConfig::mock())
.with_trigger_ratio(0.1)
.with_retain_tail_tokens(200);
let cfg = config(2_000);
let run = drive(&strategy, &cfg, 30, |i| tool_turn(i, 400)).await;
assert!(run.rounds_with_summary > 0, "expected a splice");
assert!(run.peak_tokens <= 2_000, "the budget must hold throughout");
for msg in &run.messages {
assert_eq!(
&context::truncate_tool_output(msg.clone(), &cfg),
msg,
"spliced history must already be Level-1 stable"
);
}
assert!(
orphaned_tool_calls(&run.messages).is_empty(),
"spliced tool history must stay structurally valid"
);
}
struct ScriptedProvider {
calls: Arc<std::sync::atomic::AtomicUsize>,
text: String,
}
impl ScriptedProvider {
fn new(text: impl Into<String>) -> (Arc<Self>, Arc<std::sync::atomic::AtomicUsize>) {
let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
(
Arc::new(Self {
calls: Arc::clone(&calls),
text: text.into(),
}),
calls,
)
}
}
#[async_trait::async_trait]
impl StreamProvider for ScriptedProvider {
async fn stream(
&self,
_config: StreamConfig,
_tx: tokio::sync::mpsc::UnboundedSender<crate::provider::StreamEvent>,
_cancel: tokio_util::sync::CancellationToken,
) -> Result<Message, crate::provider::ProviderError> {
self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
Ok(Message::assistant(
vec![Content::Text {
text: self.text.clone(),
}],
StopReason::Stop,
"mock",
"mock",
Usage::default(),
))
}
}
fn orphaned_tool_calls(messages: &[AgentMessage]) -> Vec<String> {
let (mut opened, mut answered) = (Vec::new(), Vec::new());
for msg in messages {
match msg {
AgentMessage::Llm(Message::Assistant { content, .. }) => {
for c in content {
if let Content::ToolCall { id, .. } = c {
opened.push(id.clone());
}
}
}
AgentMessage::Llm(Message::ToolResult { tool_call_id, .. }) => {
answered.push(tool_call_id.clone())
}
_ => {}
}
}
opened
.into_iter()
.filter(|i| !answered.contains(i))
.collect()
}
fn parallel_tool_turn(i: usize) -> Vec<AgentMessage> {
let (a, b) = (format!("call-{i}a"), format!("call-{i}b"));
vec![
AgentMessage::Llm(Message::User {
content: vec![Content::Text {
text: format!("do {i}"),
}],
timestamp: i as u64,
}),
AgentMessage::Llm(
Message::assistant(
vec![
Content::tool_call(a.clone(), "bash", serde_json::json!({"c": i})),
Content::tool_call(b.clone(), "bash", serde_json::json!({"c": i})),
],
StopReason::ToolUse,
"mock",
"mock",
Usage::default(),
)
.with_timestamp(i as u64),
),
AgentMessage::Llm(Message::ToolResult {
tool_call_id: a,
tool_name: "bash".into(),
content: vec![Content::Text {
text: format!("out a {i}: {}", "z".repeat(300)),
}],
is_error: false,
timestamp: i as u64,
}),
AgentMessage::Llm(Message::ToolResult {
tool_call_id: b,
tool_name: "bash".into(),
content: vec![Content::Text {
text: format!("out b {i}: {}", "z".repeat(300)),
}],
is_error: false,
timestamp: i as u64,
}),
]
}
#[tokio::test(flavor = "multi_thread")]
async fn feeding_the_result_back_still_splices() {
for turn_chars in [200usize, 800, 2000] {
let (provider, calls) = ScriptedProvider::new("## Goal\nThe briefing.");
let strategy = LlmCompaction::from_provider(provider, ModelConfig::mock())
.with_retain_tail_tokens(200);
let cfg = config(2_000);
let mut messages: Vec<AgentMessage> = Vec::new();
let mut spliced_rounds = 0usize;
for i in 0..25 {
messages.extend(turn(i, turn_chars));
messages = strategy.compact(std::mem::take(&mut messages), &cfg);
if has_summary(&messages) {
spliced_rounds += 1;
}
for _ in 0..25 {
tokio::time::sleep(std::time::Duration::from_millis(1)).await;
}
}
let requests = calls.load(std::sync::atomic::Ordering::SeqCst);
assert!(
spliced_rounds > 0 || requests == 0,
"turn_chars={turn_chars}: {requests} summarization requests paid for and never \
spliced — the fallback is invalidating its own pending summary"
);
}
}
#[tokio::test(flavor = "multi_thread")]
async fn a_session_that_starts_over_budget_recovers() {
let (provider, calls) = ScriptedProvider::new("## Goal\nRecovered.");
let strategy = LlmCompaction::from_provider(provider, ModelConfig::mock())
.with_retain_tail_tokens(200);
let cfg = config(2_000);
let mut messages = history(30, 400);
assert!(total_tokens(&messages) > 2_000, "must start over budget");
let mut spliced_ever = false;
for i in 100..130 {
messages.extend(turn(i, 400));
messages = strategy.compact(std::mem::take(&mut messages), &cfg);
if has_summary(&messages) {
spliced_ever = true;
}
for _ in 0..25 {
tokio::time::sleep(std::time::Duration::from_millis(1)).await;
}
}
assert!(
spliced_ever,
"{} requests issued, never spliced",
calls.load(std::sync::atomic::Ordering::SeqCst)
);
}
#[tokio::test(flavor = "multi_thread")]
async fn the_safety_net_preserves_the_briefing_it_paid_for() {
let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
let (provider, _) = ScriptedProvider::new(format!("## Goal\n{}", "verbose ".repeat(500)));
let strategy = LlmCompaction::from_provider(provider, ModelConfig::mock())
.with_trigger_ratio(0.1)
.with_retain_tail_tokens(200)
.with_event_sender(tx);
let cfg = config(2_000);
let run = drive(&strategy, &cfg, 30, |i| turn(i, 400)).await;
assert!(run.rounds_with_summary > 0, "expected at least one splice");
assert!(run.peak_tokens <= 2_000, "the budget must hold throughout");
let mut saw_summarized = false;
while let Ok(event) = rx.try_recv() {
if let AgentEvent::ContextCompacted {
method,
summary,
tokens_after,
..
} = event
{
if method == CompactionMethod::Summarized {
saw_summarized = true;
assert!(
summary.is_some_and(|s| s.messages_summarized > 0),
"a Summarized event must report a real span"
);
assert!(tokens_after <= 2_000);
}
}
}
assert!(
saw_summarized,
"the briefing was paid for but never reported as spliced"
);
}
#[tokio::test(flavor = "multi_thread")]
async fn splice_never_orphans_a_parallel_tool_call() {
for keep_first in 1..=6usize {
let (provider, _) = ScriptedProvider::new("## Goal\nX.");
let strategy = LlmCompaction::from_provider(provider, ModelConfig::mock())
.with_trigger_ratio(0.1)
.with_retain_tail_tokens(300);
let cfg = ContextConfig {
max_context_tokens: 2_000,
system_prompt_tokens: 0,
keep_first,
keep_recent: 2,
..Default::default()
};
let run = drive(&strategy, &cfg, 30, parallel_tool_turn).await;
assert!(
run.rounds_with_summary > 0,
"keep_first={keep_first}: expected a splice"
);
let orphans = orphaned_tool_calls(&run.messages);
assert!(
orphans.is_empty(),
"keep_first={keep_first}: orphaned tool_use ids {orphans:?} — a provider \
would reject this outright"
);
}
}
struct StoppedProvider {
reason: StopReason,
}
#[async_trait::async_trait]
impl StreamProvider for StoppedProvider {
async fn stream(
&self,
_config: StreamConfig,
_tx: tokio::sync::mpsc::UnboundedSender<crate::provider::StreamEvent>,
_cancel: tokio_util::sync::CancellationToken,
) -> Result<Message, crate::provider::ProviderError> {
Ok(Message::assistant(
vec![Content::Text {
text: "## Goal\nTruncated mid-".into(),
}],
self.reason.clone(),
"mock",
"mock",
Usage::default(),
))
}
}
struct HangingProvider;
#[async_trait::async_trait]
impl StreamProvider for HangingProvider {
async fn stream(
&self,
_config: StreamConfig,
_tx: tokio::sync::mpsc::UnboundedSender<crate::provider::StreamEvent>,
_cancel: tokio_util::sync::CancellationToken,
) -> Result<Message, crate::provider::ProviderError> {
std::future::pending().await
}
}
#[tokio::test(flavor = "multi_thread")]
async fn a_non_stop_summary_is_rejected_not_spliced() {
for reason in [StopReason::Length, StopReason::Refusal, StopReason::Error] {
let strategy = LlmCompaction::from_provider(
Arc::new(StoppedProvider {
reason: reason.clone(),
}),
ModelConfig::mock(),
)
.with_trigger_ratio(0.1)
.with_retain_tail_tokens(200);
let cfg = config(2_000);
let run = drive(&strategy, &cfg, 15, |i| turn(i, 400)).await;
assert_eq!(
run.rounds_with_summary, 0,
"{reason:?} must never be spliced into history"
);
assert!(
lock(&strategy.state).phase.is_idle(),
"{reason:?}: a rejected summary must leave the slot idle"
);
}
}
#[tokio::test(flavor = "multi_thread")]
async fn a_hung_request_releases_the_in_flight_slot() {
let strategy = LlmCompaction::from_provider(Arc::new(HangingProvider), ModelConfig::mock())
.with_trigger_ratio(0.1)
.with_retain_tail_tokens(200)
.with_timeout(std::time::Duration::from_millis(50))
.with_retry_config(crate::retry::RetryConfig::none());
let cfg = config(2_000);
let messages = history(30, 400);
strategy.compact(messages.clone(), &cfg);
for _ in 0..60 {
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
if lock(&strategy.state).phase.is_idle() {
break;
}
}
assert!(
lock(&strategy.state).phase.is_idle(),
"a hung request must not pin the slot in flight forever"
);
}
#[test]
fn transcript_is_capped_in_total_not_just_per_block() {
let huge: Vec<AgentMessage> = (0..4_000).flat_map(|i| turn(i, 1_000)).collect();
let transcript = serialize_transcript(&huge);
let ceiling = TRANSCRIPT_TOTAL_BYTES + 4 * TRANSCRIPT_PER_BLOCK_BYTES;
assert!(
transcript.len() <= ceiling,
"transcript must be bounded in total, got {} bytes (ceiling {ceiling})",
transcript.len()
);
assert!(transcript.contains("transcript size cap"));
}
#[tokio::test(flavor = "multi_thread")]
async fn a_short_history_does_not_burn_the_inert_warning() {
let strategy = mock("unused").with_trigger_ratio(0.1);
let cfg = config(2_000);
let messages = history(1, 400);
strategy.compact(messages, &cfg);
assert!(
!lock(&strategy.state).warned.inert,
"a merely-short history is transient and must not spend the one-shot warning"
);
}
struct FailingProvider;
#[async_trait::async_trait]
impl StreamProvider for FailingProvider {
async fn stream(
&self,
_config: StreamConfig,
_tx: tokio::sync::mpsc::UnboundedSender<crate::provider::StreamEvent>,
_cancel: tokio_util::sync::CancellationToken,
) -> Result<Message, crate::provider::ProviderError> {
Err(crate::provider::ProviderError::Network("down".into()))
}
}
#[tokio::test(flavor = "multi_thread")]
async fn provider_failure_falls_back_deterministically() {
let strategy = LlmCompaction::from_provider(Arc::new(FailingProvider), ModelConfig::mock())
.with_trigger_ratio(0.1)
.with_retain_tail_tokens(200);
let messages = history(30, 400);
let cfg = config(2_000);
let out = strategy.compact(messages.clone(), &cfg);
assert!(total_tokens(&out) <= 2_000);
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
assert!(
lock(&strategy.state).phase.is_idle(),
"a failed request must leave the slot idle, not pinned in flight"
);
}
}