pub mod handoff;
pub mod normalize;
pub mod rehydrate;
pub mod stub;
pub mod summarize;
pub(crate) mod supersede;
use std::collections::{HashMap, HashSet};
use std::path::PathBuf;
use serde::{Deserialize, Serialize};
use supercode_interchange::ChatMessage;
use supercode_interchange::{estimate_view_tokens, format_commas, Role};
pub use supercode_interchange::{
is_tool_error, mark_tool_error, mark_tool_outcome_unknown, tool_outcome, ToolOutcome,
TOOL_ERROR_METADATA_KEY, TOOL_OUTCOME_UNKNOWN_METADATA_KEY,
};
use crate::{ReductionError as Error, Result};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub struct MessageAddr {
pub index: usize,
pub role: Role,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct SidecarPtr {
pub addr: MessageAddr,
pub span: Option<(usize, usize)>,
pub content_hash: String,
}
impl SidecarPtr {
pub fn verify(&self, candidate: &[u8]) -> Result<()> {
self.verify_hash(&content_hash(candidate))
}
pub fn verify_hash(&self, actual: &str) -> Result<()> {
if actual == self.content_hash {
Ok(())
} else {
Err(Error::new(format!(
"sidecar pointer hash mismatch: expected {}, got {actual}",
self.content_hash
)))
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub enum ReductionKind {
ToolOutputTruncated {
original_bytes: usize,
kept_bytes: usize,
},
FileReadElided {
path: PathBuf,
read_log: ReadLogEntry,
},
ImageRedacted {
part_index: usize,
},
TurnsCleared {
first: usize,
last: usize,
summary: Option<SpanSummary>,
},
ToolInputElided {
original_bytes: usize,
path: Option<PathBuf>,
content_hash: String,
call_id: String,
field: String,
},
OutputNormalized {
original_bytes: usize,
normalized_bytes: usize,
},
FileReadDiffed {
path: PathBuf,
base: MessageAddr,
base_hash: ContentHash,
new_hash: ContentHash,
original_bytes: usize,
diff_bytes: usize,
},
DuplicateOutput {
canonical: MessageAddr,
original_bytes: usize,
},
Superseded {
by: MessageAddr,
original_bytes: usize,
},
}
pub type ContentHash = String;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct SpanSummary {
pub model_id: String,
pub prompt_version: String,
pub summary_hash: ContentHash,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ReadLogEntry {
pub path: PathBuf,
pub addr: MessageAddr,
pub content_hash: String,
pub mtime: Option<i64>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Reduction {
pub id: String,
pub kind: ReductionKind,
pub ptr: SidecarPtr,
pub placeholder: String,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct ReductionPassAttribution {
pub kind: String,
pub enabled: bool,
pub candidate_count: usize,
pub candidate_original_bytes: u64,
pub applied_count: usize,
pub applied_original_bytes: u64,
pub suppressed_by_later_pass_count: usize,
pub suppressed_by_later_pass_bytes: u64,
pub standalone_saved_bytes: u64,
pub standalone_saved_tokens: u64,
pub marginal_saved_bytes: u64,
pub marginal_saved_tokens: u64,
pub suppressed_bytes: u64,
pub suppressed_tokens: u64,
pub retained_bytes: u64,
pub retained_tokens: u64,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct ReductionAttribution {
pub full_bytes: u64,
pub view_bytes: u64,
pub full_tokens: u64,
pub view_tokens: u64,
pub aggregate_saved_bytes: u64,
pub aggregate_saved_tokens: u64,
pub standalone_saved_bytes: u64,
pub standalone_saved_tokens: u64,
pub overlap_suppressed_bytes: u64,
pub overlap_suppressed_tokens: u64,
pub passes: Vec<ReductionPassAttribution>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct ReductionLog {
pub reductions: Vec<Reduction>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub expanded: Vec<Reduction>,
pub read_log: Vec<ReadLogEntry>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub attribution: Option<ReductionAttribution>,
}
pub const REDUCTION_SENTINEL: &str = "[sc-reduced";
pub const REDUCTION_METADATA_KEY: &str = "sc.reduction";
pub fn set_reduction_id(msg: &mut ChatMessage, id: &str) {
msg.metadata
.insert(REDUCTION_METADATA_KEY.to_string(), id.to_string());
}
pub fn reduction_id(msg: &ChatMessage) -> Option<&str> {
msg.metadata.get(REDUCTION_METADATA_KEY).map(String::as_str)
}
pub fn content_hash(bytes: &[u8]) -> String {
blake3::hash(bytes).to_hex().to_string()
}
pub fn make_id(ordinal: usize, hash: &str) -> String {
let ord = ordinal % 10_000;
let prefix: String = hash.chars().take(4).collect();
format!("r{ord:04}-{prefix}")
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ReductionPolicy {
pub tool_output_keep_bytes: usize,
pub tool_output_trigger_bytes: usize,
pub protect_last_n_tool_results: usize,
pub elide_stale_reads: bool,
pub redact_images: bool,
pub image_redact_min_bytes: usize,
pub clear_turns_older_than: Option<usize>,
pub read_freshness: ReadFreshness,
pub diff_rereads: bool,
pub diff_max_percent: u32,
pub protect_imported_prefix: Option<usize>,
pub elide_tool_inputs: bool,
pub tool_input_trigger_bytes: usize,
pub tool_input_elidable_fields: HashMap<String, String>,
pub normalize_terminal_output: bool,
pub terminal_output_min_savings: usize,
pub duplicate_output_min_bytes: usize,
pub deduplicate_outputs: bool,
pub supersede_enabled: bool,
pub supersede_protect_last_n: usize,
pub supersede_min_bytes: usize,
pub supersede_command_fields: HashMap<String, String>,
pub prune_errored_inputs: bool,
pub errored_input_prune_after_turns: usize,
pub summarize_cleared_turns: bool,
pub expected_summary_bytes: usize,
pub summary_cost_floor_multiple: usize,
pub cleared_turns_summary: Option<PreparedClearSummary>,
}
impl Default for ReductionPolicy {
fn default() -> Self {
ReductionPolicy {
tool_output_keep_bytes: 4096,
tool_output_trigger_bytes: 8192,
protect_last_n_tool_results: 3,
elide_stale_reads: false,
redact_images: true,
image_redact_min_bytes: 8192,
clear_turns_older_than: None,
read_freshness: ReadFreshness::default(),
diff_rereads: true,
diff_max_percent: 50,
protect_imported_prefix: None,
elide_tool_inputs: true,
tool_input_trigger_bytes: 8192,
tool_input_elidable_fields: default_tool_input_elidable_fields(),
normalize_terminal_output: true,
terminal_output_min_savings: normalize::DEFAULT_MIN_SAVINGS,
duplicate_output_min_bytes: 256,
deduplicate_outputs: true,
supersede_enabled: true,
supersede_protect_last_n: 3,
supersede_min_bytes: 256,
supersede_command_fields: supersede::default_command_fields(),
prune_errored_inputs: true,
errored_input_prune_after_turns: 3,
summarize_cleared_turns: false,
expected_summary_bytes: 400,
summary_cost_floor_multiple: 4,
cleared_turns_summary: None,
}
}
}
fn default_tool_input_elidable_fields() -> HashMap<String, String> {
let mut m = HashMap::new();
m.insert("write_file".to_string(), "content".to_string());
m.insert("Write".to_string(), "content".to_string());
m
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PreparedClearSummary {
pub first: usize,
pub last: usize,
pub text: String,
pub model_id: String,
}
fn compute_clear_range(msgs: &[ChatMessage], policy: &ReductionPolicy) -> Option<(usize, usize)> {
let threshold = policy.clear_turns_older_than?;
if msgs.len() <= threshold {
return None;
}
let mut first = 0;
while first < msgs.len() && msgs[first].role == Role::System {
first += 1;
}
if let Some(protected) = policy.protect_imported_prefix {
first = first.max(protected);
}
let keep_recent = (threshold / 2).max(2);
let mut cut = msgs.len().saturating_sub(keep_recent);
while cut < msgs.len() && msgs[cut].role == Role::Tool {
cut += 1;
}
if cut > first && cut < msgs.len() {
Some((first, cut - 1))
} else {
None
}
}
fn render_span_text(msgs: &[ChatMessage]) -> String {
let mut out = String::new();
for m in msgs {
let role = match m.role {
Role::System => "system",
Role::User => "user",
Role::Assistant => "assistant",
Role::Tool => "tool",
};
out.push_str(role);
out.push_str(": ");
out.push_str(m.content.as_deref().unwrap_or(""));
out.push('\n');
}
out
}
pub fn prepare_cleared_turns_summary(
msgs: &[ChatMessage],
policy: &ReductionPolicy,
prior: &ReductionLog,
summarizer: &dyn summarize::SpanSummarizer,
) -> Option<PreparedClearSummary> {
if !policy.summarize_cleared_turns {
return None;
}
if prior
.reductions
.iter()
.any(|r| matches!(r.kind, ReductionKind::TurnsCleared { .. }))
{
return None; }
let (first, last) = compute_clear_range(msgs, policy)?;
let range = &msgs[first..=last];
let (_hash, range_bytes) = hash_turns_range(range).ok()?;
let floor = policy
.expected_summary_bytes
.saturating_mul(policy.summary_cost_floor_multiple);
if range_bytes <= floor {
return None;
}
let span_text = render_span_text(range);
let text = match summarizer.summarize(&span_text) {
Ok(t) if !t.trim().is_empty() => t,
_ => return None, };
let sanitized = text
.split_whitespace()
.collect::<Vec<_>>()
.join(" ")
.replace(']', ")");
if sanitized.is_empty() {
return None;
}
Some(PreparedClearSummary {
first,
last,
text: sanitized,
model_id: summarizer.model_id().to_string(),
})
}
pub const READ_TOOLS: &[&str] = &["read_file"];
#[derive(Debug, Clone)]
struct DetectedRead {
index: usize,
path: PathBuf,
windowed: bool,
}
fn detect_reads(msgs: &[ChatMessage]) -> Vec<DetectedRead> {
let mut out = Vec::new();
for (i, msg) in msgs.iter().enumerate() {
if msg.role != Role::Tool {
continue;
}
let Some(call_id) = msg.tool_call_id.as_deref() else {
continue;
};
let call = msgs[..i].iter().rev().find_map(|m| {
if m.role != Role::Assistant {
return None;
}
m.tool_calls().iter().find(|c| c.id == call_id).cloned()
});
let Some(call) = call else {
continue;
};
if !READ_TOOLS.contains(&call.function.name.as_str()) {
continue;
}
let Ok(args) = call.function.parsed_arguments() else {
continue;
};
let Some(path) = args.get("path").and_then(|v| v.as_str()) else {
continue;
};
let windowed = args.get("offset").is_some_and(|v| !v.is_null())
|| args.get("limit").is_some_and(|v| !v.is_null());
out.push(DetectedRead {
index: i,
path: PathBuf::from(path),
windowed,
});
}
out
}
fn detect_normalize_candidates(msgs: &[ChatMessage]) -> Vec<usize> {
let mut out = Vec::new();
for (i, msg) in msgs.iter().enumerate() {
if msg.role != Role::Tool {
continue;
}
let Some(call_id) = msg.tool_call_id.as_deref() else {
continue;
};
let named = msgs[..i].iter().rev().find_map(|m| {
if m.role != Role::Assistant {
return None;
}
m.tool_calls()
.iter()
.find(|c| c.id == call_id)
.map(|c| c.function.name.clone())
});
if named.is_some_and(|name| normalize::NORMALIZE_TOOLS.contains(&name.as_str())) {
out.push(i);
}
}
out
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct FreshEntry {
fresh: bool,
mtime: Option<i64>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct ReadFreshness {
entries: HashMap<usize, FreshEntry>,
}
pub fn probe_read_freshness(msgs: &[ChatMessage]) -> ReadFreshness {
let mut entries = HashMap::new();
for d in detect_reads(msgs) {
let recorded = msgs[d.index].content.as_deref().unwrap_or("");
let mtime = std::fs::metadata(&d.path)
.ok()
.and_then(|m| m.modified().ok())
.and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
.map(|dur| dur.as_secs() as i64);
let fresh = match std::fs::read(&d.path) {
Err(_) => false, Ok(bytes) => {
let text = String::from_utf8_lossy(&bytes);
content_hash(text.as_bytes()) == content_hash(recorded.as_bytes())
}
};
entries.insert(d.index, FreshEntry { fresh, mtime });
}
ReadFreshness { entries }
}
pub fn prepare_read_freshness(policy: &mut ReductionPolicy, msgs: &[ChatMessage]) {
if policy.elide_stale_reads {
policy.read_freshness = probe_read_freshness(msgs);
}
}
fn parse_data_url_mime(url: &str) -> Option<String> {
let rest = url.strip_prefix("data:")?;
let end = rest.find([';', ',']).unwrap_or(rest.len());
let mime = &rest[..end];
Some(if mime.is_empty() {
"application/octet-stream".to_string()
} else {
mime.to_string()
})
}
#[derive(Debug, Clone)]
struct DetectedImage {
msg_index: usize,
part_index: usize,
mime: String,
url_len: usize,
}
fn detect_images(msgs: &[ChatMessage]) -> Vec<DetectedImage> {
let mut out = Vec::new();
for (mi, msg) in msgs.iter().enumerate() {
let Some(parts) = msg.content_parts.as_ref() else {
continue;
};
for (pi, part) in parts.iter().enumerate() {
if part.get("type").and_then(|t| t.as_str()) != Some("image_url") {
continue;
}
let Some(url) = part
.get("image_url")
.and_then(|iu| iu.get("url"))
.and_then(|u| u.as_str())
else {
continue;
};
let Some(mime) = parse_data_url_mime(url) else {
continue; };
out.push(DetectedImage {
msg_index: mi,
part_index: pi,
mime,
url_len: url.len(),
});
}
}
out
}
#[derive(Debug, Clone)]
struct DetectedToolInput {
msg_index: usize,
call_id: String,
tool_name: String,
field: String,
path: Option<PathBuf>,
value: String,
}
fn detect_tool_inputs(
msgs: &[ChatMessage],
fields: &HashMap<String, String>,
) -> Vec<DetectedToolInput> {
let mut out = Vec::new();
for (i, msg) in msgs.iter().enumerate() {
if msg.role != Role::Assistant {
continue;
}
for call in msg.tool_calls() {
let Some(field) = fields.get(&call.function.name) else {
continue;
};
let Ok(args) = call.function.parsed_arguments() else {
continue;
};
let Some(value) = args.get(field.as_str()).and_then(|v| v.as_str()) else {
continue;
};
let Some(result) = msgs[i + 1..].iter().find(|m| {
m.role == Role::Tool && m.tool_call_id.as_deref() == Some(call.id.as_str())
}) else {
continue; };
if tool_outcome(result) != ToolOutcome::KnownSuccess {
continue; }
let path = args
.get("path")
.or_else(|| args.get("file_path"))
.and_then(|v| v.as_str())
.map(PathBuf::from);
out.push(DetectedToolInput {
msg_index: i,
call_id: call.id.clone(),
tool_name: call.function.name.clone(),
field: field.clone(),
path,
value: value.to_string(),
});
}
}
out
}
fn detect_errored_tool_inputs(
msgs: &[ChatMessage],
fields: &HashMap<String, String>,
) -> Vec<DetectedToolInput> {
let mut out = Vec::new();
for (i, msg) in msgs.iter().enumerate() {
if msg.role != Role::Assistant {
continue;
}
for call in msg.tool_calls() {
let Some(field) = fields.get(&call.function.name) else {
continue;
};
let Ok(args) = call.function.parsed_arguments() else {
continue;
};
let Some(value) = args.get(field.as_str()).and_then(|v| v.as_str()) else {
continue;
};
let Some(result) = msgs[i + 1..].iter().find(|m| {
m.role == Role::Tool && m.tool_call_id.as_deref() == Some(call.id.as_str())
}) else {
continue; };
if tool_outcome(result) != ToolOutcome::KnownError {
continue; }
let path = args
.get("path")
.or_else(|| args.get("file_path"))
.and_then(|v| v.as_str())
.map(PathBuf::from);
out.push(DetectedToolInput {
msg_index: i,
call_id: call.id.clone(),
tool_name: call.function.name.clone(),
field: field.clone(),
path,
value: value.to_string(),
});
}
}
out
}
fn assistant_turns_since(msgs: &[ChatMessage], index: usize) -> usize {
msgs.get(index + 1..)
.map(|rest| rest.iter().filter(|m| m.role == Role::Assistant).count())
.unwrap_or(0)
}
fn skip_ws(b: &[u8], mut i: usize) -> usize {
while i < b.len() && b[i].is_ascii_whitespace() {
i += 1;
}
i
}
fn parse_json_string(b: &[u8], i: usize) -> Option<(usize, usize, usize)> {
if i >= b.len() || b[i] != b'"' {
return None;
}
let content_start = i + 1;
let mut j = content_start;
while j < b.len() {
match b[j] {
b'\\' => j += 2,
b'"' => return Some((content_start, j, j + 1)),
_ => j += 1,
}
}
None }
fn skip_json_value(b: &[u8], i: usize) -> Option<usize> {
let i = skip_ws(b, i);
if i >= b.len() {
return None;
}
match b[i] {
b'"' => parse_json_string(b, i).map(|(_, _, end)| end),
b'{' | b'[' => {
let open = b[i];
let close = if open == b'{' { b'}' } else { b']' };
let mut depth = 0usize;
let mut j = i;
loop {
if j >= b.len() {
return None;
}
match b[j] {
b'"' => {
let (_, _, end) = parse_json_string(b, j)?;
j = end;
}
c if c == open => {
depth += 1;
j += 1;
}
c if c == close => {
depth -= 1;
j += 1;
if depth == 0 {
return Some(j);
}
}
_ => j += 1,
}
}
}
_ => {
let mut j = i;
while j < b.len() && !matches!(b[j], b',' | b'}' | b']') && !b[j].is_ascii_whitespace()
{
j += 1;
}
Some(j)
}
}
}
fn find_top_level_string_field(json: &str, field: &str) -> Option<(usize, usize)> {
let b = json.as_bytes();
let mut i = skip_ws(b, 0);
if i >= b.len() || b[i] != b'{' {
return None;
}
i += 1;
loop {
i = skip_ws(b, i);
if i >= b.len() {
return None;
}
if b[i] == b'}' {
return None; }
let (key_start, key_end, after_key) = parse_json_string(b, i)?;
let key = &json[key_start..key_end];
i = skip_ws(b, after_key);
if i >= b.len() || b[i] != b':' {
return None;
}
i = skip_ws(b, i + 1);
if i >= b.len() {
return None;
}
if key == field {
return if b[i] == b'"' {
let (val_start, val_end, _) = parse_json_string(b, i)?;
Some((val_start, val_end))
} else {
None };
}
i = skip_json_value(b, i)?;
i = skip_ws(b, i);
match b.get(i) {
Some(b',') => {
i += 1;
continue;
}
Some(b'}') => return None, _ => return None, }
}
}
fn json_escape_content(s: &str) -> String {
let quoted = serde_json::to_string(s).unwrap_or_default();
let len = quoted.len();
if len >= 2 {
quoted[1..len - 1].to_string()
} else {
String::new()
}
}
fn replace_top_level_string_field(json: &str, field: &str, new_value: &str) -> Option<String> {
let (start, end) = find_top_level_string_field(json, field)?;
let mut out = String::with_capacity(json.len() + new_value.len());
out.push_str(&json[..start]);
out.push_str(&json_escape_content(new_value));
out.push_str(&json[end..]);
Some(out)
}
fn resolve_tool_input_value(
ptr: &SidecarPtr,
call_id: &str,
field: &str,
messages: &[ChatMessage],
) -> Result<String> {
let msg = messages.get(ptr.addr.index).ok_or_else(|| {
Error::new(format!(
"invert: sidecar has no message at index {} (reduction pointer unresolvable)",
ptr.addr.index
))
})?;
if msg.role != ptr.addr.role {
return Err(Error::new(format!(
"invert: role mismatch at sidecar index {}: pointer expects {:?}, sidecar has {:?}",
ptr.addr.index, ptr.addr.role, msg.role
)));
}
let call = msg
.tool_calls()
.iter()
.find(|c| c.id == call_id)
.ok_or_else(|| {
Error::new(format!(
"invert: sidecar message at index {} has no tool_call with id {call_id}",
ptr.addr.index
))
})?;
let parsed = call.function.parsed_arguments().map_err(|e| {
Error::new(format!(
"invert: tool_call {call_id} arguments are not valid JSON: {e}"
))
})?;
let value = parsed
.get(field)
.and_then(|v| v.as_str())
.ok_or_else(|| {
Error::new(format!(
"invert: tool_call {call_id} has no string field `{field}`"
))
})?
.to_string();
ptr.verify(value.as_bytes())?;
Ok(value)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum EscalationAction {
KeepStub,
RehydrateFromSidecar,
}
pub fn tool_input_escalation_action(fresh: bool) -> EscalationAction {
if fresh {
EscalationAction::KeepStub
} else {
EscalationAction::RehydrateFromSidecar
}
}
pub fn probe_tool_input_fresh(path: &std::path::Path, content_hash_hex: &str) -> bool {
match std::fs::read(path) {
Err(_) => false,
Ok(bytes) => content_hash(&bytes) == content_hash_hex,
}
}
fn char_boundary_floor(s: &str, target: usize) -> usize {
let mut end = target.min(s.len());
while end > 0 && !s.is_char_boundary(end) {
end -= 1;
}
end
}
fn sanitize_summary_fragment(s: &str) -> String {
s.chars()
.map(|c| if c == ']' || c.is_control() { '_' } else { c })
.collect()
}
fn rebuild_truncated_content(original: &str, r: &Reduction) -> String {
let kept = r.ptr.span.map(|(kept, _total)| kept).unwrap_or(0);
let kept = kept.min(original.len());
let mut s = original[..kept].to_string();
s.push_str("\n\n");
s.push_str(&r.placeholder);
s
}
fn rebuild_normalized_content(original: &str, r: &Reduction) -> String {
let mut s = normalize::normalize(original);
s.push_str("\n\n");
s.push_str(&r.placeholder);
s
}
fn rebuild_diffed_content(base_text: &str, new_text: &str, r: &Reduction) -> String {
let diff = diffy::create_patch(base_text, new_text);
let mut s = r.placeholder.clone();
s.push('\n');
s.push_str(&diff.to_string());
s
}
fn reapply_reduction(view: &mut Vec<ChatMessage>, r: &Reduction, msgs: &[ChatMessage]) {
match &r.kind {
ReductionKind::ToolOutputTruncated { .. } => {
let idx = r.ptr.addr.index;
let Some(msg) = view.get_mut(idx) else {
return; };
if let Some(original) = msg.content.clone() {
msg.content = Some(rebuild_truncated_content(&original, r));
}
set_reduction_id(msg, &r.id);
}
ReductionKind::OutputNormalized { .. } => {
let idx = r.ptr.addr.index;
let Some(msg) = view.get_mut(idx) else {
return; };
if let Some(original) = msg.content.clone() {
msg.content = Some(rebuild_normalized_content(&original, r));
}
set_reduction_id(msg, &r.id);
}
ReductionKind::FileReadElided { .. } => {
let idx = r.ptr.addr.index;
let Some(msg) = view.get_mut(idx) else {
return; };
msg.content = Some(r.placeholder.clone());
set_reduction_id(msg, &r.id);
}
ReductionKind::FileReadDiffed { base, .. } => {
let idx = r.ptr.addr.index;
if view.get(idx).is_none() {
return; }
let (Some(base_text), Some(new_text)) = (
msgs.get(base.index).and_then(|m| m.content.as_deref()),
msgs.get(idx).and_then(|m| m.content.as_deref()),
) else {
return; };
let content = rebuild_diffed_content(base_text, new_text, r);
let msg = &mut view[idx];
msg.content = Some(content);
set_reduction_id(msg, &r.id);
}
ReductionKind::TurnsCleared { first, last, .. } => {
if *first > *last || *last >= view.len() {
return; }
let mut placeholder = ChatMessage::system(r.placeholder.clone());
set_reduction_id(&mut placeholder, &r.id);
view.splice(*first..=*last, std::iter::once(placeholder));
}
ReductionKind::ImageRedacted { part_index } => {
let idx = r.ptr.addr.index;
let Some(msg) = view.get_mut(idx) else {
return; };
if let Some(parts) = msg.content_parts.as_mut() {
if let Some(part) = parts.get_mut(*part_index) {
*part = serde_json::json!({"type": "text", "text": r.placeholder});
}
}
set_reduction_id(msg, &r.id);
}
ReductionKind::ToolInputElided { call_id, field, .. } => {
let idx = r.ptr.addr.index;
let Some(spliced) = view
.get(idx)
.and_then(|m| m.tool_calls.as_ref())
.and_then(|calls| calls.iter().find(|c| &c.id == call_id))
.and_then(|call| {
replace_top_level_string_field(&call.function.arguments, field, &r.placeholder)
})
else {
return; };
let Some(msg) = view.get_mut(idx) else {
return;
};
if let Some(calls) = msg.tool_calls.as_mut() {
if let Some(call) = calls.iter_mut().find(|c| &c.id == call_id) {
call.function.arguments = spliced;
}
}
set_reduction_id(msg, &r.id);
}
ReductionKind::DuplicateOutput { .. } => {
let idx = r.ptr.addr.index;
let Some(msg) = view.get_mut(idx) else {
return; };
msg.content = Some(r.placeholder.clone());
set_reduction_id(msg, &r.id);
}
ReductionKind::Superseded { .. } => {
let idx = r.ptr.addr.index;
let Some(msg) = view.get_mut(idx) else {
return; };
msg.content = Some(r.placeholder.clone());
set_reduction_id(msg, &r.id);
}
}
}
fn count_roles(msgs: &[ChatMessage]) -> (usize, usize, usize) {
let mut user = 0;
let mut assistant = 0;
let mut tool = 0;
for m in msgs {
match m.role {
Role::User => user += 1,
Role::Assistant => assistant += 1,
Role::Tool => tool += 1,
Role::System => {}
}
}
(user, assistant, tool)
}
fn hash_turns_range(msgs: &[ChatMessage]) -> Result<(String, usize)> {
let mut combined = Vec::new();
for m in msgs {
let bytes = serde_json::to_vec(m)
.map_err(|e| Error::new(format!("failed to serialize message: {e}")))?;
combined.extend_from_slice(&bytes);
}
Ok((content_hash(&combined), combined.len()))
}
pub fn project_messages(
msgs: &[ChatMessage],
policy: &ReductionPolicy,
prior: &ReductionLog,
) -> (Vec<ChatMessage>, ReductionLog) {
let mut view: Vec<ChatMessage> = msgs.to_vec();
let mut log = prior.clone();
for r in &prior.reductions {
if !matches!(r.kind, ReductionKind::TurnsCleared { .. }) {
reapply_reduction(&mut view, r, msgs);
}
}
let mut existing_clears: Vec<(usize, usize)> = prior
.reductions
.iter()
.chain(prior.expanded.iter())
.filter_map(|r| match r.kind {
ReductionKind::TurnsCleared { first, last, .. } => Some((first, last)),
_ => None,
})
.collect();
existing_clears.sort_by_key(|&(f, _)| f);
let in_existing_clear = |i: usize| existing_clears.iter().any(|&(f, l)| i >= f && i <= l);
let already_reduced: HashSet<usize> = prior
.reductions
.iter()
.chain(prior.expanded.iter())
.filter(|r| !matches!(r.kind, ReductionKind::TurnsCleared { .. }))
.map(|r| r.ptr.addr.index)
.collect();
let protected: HashSet<usize> = view
.iter()
.enumerate()
.filter(|(_, m)| m.role == Role::Tool)
.map(|(i, _)| i)
.rev()
.take(policy.protect_last_n_tool_results)
.collect();
let mut reduced_this_run: HashSet<usize> = HashSet::new();
let mut ordinal = next_reduction_ordinal(
prior
.reductions
.iter()
.chain(prior.expanded.iter())
.map(|r| r.id.as_str()),
);
let read_indices: HashSet<usize> = {
let detected = detect_reads(msgs);
let mut seen_paths: HashSet<&std::path::Path> = HashSet::new();
detected
.iter()
.filter(|d| !seen_paths.insert(d.path.as_path()))
.map(|d| d.index)
.collect()
};
let mut first_seen: HashMap<String, usize> = HashMap::new();
for (i, m) in msgs.iter().enumerate() {
if !policy.deduplicate_outputs || m.role != Role::Tool || read_indices.contains(&i) {
continue;
}
let Some(content) = m.content.as_ref() else {
continue;
};
if content.len() < policy.duplicate_output_min_bytes {
continue;
}
let hash = content_hash(content.as_bytes());
let Some(&canonical_idx) = first_seen.get(&hash) else {
if !already_reduced.contains(&i) && !in_existing_clear(i) {
first_seen.insert(hash, i);
}
continue;
};
if already_reduced.contains(&i) || protected.contains(&i) || in_existing_clear(i) {
continue;
}
let original_bytes = content.len();
let id = make_id(ordinal, &hash);
let tool_name = sanitize_summary_fragment(m.name.as_deref().unwrap_or("tool"));
let summary = format!(
"{tool_name} output duplicates msg #{canonical_idx} ({}B) — identical to an \
earlier tool result, full output in session sidecar",
format_commas(original_bytes),
);
let placeholder = stub::format(stub::Kind::Duplicate, &id, &summary);
if placeholder.len() >= original_bytes {
continue;
}
ordinal += 1;
let reduction = Reduction {
id: id.clone(),
kind: ReductionKind::DuplicateOutput {
canonical: MessageAddr {
index: canonical_idx,
role: Role::Tool,
},
original_bytes,
},
ptr: SidecarPtr {
addr: MessageAddr {
index: i,
role: Role::Tool,
},
span: None,
content_hash: hash,
},
placeholder,
};
view[i].content = Some(reduction.placeholder.clone());
set_reduction_id(&mut view[i], &reduction.id);
reduced_this_run.insert(i);
log.reductions.push(reduction);
}
if policy.supersede_enabled {
let supersede_protected: HashSet<usize> = view
.iter()
.enumerate()
.filter(|(_, m)| m.role == Role::Tool)
.map(|(i, _)| i)
.rev()
.take(policy.supersede_protect_last_n)
.collect();
let occurrences = supersede::detect(msgs, &read_indices, &policy.supersede_command_fields);
let mut by_key: HashMap<&str, Vec<usize>> = HashMap::new();
for c in &occurrences {
by_key.entry(c.key.as_str()).or_default().push(c.index);
}
let mut recurring_hashes: HashSet<String> = HashSet::new();
{
let mut seen: HashSet<String> = HashSet::new();
for (i, m) in msgs.iter().enumerate() {
if m.role != Role::Tool || read_indices.contains(&i) {
continue;
}
let Some(content) = m.content.as_ref() else {
continue;
};
let h = content_hash(content.as_bytes());
if !seen.insert(h.clone()) {
recurring_hashes.insert(h);
}
}
}
let mut mint_candidates: Vec<(usize, usize)> = Vec::new(); for indices in by_key.values() {
if indices.len() < 2 {
continue; }
let mut sorted = indices.clone();
sorted.sort_unstable();
let newest = *sorted.last().expect("checked len >= 2 above");
for &idx in &sorted[..sorted.len() - 1] {
mint_candidates.push((idx, newest));
}
}
mint_candidates.sort_by_key(|&(idx, _)| idx);
for (idx, successor_idx) in mint_candidates {
if already_reduced.contains(&idx)
|| reduced_this_run.contains(&idx)
|| supersede_protected.contains(&idx)
|| in_existing_clear(idx)
{
continue;
}
let original = view[idx].content.clone().unwrap_or_default();
let original_bytes = original.len();
if original_bytes < policy.supersede_min_bytes {
continue; }
let hash = content_hash(original.as_bytes());
if recurring_hashes.contains(&hash) {
continue; }
let id = make_id(ordinal, &hash);
let tool_name = sanitize_summary_fragment(
occurrences
.iter()
.find(|c| c.index == idx)
.map(|c| c.tool_name.as_str())
.unwrap_or("tool"),
);
let summary = format!(
"{tool_name} superseded by newer result at msg #{successor_idx} ({}B) — \
expand_reduction(\"{id}\") to restore",
format_commas(original_bytes),
);
let placeholder = stub::format(stub::Kind::Superseded, &id, &summary);
if placeholder.len() >= original_bytes {
continue;
}
ordinal += 1;
let reduction = Reduction {
id: id.clone(),
kind: ReductionKind::Superseded {
by: MessageAddr {
index: successor_idx,
role: Role::Tool,
},
original_bytes,
},
ptr: SidecarPtr {
addr: MessageAddr {
index: idx,
role: Role::Tool,
},
span: None,
content_hash: hash,
},
placeholder,
};
view[idx].content = Some(reduction.placeholder.clone());
set_reduction_id(&mut view[idx], &reduction.id);
reduced_this_run.insert(idx);
log.reductions.push(reduction);
}
}
if policy.normalize_terminal_output {
let candidates: Vec<usize> = detect_normalize_candidates(&view)
.into_iter()
.filter(|i| {
!already_reduced.contains(i)
&& !reduced_this_run.contains(i)
&& !in_existing_clear(*i)
})
.collect();
for idx in candidates {
let original = view[idx].content.clone().unwrap_or_default();
let original_bytes = original.len();
let normalized = normalize::normalize(&original);
let normalized_bytes = normalized.len();
if original_bytes.saturating_sub(normalized_bytes) < policy.terminal_output_min_savings
{
continue; }
if normalized_bytes > policy.tool_output_trigger_bytes {
continue;
}
let hash = content_hash(original.as_bytes());
let id = make_id(ordinal, &hash);
ordinal += 1;
let summary = normalize::summary(original_bytes, normalized_bytes);
let placeholder = stub::format(stub::Kind::OutputNormalized, &id, &summary);
let reduction = Reduction {
id: id.clone(),
kind: ReductionKind::OutputNormalized {
original_bytes,
normalized_bytes,
},
ptr: SidecarPtr {
addr: MessageAddr {
index: idx,
role: view[idx].role,
},
span: None,
content_hash: hash,
},
placeholder,
};
let mut new_content = normalized;
new_content.push_str("\n\n");
new_content.push_str(&reduction.placeholder);
view[idx].content = Some(new_content);
set_reduction_id(&mut view[idx], &reduction.id);
reduced_this_run.insert(idx);
log.reductions.push(reduction);
}
}
let mut candidates: Vec<usize> = view
.iter()
.enumerate()
.filter(|(i, m)| {
m.role == Role::Tool
&& !already_reduced.contains(i)
&& !reduced_this_run.contains(i)
&& !protected.contains(i)
&& !in_existing_clear(*i)
&& m.content.as_ref().map(|c| c.len()).unwrap_or(0)
> policy.tool_output_trigger_bytes
})
.map(|(i, _)| i)
.collect();
let byte_len = |i: usize| view[i].content.as_ref().map(|c| c.len()).unwrap_or(0);
candidates.sort_by(|&a, &b| byte_len(b).cmp(&byte_len(a)).then(a.cmp(&b)));
for idx in candidates {
let original = view[idx].content.clone().unwrap_or_default();
let original_bytes = original.len();
let hash = content_hash(original.as_bytes());
let id = make_id(ordinal, &hash);
ordinal += 1;
let kept_bytes = char_boundary_floor(&original, policy.tool_output_keep_bytes);
let tool_name = sanitize_summary_fragment(view[idx].name.as_deref().unwrap_or("tool"));
let summary = format!(
"{tool_name} output truncated {}B, kept {}B — full output in session sidecar",
format_commas(original_bytes),
format_commas(kept_bytes),
);
let placeholder = stub::format(stub::Kind::ToolOutput, &id, &summary);
let reduction = Reduction {
id: id.clone(),
kind: ReductionKind::ToolOutputTruncated {
original_bytes,
kept_bytes,
},
ptr: SidecarPtr {
addr: MessageAddr {
index: idx,
role: view[idx].role,
},
span: Some((kept_bytes, original_bytes)),
content_hash: hash,
},
placeholder,
};
let mut new_content = original[..kept_bytes].to_string();
new_content.push_str("\n\n");
new_content.push_str(&reduction.placeholder);
view[idx].content = Some(new_content);
set_reduction_id(&mut view[idx], &reduction.id);
reduced_this_run.insert(idx);
log.reductions.push(reduction);
}
if policy.elide_stale_reads || policy.diff_rereads {
for d in detect_reads(&view) {
let idx = d.index;
if already_reduced.contains(&idx)
|| reduced_this_run.contains(&idx)
|| protected.contains(&idx)
|| in_existing_clear(idx)
{
continue;
}
let original = msgs[idx].content.clone().unwrap_or_default();
let hash = content_hash(original.as_bytes());
let mtime = policy
.read_freshness
.entries
.get(&idx)
.and_then(|e| e.mtime);
let addr = MessageAddr {
index: idx,
role: view[idx].role,
};
let prior_read: Option<ReadLogEntry> = log
.read_log
.iter()
.filter(|e| e.path == d.path && e.addr.index < idx)
.max_by_key(|e| e.addr.index)
.cloned();
if !log.read_log.iter().any(|e| e.addr.index == idx) {
log.read_log.push(ReadLogEntry {
path: d.path.clone(),
addr,
content_hash: hash.clone(),
mtime,
});
}
let mut claimed_by_diff = false;
if policy.diff_rereads && !d.windowed {
if let Some(prior) = &prior_read {
if prior.content_hash != hash {
claimed_by_diff = true;
if let Some(base_text) = msgs
.get(prior.addr.index)
.and_then(|m| m.content.as_deref())
{
let diff_text = diffy::create_patch(base_text, &original).to_string();
let diff_bytes = diff_text.len();
let original_bytes = original.len();
let within_guard = (diff_bytes as u128).saturating_mul(100)
<= (original_bytes as u128) * policy.diff_max_percent as u128;
if within_guard {
let id = make_id(ordinal, &hash);
ordinal += 1;
#[allow(clippy::manual_checked_ops)]
let percent = if original_bytes == 0 {
0
} else {
diff_bytes * 100 / original_bytes
};
let summary = format!(
"read {} diffed vs prior read at msg #{} — {}B diff, {}B full ({percent}%)",
d.path.display(),
prior.addr.index,
format_commas(diff_bytes),
format_commas(original_bytes),
);
let placeholder =
stub::format(stub::Kind::FileReadDiffed, &id, &summary);
let reduction = Reduction {
id: id.clone(),
kind: ReductionKind::FileReadDiffed {
path: d.path.clone(),
base: prior.addr,
base_hash: prior.content_hash.clone(),
new_hash: hash.clone(),
original_bytes,
diff_bytes,
},
ptr: SidecarPtr {
addr,
span: None,
content_hash: hash.clone(),
},
placeholder,
};
let mut new_content = reduction.placeholder.clone();
new_content.push('\n');
new_content.push_str(&diff_text);
view[idx].content = Some(new_content);
set_reduction_id(&mut view[idx], &reduction.id);
reduced_this_run.insert(idx);
log.reductions.push(reduction);
}
}
}
}
}
if claimed_by_diff {
continue; }
if !policy.elide_stale_reads {
continue;
}
let fresh = policy
.read_freshness
.entries
.get(&idx)
.is_some_and(|e| e.fresh);
if !fresh {
continue; }
let id = make_id(ordinal, &hash);
ordinal += 1;
let summary = format!(
"read {} elided — file unchanged on disk, re-read on demand",
d.path.display()
);
let placeholder = stub::format(stub::Kind::FileRead, &id, &summary);
let reduction = Reduction {
id: id.clone(),
kind: ReductionKind::FileReadElided {
path: d.path.clone(),
read_log: ReadLogEntry {
path: d.path.clone(),
addr,
content_hash: hash.clone(),
mtime,
},
},
ptr: SidecarPtr {
addr,
span: None,
content_hash: hash,
},
placeholder,
};
view[idx].content = Some(reduction.placeholder.clone());
set_reduction_id(&mut view[idx], &reduction.id);
reduced_this_run.insert(idx);
log.reductions.push(reduction);
}
}
if policy.redact_images {
for img in detect_images(&view) {
if img.url_len < policy.image_redact_min_bytes || in_existing_clear(img.msg_index) {
continue;
}
let idx = img.msg_index;
let part_index = img.part_index;
let original_part = view[idx].content_parts.as_ref().unwrap()[part_index].clone();
let serialized = serde_json::to_vec(&original_part)
.expect("a content part is always representable as JSON");
let hash = content_hash(&serialized);
let id = make_id(ordinal, &hash);
ordinal += 1;
let size_kb = (img.url_len + 512) / 1024;
let summary = format!("image redacted ({}, {size_kb}KB)", img.mime);
let placeholder = stub::format(stub::Kind::Image, &id, &summary);
let reduction = Reduction {
id: id.clone(),
kind: ReductionKind::ImageRedacted { part_index },
ptr: SidecarPtr {
addr: MessageAddr {
index: idx,
role: view[idx].role,
},
span: None,
content_hash: hash,
},
placeholder,
};
let parts = view[idx].content_parts.as_mut().unwrap();
parts[part_index] = serde_json::json!({"type": "text", "text": reduction.placeholder});
set_reduction_id(&mut view[idx], &reduction.id);
reduced_this_run.insert(idx);
log.reductions.push(reduction);
}
}
if policy.elide_tool_inputs {
let already_call_ids: HashSet<&str> = prior
.reductions
.iter()
.filter_map(|r| match &r.kind {
ReductionKind::ToolInputElided { call_id, .. } => Some(call_id.as_str()),
_ => None,
})
.collect();
let mut candidates: Vec<DetectedToolInput> =
detect_tool_inputs(&view, &policy.tool_input_elidable_fields)
.into_iter()
.filter(|d| {
!already_call_ids.contains(d.call_id.as_str())
&& !in_existing_clear(d.msg_index)
&& d.value.len() > policy.tool_input_trigger_bytes
})
.collect();
candidates.sort_by(|a, b| {
b.value
.len()
.cmp(&a.value.len())
.then(a.call_id.cmp(&b.call_id))
});
for d in candidates {
let hash = content_hash(d.value.as_bytes());
let id = make_id(ordinal, &hash);
ordinal += 1;
let original_bytes = d.value.len();
let path_clause = d
.path
.as_ref()
.map(|p| format!(", on disk at {}", p.display()))
.unwrap_or_default();
let hash_prefix: String = hash.chars().take(8).collect();
let summary = format!(
"{} input elided: `{}` field, {}B{path_clause}, blake3={hash_prefix}... — full \
args in session sidecar",
d.tool_name,
d.field,
format_commas(original_bytes),
);
let placeholder = stub::format(stub::Kind::ToolInput, &id, &summary);
let Some(original_args) = view
.get(d.msg_index)
.and_then(|m| m.tool_calls.as_ref())
.and_then(|calls| calls.iter().find(|c| c.id == d.call_id))
.map(|call| call.function.arguments.clone())
else {
continue; };
let Some(spliced) =
replace_top_level_string_field(&original_args, &d.field, &placeholder)
else {
continue; };
let role = view[d.msg_index].role;
let calls = view[d.msg_index]
.tool_calls
.as_mut()
.expect("checked above: this message has tool_calls");
let call = calls
.iter_mut()
.find(|c| c.id == d.call_id)
.expect("checked above: this call_id is present");
call.function.arguments = spliced;
set_reduction_id(&mut view[d.msg_index], &id);
let reduction = Reduction {
id: id.clone(),
kind: ReductionKind::ToolInputElided {
original_bytes,
path: d.path.clone(),
content_hash: hash.clone(),
call_id: d.call_id.clone(),
field: d.field.clone(),
},
ptr: SidecarPtr {
addr: MessageAddr {
index: d.msg_index,
role,
},
span: None,
content_hash: hash,
},
placeholder,
};
log.reductions.push(reduction);
}
}
if policy.prune_errored_inputs {
let already_call_ids: HashSet<&str> = prior
.reductions
.iter()
.filter_map(|r| match &r.kind {
ReductionKind::ToolInputElided { call_id, .. } => Some(call_id.as_str()),
_ => None,
})
.collect();
let mut candidates: Vec<DetectedToolInput> =
detect_errored_tool_inputs(&view, &policy.tool_input_elidable_fields)
.into_iter()
.filter(|d| {
!already_call_ids.contains(d.call_id.as_str())
&& !in_existing_clear(d.msg_index)
&& d.value.len() > policy.tool_input_trigger_bytes
&& assistant_turns_since(&view, d.msg_index)
>= policy.errored_input_prune_after_turns
})
.collect();
candidates.sort_by(|a, b| {
b.value
.len()
.cmp(&a.value.len())
.then(a.call_id.cmp(&b.call_id))
});
for d in candidates {
let hash = content_hash(d.value.as_bytes());
let id = make_id(ordinal, &hash);
ordinal += 1;
let original_bytes = d.value.len();
let turns = assistant_turns_since(&view, d.msg_index);
let path_clause = d
.path
.as_ref()
.map(|p| format!(", on disk at {}", p.display()))
.unwrap_or_default();
let hash_prefix: String = hash.chars().take(8).collect();
let summary = format!(
"{} input elided (errored call, {turns} turns old): `{}` field, {}B{path_clause}, \
blake3={hash_prefix}... — full args in session sidecar",
d.tool_name,
d.field,
format_commas(original_bytes),
);
let placeholder = stub::format(stub::Kind::ToolInput, &id, &summary);
let Some(original_args) = view
.get(d.msg_index)
.and_then(|m| m.tool_calls.as_ref())
.and_then(|calls| calls.iter().find(|c| c.id == d.call_id))
.map(|call| call.function.arguments.clone())
else {
continue; };
let Some(spliced) =
replace_top_level_string_field(&original_args, &d.field, &placeholder)
else {
continue; };
let role = view[d.msg_index].role;
let calls = view[d.msg_index]
.tool_calls
.as_mut()
.expect("checked above: this message has tool_calls");
let call = calls
.iter_mut()
.find(|c| c.id == d.call_id)
.expect("checked above: this call_id is present");
call.function.arguments = spliced;
set_reduction_id(&mut view[d.msg_index], &id);
let reduction = Reduction {
id: id.clone(),
kind: ReductionKind::ToolInputElided {
original_bytes,
path: d.path.clone(),
content_hash: hash.clone(),
call_id: d.call_id.clone(),
field: d.field.clone(),
},
ptr: SidecarPtr {
addr: MessageAddr {
index: d.msg_index,
role,
},
span: None,
content_hash: hash,
},
placeholder,
};
log.reductions.push(reduction);
}
}
if !existing_clears.is_empty() {
let mut to_reapply: Vec<Reduction> = log
.reductions
.iter()
.filter(|r| matches!(r.kind, ReductionKind::TurnsCleared { .. }))
.cloned()
.collect();
to_reapply.sort_by_key(|r| match r.kind {
ReductionKind::TurnsCleared { first, .. } => std::cmp::Reverse(first),
_ => unreachable!("filtered to TurnsCleared above"),
});
for r in &to_reapply {
reapply_reduction(&mut view, r, msgs);
}
} else if let Some((first, last)) = compute_clear_range(&view, policy) {
let range = &msgs[first..=last];
let (hash, range_bytes) =
hash_turns_range(range).expect("ChatMessage always serializes to JSON");
let (user, assistant, tool) = count_roles(range);
let id = make_id(ordinal, &hash);
let deterministic_summary = format!(
"turns {first}..{} cleared ({} messages: {user} user, {assistant} \
assistant, {tool} tool; {}B) — full turns in session sidecar",
last + 1,
format_commas(range.len()),
format_commas(range_bytes),
);
let prepared = policy
.summarize_cleared_turns
.then_some(policy.cleared_turns_summary.as_ref())
.flatten()
.filter(|p| p.first == first && p.last == last);
let (summary_text, summary_audit) = match prepared {
Some(p) => {
let banner = format!(
"sc-summary of {id}, original {} messages in sidecar — \
expand_reduction(\"{id}\") for verbatim",
format_commas(range.len()),
);
let text = format!("{} ({banner})", p.text);
let audit = SpanSummary {
model_id: p.model_id.clone(),
prompt_version: summarize::PROMPT_VERSION.to_string(),
summary_hash: content_hash(p.text.as_bytes()),
};
(text, Some(audit))
}
None => (deterministic_summary, None),
};
let placeholder = stub::format(stub::Kind::TurnsCleared, &id, &summary_text);
log.reductions
.retain(|r| !(r.ptr.addr.index >= first && r.ptr.addr.index <= last));
let reduction = Reduction {
id: id.clone(),
kind: ReductionKind::TurnsCleared {
first,
last,
summary: summary_audit,
},
ptr: SidecarPtr {
addr: MessageAddr {
index: first,
role: range[0].role,
},
span: None,
content_hash: hash,
},
placeholder,
};
let mut stub_msg = ChatMessage::system(reduction.placeholder.clone());
set_reduction_id(&mut stub_msg, &reduction.id);
view.splice(first..=last, std::iter::once(stub_msg));
log.reductions.push(reduction);
}
(view, log)
}
fn next_reduction_ordinal<'a>(ids: impl Iterator<Item = &'a str>) -> usize {
ids.filter_map(|id| {
let digits = id.strip_prefix('r')?.get(..4)?;
digits.parse::<usize>().ok()
})
.max()
.map_or(0, |max| max.saturating_add(1))
}
#[cfg(test)]
mod ordinal_tests {
use super::next_reduction_ordinal;
#[test]
fn sparse_legacy_ids_advance_past_the_greatest_live_ordinal() {
let ids = ["r0001-dead", "r0007-beef"];
assert_eq!(next_reduction_ordinal(ids.into_iter()), 8);
}
#[test]
fn malformed_ids_cannot_force_reuse_of_a_valid_live_ordinal() {
let ids = ["legacy", "r0003-cafe", "rxxxx-nope"];
assert_eq!(next_reduction_ordinal(ids.into_iter()), 4);
}
}
const MAX_AGGRESSIVE_LEVELS: u32 = 5;
fn tighten(base: &ReductionPolicy, level: u32) -> ReductionPolicy {
let shift = level.min(5);
let shrink = |n: usize, floor: usize| -> usize { (n >> shift).max(floor) };
ReductionPolicy {
tool_output_keep_bytes: shrink(base.tool_output_keep_bytes, 128),
tool_output_trigger_bytes: shrink(base.tool_output_trigger_bytes, 256),
protect_last_n_tool_results: base
.protect_last_n_tool_results
.saturating_sub(level as usize),
image_redact_min_bytes: shrink(base.image_redact_min_bytes, 256),
tool_input_trigger_bytes: shrink(base.tool_input_trigger_bytes, 256),
duplicate_output_min_bytes: shrink(base.duplicate_output_min_bytes, 16),
supersede_min_bytes: shrink(base.supersede_min_bytes, 16),
supersede_protect_last_n: base.supersede_protect_last_n.saturating_sub(level as usize),
errored_input_prune_after_turns: base
.errored_input_prune_after_turns
.saturating_sub(level as usize),
..base.clone()
}
}
pub fn reduce_to_fit<F>(
full_msgs: &[ChatMessage],
base_policy: &ReductionPolicy,
prior_log: &ReductionLog,
fits: F,
) -> (Vec<ChatMessage>, ReductionLog, ReductionPolicy)
where
F: Fn(&[ChatMessage]) -> bool,
{
let (mut best_view, mut best_log) = project_messages(full_msgs, base_policy, prior_log);
let mut best_policy = base_policy.clone();
let mut best_tokens = estimate_view_tokens(&best_view);
let mut level = 1;
while !fits(&best_view) && level <= MAX_AGGRESSIVE_LEVELS {
let candidate_policy = tighten(base_policy, level);
let (view, log) = project_messages(full_msgs, &candidate_policy, prior_log);
let view_tokens = estimate_view_tokens(&view);
if view_tokens < best_tokens {
best_view = view;
best_log = log;
best_policy = candidate_policy;
best_tokens = view_tokens;
}
level += 1;
}
if !fits(&best_view) && base_policy.clear_turns_older_than.is_none() {
let mut threshold = full_msgs.len();
loop {
if threshold <= MIN_CLEAR_TURNS_WINDOW {
break;
}
threshold = (threshold / 2).max(MIN_CLEAR_TURNS_WINDOW);
let mut candidate_policy = best_policy.clone();
candidate_policy.clear_turns_older_than = Some(threshold);
let (view, log) = project_messages(full_msgs, &candidate_policy, prior_log);
let view_tokens = estimate_view_tokens(&view);
if view_tokens < best_tokens {
best_view = view;
best_log = log;
best_policy = candidate_policy;
best_tokens = view_tokens;
}
if fits(&best_view) {
break;
}
}
}
(best_view, best_log, best_policy)
}
const MIN_CLEAR_TURNS_WINDOW: usize = 4;
fn resolve_original_content(ptr: &SidecarPtr, messages: &[ChatMessage]) -> Result<String> {
let msg = messages.get(ptr.addr.index).ok_or_else(|| {
Error::new(format!(
"invert: sidecar has no message at index {} (reduction pointer unresolvable)",
ptr.addr.index
))
})?;
if msg.role != ptr.addr.role {
return Err(Error::new(format!(
"invert: role mismatch at sidecar index {}: pointer expects {:?}, sidecar has {:?}",
ptr.addr.index, ptr.addr.role, msg.role
)));
}
let content = msg.content.clone().unwrap_or_default();
ptr.verify(content.as_bytes())?;
Ok(content)
}
fn resolve_image_part(
ptr: &SidecarPtr,
part_index: usize,
messages: &[ChatMessage],
) -> Result<serde_json::Value> {
let msg = messages.get(ptr.addr.index).ok_or_else(|| {
Error::new(format!(
"invert: sidecar has no message at index {} (reduction pointer unresolvable)",
ptr.addr.index
))
})?;
if msg.role != ptr.addr.role {
return Err(Error::new(format!(
"invert: role mismatch at sidecar index {}: pointer expects {:?}, sidecar has {:?}",
ptr.addr.index, ptr.addr.role, msg.role
)));
}
let part = msg
.content_parts
.as_ref()
.and_then(|parts| parts.get(part_index))
.ok_or_else(|| {
Error::new(format!(
"invert: sidecar message at index {} has no content part {part_index}",
ptr.addr.index
))
})?;
let serialized = serde_json::to_vec(part)
.map_err(|e| Error::new(format!("invert: failed to serialize image part: {e}")))?;
ptr.verify(&serialized)?;
Ok(part.clone())
}
fn resolve_turns_range(
ptr: &SidecarPtr,
first: usize,
last: usize,
messages: &[ChatMessage],
) -> Result<Vec<ChatMessage>> {
let mut msgs = Vec::with_capacity(last.saturating_sub(first) + 1);
for i in first..=last {
let msg = messages.get(i).ok_or_else(|| {
Error::new(format!(
"invert: sidecar has no message at index {i} (turns-cleared range unresolvable)"
))
})?;
msgs.push(msg.clone());
}
if let Some(first_msg) = msgs.first() {
if first_msg.role != ptr.addr.role {
return Err(Error::new(format!(
"invert: role mismatch at sidecar index {first}: pointer expects {:?}, sidecar has {:?}",
ptr.addr.role, first_msg.role
)));
}
}
let (hash, _bytes) = hash_turns_range(&msgs)?;
ptr.verify_hash(&hash)?;
Ok(msgs)
}
fn invert_one_reduction(
out: &mut Vec<ChatMessage>,
r: &Reduction,
sidecar_messages: &[ChatMessage],
) -> Result<()> {
let pos = out
.iter()
.position(|m| reduction_id(m) == Some(r.id.as_str()))
.ok_or_else(|| {
Error::new(format!(
"invert: no message in the reduced view carries reduction id {}",
r.id
))
})?;
match &r.kind {
ReductionKind::ToolOutputTruncated { .. }
| ReductionKind::FileReadElided { .. }
| ReductionKind::OutputNormalized { .. }
| ReductionKind::FileReadDiffed { .. }
| ReductionKind::DuplicateOutput { .. }
| ReductionKind::Superseded { .. } => {
let original = resolve_original_content(&r.ptr, sidecar_messages)?;
out[pos].content = Some(original);
out[pos].metadata.remove(REDUCTION_METADATA_KEY);
}
ReductionKind::ImageRedacted { part_index } => {
let part = resolve_image_part(&r.ptr, *part_index, sidecar_messages)?;
let parts = out[pos].content_parts.get_or_insert_with(Vec::new);
if *part_index < parts.len() {
parts[*part_index] = part;
} else {
parts.push(part);
}
out[pos].metadata.remove(REDUCTION_METADATA_KEY);
}
ReductionKind::TurnsCleared { first, last, .. } => {
let msgs = resolve_turns_range(&r.ptr, *first, *last, sidecar_messages)?;
out.splice(pos..=pos, msgs);
}
ReductionKind::ToolInputElided { call_id, field, .. } => {
let original_value =
resolve_tool_input_value(&r.ptr, call_id, field, sidecar_messages)?;
let current_args = out[pos]
.tool_calls
.as_ref()
.and_then(|calls| calls.iter().find(|c| &c.id == call_id))
.map(|call| call.function.arguments.clone())
.ok_or_else(|| {
Error::new(format!(
"invert: reduced message at position {pos} has no tool_call with id \
{call_id}"
))
})?;
let restored = replace_top_level_string_field(¤t_args, field, &original_value)
.ok_or_else(|| {
Error::new(format!(
"invert: reduced tool_call {call_id} arguments do not contain field \
`{field}` to restore"
))
})?;
let calls = out[pos]
.tool_calls
.as_mut()
.expect("checked above: tool_calls present");
let call = calls
.iter_mut()
.find(|c| &c.id == call_id)
.expect("checked above: call_id present");
call.function.arguments = restored;
out[pos].metadata.remove(REDUCTION_METADATA_KEY);
}
}
Ok(())
}
pub fn invert_messages(
reduced: &[ChatMessage],
log: &ReductionLog,
sidecar_messages: &[ChatMessage],
) -> Result<Vec<ChatMessage>> {
let mut out: Vec<ChatMessage> = reduced.to_vec();
let by_id: HashMap<&str, &Reduction> =
log.reductions.iter().map(|r| (r.id.as_str(), r)).collect();
for msg in &out {
if let Some(id) = reduction_id(msg) {
if !by_id.contains_key(id) {
return Err(Error::new(format!(
"invert: reduced message carries reduction id {id} with no matching entry \
in the reduction log — unresolvable pointer"
)));
}
}
}
for r in &log.reductions {
invert_one_reduction(&mut out, r, sidecar_messages)?;
}
for msg in out.iter_mut() {
msg.metadata.remove(REDUCTION_METADATA_KEY);
}
Ok(out)
}
pub fn verify_log_messages(log: &ReductionLog, sidecar_messages: &[ChatMessage]) -> Result<()> {
for r in log.reductions.iter().chain(log.expanded.iter()) {
let resolved = match &r.kind {
ReductionKind::ToolOutputTruncated { .. }
| ReductionKind::FileReadElided { .. }
| ReductionKind::OutputNormalized { .. }
| ReductionKind::FileReadDiffed { .. }
| ReductionKind::DuplicateOutput { .. }
| ReductionKind::Superseded { .. } => {
resolve_original_content(&r.ptr, sidecar_messages).map(|_| ())
}
ReductionKind::ImageRedacted { part_index } => {
resolve_image_part(&r.ptr, *part_index, sidecar_messages).map(|_| ())
}
ReductionKind::TurnsCleared { first, last, .. } => {
resolve_turns_range(&r.ptr, *first, *last, sidecar_messages).map(|_| ())
}
ReductionKind::ToolInputElided { call_id, field, .. } => {
resolve_tool_input_value(&r.ptr, call_id, field, sidecar_messages).map(|_| ())
}
};
if let Err(e) = resolved {
return Err(Error::new(format!(
"reduction {} unresolvable against the sidecar: {e}",
r.id
)));
}
}
Ok(())
}
pub fn invert_one_messages(
reduced: &[ChatMessage],
log: &ReductionLog,
id: &str,
sidecar_messages: &[ChatMessage],
) -> Result<(Vec<ChatMessage>, ReductionLog)> {
let r = log
.reductions
.iter()
.find(|r| r.id == id)
.cloned()
.ok_or_else(|| Error::new(format!("invert_one: no reduction with id {id} in the log")))?;
let mut out: Vec<ChatMessage> = reduced.to_vec();
invert_one_reduction(&mut out, &r, sidecar_messages)?;
let mut new_log = log.clone();
new_log.reductions.retain(|x| x.id != id);
if !new_log.expanded.iter().any(|x| x.id == r.id) {
new_log.expanded.push(r);
}
Ok((out, new_log))
}
#[cfg(test)]
mod tool_input_splice_tests {
use super::*;
#[test]
fn finds_and_replaces_only_the_named_fields_value() {
let json = r#"{"path":"src/foo.rs","content":"hello world","flag":true}"#;
let (start, end) = find_top_level_string_field(json, "content").unwrap();
assert_eq!(&json[start..end], "hello world");
let replaced = replace_top_level_string_field(json, "content", "STUB").unwrap();
assert_eq!(
replaced,
r#"{"path":"src/foo.rs","content":"STUB","flag":true}"#
);
assert!(replaced.contains(r#""path":"src/foo.rs""#));
assert!(replaced.contains(r#""flag":true"#));
}
#[test]
fn preserves_whitespace_and_key_order_around_the_replaced_field() {
let json = "{ \"content\" : \"big\", \"path\":\"a/b.rs\" }";
let replaced = replace_top_level_string_field(json, "content", "X").unwrap();
assert_eq!(replaced, "{ \"content\" : \"X\", \"path\":\"a/b.rs\" }");
}
#[test]
fn handles_escaped_quotes_backslashes_and_unicode_in_the_value() {
let original_value = "line1\nline2 \"quoted\" \\ and unicode caf\u{e9}";
let json = serde_json::json!({"path": "p", "content": original_value}).to_string();
let (start, end) = find_top_level_string_field(&json, "content").unwrap();
let raw = format!("\"{}\"", &json[start..end]);
let decoded: String = serde_json::from_str(&raw).unwrap();
assert_eq!(decoded, original_value);
let new_value = "replacement with \"quotes\" and \\ backslash and \u{1f600}";
let replaced = replace_top_level_string_field(&json, "content", new_value).unwrap();
let reparsed: serde_json::Value = serde_json::from_str(&replaced).unwrap();
assert_eq!(reparsed["content"], new_value);
assert_eq!(reparsed["path"], "p");
}
#[test]
fn skips_nested_objects_and_arrays_in_sibling_fields() {
let json = r#"{"meta":{"a":[1,2,{"b":"}}}"}]},"content":"payload","tags":["x","y"]}"#;
let (start, end) = find_top_level_string_field(json, "content").unwrap();
assert_eq!(&json[start..end], "payload");
let replaced = replace_top_level_string_field(json, "content", "NEW").unwrap();
let reparsed: serde_json::Value = serde_json::from_str(&replaced).unwrap();
assert_eq!(reparsed["content"], "NEW");
assert_eq!(reparsed["tags"][0], "x");
assert_eq!(reparsed["meta"]["a"][2]["b"], "}}}");
}
#[test]
fn returns_none_when_field_absent_or_not_a_string_or_not_an_object() {
assert_eq!(
find_top_level_string_field(r#"{"path":"a"}"#, "content"),
None
);
assert_eq!(
find_top_level_string_field(r#"{"content":42}"#, "content"),
None
);
assert_eq!(
find_top_level_string_field(r#"["not","an","object"]"#, "content"),
None
);
assert_eq!(
find_top_level_string_field("not json at all", "content"),
None
);
assert_eq!(
replace_top_level_string_field(r#"{"path":"a"}"#, "content", "x"),
None
);
}
}