use crate::prelude::*;
use regex::Regex;
use std::io::{self, Read};
use std::path::Path;
use std::collections::HashSet;
lazy_static::lazy_static! {
static ref DIFF_LINE_RE: Regex = Regex::new(r"^\s*\d+\s([+\- ])\s?(.*)$").unwrap();
static ref UPDATE_HEADER_RE: Regex = Regex::new(r"^\s*[⏺⎿]\s+Update\((.+)\)\s*$").unwrap();
static ref STATUS_LINE_RE: Regex = Regex::new(r"^\s*[⏺⎿]\s*(.*)$").unwrap();
static ref SPACES_RE: Regex = Regex::new(r"\s+").unwrap();
static ref NUMBER_ONLY_RE: Regex = Regex::new(r"^\s*\d+\s*$").unwrap();
static ref LIST_ITEM_RE: Regex = Regex::new(r"^\s*(\d+\.)\s+(.*)$").unwrap();
static ref THINKING_STATUS_RE: Regex = Regex::new(
r"^\s*[✻*]\s+(?:Crunched|Brewed) for\s+\d+\w*(?:\s+\d+\w*)*\s*$"
).unwrap();
static ref BARE_TOOL_ACTION_RE: Regex = Regex::new(
r"^\s*(?:Explore|Read|Search|Find|Grep|Open|Edit|Write|Update|Bash|Shell|Run|Agent)\(.*\)(?:\s+.+)?\s*$"
).unwrap();
static ref NUMBERED_TOOL_ACTION_RE: Regex = Regex::new(
r"^\s*\d+\s+(?:Explore|Read|Search|Find|Grep|Open|Edit|Write|Update|Bash|Shell|Run|Agent)\(.*\)(?:\s+.+)?\s*$"
).unwrap();
static ref TRANSCRIPT_ROW_NUMBER_RE: Regex = Regex::new(r"^\s*\d+\s*$").unwrap();
static ref BACKGROUND_HINT_RE: Regex = Regex::new(r"^\s*ctrl\+[a-z]\s+to run in background\s*$").unwrap();
static ref NUMBERED_BACKGROUND_HINT_RE: Regex = Regex::new(
r"^\s*\d+\s+ctrl\+[a-z]\s+to run in background\s*$"
).unwrap();
static ref BASH_ERROR_RE: Regex = Regex::new(
r"^\s*Bash\([^)]+\)\s+Error:\s+Permission to use Bash with command .+ has been denied\.\s*$"
).unwrap();
static ref FILLER_RE: Regex = Regex::new(
r"^(?:Good\.|There it is\.|Let me check:|Let me verify:|Now let me check:|Now let me verify:|Let me search for .+|Now let me also check[—:].*|Let me also verify:)$"
).unwrap();
static ref WORD_RE: Regex = Regex::new(r"[A-Za-z]+(?:'[A-Za-z]+)?").unwrap();
static ref SPACE_BEFORE_PUNCT_RE: Regex = Regex::new(r"\s+([,.;:!?])").unwrap();
static ref SPACE_AFTER_OPEN_RE: Regex = Regex::new(r"([(\[{])\s+").unwrap();
static ref SPACE_BEFORE_CLOSE_RE: Regex = Regex::new(r"\s+([)\]}])").unwrap();
static ref DECORATIVE_SYMBOL_RE: Regex = Regex::new(r"^[•◦▪▫◆◇○●■□△▲▽▼★☆※→←↑↓➜➤➔◆◉◎◈◇]+$").unwrap();
static ref ADDED_LINES_RE: Regex = Regex::new(r"\bAdded \d+ lines?\b").unwrap();
static ref REMOVED_LINES_RE: Regex = Regex::new(r"(?i)\bremoved \d+ lines?\b").unwrap();
}
const STATUS_PREFIXES: &[&str] = &[
"Read ",
"Searched ",
"Search ",
"Find ",
"Opened ",
"Context limit reached",
"Let me mark ",
"Let me do ",
"Bash(",
"Error: Permission to use Bash with command ",
];
const STOP_WORDS: &[&str] = &[
"a", "an", "and", "are", "as", "at", "be", "been", "being", "but", "by", "for", "from", "had",
"has", "have", "he", "her", "here", "hers", "him", "his", "i", "if", "in", "into", "is", "it",
"its", "itself", "let", "me", "my", "of", "on", "or", "our", "ours", "she", "so", "than",
"that", "the", "their", "theirs", "them", "then", "there", "these", "they", "this", "those",
"to", "too", "up", "us", "very", "was", "we", "were", "what", "when", "where", "which", "who",
"why", "with", "you", "your", "yours",
];
const LOW_VALUE_TOKENS: &[&str] = &[
"actually",
"also",
"basically",
"carefully",
"clearly",
"correct",
"differently",
"fine",
"good",
"hmm",
"just",
"look",
"maybe",
"more",
"now",
"obviously",
"ok",
"okay",
"quite",
"really",
"reconsider",
"rethink",
"right",
"simply",
"still",
"wait",
"well",
];
const LOW_VALUE_SYMBOL_TOKENS: &[&str] = &[">", "⏺", "⎿", "❯", "✻", "─"];
fn stop_words_set() -> HashSet<&'static str> {
STOP_WORDS.iter().copied().collect()
}
fn low_value_tokens_set() -> HashSet<&'static str> {
LOW_VALUE_TOKENS.iter().copied().collect()
}
fn low_value_symbol_set() -> HashSet<&'static str> {
LOW_VALUE_SYMBOL_TOKENS.iter().copied().collect()
}
fn low_value_pairs_set() -> HashSet<(&'static str, &'static str)> {
[("let", "me"), ("now", "let"), ("but", "wait")]
.into_iter()
.collect()
}
fn is_disposable_status(text: &str) -> bool {
let stripped = text.trim();
if stripped.is_empty() {
return false;
}
if STATUS_PREFIXES.iter().any(|p| stripped.starts_with(p)) {
return true;
}
if THINKING_STATUS_RE.is_match(stripped) {
return true;
}
if BARE_TOOL_ACTION_RE.is_match(stripped) {
return true;
}
if NUMBERED_TOOL_ACTION_RE.is_match(stripped) {
return true;
}
if TRANSCRIPT_ROW_NUMBER_RE.is_match(stripped) {
return true;
}
if BACKGROUND_HINT_RE.is_match(stripped) {
return true;
}
if NUMBERED_BACKGROUND_HINT_RE.is_match(stripped) {
return true;
}
if BASH_ERROR_RE.is_match(stripped) {
return true;
}
if FILLER_RE.is_match(stripped) {
return true;
}
if stripped.contains("ctrl+o to expand") {
return true;
}
if ADDED_LINES_RE.is_match(stripped) {
return true;
}
if REMOVED_LINES_RE.is_match(stripped) {
return true;
}
false
}
fn normalize_text_line(text: &str) -> String {
SPACES_RE.replace_all(text.trim(), " ").to_string()
}
fn flush_paragraph(buffer: &mut Vec<String>, out: &mut Vec<String>) {
if buffer.is_empty() {
return;
}
let merged = normalize_text_line(&buffer.join(" "));
if !merged.is_empty() {
out.push(merged);
}
buffer.clear();
}
fn flush_code(buffer: &mut Vec<String>, out: &mut Vec<String>) {
if buffer.is_empty() {
return;
}
out.push("```diff".to_string());
out.extend(buffer.iter().cloned());
out.push("```".to_string());
buffer.clear();
}
fn join_fragment(base: &str, fragment: &str) -> String {
let fragment = normalize_text_line(fragment);
if fragment.is_empty() {
return base.to_string();
}
if base.is_empty() {
return fragment;
}
let separators = Regex::new(r"[(/\-]$").unwrap();
if WORD_RE.is_match(base) && fragment.starts_with(|c: char| c == '+' || c == '-') {
let second_char = fragment.chars().nth(1);
if second_char.map_or(false, |c| c.is_ascii_alphabetic()) {
return format!("{} {}", base, &fragment[1..]);
}
}
if separators.is_match(base) {
format!("{}{}", base, fragment)
} else {
format!("{} {}", base, fragment)
}
}
fn strip_stop_words_from_line(line: &str) -> String {
let stop_set = stop_words_set();
let low_set = low_value_tokens_set();
let symbol_set = low_value_symbol_set();
let pairs_set = low_value_pairs_set();
let token_re = Regex::new(r"[A-Za-z]+(?:'[A-Za-z]+)?|[0-9]+|[^\w\s]|[\s]+").unwrap();
let tokens: Vec<&str> = token_re.find_iter(line).map(|m| m.as_str()).collect();
let mut kept: Vec<String> = Vec::new();
let mut prev: Option<&str> = None;
let mut pending_sp = false;
for token in tokens {
if token.chars().all(|c| c.is_whitespace()) {
pending_sp = kept.last().is_some();
continue;
}
if WORD_RE.is_match(token) {
let lowered = token.to_lowercase();
if stop_set.contains(lowered.as_str()) || low_set.contains(lowered.as_str()) {
prev = Some(token);
continue;
}
if let Some(pw) = prev {
if pairs_set.contains(&(pw, lowered.as_str())) {
prev = Some(token);
continue;
}
}
if pending_sp && !kept.is_empty() && !kept.last().unwrap().ends_with(' ') {
kept.push(" ".to_string());
}
kept.push(token.to_string());
prev = Some(token);
pending_sp = false;
continue;
}
if symbol_set.contains(token) {
prev = None;
pending_sp = false;
continue;
}
if DECORATIVE_SYMBOL_RE.is_match(token) {
prev = None;
pending_sp = false;
continue;
}
if pending_sp && !kept.is_empty() && !kept.last().unwrap().ends_with(' ') {
kept.push(" ".to_string());
}
kept.push(token.to_string());
prev = None;
pending_sp = false;
}
let text = kept.join("").trim().to_string();
if text.is_empty() {
return String::new();
}
let text = SPACE_BEFORE_PUNCT_RE.replace_all(&text, "$1").to_string();
let text = SPACE_AFTER_OPEN_RE.replace_all(&text, "$1").to_string();
SPACE_BEFORE_CLOSE_RE.replace_all(&text, "$1").to_string()
}
fn apply_token_mode(lines: &[String]) -> Vec<String> {
let mut out: Vec<String> = Vec::new();
let mut in_code_block = false;
for line in lines {
if line.starts_with("```") {
in_code_block = !in_code_block;
out.push(line.clone());
continue;
}
if in_code_block || line.is_empty() {
out.push(line.clone());
continue;
}
let stripped = strip_stop_words_from_line(line);
if !stripped.is_empty() {
out.push(stripped);
}
}
let mut cleaned: Vec<String> = Vec::new();
let mut previous_blank = false;
for line in out {
let is_blank = line.is_empty();
if is_blank && previous_blank {
continue;
}
cleaned.push(line);
previous_blank = is_blank;
}
cleaned
}
pub fn clean_lines(lines: &[String]) -> Vec<String> {
let mut out: Vec<String> = Vec::new();
let mut paragraph: Vec<String> = Vec::new();
let mut code: Vec<String> = Vec::new();
let mut list_item: Option<String> = None;
let mut in_update_block = false;
let mut saw_diff_in_update = false;
for raw in lines {
let line = raw.trim_end_matches('\n');
let stripped = line.trim();
if UPDATE_HEADER_RE.is_match(line) {
flush_paragraph(&mut paragraph, &mut out);
if let Some(li) = list_item.take() {
out.push(li);
}
flush_code(&mut code, &mut out);
in_update_block = true;
saw_diff_in_update = false;
continue;
}
if let Some(caps) = STATUS_LINE_RE.captures(line) {
let text = caps.get(1).unwrap().as_str();
if is_disposable_status(text) {
flush_paragraph(&mut paragraph, &mut out);
if let Some(li) = list_item.take() {
out.push(li);
}
flush_code(&mut code, &mut out);
continue;
}
let effective_line = text.trim().to_string();
let effective_stripped = effective_line.trim();
if in_update_block {
if let Some(diff_caps) = DIFF_LINE_RE.captures(&effective_line) {
flush_paragraph(&mut paragraph, &mut out);
if let Some(li) = list_item.take() {
out.push(li);
}
let sign = diff_caps.get(1).unwrap().as_str();
let content = diff_caps.get(2).unwrap().as_str().trim_end();
if (sign == "+" || sign == "-") && !content.is_empty() {
code.push(format!("{} {}", sign, content));
saw_diff_in_update = true;
}
continue;
}
if effective_stripped.is_empty() {
flush_code(&mut code, &mut out);
in_update_block = false;
saw_diff_in_update = false;
continue;
}
if NUMBER_ONLY_RE.is_match(&effective_line) {
continue;
}
if !saw_diff_in_update {
continue;
}
let continuation = normalize_text_line(&effective_line);
if !code.is_empty()
&& !continuation.is_empty()
&& !continuation.contains("ctrl+o to expand")
{
let joined = join_fragment(code.last().unwrap(), &continuation);
*code.last_mut().unwrap() = joined;
continue;
}
}
if effective_stripped.is_empty() {
flush_paragraph(&mut paragraph, &mut out);
if let Some(li) = list_item.take() {
out.push(li);
}
flush_code(&mut code, &mut out);
if !out.is_empty() && out.last().map_or(true, |l| !l.is_empty()) {
out.push(String::new());
}
continue;
}
if is_disposable_status(effective_stripped) {
flush_paragraph(&mut paragraph, &mut out);
if let Some(li) = list_item.take() {
out.push(li);
}
flush_code(&mut code, &mut out);
continue;
}
if let Some(list_caps) = LIST_ITEM_RE.captures(effective_stripped) {
flush_paragraph(&mut paragraph, &mut out);
if let Some(li) = list_item.take() {
out.push(li);
}
let marker = list_caps.get(1).unwrap().as_str();
let content = list_caps.get(2).unwrap().as_str();
list_item = Some(format!("{} {}", marker, normalize_text_line(content)));
continue;
}
if let Some(ref li) = list_item {
if effective_line.starts_with(' ') {
list_item = Some(join_fragment(li, effective_stripped));
continue;
}
}
if list_item.is_some() {
out.push(list_item.take().unwrap());
}
let numbered_re = Regex::new(r"^\d+\.\s").unwrap();
if numbered_re.is_match(effective_stripped) {
flush_paragraph(&mut paragraph, &mut out);
flush_code(&mut code, &mut out);
out.push(normalize_text_line(effective_stripped));
continue;
}
if effective_stripped.starts_with("---") {
flush_paragraph(&mut paragraph, &mut out);
flush_code(&mut code, &mut out);
if !out.is_empty() && out.last().map_or(true, |l| !l.is_empty()) {
out.push(String::new());
}
continue;
}
paragraph.push(effective_line);
continue;
}
if in_update_block {
if let Some(diff_caps) = DIFF_LINE_RE.captures(line) {
flush_paragraph(&mut paragraph, &mut out);
if let Some(li) = list_item.take() {
out.push(li);
}
let sign = diff_caps.get(1).unwrap().as_str();
let content = diff_caps.get(2).unwrap().as_str().trim_end();
if (sign == "+" || sign == "-") && !content.is_empty() {
code.push(format!("{} {}", sign, content));
saw_diff_in_update = true;
}
continue;
}
if stripped.is_empty() {
flush_code(&mut code, &mut out);
in_update_block = false;
saw_diff_in_update = false;
continue;
}
if NUMBER_ONLY_RE.is_match(line) {
continue;
}
if !saw_diff_in_update {
continue;
}
let continuation = normalize_text_line(stripped);
if !code.is_empty()
&& !continuation.is_empty()
&& !continuation.contains("ctrl+o to expand")
{
let joined = join_fragment(code.last().unwrap(), &continuation);
*code.last_mut().unwrap() = joined;
continue;
}
}
if stripped.is_empty() {
flush_paragraph(&mut paragraph, &mut out);
if let Some(li) = list_item.take() {
out.push(li);
}
flush_code(&mut code, &mut out);
if !out.is_empty() && out.last().map_or(true, |l| !l.is_empty()) {
out.push(String::new());
}
continue;
}
if is_disposable_status(stripped) {
flush_paragraph(&mut paragraph, &mut out);
if let Some(li) = list_item.take() {
out.push(li);
}
flush_code(&mut code, &mut out);
continue;
}
if let Some(list_caps) = LIST_ITEM_RE.captures(stripped) {
flush_paragraph(&mut paragraph, &mut out);
if let Some(li) = list_item.take() {
out.push(li);
}
let marker = list_caps.get(1).unwrap().as_str();
let content = list_caps.get(2).unwrap().as_str();
list_item = Some(format!("{} {}", marker, normalize_text_line(content)));
continue;
}
if let Some(ref li) = list_item {
if line.starts_with(' ') {
list_item = Some(join_fragment(li, stripped));
continue;
}
}
if list_item.is_some() {
out.push(list_item.take().unwrap());
}
let numbered_re = Regex::new(r"^\d+\.\s").unwrap();
if numbered_re.is_match(stripped) {
flush_paragraph(&mut paragraph, &mut out);
flush_code(&mut code, &mut out);
out.push(normalize_text_line(stripped));
continue;
}
if stripped.starts_with("---") {
flush_paragraph(&mut paragraph, &mut out);
flush_code(&mut code, &mut out);
if !out.is_empty() && out.last().map_or(true, |l| !l.is_empty()) {
out.push(String::new());
}
continue;
}
paragraph.push(line.to_string());
}
flush_paragraph(&mut paragraph, &mut out);
if let Some(li) = list_item {
out.push(li);
}
flush_code(&mut code, &mut out);
while out.last().map_or(false, |l| l.is_empty()) {
out.pop();
}
let mut cleaned: Vec<String> = Vec::new();
let mut previous_blank = false;
for line in out {
let is_blank = line.is_empty();
if is_blank && previous_blank {
continue;
}
cleaned.push(line);
previous_blank = is_blank;
}
cleaned
}
pub fn compact_file(input_path: &Path, token_mode: bool) -> Result<String> {
let content = std::fs::read_to_string(input_path)
.with_context(|| format!("Failed to read file: {}", input_path.display()))?;
let lines: Vec<String> = content.lines().map(|l| l.to_string()).collect();
let cleaned = clean_lines(&lines);
let result = if token_mode { apply_token_mode(&cleaned) } else { cleaned };
Ok(result.join("\n"))
}
pub fn compact_stdin(token_mode: bool) -> Result<String> {
let mut content = String::new();
io::stdin()
.read_to_string(&mut content)
.with_context(|| "Failed to read from stdin")?;
let lines: Vec<String> = content.lines().map(|l| l.to_string()).collect();
let cleaned = clean_lines(&lines);
let result = if token_mode { apply_token_mode(&cleaned) } else { cleaned };
Ok(result.join("\n"))
}
fn reformat_speaker_headers(lines: Vec<String>) -> Vec<String> {
let mut sections: Vec<(Option<String>, Vec<String>)> = Vec::new();
let mut current_header: Option<String> = None;
let mut current_body: Vec<String> = Vec::new();
for line in lines {
let trimmed = line.trim();
if trimmed == "[User]" || trimmed == "[Assistant]" {
sections.push((current_header.take(), current_body.drain(..).collect()));
current_header = Some(if trimmed == "[User]" {
"USER:".to_string()
} else {
"ASSIST:".to_string()
});
} else {
current_body.push(line);
}
}
sections.push((current_header, current_body));
let mut out: Vec<String> = Vec::new();
for (header, body) in sections {
let has_content = body.iter().any(|l| !l.trim().is_empty());
match header {
None => {
for line in body {
out.push(line);
}
}
Some(hdr) => {
if !has_content {
continue;
}
if !out.is_empty() {
out.push(String::new());
}
out.push(hdr);
for line in body {
out.push(line);
}
}
}
}
let mut cleaned: Vec<String> = Vec::new();
let mut prev_blank = false;
for line in out {
let blank = line.trim().is_empty();
if blank && prev_blank {
continue;
}
cleaned.push(line);
prev_blank = blank;
}
while cleaned.last().map_or(false, |l: &String| l.trim().is_empty()) {
cleaned.pop();
}
cleaned
}
pub fn compact_text(input: &str, token_mode: bool) -> String {
let lines: Vec<String> = input.lines().map(|l| l.to_string()).collect();
let cleaned = clean_lines(&lines);
if token_mode {
apply_token_mode(&cleaned).join("\n")
} else {
cleaned.join("\n")
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_disposable_status_read() {
assert!(is_disposable_status("Read 5 files"));
assert!(is_disposable_status("Searched for patterns in src/"));
assert!(is_disposable_status("Bash(cargo test)"));
}
#[test]
fn test_disposable_status_thinking() {
assert!(is_disposable_status("✻ Crunched for 3s"));
assert!(is_disposable_status("✻ Brewed for 10s 5ms"));
}
#[test]
fn test_disposable_status_background() {
assert!(is_disposable_status("ctrl+c to run in background"));
assert!(is_disposable_status("5 ctrl+c to run in background"));
}
#[test]
fn test_not_disposable() {
assert!(!is_disposable_status("I found the bug in the parser"));
assert!(!is_disposable_status("The fix is to update the regex"));
}
#[test]
fn test_clean_lines_removes_tool_actions() {
let input = vec![
"⏺ Read file.rs".to_string(),
"I found the issue".to_string(),
"⏺ Searched for pattern".to_string(),
"Let me fix it now".to_string(),
];
let result = clean_lines(&input);
let combined = result.join("|");
assert!(!combined.contains("Read file") && !combined.contains("Searched for pattern"));
assert!(combined.contains("issue"));
}
#[test]
fn test_clean_lines_preserves_diff_blocks() {
let input = vec![
"⏺ Update(src/main.rs)".to_string(),
" 5 - let old = true;".to_string(),
" 5 + let new = false;".to_string(),
"".to_string(),
"I updated the file".to_string(),
];
let result = clean_lines(&input);
assert!(result.iter().any(|l| l.contains("```diff")));
assert!(result.iter().any(|l| l.contains("- let old")));
assert!(result.iter().any(|l| l.contains("+ let new")));
}
#[test]
fn test_clean_lines_collapse_blanks() {
let input = vec![
"First paragraph".to_string(),
"".to_string(),
"".to_string(),
"".to_string(),
"Second paragraph".to_string(),
];
let result = clean_lines(&input);
let blank_count = result.iter().filter(|l| l.is_empty()).count();
assert!(
blank_count <= 1,
"Should collapse consecutive blank lines to at most one"
);
}
#[test]
fn test_normalize_text_line() {
assert_eq!(normalize_text_line(" hello world "), "hello world");
assert_eq!(normalize_text_line("already normal"), "already normal");
}
#[test]
fn test_compact_text_basic() {
let input = "⏺ Read 5 files\n\nI analyzed the code and found the bug.\n\n⏺ Searched for TODO\n\nHere is the fix.";
let result = compact_text(input, false);
assert!(!result.contains("Read 5 files"));
assert!(!result.contains("Searched for TODO"));
assert!(result.contains("bug"));
assert!(result.contains("fix"));
}
#[test]
fn test_compact_text_no_duplicate() {
let input = "⏺ Read 5 files\n\nI found the issue in the parser.\n\n⏺ Bash(cargo test)\n\nTests pass.";
let result = compact_text(input, false);
let issue_count = result.matches("issue").count();
assert_eq!(issue_count, 1, "Should not duplicate non-disposable text");
}
#[test]
fn test_token_mode_strips_fillers() {
let input = "I actually just want to basically verify the result";
let result = compact_text(input, true);
assert!(!result.contains("actually"));
assert!(!result.contains("just"));
assert!(!result.contains("basically"));
assert!(result.contains("want"));
assert!(result.contains("verify"));
assert!(result.contains("result"));
}
}