goosedump 0.12.33

// SPDX-License-Identifier: LGPL-2.1-or-later
// Copyright (C) Jarkko Sakkinen 2026

use std::borrow::Cow;
use std::collections::{BTreeSet, HashSet};
use std::ops::Range;
use std::time::Instant;

use anyhow::{Result, ensure};

use crate::engine::display;
use crate::engine::message::{
    BashOutput, ConversationMessage, MessageKind, MessageView, Part, ToolCall,
};
use crate::engine::model::{Embedder, Embedding, Similarity, TextGen};
use crate::engine::text;

const BRIEF_MAX_LINES: usize = 120;
const BRIEF_HEAD_LINES: usize = 24;
const BRIEF_TAIL_LINES: usize = 48;
const BRIEF_FORMAT_LINES: usize = 3;
const BRIEF_RANKED_FORMAT_LINES: usize = 4;
const BRIEF_RELEVANT_LINES: usize =
    BRIEF_MAX_LINES - BRIEF_HEAD_LINES - BRIEF_TAIL_LINES - BRIEF_RANKED_FORMAT_LINES - 1;
const TEXT_LINE_MAX_CHARS: usize = 240;
const ASSISTANT_LINE_MAX_CHARS: usize = 200;
const BASH_LINE_MAX_CHARS: usize = 120;
const TOOL_CALLS_PER_TURN: usize = 8;
const GOAL_LIMIT: usize = 8;
const COMMIT_LIMIT: usize = 8;
const COMMIT_SUBJECT_MAX_CHARS: usize = 120;
const PREFERENCE_LINE_MAX_CHARS: usize = 180;
const OUTSTANDING_LINE_MAX_CHARS: usize = 150;
const DIRECTIVE_PROMPT_MAX_CHARS: usize = 80;
const DIRECTIVE_SEMANTIC_SIMILARITY: f32 = 0.84;
const DIRECTIVE_CONTENT_SIMILARITY: f32 = 0.80;
const PREFERENCE_LIMIT: usize = 15;
const SUMMARY_BRIEF_HEAD_LINES: usize = 4;
const SUMMARY_BRIEF_TAIL_LINES: usize = 8;
const SECTION_GOAL: &str = "## Goal";
const SECTION_CONSTRAINTS: &str = "## Constraints & Preferences";
const SECTION_PROGRESS: &str = "## Progress";
const SECTION_DONE: &str = "### Done";
const SECTION_IN_PROGRESS: &str = "### In Progress";
const SECTION_CRITICAL_CONTEXT: &str = "## Critical Context";
const TAG_READ_FILES: &str = "<read-files>";
const TAG_MODIFIED_FILES: &str = "<modified-files>";

#[derive(Debug, Default)]
struct ExtractedContext {
    goals: Vec<String>,
    file_activity: FileActivity,
    commits: Vec<String>,
    outstanding: Vec<String>,
    preferences: Vec<String>,
    brief: String,
}

#[derive(Debug, Default)]
struct OrderedPaths {
    values: Vec<String>,
    seen: HashSet<String>,
}

impl OrderedPaths {
    fn contains(&self, path: &str) -> bool {
        self.seen.contains(path)
    }

    fn insert(&mut self, path: String) {
        if self.seen.insert(path.clone()) {
            self.values.push(path);
        }
    }

    fn iter(&self) -> impl Iterator<Item = &String> {
        self.values.iter()
    }

    fn map_paths(&mut self, prefix: &str) {
        let mut next = Self::default();
        for path in &self.values {
            next.insert(path.strip_prefix(prefix).unwrap_or(path).to_string());
        }
        *self = next;
    }
}

#[derive(Debug, Default)]
struct FileActivity {
    read: OrderedPaths,
    modified: OrderedPaths,
    created: OrderedPaths,
}

#[derive(Debug)]
struct BriefSection {
    header: &'static str,
    lines: Vec<String>,
}

/// The extracted compaction summary, exposed field-by-field so the per-client
/// emitters can map it into each client's own summary text and record shape.
pub struct Summary {
    pub goals: Vec<String>,
    pub read: Vec<String>,
    pub modified: Vec<String>,
    pub created: Vec<String>,
    pub commits: Vec<String>,
    pub outstanding: Vec<String>,
    pub preferences: Vec<String>,
    pub brief: String,
    /// Estimated token count of the messages the summary replaces, for
    /// clients whose compaction records carry one (e.g. Pi's `tokensBefore`).
    pub tokens_before: u64,
}

#[derive(Debug, Default)]
struct SummaryRelevance {
    read: Vec<Similarity>,
    modified: Vec<Similarity>,
    created: Vec<Similarity>,
    brief: Vec<Similarity>,
}

impl Summary {
    fn split_relevance(&self, scores: Vec<Similarity>) -> Result<SummaryRelevance> {
        let lengths = [
            self.read.len(),
            self.modified.len(),
            self.created.len(),
            self.brief.lines().count(),
        ];
        let expected = lengths.into_iter().try_fold(0_usize, |total, length| {
            total
                .checked_add(length)
                .ok_or_else(|| anyhow::anyhow!("summary relevance count overflow"))
        })?;
        ensure!(
            scores.len() == expected,
            "summary item and relevance counts differ"
        );

        let mut scores = scores.into_iter();
        Ok(SummaryRelevance {
            read: scores.by_ref().take(lengths[0]).collect(),
            modified: scores.by_ref().take(lengths[1]).collect(),
            created: scores.by_ref().take(lengths[2]).collect(),
            brief: scores.collect(),
        })
    }
}

struct CompactProfiler {
    print: bool,
}

impl CompactProfiler {
    fn new(print: bool) -> Self {
        Self { print }
    }

    fn record(&mut self, name: &'static str, started: Instant) {
        let milliseconds = started.elapsed().as_secs_f64() * 1000.0;
        if self.print {
            eprintln!("[compact-profile] {name}: {milliseconds:.1} ms");
        }
    }
}

fn compact_profile_enabled() -> bool {
    std::env::var_os("GOOSEDUMP_PROFILE_COMPACT").is_some()
}

pub fn summarize_with_previous_and_budget(
    messages: &[ConversationMessage],
    previous_summary: Option<&str>,
    max_tokens: usize,
) -> Result<Summary> {
    let previous_summary = previous_summary.filter(|summary| !summary.trim().is_empty());
    summarize_inner(
        messages,
        previous_summary,
        max_tokens,
        compact_profile_enabled(),
    )
}

fn summarize_inner(
    messages: &[ConversationMessage],
    previous_summary: Option<&str>,
    max_tokens: usize,
    print_profile: bool,
) -> Result<Summary> {
    let started = Instant::now();
    let mut profiler = CompactProfiler::new(print_profile);
    let mut embedder = None;
    let ctx = extract_context(messages, previous_summary, &mut embedder, &mut profiler)?;
    let mut summary = Summary {
        goals: ctx.goals,
        read: ctx.file_activity.read.values,
        modified: ctx.file_activity.modified.values,
        created: ctx.file_activity.created.values,
        commits: ctx.commits,
        outstanding: ctx.outstanding,
        preferences: ctx.preferences,
        brief: ctx.brief,
        tokens_before: estimate_tokens(messages)
            + previous_summary.map_or(0, |summary| (summary.chars().count() / 4) as u64),
    };
    bound_summary(&mut summary, max_tokens, &mut embedder, &mut profiler)?;
    profiler.record("summarize total", started);
    Ok(summary)
}

/// Rough token estimate (chars/4 over the searchable text), matching the
/// heuristic granularity clients use for their own pre-compaction counts.
fn estimate_tokens(messages: &[ConversationMessage]) -> u64 {
    let chars: usize = messages
        .iter()
        .map(|msg| display::searchable_text(msg).chars().count())
        .sum();
    (chars / 4) as u64
}

pub fn summary_token_estimate(summary: &Summary) -> usize {
    crate::engine::emit::pi_summary_text(summary)
        .chars()
        .count()
        .div_ceil(4)
}

fn load_embedder<'a>(
    embedder: &'a mut Option<Embedder>,
    profiler: &mut CompactProfiler,
) -> Result<&'a Embedder> {
    if embedder.is_none() {
        let started = Instant::now();
        *embedder = Some(Embedder::load()?);
        profiler.record("bge load", started);
    }
    embedder
        .as_ref()
        .ok_or_else(|| anyhow::anyhow!("BGE embedder is unavailable after loading"))
}

fn bound_summary(
    summary: &mut Summary,
    max_tokens: usize,
    embedder: &mut Option<Embedder>,
    profiler: &mut CompactProfiler,
) -> Result<()> {
    if max_tokens == 0 || summary_token_estimate(summary) <= max_tokens {
        return Ok(());
    }

    let references = directive_references(&summary.goals, &summary.outstanding);
    let has_candidates = summary.read.len() > 1
        || summary.modified.len() > 1
        || summary.created.len() > 1
        || summary.brief.lines().count() > SUMMARY_BRIEF_HEAD_LINES + SUMMARY_BRIEF_TAIL_LINES;
    let relevance = if references.is_empty() || !has_candidates {
        None
    } else {
        let embedder = load_embedder(embedder, profiler)?;
        let started = Instant::now();
        let relevance = summary_relevance(summary, &references, embedder)?;
        profiler.record("bge budget relevance", started);
        Some(relevance)
    };
    bound_summary_ranked(summary, max_tokens, relevance.as_ref())
}

fn directive_references<'a>(goals: &'a [String], outstanding: &'a [String]) -> Vec<&'a str> {
    goals
        .iter()
        .chain(outstanding)
        .map(|item| strip_entry_ref(item))
        .filter(|item| !item.trim().is_empty())
        .collect()
}

fn summary_relevance(
    summary: &Summary,
    references: &[&str],
    embedder: &Embedder,
) -> Result<SummaryRelevance> {
    let brief = summary.brief.lines().map(str::to_owned);
    let path_text = |path: &str| {
        path.chars()
            .map(|ch| match ch {
                '/' | '\\' | '.' | '_' | '-' => ' ',
                _ => ch,
            })
            .collect::<String>()
    };
    let mut candidates = Vec::with_capacity(
        summary.read.len()
            + summary.modified.len()
            + summary.created.len()
            + summary.brief.lines().count(),
    );
    candidates.extend(summary.read.iter().map(|path| path_text(path)));
    candidates.extend(summary.modified.iter().map(|path| path_text(path)));
    candidates.extend(summary.created.iter().map(|path| path_text(path)));
    candidates.extend(brief);
    let candidate_refs = candidates.iter().map(String::as_str).collect::<Vec<_>>();
    let scores = embedder.relevance(references, &candidate_refs)?;
    summary.split_relevance(scores)
}

fn bound_summary_ranked(
    summary: &mut Summary,
    max_tokens: usize,
    relevance: Option<&SummaryRelevance>,
) -> Result<()> {
    trim_items_to_budget(
        summary,
        max_tokens,
        "read-only paths",
        |value| &mut value.read,
        relevance.map(|value| value.read.as_slice()),
    )?;
    trim_items_to_budget(
        summary,
        max_tokens,
        "completed items",
        |value| &mut value.commits,
        None,
    )?;
    trim_brief_to_budget(
        summary,
        max_tokens,
        relevance.map(|value| value.brief.as_slice()),
    )?;
    trim_items_to_budget(
        summary,
        max_tokens,
        "modified paths",
        |value| &mut value.modified,
        relevance.map(|value| value.modified.as_slice()),
    )?;
    trim_items_to_budget(
        summary,
        max_tokens,
        "created paths",
        |value| &mut value.created,
        relevance.map(|value| value.created.as_slice()),
    )?;
    Ok(())
}

fn trim_items_to_budget(
    summary: &mut Summary,
    max_tokens: usize,
    label: &str,
    items: fn(&mut Summary) -> &mut Vec<String>,
    relevance: Option<&[Similarity]>,
) -> Result<()> {
    if summary_token_estimate(summary) <= max_tokens {
        return Ok(());
    }

    let before = summary_token_estimate(summary);
    let original = items(summary).clone();
    let mut scores = relevance.map(|scores| scores.iter().copied().map(Some).collect::<Vec<_>>());
    if let Some(scores) = &scores {
        ensure!(
            scores.len() == original.len(),
            "summary item and relevance counts differ"
        );
    }
    while summary_token_estimate(summary) > max_tokens {
        let trimmed = if let Some(scores) = &mut scores {
            trim_least_relevant_item(items(summary), scores, 1, label)?
        } else {
            trim_oldest_item(items(summary), 1, label)
        };
        if !trimmed {
            break;
        }
    }
    if summary_token_estimate(summary) >= before {
        *items(summary) = original;
    }
    Ok(())
}

fn trim_brief_to_budget(
    summary: &mut Summary,
    max_tokens: usize,
    relevance: Option<&[Similarity]>,
) -> Result<()> {
    if summary_token_estimate(summary) <= max_tokens {
        return Ok(());
    }

    let before = summary_token_estimate(summary);
    let original = summary.brief.clone();
    let mut scores = relevance.map(|scores| scores.iter().copied().map(Some).collect::<Vec<_>>());
    while summary_token_estimate(summary) > max_tokens
        && shrink_brief_once(&mut summary.brief, scores.as_mut())?
    {}
    if summary_token_estimate(summary) >= before {
        summary.brief = original;
    }
    Ok(())
}

fn trim_oldest_item(items: &mut Vec<String>, min_kept: usize, label: &str) -> bool {
    let marker_index = items.iter().position(|item| is_omission_marker(item));
    let mut omitted = marker_index
        .and_then(|index| items.get(index))
        .and_then(|item| parse_omission_count(item))
        .unwrap_or(0);
    if let Some(index) = marker_index {
        items.remove(index);
    }
    if items.len() <= min_kept {
        if omitted > 0 {
            items.insert(0, omission_marker(omitted, label));
        }
        return false;
    }

    items.remove(0);
    omitted += 1;
    items.insert(0, omission_marker(omitted, label));
    true
}

fn trim_least_relevant_item(
    items: &mut Vec<String>,
    relevance: &mut Vec<Option<Similarity>>,
    min_kept: usize,
    label: &str,
) -> Result<bool> {
    ensure!(
        items.len() == relevance.len(),
        "summary item and relevance counts differ"
    );
    let marker_index = items.iter().position(|item| is_omission_marker(item));
    let mut omitted = marker_index
        .and_then(|index| items.get(index))
        .and_then(|item| parse_omission_count(item))
        .unwrap_or(0);
    if let Some(index) = marker_index {
        items.remove(index);
        relevance.remove(index);
    }
    if items.len() <= min_kept {
        if omitted > 0 {
            items.insert(0, omission_marker(omitted, label));
            relevance.insert(0, None);
        }
        return Ok(false);
    }

    let remove_index = relevance
        .iter()
        .enumerate()
        .filter_map(|(index, score)| score.map(|score| (index, score)))
        .min_by_key(|(_, score)| *score)
        .map(|(index, _)| index)
        .ok_or_else(|| anyhow::anyhow!("summary relevance scores are empty"))?;
    items.remove(remove_index);
    relevance.remove(remove_index);
    omitted += 1;
    items.insert(0, omission_marker(omitted, label));
    relevance.insert(0, None);
    Ok(true)
}

fn shrink_brief_once(
    brief: &mut String,
    relevance: Option<&mut Vec<Option<Similarity>>>,
) -> Result<bool> {
    let source: Vec<&str> = brief.lines().collect();
    if let Some(scores) = relevance.as_deref() {
        ensure!(
            source.len() == scores.len(),
            "summary brief and relevance counts differ"
        );
    }
    let ranked = relevance.is_some();
    let mut omitted = 0;
    let mut lines = Vec::with_capacity(source.len());
    for (index, line) in source.into_iter().enumerate() {
        if is_brief_omission_marker(line.trim()) {
            omitted += parse_omission_count(line.trim()).unwrap_or(0);
            continue;
        }
        let score = relevance
            .as_deref()
            .and_then(|scores| scores.get(index))
            .copied()
            .flatten();
        lines.push((line.to_string(), score));
    }

    let minimum_lines = SUMMARY_BRIEF_HEAD_LINES + SUMMARY_BRIEF_TAIL_LINES;
    if lines.len() <= minimum_lines {
        return Ok(false);
    }
    let removable = lines.len() - SUMMARY_BRIEF_TAIL_LINES;
    let remove_index = if ranked {
        lines
            .iter()
            .enumerate()
            .skip(SUMMARY_BRIEF_HEAD_LINES)
            .take(removable - SUMMARY_BRIEF_HEAD_LINES)
            .filter(|(_, (line, _))| {
                !(line.is_empty() || line.starts_with('[') && line.ends_with(']'))
            })
            .filter_map(|(index, (_, score))| score.map(|score| (index, score)))
            .min_by_key(|(_, score)| *score)
            .map(|(index, _)| index)
    } else {
        Some(SUMMARY_BRIEF_HEAD_LINES)
    };
    let Some(remove_index) = remove_index else {
        return Ok(false);
    };
    lines.remove(remove_index);
    omitted += 1;
    lines.insert(
        SUMMARY_BRIEF_HEAD_LINES,
        (
            format!("...({omitted} earlier lines omitted; use goose_search for details)"),
            None,
        ),
    );
    *brief = lines
        .iter()
        .map(|(line, _)| line.as_str())
        .collect::<Vec<_>>()
        .join("\n");
    if let Some(scores) = relevance {
        *scores = lines.into_iter().map(|(_, score)| score).collect();
    }
    Ok(true)
}

fn omission_marker(count: usize, label: &str) -> String {
    format!("...({count} older {label} omitted; use goose_search for details)")
}

fn is_omission_marker(value: &str) -> bool {
    value.starts_with("...(") && value.contains(" omitted; use goose_search for details)")
}

fn is_brief_omission_marker(value: &str) -> bool {
    value.starts_with("...(") && value.contains(" earlier lines omitted")
}

fn parse_omission_count(value: &str) -> Option<usize> {
    value
        .strip_prefix("...(")?
        .split_whitespace()
        .next()?
        .parse()
        .ok()
}

fn extract_recent(items: &mut Vec<String>, limit: usize, keep_first: bool) {
    if items.len() <= limit {
        return;
    }
    if keep_first {
        let tail_start = items.len() - (limit - 1);
        let mut capped = Vec::with_capacity(limit);
        capped.push(items[0].clone());
        capped.extend_from_slice(&items[tail_start..]);
        *items = capped;
    } else {
        let excess = items.len() - limit;
        items.drain(0..excess);
    }
}

fn extract_context(
    messages: &[ConversationMessage],
    previous_summary: Option<&str>,
    embedder: &mut Option<Embedder>,
    profiler: &mut CompactProfiler,
) -> Result<ExtractedContext> {
    let started = Instant::now();
    let mut ctx = ExtractedContext::default();

    if let Some(summary) = previous_summary {
        merge_prior_summary(summary, &mut ctx);
    }
    collect_prior_summaries(messages, &mut ctx, previous_summary.is_some());
    for message in messages {
        collect_goals(message, &mut ctx.goals);
        collect_preferences(message, &mut ctx.preferences);
        collect_commits(message, &mut ctx.commits);
    }
    extract_recent(&mut ctx.commits, COMMIT_LIMIT, false);
    collect_file_activity(messages, &mut ctx.file_activity);

    if ctx.goals.len() >= 2 || ctx.preferences.len() >= 2 {
        let embedder = load_embedder(embedder, profiler)?;
        let dedup_started = Instant::now();
        ctx.deduplicate_directives(embedder)?;
        profiler.record("bge directive dedup", dedup_started);
    }
    extract_recent(&mut ctx.goals, GOAL_LIMIT, true);
    extract_recent(&mut ctx.preferences, PREFERENCE_LIMIT, false);
    if ctx.goals.len() >= 2 || ctx.preferences.len() >= 2 {
        let textgen_started = Instant::now();
        let mut textgen = TextGen::load()?;
        profiler.record("textgen load", textgen_started);

        let refine_started = Instant::now();
        if ctx.goals.len() >= 2 && ctx.preferences.len() >= 2 {
            refine_directive_groups(&mut ctx.goals, &mut ctx.preferences, &mut textgen)?;
        } else {
            refine_directives(&mut ctx.goals, "session goals", &mut textgen)?;
            refine_directives(&mut ctx.preferences, "user preferences", &mut textgen)?;
        }
        profiler.record("textgen refine", refine_started);
    }
    trim_file_activity(&mut ctx.file_activity);
    let todos = todo_snapshot(messages);
    if !todos.is_empty() {
        ctx.outstanding.retain(|item| !item.starts_with("[todo] "));
    }
    extend_unique(&mut ctx.outstanding, todos);
    extend_unique(&mut ctx.outstanding, conversation_outstanding(messages));

    let brief_started = Instant::now();
    let recent_brief = conversation_brief(messages);
    if !recent_brief.is_empty() {
        ctx.brief = if ctx.brief.is_empty() {
            recent_brief
        } else {
            format!("{}\n\n{}", ctx.brief, recent_brief)
        };
    }
    if ctx.brief.lines().count() > BRIEF_MAX_LINES {
        let references = directive_references(&ctx.goals, &ctx.outstanding);
        ctx.brief = if references.is_empty() {
            cap_brief(&ctx.brief)
        } else {
            let embedder = load_embedder(embedder, profiler)?;
            let relevance_started = Instant::now();
            let brief = cap_brief_ranked(&ctx.brief, &references, embedder)?;
            profiler.record("bge brief sampling", relevance_started);
            brief
        };
    }
    profiler.record("conversation brief", brief_started);

    if ctx.goals.is_empty() {
        ctx.goals.push("Ongoing development work".to_string());
    }

    profiler.record("extract context", started);
    Ok(ctx)
}

fn collect_prior_summaries(
    messages: &[ConversationMessage],
    ctx: &mut ExtractedContext,
    has_explicit_previous: bool,
) {
    let latest_pi_compaction = messages.iter().rposition(|message| {
        matches!(&message.kind, MessageKind::PiCompaction { .. })
            && compact_summary_text(message).is_some()
    });

    for (index, message) in messages.iter().enumerate() {
        if matches!(&message.kind, MessageKind::PiCompaction { .. })
            && (has_explicit_previous || Some(index) != latest_pi_compaction)
        {
            continue;
        }
        let Some(text) = compact_summary_text(message) else {
            continue;
        };
        merge_prior_summary(&text, ctx);
    }
}

fn compact_summary_text(message: &ConversationMessage) -> Option<Cow<'_, str>> {
    let text = match &message.kind {
        MessageKind::PiCompaction { summary, .. }
        | MessageKind::PiBranchSummary { summary, .. } => Cow::Borrowed(summary.as_str()),
        _ => match message.view() {
            MessageView::Text { text, .. } | MessageView::Assistant { text, .. } => {
                Cow::Owned(text)
            }
            MessageView::ToolResult(_) | MessageView::Bash(_) => return None,
        },
    };
    is_compact_summary(&text).then_some(text)
}

fn is_compaction_record(message: &ConversationMessage) -> bool {
    matches!(
        &message.kind,
        MessageKind::PiCompaction { .. } | MessageKind::PiBranchSummary { .. }
    )
}

fn is_compact_summary(text: &str) -> bool {
    text.lines()
        .any(|line| matches_compact_section(line.trim()))
}

fn matches_compact_section(line: &str) -> bool {
    matches!(
        line,
        SECTION_GOAL
            | SECTION_CONSTRAINTS
            | SECTION_PROGRESS
            | SECTION_DONE
            | SECTION_IN_PROGRESS
            | SECTION_CRITICAL_CONTEXT
            | TAG_READ_FILES
            | TAG_MODIFIED_FILES
    )
}

fn merge_prior_summary(text: &str, ctx: &mut ExtractedContext) {
    merge_anchored_summary(text, ctx);
}

fn merge_anchored_summary(text: &str, ctx: &mut ExtractedContext) {
    let mut section = None;
    let mut brief = Vec::new();

    for raw_line in text.lines() {
        let line = raw_line.trim();
        if section == Some(SECTION_CRITICAL_CONTEXT)
            && !matches!(line, TAG_READ_FILES | TAG_MODIFIED_FILES)
        {
            if !line.is_empty() {
                brief.push(raw_line.to_string());
            }
            continue;
        }

        match line {
            SECTION_GOAL
            | SECTION_CONSTRAINTS
            | SECTION_DONE
            | SECTION_IN_PROGRESS
            | SECTION_CRITICAL_CONTEXT
            | TAG_READ_FILES
            | TAG_MODIFIED_FILES => {
                section = Some(line);
                continue;
            }
            SECTION_PROGRESS | "</read-files>" | "</modified-files>" => {
                section = None;
                continue;
            }
            _ => {}
        }

        match section {
            Some(SECTION_GOAL) if !line.is_empty() => {
                let item = line.strip_prefix("- ").unwrap_or(line);
                push_unique(&mut ctx.goals, item.to_string());
            }
            Some(SECTION_CONSTRAINTS) => {
                merge_markdown_item(line, "- ", &mut ctx.preferences);
            }
            Some(SECTION_DONE) => merge_markdown_item(line, "- [x] ", &mut ctx.commits),
            Some(SECTION_IN_PROGRESS) => {
                let item = line
                    .strip_prefix("- [ ] ")
                    .or_else(|| line.strip_prefix("- "));
                if let Some(item) = item {
                    push_unique(&mut ctx.outstanding, item.to_string());
                }
            }
            Some(TAG_READ_FILES) if !line.is_empty() => {
                ctx.file_activity.read.insert(line.to_string());
            }
            Some(TAG_MODIFIED_FILES) if !line.is_empty() => {
                ctx.file_activity.modified.insert(line.to_string());
            }
            _ => {}
        }
    }

    let prior_brief = brief.join("\n");
    if !prior_brief.is_empty() {
        ctx.brief = if ctx.brief.is_empty() {
            prior_brief
        } else {
            cap_brief(&format!("{}\n\n{}", ctx.brief, prior_brief))
        };
    }
}

fn merge_markdown_item(line: &str, prefix: &str, items: &mut Vec<String>) {
    if let Some(item) = line.strip_prefix(prefix) {
        push_unique(items, item.to_string());
    }
}
fn collect_goals(message: &ConversationMessage, goals: &mut Vec<String>) {
    if is_compaction_record(message) || compact_summary_text(message).is_some() {
        return;
    }
    let MessageView::Text { role, text } = message.view() else {
        return;
    };
    if role != "user" {
        return;
    }

    for line in non_empty_lines(&text) {
        let lower = line.to_ascii_lowercase();
        let is_first_goal = goals.is_empty();
        let is_scope_change = contains_any(
            &lower,
            &[
                "also ", "instead", "change ", "switch ", "update ", "fix ", "add ", "remove ",
                "don't ", "do not ",
            ],
        );
        if is_first_goal || is_scope_change {
            push_unique(
                goals,
                referenced_sentence(&line, TEXT_LINE_MAX_CHARS, &message.entry_id),
            );
        }
    }
}
fn collect_commits(message: &ConversationMessage, commits: &mut Vec<String>) {
    if is_compaction_record(message) || compact_summary_text(message).is_some() {
        return;
    }
    // Provenance: a commit is only trusted from tool-produced output, where git
    // prints its porcelain `[branch hash] subject` line. Assistant/user prose
    // mentions hashes unreliably (plans, quotes, reverts) and is not a source.
    let text_value = match message.view() {
        MessageView::ToolResult(result) => result.content.as_str(),
        MessageView::Bash(output) => output.output.as_str(),
        MessageView::Text { .. } | MessageView::Assistant { .. } => return,
    };

    for line in non_empty_lines(text_value) {
        if let Some(commit) = (|| {
            let (inside, after) = line.trim().strip_prefix('[')?.split_once(']')?;
            let hash = inside.split_whitespace().next_back()?;
            if !(7..=40).contains(&hash.len()) || !hash.chars().all(|ch| ch.is_ascii_hexdigit()) {
                return None;
            }
            let subject = after.trim();
            let subject = if subject.is_empty() {
                line.as_str()
            } else {
                subject
            };
            Some(format!(
                "{}: {}",
                &hash[..hash.len().min(12)],
                clean_sentence(subject, COMMIT_SUBJECT_MAX_CHARS)
            ))
        })() {
            push_unique(
                commits,
                format!("{} ({})", commit, entry_ref(&message.entry_id)),
            );
        }
        if commits.len() >= COMMIT_LIMIT {
            break;
        }
    }
}

fn collect_preferences(message: &ConversationMessage, preferences: &mut Vec<String>) {
    if is_compaction_record(message) || compact_summary_text(message).is_some() {
        return;
    }
    let MessageView::Text { role, text } = message.view() else {
        return;
    };
    if role != "user" {
        return;
    }

    for line in non_empty_lines(&text) {
        let lower = line.to_ascii_lowercase();
        if contains_any(
            &lower,
            &[
                "prefer ", "always ", "never ", "don't ", "do not ", "must ", "should ",
            ],
        ) {
            push_unique(
                preferences,
                referenced_sentence(&line, PREFERENCE_LINE_MAX_CHARS, &message.entry_id),
            );
        }
    }
}

/// Collect file activity from tool calls across the whole stream. A write whose
/// tool result errored is ignored (a failed edit is not a change), and a path is
/// `created` only when it was not already known to exist (read or written
/// earlier); otherwise the write is a modification.
fn collect_file_activity(messages: &[ConversationMessage], activity: &mut FileActivity) {
    let errored: HashSet<String> = messages
        .iter()
        .flat_map(|message| message.parts.iter())
        .filter_map(|part| match part {
            Part::ToolResult(result) if result.is_error && !result.call_id.is_empty() => {
                Some(result.call_id.clone())
            }
            _ => None,
        })
        .collect();
    let mut seen: HashSet<String> = HashSet::new();

    for message in messages {
        if !message.is_assistant() {
            continue;
        }
        for tool_call in message.tool_calls() {
            if !tool_call.id.is_empty() && errored.contains(tool_call.id.as_str()) {
                continue;
            }
            let Some(path) = display::path_argument(&tool_call.arguments) else {
                continue;
            };
            if is_read_tool(&tool_call.name) {
                activity.read.insert(path.clone());
                seen.insert(path);
            } else if is_create_tool(&tool_call.name) {
                if !activity.created.contains(&path) {
                    if seen.contains(&path) {
                        activity.modified.insert(path.clone());
                    } else {
                        activity.created.insert(path.clone());
                    }
                }
                seen.insert(path);
            } else if is_write_tool(&tool_call.name) {
                if !activity.created.contains(&path) {
                    activity.modified.insert(path.clone());
                }
                seen.insert(path);
            }
        }
    }
}

fn is_read_tool(name: &str) -> bool {
    matches!(name, "read" | "Read" | "read_file" | "View")
}

fn is_write_tool(name: &str) -> bool {
    matches!(
        name,
        "edit" | "Edit" | "write" | "Write" | "edit_file" | "write_file" | "MultiEdit"
    )
}

fn is_create_tool(name: &str) -> bool {
    matches!(name, "write" | "Write" | "write_file")
}

fn trim_file_activity(activity: &mut FileActivity) {
    let all: Vec<String> = activity
        .read
        .iter()
        .chain(activity.modified.iter())
        .chain(activity.created.iter())
        .cloned()
        .collect();
    let prefix = longest_common_dir_prefix(&all);
    if prefix.is_empty() {
        return;
    }

    activity.read.map_paths(&prefix);
    activity.modified.map_paths(&prefix);
    activity.created.map_paths(&prefix);
}

fn longest_common_dir_prefix(paths: &[String]) -> String {
    let absolute: Vec<&str> = paths
        .iter()
        .filter_map(|path| path.starts_with('/').then_some(path.as_str()))
        .collect();
    if absolute.len() < 2 {
        return String::new();
    }

    let split: Vec<Vec<&str>> = absolute
        .iter()
        .map(|path| path.split('/').collect())
        .collect();
    let min_len = split.iter().map(Vec::len).min().unwrap_or(0);
    let mut idx = 0;
    while idx + 1 < min_len {
        let segment = split[0][idx];
        if !split.iter().all(|parts| parts[idx] == segment) {
            break;
        }
        idx += 1;
    }

    if idx < 2 {
        String::new()
    } else {
        format!("{}/", split[0][..idx].join("/"))
    }
}

#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum DirectivePolarity {
    Affirmative,
    Negative,
}

#[derive(Debug)]
struct Directive {
    text: String,
    words: Vec<String>,
    polarity: DirectivePolarity,
}

impl From<&str> for Directive {
    fn from(value: &str) -> Self {
        let text = strip_entry_ref(value).to_string();
        let words = text::split_words(&text);
        let normalized = text.replace('’', "'");
        let contracted = normalized
            .split(|ch: char| !(ch.is_alphanumeric() || ch == '\''))
            .any(|word| word.ends_with("n't"));
        let negative_action = words
            .iter()
            .find(|word| {
                !matches!(
                    word.as_str(),
                    "all"
                        | "always"
                        | "cannot"
                        | "cant"
                        | "couldnt"
                        | "didnt"
                        | "do"
                        | "doesnt"
                        | "dont"
                        | "ever"
                        | "longer"
                        | "must"
                        | "mustnt"
                        | "never"
                        | "no"
                        | "not"
                        | "one"
                        | "please"
                        | "should"
                        | "shouldnt"
                        | "the"
                        | "user"
                        | "users"
                        | "we"
                        | "wont"
                        | "wouldnt"
                        | "you"
                )
            })
            .is_some_and(|word| {
                matches!(
                    word.as_str(),
                    "avoid"
                        | "avoids"
                        | "avoiding"
                        | "cease"
                        | "ceases"
                        | "ceasing"
                        | "disable"
                        | "disables"
                        | "disabled"
                        | "disallow"
                        | "disallows"
                        | "disallowed"
                        | "forbid"
                        | "forbids"
                        | "forbidden"
                        | "prohibit"
                        | "prohibits"
                        | "prohibited"
                        | "refrain"
                        | "refrains"
                        | "refraining"
                        | "stop"
                        | "stops"
                        | "stopping"
                )
            });
        let explicit_negative = contracted
            || words.iter().any(|word| {
                matches!(
                    word.as_str(),
                    "never"
                        | "no"
                        | "not"
                        | "cannot"
                        | "dont"
                        | "doesnt"
                        | "didnt"
                        | "cant"
                        | "couldnt"
                        | "wont"
                        | "wouldnt"
                        | "shouldnt"
                        | "mustnt"
                )
            });
        let negative = explicit_negative != negative_action;
        let polarity = if negative {
            DirectivePolarity::Negative
        } else {
            DirectivePolarity::Affirmative
        };
        Self {
            text,
            words,
            polarity,
        }
    }
}

impl Directive {
    fn content_text(&self) -> String {
        let content = self
            .words
            .iter()
            .map(String::as_str)
            .filter(|word| !matches!(*word, "always" | "never" | "must" | "should" | "do" | "not"))
            .collect::<Vec<_>>()
            .join(" ");
        if content.is_empty() {
            self.text.clone()
        } else {
            content
        }
    }

    fn distinctive_tokens(&self) -> BTreeSet<&str> {
        self.words
            .iter()
            .map(|word| word.trim_end_matches('.'))
            .map(|word| match word {
                "changes" => "change",
                "commits" => "commit",
                word => word,
            })
            .filter(|word| {
                !text::STOP_WORDS.contains(word) && !matches!(*word, "always" | "never" | "please")
            })
            .collect()
    }

    fn has_conflicting_tokens(&self, other: &Self) -> bool {
        let left = self.distinctive_tokens();
        let right = other.distinctive_tokens();
        let left_only = left.difference(&right).copied().collect::<Vec<_>>();
        let right_only = right.difference(&left).copied().collect::<Vec<_>>();
        let substitution = left_only.len() == 1 && right_only.len() == 1;
        let has_word = |directive: &Self, expected: &str| {
            directive
                .words
                .iter()
                .any(|word| word.trim_end_matches('.') == expected)
        };
        let push_object_synonyms = substitution
            && (left_only.as_slice() == ["change"] && right_only.as_slice() == ["commit"]
                || left_only.as_slice() == ["commit"] && right_only.as_slice() == ["change"])
            && has_word(self, "push")
            && has_word(other, "push");
        let with_without = has_word(self, "with") && has_word(other, "without")
            || has_word(self, "without") && has_word(other, "with");
        (substitution && !push_object_synonyms) || with_without
    }
}

#[derive(Clone, Copy)]
struct EmbeddedDirective<'a> {
    directive: &'a Directive,
    semantic: &'a Embedding,
    content: &'a Embedding,
}

impl EmbeddedDirective<'_> {
    fn matches(
        self,
        other: Self,
        semantic_threshold: Similarity,
        content_threshold: Similarity,
    ) -> bool {
        if self.directive.polarity != other.directive.polarity
            || self.directive.has_conflicting_tokens(other.directive)
        {
            return false;
        }
        if self.semantic.similarity(other.semantic) < semantic_threshold {
            return false;
        }
        self.content.similarity(other.content) >= content_threshold
    }
}

impl ExtractedContext {
    /// Remove high-confidence semantic duplicates while preserving the newest
    /// verbatim candidate and its provenance reference.
    fn deduplicate_directives(&mut self, embedder: &Embedder) -> Result<()> {
        let semantic_threshold = Similarity::try_from(DIRECTIVE_SEMANTIC_SIMILARITY)?;
        let content_threshold = Similarity::try_from(DIRECTIVE_CONTENT_SIMILARITY)?;
        for items in [&mut self.goals, &mut self.preferences] {
            if items.len() < 2 {
                continue;
            }

            let directives = items
                .iter()
                .map(|item| Directive::from(item.as_str()))
                .collect::<Vec<_>>();
            let content_texts = directives
                .iter()
                .map(Directive::content_text)
                .collect::<Vec<_>>();
            let inputs = directives
                .iter()
                .map(|directive| directive.text.as_str())
                .chain(content_texts.iter().map(String::as_str))
                .collect::<Vec<_>>();
            let embeddings = embedder.embed_batch(&inputs)?;
            ensure!(
                embeddings.len() == inputs.len(),
                "directive embedding count differs"
            );
            let (semantic_embeddings, content_embeddings) = embeddings.split_at(directives.len());
            let candidates = directives
                .iter()
                .zip(semantic_embeddings)
                .zip(content_embeddings)
                .map(|((directive, semantic), content)| EmbeddedDirective {
                    directive,
                    semantic,
                    content,
                })
                .collect::<Vec<_>>();
            let mut keep = vec![false; candidates.len()];
            let mut retained = Vec::with_capacity(candidates.len());
            for (index, candidate) in candidates.into_iter().enumerate().rev() {
                let mut duplicate = false;
                for existing in &retained {
                    if candidate.matches(*existing, semantic_threshold, content_threshold) {
                        duplicate = true;
                        break;
                    }
                }
                if !duplicate {
                    keep[index] = true;
                    retained.push(candidate);
                }
            }

            *items = std::mem::take(items)
                .into_iter()
                .zip(keep)
                .filter_map(|(item, retain)| retain.then_some(item))
                .collect();
        }
        Ok(())
    }
}

/// Ask the text model which candidate lines are genuine, still-current
/// directives. The model only selects among the deterministic candidates —
/// emitted text stays verbatim, provenance refs intact — and an unparseable,
/// empty, or all-of-them answer keeps the list unchanged, so the model can
/// only sharpen the list, never invent or empty it.
fn directive_answer_token_budget(len: usize) -> usize {
    // Each answer line is roughly three BPE tokens for a single-digit id
    // (`g` + `1` + `\n`) and closer to four for a double-digit one (`p15\n`).
    // `TextGen::complete` hard-caps at this value and only stops early on EOS,
    // so the budget must fit the worst case — listing every id when the model
    // judges all directives current — or the tail is truncated and
    // `apply_keep_set` drops ids the model wanted to keep.
    (len.saturating_mul(4) + 8).max(16)
}

fn parse_prefixed_keep_set(answer: &str, prefix: char, len: usize) -> Vec<usize> {
    let mut keep: Vec<usize> = answer
        .split(|ch: char| !ch.is_ascii_alphanumeric())
        .filter(|part| !part.is_empty())
        .filter_map(|part| {
            let mut chars = part.chars();
            let head = chars.next()?.to_ascii_lowercase();
            (head == prefix).then_some(chars.as_str())
        })
        .filter(|digits| !digits.is_empty())
        .filter_map(|digits| digits.parse().ok())
        .filter(|idx| (1..=len).contains(idx))
        .collect();
    keep.sort_unstable();
    keep.dedup();
    keep
}

fn apply_keep_set(items: &mut Vec<String>, keep: &[usize]) {
    if keep.is_empty() || keep.len() == items.len() {
        return;
    }
    *items = keep
        .iter()
        .filter_map(|&idx| items.get(idx - 1).cloned())
        .collect();
}

fn directive_prompt_text(item: &str) -> String {
    clean_sentence(strip_entry_ref(item), DIRECTIVE_PROMPT_MAX_CHARS)
}

fn refine_directive_groups(
    goals: &mut Vec<String>,
    preferences: &mut Vec<String>,
    textgen: &mut TextGen,
) -> Result<()> {
    use std::fmt::Write as _;

    let mut numbered = String::new();
    let _ = writeln!(numbered, "[Session goals]");
    for (idx, item) in goals.iter().enumerate() {
        let _ = writeln!(numbered, "g{}. {}", idx + 1, directive_prompt_text(item));
    }
    let _ = writeln!(numbered, "\n[User preferences]");
    for (idx, item) in preferences.iter().enumerate() {
        let _ = writeln!(numbered, "p{}. {}", idx + 1, directive_prompt_text(item));
    }

    let system = "Return only current ids, one per line, no prose. Use gN for goals and pN for preferences. If all ids in a section are current, list them all.";
    let answer = textgen.complete(
        system,
        &numbered,
        directive_answer_token_budget(goals.len().saturating_add(preferences.len())),
    )?;
    let goal_keep = parse_prefixed_keep_set(&answer, 'g', goals.len());
    apply_keep_set(goals, &goal_keep);
    let pref_keep = parse_prefixed_keep_set(&answer, 'p', preferences.len());
    apply_keep_set(preferences, &pref_keep);
    Ok(())
}
fn refine_directives(items: &mut Vec<String>, kind: &str, textgen: &mut TextGen) -> Result<()> {
    use std::fmt::Write as _;
    if items.len() < 2 {
        return Ok(());
    }
    let mut numbered = String::new();
    for (idx, item) in items.iter().enumerate() {
        let _ = writeln!(numbered, "{}. {}", idx + 1, directive_prompt_text(item));
    }
    let system = format!(
        "Return only current {kind} ids, one per line, no prose. If all ids are current, list them all."
    );
    let answer = textgen.complete(
        &system,
        &numbered,
        directive_answer_token_budget(items.len()),
    )?;
    let keep = parse_keep_set(&answer, items.len());
    apply_keep_set(items, &keep);
    Ok(())
}

/// Parse the model's comma/whitespace-separated 1-based answer into a sorted,
/// deduplicated list of indices into a list of `len`. Non-numeric tokens and
/// numbers outside `1..=len` are dropped. An empty result (or one listing every
/// line) is what [`apply_keep_set`] treats as a no-op, so the model can only
/// sharpen the list, never invent or empty it.
fn parse_keep_set(answer: &str, len: usize) -> Vec<usize> {
    let mut keep: Vec<usize> = answer
        .split(|ch: char| !ch.is_ascii_digit())
        .filter(|part| !part.is_empty())
        .filter_map(|part| part.parse().ok())
        .filter(|idx| (1..=len).contains(idx))
        .collect();
    keep.sort_unstable();
    keep.dedup();
    keep
}

/// The last todo/plan tool snapshot, as `[todo] title (status)` items for every
/// entry not yet completed. The todo tool is a data source here, not noise: its
/// final state is exactly the outstanding work. Two shapes are understood —
/// snapshot-style tool calls carrying the whole list in their arguments
/// (`todos`/`plan` arrays), and event-style tool results printing
/// `Created #N: title (status)` / `Updated #N (from -> to)` lines.
fn todo_snapshot(messages: &[ConversationMessage]) -> Vec<String> {
    let mut snapshot: Vec<(String, String)> = Vec::new();
    let mut events: std::collections::BTreeMap<String, (String, String)> =
        std::collections::BTreeMap::new();
    let mut entry_id = String::new();

    for message in messages {
        if message.is_assistant() {
            for tool_call in message.tool_calls() {
                if !is_internal_tool_name(&tool_call.name) {
                    continue;
                }
                if let Some(items) = todo_items_from_arguments(&tool_call.arguments) {
                    snapshot = items;
                    events.clear();
                    entry_id.clone_from(&message.entry_id);
                }
            }
        }
        if let MessageView::ToolResult(result) = message.view()
            && is_internal_tool_name(&result.tool_name)
        {
            let mut changed = false;
            for line in non_empty_lines(&result.content) {
                changed |= apply_todo_event(&line, &mut events);
            }
            if changed {
                entry_id.clone_from(&message.entry_id);
            }
        }
    }

    if !events.is_empty() {
        snapshot = events.into_values().collect();
    }
    snapshot
        .into_iter()
        .filter(|(_, status)| {
            !matches!(
                status.as_str(),
                "completed" | "done" | "cancelled" | "canceled"
            )
        })
        .map(|(title, status)| format!("[todo] {title} ({status}) ({})", entry_ref(&entry_id)))
        .collect()
}

/// Extract `(title, status)` items from snapshot-style todo/plan tool-call
/// arguments (`todos` or `plan` arrays of objects).
fn todo_items_from_arguments(arguments: &serde_json::Value) -> Option<Vec<(String, String)>> {
    let obj = arguments.as_object()?;
    let list = ["todos", "plan"]
        .iter()
        .find_map(|key| obj.get(*key).and_then(serde_json::Value::as_array))?;

    let mut items = Vec::new();
    for entry in list {
        let entry = entry.as_object()?;
        let title = ["content", "step", "text", "title"]
            .iter()
            .find_map(|key| entry.get(*key).and_then(serde_json::Value::as_str))?;
        let status = entry
            .get("status")
            .and_then(serde_json::Value::as_str)
            .unwrap_or("pending");
        items.push((title.to_string(), status.to_string()));
    }
    (!items.is_empty()).then_some(items)
}

/// Apply one event-style todo line to the state map. `Created #N: title (status)`
/// inserts an item; `Updated #N (from -> to)` moves its status. Anything else in
/// the result (listings, snippets) is not a todo event and is ignored.
fn apply_todo_event(
    line: &str,
    items: &mut std::collections::BTreeMap<String, (String, String)>,
) -> bool {
    if let Some(rest) = line.strip_prefix("Created #")
        && let Some((id, rest)) = rest.split_once(':')
    {
        let rest = rest.trim();
        let (title, status) = match rest.rsplit_once(" (") {
            Some((title, status)) if status.ends_with(')') => (title, status.trim_end_matches(')')),
            _ => (rest, "pending"),
        };
        items.insert(id.to_string(), (title.to_string(), status.to_string()));
        return true;
    }
    if let Some(rest) = line.strip_prefix("Updated #")
        && let Some((id, transition)) = rest.split_once(" (")
        && let Some((_, to)) = transition.trim_end_matches(')').rsplit_once("-> ")
        && let Some(item) = items.get_mut(id)
    {
        item.1 = to.trim().to_string();
        return true;
    }
    false
}

fn conversation_resolved(messages: &[ConversationMessage]) -> bool {
    for message in messages.iter().rev() {
        let text_value = match message.view() {
            MessageView::ToolResult(result) => result.content.as_str(),
            MessageView::Bash(output) => output.output.as_str(),
            _ => continue,
        };
        for line in non_empty_lines(text_value) {
            let lower = line.to_ascii_lowercase();
            if is_success_line(&lower) {
                return true;
            }
            if is_tool_failure_line(&lower) {
                return false;
            }
        }
    }
    false
}

fn conversation_outstanding(messages: &[ConversationMessage]) -> Vec<String> {
    let conversation_resolved = conversation_resolved(messages);
    let mut items = Vec::new();

    for message in messages.iter().rev() {
        if is_compaction_record(message) || compact_summary_text(message).is_some() {
            continue;
        }
        match message.view() {
            MessageView::Text { role, text } => {
                if role != "user" && role != "assistant" {
                    continue;
                }
                collect_outstanding_lines(&text, role, &message.entry_id, &mut items);
            }
            MessageView::Assistant { text, .. } => {
                collect_outstanding_lines(&text, "assistant", &message.entry_id, &mut items);
            }
            MessageView::ToolResult(result) => {
                if result.is_error && !conversation_resolved {
                    collect_tool_outstanding_lines(&result.content, &message.entry_id, &mut items);
                }
            }
            MessageView::Bash(output) => {
                if !conversation_resolved {
                    collect_tool_outstanding_lines(&output.output, &message.entry_id, &mut items);
                }
            }
        }
        if items.len() >= 5 {
            break;
        }
    }

    items.reverse();
    items
}

fn collect_outstanding_lines(
    text_value: &str,
    role: &str,
    entry_id: &str,
    items: &mut Vec<String>,
) {
    for line in non_empty_lines(text_value) {
        let lower = line.to_ascii_lowercase();
        if is_resolved_line(&lower) {
            continue;
        }
        if !contains_any(
            &lower,
            &[
                "fail",
                "failure",
                "error",
                "broken",
                "cannot",
                "can't",
                "won't work",
                "does not work",
                "doesn't work",
                "blocked",
                "blocker",
                "not fixed",
                "not resolved",
                "crash",
                "todo",
                "pending",
                "remaining",
            ],
        ) {
            continue;
        }
        if is_success_line(&lower) {
            continue;
        }
        if is_short_or_omitted_line(&line) {
            continue;
        }
        let item = if role == "user" {
            format!(
                "[user] {}",
                referenced_sentence(&line, OUTSTANDING_LINE_MAX_CHARS, entry_id)
            )
        } else {
            referenced_sentence(&line, OUTSTANDING_LINE_MAX_CHARS, entry_id)
        };
        push_unique(items, item);
        break;
    }
}

fn collect_tool_outstanding_lines(text_value: &str, entry_id: &str, items: &mut Vec<String>) {
    for line in non_empty_lines(text_value) {
        let lower = line.to_ascii_lowercase();
        if !is_tool_failure_line(&lower)
            || is_success_line(&lower)
            || is_short_or_omitted_line(&line)
        {
            continue;
        }
        push_unique(
            items,
            referenced_sentence(&line, OUTSTANDING_LINE_MAX_CHARS, entry_id),
        );
        break;
    }
}

fn is_short_or_omitted_line(line: &str) -> bool {
    line.len() < 12 || line.starts_with("...")
}

fn is_tool_failure_line(line: &str) -> bool {
    contains_any(
        line,
        &[
            "error:",
            "error ",
            "failed",
            "failure",
            "panic",
            "traceback",
            "exception",
            "command not found",
            "no such file",
            "permission denied",
        ],
    )
}

fn is_resolved_line(line: &str) -> bool {
    contains_any(
        line,
        &[
            "fixed",
            "resolved",
            "passing",
            "passes",
            "now works",
            "no longer",
            "done",
            "completed",
        ],
    ) && !contains_any(line, &["not fixed", "not resolved", "unresolved"])
}

fn is_success_line(line: &str) -> bool {
    line.contains("test result: ok")
        || line.contains(" 0 failed")
        || line.contains("fail=0")
        || line.contains("failed 0")
        || line.contains("error=0")
}

fn conversation_brief(messages: &[ConversationMessage]) -> String {
    let mut sections: Vec<BriefSection> = Vec::new();

    for message in messages {
        if is_compaction_record(message) {
            continue;
        }
        match message.view() {
            MessageView::Text { role, text } => {
                if is_compact_summary(&text) {
                    continue;
                }
                if text.trim().is_empty() {
                    continue;
                }
                let header = match role {
                    "user" => "[user]",
                    "assistant" => "[assistant]",
                    _ => continue,
                };
                push_brief(
                    &mut sections,
                    header,
                    format!(
                        "{} ({})",
                        clean_sentence(&text, brief_limit(header)),
                        entry_ref(&message.entry_id)
                    ),
                );
            }
            MessageView::Assistant {
                text, tool_calls, ..
            } => {
                if is_compact_summary(&text) {
                    continue;
                }
                if !text.trim().is_empty() {
                    push_brief(
                        &mut sections,
                        "[assistant]",
                        format!(
                            "{} ({})",
                            clean_sentence(&strip_self_talk(&text), ASSISTANT_LINE_MAX_CHARS),
                            entry_ref(&message.entry_id)
                        ),
                    );
                }
                push_tool_calls(&mut sections, &tool_calls, &message.entry_id);
            }
            MessageView::Bash(output) => {
                let cmd = compress_bash(output);
                if !cmd.is_empty() {
                    push_brief(
                        &mut sections,
                        "[user]",
                        format!("$ {} ({})", cmd, entry_ref(&message.entry_id)),
                    );
                }
            }
            MessageView::ToolResult(_) => {}
        }
    }

    lexical_dedup_sections(&mut sections);
    stringify_brief(&mut sections)
}

/// Fold exact-normalized duplicate brief lines so near-restatements of an
/// earlier kept line are dropped. Tool-call lines (`* …`) already have
/// signature dedup and are left untouched.
fn lexical_dedup_sections(sections: &mut [BriefSection]) {
    let mut seen: HashSet<String> = HashSet::new();
    for section in sections.iter_mut() {
        section.lines.retain(|line| {
            if line.starts_with("* ") {
                return true;
            }
            let normalized = strip_entry_ref(line).to_lowercase();
            normalized.trim().is_empty() || seen.insert(normalized)
        });
    }
}

/// Strip a trailing ` (#entry-id)` reference so dedup compares content, not
/// volatile ids.
fn strip_entry_ref(line: &str) -> &str {
    match line.rfind(" (#") {
        Some(index) if line.ends_with(')') => &line[..index],
        _ => line,
    }
}

fn push_tool_calls(sections: &mut Vec<BriefSection>, tool_calls: &[&ToolCall], entry_id: &str) {
    let visible_tool_calls: Vec<&ToolCall> = tool_calls
        .iter()
        .copied()
        .filter(|tool_call| is_visible_tool_call(tool_call))
        .collect();
    if visible_tool_calls.is_empty() {
        return;
    }

    let omitted = visible_tool_calls.len().saturating_sub(TOOL_CALLS_PER_TURN);
    if omitted > 0 {
        push_brief(
            sections,
            "[assistant]",
            format!("* ({omitted} earlier tool-call entries omitted)"),
        );
    }

    for tool_call in visible_tool_calls.into_iter().skip(omitted) {
        push_brief(
            sections,
            "[assistant]",
            format!("{} ({})", tool_one_liner(tool_call), entry_ref(entry_id)),
        );
    }
}

fn is_visible_tool_call(tool_call: &ToolCall) -> bool {
    let name = tool_call.name.trim();
    !name.is_empty() && !is_internal_tool_name(name)
}

fn is_internal_tool_name(name: &str) -> bool {
    [
        "todo",
        "todowrite",
        "task",
        "task_status",
        "progress",
        "update_plan",
    ]
    .iter()
    .any(|internal| name.eq_ignore_ascii_case(internal))
}

fn push_brief(sections: &mut Vec<BriefSection>, header: &'static str, line: String) {
    if line.trim().is_empty() {
        return;
    }
    if sections.last().is_some_and(|last| last.header == header) {
        if let Some(last) = sections.last_mut() {
            last.lines.push(line);
        }
        return;
    }
    sections.push(BriefSection {
        header,
        lines: vec![line],
    });
}

fn stringify_brief(sections: &mut [BriefSection]) -> String {
    collapse_repeated_tool_lines(sections);
    cap_tool_lines(sections);

    let mut out = Vec::new();
    for (idx, section) in sections.iter().enumerate() {
        if idx > 0 {
            let previous = &sections[idx - 1];
            let previous_tools = previous.lines.iter().all(|line| line.starts_with("* "));
            let current_tools = section.lines.iter().all(|line| line.starts_with("* "));
            if !(previous.header == "[assistant]"
                && section.header == "[assistant]"
                && previous_tools
                && current_tools)
            {
                out.push(String::new());
            }
        }
        out.push(section.header.to_string());
        out.extend(section.lines.iter().cloned());
    }

    out.join("\n")
}

fn collapse_repeated_tool_lines(sections: &mut [BriefSection]) {
    let mut seen: HashSet<String> = HashSet::new();
    for section in sections {
        if section.header != "[assistant]" {
            continue;
        }
        section
            .lines
            .retain(|line| match tool_line_signature(line) {
                Some(signature) => seen.insert(signature.to_string()),
                None => true,
            });
    }
}

fn tool_line_signature(line: &str) -> Option<&str> {
    if !line.starts_with("* ") {
        return None;
    }
    line.rsplit_once(" (#").map(|(signature, _)| signature)
}

fn cap_tool_lines(sections: &mut [BriefSection]) {
    for section in sections {
        if section.header != "[assistant]" {
            continue;
        }
        let tool_indexes: Vec<usize> = section
            .lines
            .iter()
            .enumerate()
            .filter_map(|(idx, line)| line.starts_with("* ").then_some(idx))
            .collect();
        if tool_indexes.len() <= TOOL_CALLS_PER_TURN {
            continue;
        }
        let drop_count = tool_indexes.len() - TOOL_CALLS_PER_TURN;
        let drop_set: BTreeSet<usize> = tool_indexes.iter().take(drop_count).copied().collect();
        let first_kept = tool_indexes[drop_count];
        let mut next = Vec::new();
        let mut inserted = false;
        for (idx, line) in section.lines.iter().enumerate() {
            if drop_set.contains(&idx) {
                continue;
            }
            if !inserted && idx == first_kept {
                next.push(format!(
                    "* ({drop_count} earlier tool-call entries omitted)"
                ));
                inserted = true;
            }
            next.push(line.clone());
        }
        section.lines = next;
    }
}

struct BriefCandidates<'a> {
    lines: &'a [&'a str],
    ranges: Vec<Range<usize>>,
}

impl<'a> BriefCandidates<'a> {
    fn new(lines: &'a [&'a str], tail_starts_section: bool) -> Self {
        let starts = lines
            .iter()
            .enumerate()
            .filter_map(|(index, line)| is_brief_header(line).then_some(index))
            .collect::<Vec<_>>();
        let mut ranges = Vec::new();
        if let Some(&first) = starts.first()
            && first > 0
        {
            ranges.push(0..first);
        }
        ranges.extend(
            starts
                .windows(2)
                .filter_map(|window| window.first().zip(window.get(1)))
                .map(|(&start, &end)| start..end),
        );
        if tail_starts_section && let Some(&start) = starts.last() {
            ranges.push(start..lines.len());
        }
        Self { lines, ranges }
    }

    fn select(&self, references: &[&str], embedder: &Embedder) -> Result<BriefSelection> {
        let candidates = self
            .ranges
            .iter()
            .map(|range| {
                self.lines
                    .get(range.clone())
                    .map(|lines| lines.join("\n"))
                    .ok_or_else(|| anyhow::anyhow!("brief candidate range is invalid"))
            })
            .collect::<Result<Vec<_>>>()?;
        let candidate_refs = candidates.iter().map(String::as_str).collect::<Vec<_>>();
        let scores = embedder.relevance(references, &candidate_refs)?;
        ensure!(
            scores.len() == self.ranges.len(),
            "brief relevance score count differs"
        );
        let mut ranked = scores.into_iter().enumerate().collect::<Vec<_>>();
        ranked.sort_by(|(left_index, left_score), (right_index, right_score)| {
            right_score
                .cmp(left_score)
                .then_with(|| left_index.cmp(right_index))
        });

        let mut selection = BriefSelection::default();
        for (index, _) in ranked {
            let range = self
                .ranges
                .get(index)
                .ok_or_else(|| anyhow::anyhow!("brief candidate index is invalid"))?;
            let chunk_lines = range.end.saturating_sub(range.start);
            if chunk_lines <= BRIEF_RELEVANT_LINES.saturating_sub(selection.line_count) {
                selection.indexes.insert(index);
                selection.line_count += chunk_lines;
            }
        }
        Ok(selection)
    }

    fn selected_lines(&self, selection: &BriefSelection) -> Result<Vec<&'a str>> {
        let mut lines = Vec::new();
        for index in &selection.indexes {
            let range = self
                .ranges
                .get(*index)
                .ok_or_else(|| anyhow::anyhow!("selected brief index is invalid"))?;
            let chunk = self
                .lines
                .get(range.clone())
                .ok_or_else(|| anyhow::anyhow!("selected brief range is invalid"))?;
            lines.extend_from_slice(chunk);
        }
        Ok(lines)
    }
}

#[derive(Default)]
struct BriefSelection {
    indexes: BTreeSet<usize>,
    line_count: usize,
}

fn cap_brief_ranked(value: &str, references: &[&str], embedder: &Embedder) -> Result<String> {
    let lines: Vec<&str> = value.lines().collect();
    if lines.len() <= BRIEF_MAX_LINES {
        return Ok(value.to_string());
    }

    let first_header = lines
        .iter()
        .position(|line| is_brief_header(line))
        .unwrap_or(0);
    let effective = lines
        .get(first_header..)
        .ok_or_else(|| anyhow::anyhow!("brief header range is invalid"))?;
    if effective.len() <= BRIEF_MAX_LINES {
        return Ok(effective.join("\n"));
    }

    let head_len = BRIEF_HEAD_LINES.min(effective.len());
    let tail_start = effective
        .len()
        .checked_sub(BRIEF_TAIL_LINES)
        .ok_or_else(|| anyhow::anyhow!("brief tail range underflow"))?;
    let middle = effective
        .get(head_len..tail_start)
        .ok_or_else(|| anyhow::anyhow!("brief middle range is invalid"))?;
    let tail_first = effective
        .get(tail_start)
        .ok_or_else(|| anyhow::anyhow!("brief tail is empty"))?;
    let candidates = BriefCandidates::new(middle, is_brief_header(tail_first));
    if candidates.ranges.is_empty() {
        return Ok(cap_brief(value));
    }

    let selection = candidates.select(references, embedder)?;
    if selection.indexes.is_empty() {
        return Ok(cap_brief(value));
    }

    let sampled = candidates.selected_lines(&selection)?.join("\n");
    let tail_header = brief_tail_header(effective, head_len, tail_start);
    let omitted = effective
        .len()
        .saturating_sub(head_len + BRIEF_TAIL_LINES + selection.line_count)
        .saturating_sub(usize::from(tail_header.is_some()));
    let head = effective
        .get(..head_len)
        .ok_or_else(|| anyhow::anyhow!("brief head range is invalid"))?
        .join("\n");
    let tail = effective
        .get(tail_start..)
        .ok_or_else(|| anyhow::anyhow!("brief tail range is invalid"))?
        .join("\n");
    let tail_header = tail_header.map_or_else(String::new, |header| format!("{header}\n"));

    Ok(format!(
        "{head}\n\n...({omitted} earlier lines omitted; selected relevant excerpts retained)\n\n{sampled}\n\n{tail_header}{tail}"
    ))
}

fn cap_brief(value: &str) -> String {
    let lines: Vec<&str> = value.lines().collect();
    if lines.len() <= BRIEF_MAX_LINES {
        return value.to_string();
    }

    let first_header = lines
        .iter()
        .position(|line| is_brief_header(line))
        .unwrap_or(0);
    let effective = &lines[first_header..];
    if effective.len() <= BRIEF_MAX_LINES {
        return effective.join("\n");
    }

    let head_len = BRIEF_HEAD_LINES.min(effective.len());
    let available_tail = BRIEF_MAX_LINES - head_len - BRIEF_FORMAT_LINES;
    let tentative_tail_start = effective.len() - available_tail;
    let restore_header = brief_tail_header(effective, head_len, tentative_tail_start).is_some();
    let tail_len = available_tail - usize::from(restore_header);
    let tail_start = effective.len() - tail_len;
    let tail_header = brief_tail_header(effective, head_len, tail_start);
    let omitted = (tail_start - head_len).saturating_sub(usize::from(tail_header.is_some()));

    let head = effective[..head_len].join("\n");
    let tail = effective[tail_start..].join("\n");
    let tail_header = tail_header.map_or_else(String::new, |header| format!("{header}\n"));

    format!("{head}\n\n...({omitted} earlier lines omitted)\n\n{tail_header}{tail}")
}

fn is_brief_header(line: &str) -> bool {
    line.starts_with('[') && line.ends_with(']')
}

fn brief_tail_header<'a>(
    lines: &'a [&'a str],
    omitted_start: usize,
    tail_start: usize,
) -> Option<&'a str> {
    let first = *lines.get(tail_start)?;
    if is_brief_header(first) {
        return None;
    }
    lines
        .get(omitted_start..tail_start)?
        .iter()
        .rev()
        .copied()
        .find(|line| is_brief_header(line))
}

fn tool_one_liner(tool_call: &ToolCall) -> String {
    if let Some(path) = display::path_argument(&tool_call.arguments) {
        return format!("* {} \"{}\"", tool_call.name, text::clip(&path, 90));
    }

    if let Some(obj) = tool_call.arguments.as_object() {
        if (tool_call.name == "bash" || tool_call.name == "Bash")
            && let Some(command) = obj.get("command").and_then(|value| value.as_str())
        {
            return format!("* {} \"{}\"", tool_call.name, compress_command(command));
        }
        for key in ["query", "pattern", "description"] {
            if let Some(value) = obj.get(key).and_then(|value| value.as_str()) {
                return format!("* {} \"{}\"", tool_call.name, text::clip(value, 60));
            }
        }
    }

    format!("* {}", tool_call.name)
}

fn compress_bash(output: &BashOutput) -> String {
    compress_command(&output.command)
}

fn compress_command(command: &str) -> String {
    let mut cmd = command
        .lines()
        .map(str::trim)
        .find(|line| !line.is_empty())
        .unwrap_or(command)
        .to_string();

    if let Some(stripped) = strip_cd_prefix(&cmd) {
        cmd = stripped.to_string();
    }
    for pipe in [
        " | head",
        " | tail",
        " | sort",
        " | wc",
        " | column",
        " | tr",
        " | cut",
    ] {
        if let Some(idx) = cmd.rfind(pipe) {
            cmd.truncate(idx);
        }
    }

    text::clip(&cmd, BASH_LINE_MAX_CHARS)
}

fn strip_cd_prefix(command: &str) -> Option<&str> {
    let rest = command.strip_prefix("cd ")?;
    let (_, suffix) = rest.split_once(" && ")?;
    Some(suffix)
}

fn entry_ref(entry_id: &str) -> String {
    format!("#{entry_id}")
}

fn brief_limit(header: &str) -> usize {
    if header == "[assistant]" {
        ASSISTANT_LINE_MAX_CHARS
    } else {
        TEXT_LINE_MAX_CHARS
    }
}

fn strip_self_talk(value: &str) -> String {
    let mut text_value = value.trim().to_string();
    for _ in 0..2 {
        let lower = text_value.to_ascii_lowercase();
        let Some(prefix) = ["hmm", "wait", "actually", "oh", "okay", "ok", "well", "so"]
            .iter()
            .find(|prefix| lower.starts_with(**prefix))
        else {
            break;
        };
        let rest = text_value[prefix.len()..].trim_start_matches([',', '.', '!', ' ', '-']);
        if rest == text_value {
            break;
        }
        text_value = rest.to_string();
    }
    text_value
}

fn non_empty_lines(value: &str) -> Vec<String> {
    text::sanitize(value)
        .lines()
        .map(str::trim)
        .filter(|line| !line.is_empty())
        .filter(|line| !line.starts_with("<skill") && !line.starts_with("</skill"))
        .map(ToString::to_string)
        .collect()
}

fn clean_sentence(value: &str, max_chars: usize) -> String {
    let flat = value.split_whitespace().collect::<Vec<_>>().join(" ");
    text::clip(flat.trim_matches(['-', '*', ' ']), max_chars)
}

fn referenced_sentence(value: &str, max_chars: usize, entry_id: &str) -> String {
    format!(
        "{} ({})",
        clean_sentence(value, max_chars),
        entry_ref(entry_id)
    )
}

fn contains_any(value: &str, needles: &[&str]) -> bool {
    needles.iter().any(|needle| value.contains(needle))
}

fn push_unique(items: &mut Vec<String>, item: String) {
    if !item.is_empty() && !items.iter().any(|existing| existing == &item) {
        items.push(item);
    }
}

fn extend_unique(items: &mut Vec<String>, incoming: impl IntoIterator<Item = String>) {
    for item in incoming {
        push_unique(items, item);
    }
}