collet 0.1.1

use std::fmt::Write;

use crate::api::content::Content;
use crate::api::models::Message;

use super::{CompactionEvent, ConversationContext, MAX_COMPACTION_LOG};

impl ConversationContext {
    pub(super) fn compact_if_needed(&mut self) {
        let system_tokens = self.system_prompt.len() / 4 + 10;
        let estimated_total = system_tokens + self.cached_msg_tokens;
        // Use API-reported prompt tokens as ground truth when available, since
        // our heuristic estimate can underreport actual wire size by 20-30%.
        // `used_tokens()` already returns max(baseline+delta, estimate).
        let total_tokens = self.used_tokens().max(estimated_total);
        let effective_threshold = self.adaptive_threshold();
        let threshold = (self.max_context_tokens as f64 * effective_threshold as f64) as usize;

        if total_tokens <= threshold {
            return;
        }

        tracing::info!(
            "Context compaction triggered: ~{} tokens (threshold: {})",
            total_tokens,
            threshold,
        );

        let target_ratio = self.adaptive_target_ratio();
        let target_tokens = (self.max_context_tokens as f64 * target_ratio) as usize;

        self.run_compaction_pipeline(target_tokens);
    }

    /// Execute the full compaction pipeline with logging.
    pub(super) fn run_compaction_pipeline(&mut self, target_tokens: usize) {
        // Determine protected window: last 3 turns, minimum 40k tokens worth
        let protected_from = self.find_protected_boundary(3, 40_000);

        // Before any compaction, extract and preserve implementation decisions
        // from messages that are about to be compacted away.
        self.preserve_decisions_from_range(0, protected_from);

        for pass in 0..3 {
            let current_tokens = self.estimate_total_tokens();
            if current_tokens <= target_tokens {
                break;
            }

            let before_messages = self.messages.len();
            let before_tokens = current_tokens;

            // Pipeline stages (gentle → aggressive)
            // Stage 1: Observation masking — replace old tool outputs with placeholders
            if self.estimate_total_tokens() > target_tokens {
                self.mask_old_tool_outputs(protected_from);
                self.cached_msg_tokens = self
                    .messages
                    .iter()
                    .map(Self::estimate_message_tokens)
                    .sum();
            }

            // Stage 2: Dedup near-identical tool results
            if self.estimate_total_tokens() > target_tokens {
                self.dedup_tool_results();
                self.cached_msg_tokens = self
                    .messages
                    .iter()
                    .map(Self::estimate_message_tokens)
                    .sum();
            }

            // Stage 3: Smart relevance-aware compaction
            if self.estimate_total_tokens() > target_tokens {
                self.smart_compaction_pass(target_tokens);
            }

            // Stage 4: Fallback fixed-window truncation
            if self.estimate_total_tokens() > target_tokens {
                self.do_compaction_pass(target_tokens);
            }

            let after_tokens = self.estimate_total_tokens();
            let after_messages = self.messages.len();

            // Build summary preview for log
            let summary_preview = self
                .messages
                .iter()
                .find(|m| {
                    m.role == "system"
                        && m.content
                            .as_ref()
                            .is_some_and(|c| c.text_content().contains("[Compacted"))
                })
                .and_then(|m| m.content.clone())
                .map(|c| c.text_content())
                .unwrap_or_default();
            let preview = if summary_preview.len() > 200 {
                format!("{}...", crate::util::truncate_bytes(&summary_preview, 200))
            } else {
                summary_preview
            };

            if self.compaction_log.len() >= MAX_COMPACTION_LOG {
                self.compaction_log
                    .drain(..self.compaction_log.len() - MAX_COMPACTION_LOG + 1);
            }
            self.compaction_log.push(CompactionEvent {
                before_messages,
                after_messages,
                before_tokens,
                after_tokens,
                summary_preview: preview,
            });

            // Reset API-reported baseline and caches after message set changed
            self.cached_msg_tokens = self
                .messages
                .iter()
                .map(Self::estimate_message_tokens)
                .sum();
            self.cached_tool_count = self
                .messages
                .iter()
                .filter(|m| m.role == "tool" || m.tool_calls.is_some())
                .count();
            self.last_actual_prompt_tokens = None;
            self.messages_at_last_api_call = 0;

            tracing::info!(
                "Compaction pass {}: {} → {} messages, ~{} → ~{} tokens",
                pass + 1,
                before_messages,
                after_messages,
                before_tokens,
                after_tokens,
            );

            crate::telemetry::track(
                "compaction_triggered",
                serde_json::json!({
                    "pass": pass + 1,
                    "before_tokens": before_tokens,
                    "after_tokens": after_tokens,
                    "before_messages": before_messages,
                    "after_messages": after_messages,
                }),
            );

            // If compaction didn't help much, stop
            if before_messages.saturating_sub(after_messages) < 4
                && before_tokens.saturating_sub(after_tokens) < 1000
            {
                break;
            }
        }
    }

    // ── Stage 1: Observation Masking ────────────────────────────────────
    //
    // JetBrains research (500 SWE-bench instances): observation masking
    // matches or beats LLM summarization in 4/5 configurations while
    // costing 52% less. Replace old tool outputs with compact placeholders
    // while preserving the action history (which tool was called, with
    // what arguments).

    /// Replace old tool result content with a compact placeholder.
    ///
    /// Only masks tool results outside the protected window.
    /// Keeps: tool name/args (from the assistant's tool_call), whether it
    /// succeeded/failed, and first line of output for error context.
    pub(crate) fn mask_old_tool_outputs(&mut self, protected_from: usize) {
        for i in 0..protected_from.min(self.messages.len()) {
            let msg = &self.messages[i];
            if msg.role != "tool" {
                continue;
            }
            let content = match &msg.content {
                Some(c) if c.text_content().len() > 300 => c.text_content().to_string(),
                _ => continue,
            };

            // Preserve error context (first lines for errors, just a marker otherwise)
            let is_error = content.starts_with("Error:") || content.starts_with("error:");
            let masked = if is_error {
                // Keep first 500 chars of error for debugging context
                let preview = truncate_str(&content, 500);
                format!("[Tool output masked — error preserved]\n{preview}")
            } else {
                let line_count = content.lines().count();
                let char_count = content.len();
                format!(
                    "[Tool output masked — {} lines, ~{} chars]",
                    line_count, char_count
                )
            };

            self.messages[i].content = Some(Content::text(masked));
        }
    }

    /// Find the message index where the protected window starts.
    ///
    /// Protected window = last N user turns OR at least `min_tokens` worth
    /// of recent messages, whichever is larger.
    fn find_protected_boundary(&self, turns: usize, min_tokens: usize) -> usize {
        let len = self.messages.len();
        if len == 0 {
            return 0;
        }

        // Count back N user turns
        let mut user_turn_count = 0;
        let mut boundary_by_turns = len;
        for i in (0..len).rev() {
            if self.messages[i].role == "user" {
                user_turn_count += 1;
                if user_turn_count >= turns {
                    boundary_by_turns = i;
                    break;
                }
            }
        }

        // Count back at least min_tokens
        let mut token_sum = 0;
        let mut boundary_by_tokens = len;
        for i in (0..len).rev() {
            token_sum += Self::estimate_message_tokens(&self.messages[i]);
            if token_sum >= min_tokens {
                boundary_by_tokens = i;
                break;
            }
        }

        // Use the more protective (earlier) boundary
        boundary_by_turns.min(boundary_by_tokens)
    }

    // ── Stage 2: Dedup ─────────────────────────────────────────────────

    /// Replace near-duplicate tool results with a placeholder.
    ///
    /// Groups by tool name + file path (not tool_call_id which is random).
    /// Uses SimHash (hamming_distance < 8) to detect near-identical content.
    /// The older duplicate is replaced so the LLM sees only the latest version.
    fn dedup_tool_results(&mut self) {
        use std::collections::HashMap;

        // Build a mapping: message index → (tool_name, file_path) from
        // the preceding assistant tool_call message.
        let mut tool_info: HashMap<usize, String> = HashMap::new();
        for i in 0..self.messages.len() {
            let msg = &self.messages[i];
            if let Some(ref tcs) = msg.tool_calls {
                for tc in tcs {
                    let name = &tc.function.name;
                    let path = serde_json::from_str::<serde_json::Value>(&tc.function.arguments)
                        .ok()
                        .and_then(|v| v.get("path").and_then(|p| p.as_str()).map(String::from))
                        .unwrap_or_default();
                    let key = format!("{name}:{path}");
                    // Map the next tool result message to this key
                    if i + 1 < self.messages.len() && self.messages[i + 1].role == "tool" {
                        tool_info.insert(i + 1, key);
                    }
                }
            }
        }

        let mut seen: HashMap<String, (usize, u64)> = HashMap::new();
        let mut to_nullify: Vec<usize> = Vec::new();

        for i in 0..self.messages.len() {
            let msg = &self.messages[i];
            if msg.role != "tool" {
                continue;
            }
            let content = match &msg.content {
                Some(c) if c.text_content().len() >= 50 => c.text_content().to_string(),
                _ => continue,
            };

            let tokens = tokenize_for_scoring(&content);
            let hash = simhash(&tokens);

            // Use tool name + path as grouping key (not random tool_call_id)
            let key = tool_info
                .get(&i)
                .cloned()
                .unwrap_or_else(|| msg.tool_call_id.as_deref().unwrap_or("unknown").to_string());

            if let Some((prev_idx, prev_hash)) = seen.get(&key)
                && hamming_distance(hash, *prev_hash) < 8
            {
                to_nullify.push(*prev_idx);
            }
            seen.insert(key, (i, hash));
        }

        for idx in to_nullify {
            if let Some(m) = self.messages.get_mut(idx) {
                m.content = Some(Content::text("[Duplicate read — see later result]"));
            }
        }
    }

    // ── Stage 3: Smart Compaction ──────────────────────────────────────

    /// Single pass of fallback compaction (position-based).
    fn do_compaction_pass(&mut self, target_tokens: usize) {
        let keep_start = 4.min(self.messages.len());

        // Calculate how many recent messages we can keep
        let mut recent_tokens = 0;
        let mut keep_from_end = 0;
        for msg in self.messages.iter().rev() {
            let msg_tokens = Self::estimate_message_tokens(msg);
            if recent_tokens + msg_tokens > target_tokens / 2 {
                break;
            }
            recent_tokens += msg_tokens;
            keep_from_end += 1;
        }
        keep_from_end = keep_from_end.max(6); // keep at least last 6 messages

        let compact_end = self.messages.len().saturating_sub(keep_from_end);

        // If nothing to compact, bail out
        if compact_end <= keep_start {
            return;
        }

        // Build structured summary from compacted messages
        let summary = self.build_structured_summary(keep_start, compact_end);

        let start_messages = self.messages[..keep_start].to_vec();
        let end_messages = self.messages[compact_end..].to_vec();

        self.messages = start_messages;
        self.messages.push(summary);
        self.messages.extend(end_messages);
        self.cached_msg_tokens = self
            .messages
            .iter()
            .map(Self::estimate_message_tokens)
            .sum();
    }

    /// Build a structured summary from compacted messages.
    ///
    /// Based on OpenCode/Factory.ai patterns: explicit sections for
    /// goal, discoveries, files, decisions, errors, tool usage.
    fn build_structured_summary(&self, start: usize, end: usize) -> Message {
        let compacted = &self.messages[start..end];
        let compacted_count = compacted.len();

        let mut goal_statements = Vec::new();
        let mut discoveries = Vec::new();
        let mut decisions = Vec::new();
        let mut files_modified = Vec::new();
        let mut files_read = Vec::new();
        let mut errors = Vec::new();
        let mut error_resolutions = Vec::new();
        let mut tool_calls_count: u32 = 0;
        let mut tool_failures: u32 = 0;

        let mut prev_was_error = false;

        for msg in compacted {
            match msg.role.as_str() {
                "user" => {
                    if let Some(ref content) = msg.content {
                        let text = content.text_content();
                        let preview = truncate_str(&text, 300);
                        goal_statements.push(preview);
                    }
                }
                "assistant" => {
                    if let Some(ref content) = msg.content {
                        let text = content.text_content();
                        for line in text.lines().take(50) {
                            let trimmed = line.trim();
                            if trimmed.len() < 15 {
                                continue;
                            }

                            // Decision patterns (expanded from original)
                            let is_decision = trimmed.starts_with("I'll ")
                                || trimmed.starts_with("Let me ")
                                || trimmed.starts_with("I need to ")
                                || trimmed.starts_with("The fix ")
                                || trimmed.starts_with("The issue ")
                                || trimmed.starts_with("The problem ")
                                || trimmed.starts_with("This ")
                                || trimmed.starts_with("We need ")
                                || trimmed.starts_with("I should ")
                                || trimmed.starts_with("My approach ")
                                || trimmed.starts_with("First, ")
                                || trimmed.starts_with("Instead, ");

                            if is_decision {
                                decisions.push(truncate_str(trimmed, 300));
                            }

                            // Discovery patterns (findings, learnings)
                            let is_discovery = trimmed.starts_with("Found ")
                                || trimmed.starts_with("It turns out ")
                                || trimmed.starts_with("The root cause ")
                                || trimmed.starts_with("Looking at ")
                                || trimmed.starts_with("The error ")
                                || trimmed.contains("because ");

                            if is_discovery && !is_decision {
                                discoveries.push(truncate_str(trimmed, 300));
                            }

                            // Error resolution: if previous tool result was an error
                            if prev_was_error && is_decision {
                                error_resolutions.push(truncate_str(trimmed, 300));
                            }
                        }
                        prev_was_error = false;
                    }

                    // Track tool calls → file paths
                    if let Some(ref tcs) = msg.tool_calls {
                        for tc in tcs {
                            let name = &tc.function.name;
                            let args = &tc.function.arguments;
                            tool_calls_count += 1;

                            if let Ok(parsed) = serde_json::from_str::<serde_json::Value>(args)
                                && let Some(path) = parsed.get("path").and_then(|p| p.as_str())
                            {
                                let path = path.to_string();
                                match name.as_str() {
                                    "file_write" | "file_edit" | "git_patch" => {
                                        if !files_modified.contains(&path) {
                                            files_modified.push(path);
                                        }
                                    }
                                    "file_read" => {
                                        if !files_read.contains(&path) {
                                            files_read.push(path);
                                        }
                                    }
                                    _ => {}
                                }
                            }
                        }
                    }
                }
                "tool" => {
                    if let Some(ref content) = msg.content {
                        let text = content.text_content();
                        if text.starts_with("Error:") || text.starts_with("error:") {
                            tool_failures += 1;
                            errors.push(truncate_str(&text, 300));
                            prev_was_error = true;
                        } else {
                            prev_was_error = false;
                        }
                    }
                }
                _ => {}
            }

            // Generic file path extraction from content
            if let Some(ref content) = msg.content {
                let text = content.text_content();
                extract_file_paths(&text, &files_modified, &mut files_read);
            }
        }

        // Deduplicate
        files_modified.sort();
        files_modified.dedup();
        files_read.sort();
        files_read.dedup();
        files_read.retain(|f| !files_modified.contains(f));
        decisions.truncate(20);
        discoveries.truncate(12);
        errors.truncate(15);
        error_resolutions.truncate(10);

        // Build structured summary (OpenCode-style sections)
        let mut summary = String::new();
        let _ = writeln!(summary, "[Compacted {compacted_count} messages]\n");

        // § Goal
        if !goal_statements.is_empty() {
            let _ = writeln!(summary, "### Goal");
            for (i, g) in goal_statements.iter().enumerate().take(5) {
                let _ = writeln!(summary, "{}. {g}", i + 1);
            }
            summary.push('\n');
        }

        // § Discoveries
        if !discoveries.is_empty() {
            let _ = writeln!(summary, "### Discoveries");
            for d in &discoveries {
                let _ = writeln!(summary, "- {d}");
            }
            summary.push('\n');
        }

        // § Key Decisions
        if !decisions.is_empty() {
            let _ = writeln!(summary, "### Key Decisions");
            for d in &decisions {
                let _ = writeln!(summary, "- {d}");
            }
            summary.push('\n');
        }

        // § Files Modified
        if !files_modified.is_empty() {
            let _ = writeln!(summary, "### Files Modified");
            for f in files_modified.iter().take(30) {
                let _ = writeln!(summary, "- {f}");
            }
            summary.push('\n');
        }

        // § Files Read
        if !files_read.is_empty() {
            let count = files_read.len();
            let preview: Vec<_> = files_read.iter().take(15).map(|s| s.as_str()).collect();
            let _ = writeln!(summary, "### Files Read ({count})");
            let _ = writeln!(summary, "{}", preview.join(", "));
            if count > 15 {
                let _ = writeln!(summary, "... and {} more", count - 15);
            }
            summary.push('\n');
        }

        // § Tool Usage
        if tool_calls_count > 0 {
            let _ = writeln!(
                summary,
                "### Tool Usage: {tool_calls_count} calls, {tool_failures} failures\n"
            );
        }

        // § Errors & Resolutions
        if !errors.is_empty() {
            let _ = writeln!(summary, "### Errors Encountered");
            for (i, e) in errors.iter().enumerate() {
                let _ = writeln!(summary, "{}. {e}", i + 1);
                if let Some(res) = error_resolutions.get(i) {
                    let _ = writeln!(summary, "   → Resolution: {res}");
                }
            }
            summary.push('\n');
        }

        Message {
            role: "system".to_string(),
            content: Some(Content::text(summary)),
            reasoning_content: None,
            tool_calls: None,
            tool_call_id: None,
        }
    }

    /// S2C: Measure compaction quality by checking what information survived.
    ///
    /// Compares the compacted messages against the last `CompactionEvent` to
    /// measure how well file paths, decisions, and errors were preserved.
    // ── Stage 3: Relevance-Aware Smart Compaction ──────────────────────
    //
    /// Relevance-aware compaction pass.
    ///
    /// Algorithm (7 phases):
    ///   0. (dedup already ran in pipeline)
    ///   1. Group messages into turns (user-message boundaries).
    ///   2. Tokenize the most recent user message as the task query.
    ///   3. Always keep: `turn[0]`, `turn[-3]`, `turn[-2]`, `turn[-1]` (protect last 3 turns).
    ///   4. Score remaining turns (structural bonus + IDF relevance + recency).
    ///   5. Fill a verbatim budget (40% of target_tokens) with top-scoring turns.
    ///   6. Summarize the dropped turns with `build_structured_summary`.
    ///   7. Reassemble in original order: verbatim turns + summary for dropped spans.
    fn smart_compaction_pass(&mut self, target_tokens: usize) {
        if self.messages.len() < 8 {
            return;
        }

        // Phase 1
        let turns = group_into_turns(&self.messages);
        if turns.len() < 4 {
            return;
        }

        // Phase 2: query = most recent user message
        let query_tokens: std::collections::HashSet<String> = self
            .messages
            .iter()
            .rev()
            .find(|m| m.role == "user")
            .and_then(|m| m.content.as_ref().map(|c| c.text_content()))
            .map(|c| tokenize_for_scoring(&c).into_iter().collect())
            .unwrap_or_default();

        // Phase 3: always-keep set (first turn + last 3 turns)
        let always_keep: std::collections::HashSet<usize> = {
            let mut s = std::collections::HashSet::new();
            s.insert(0);
            let n = turns.len();
            for i in n.saturating_sub(3)..n {
                s.insert(i);
            }
            s
        };

        // Phase 4: score middle turns (IDF-weighted relevance)
        let idf_weights = build_idf_weights(&self.messages);
        let mut scored: Vec<(usize, f32)> = turns
            .iter()
            .enumerate()
            .filter(|(i, _)| !always_keep.contains(i))
            .map(|(i, t)| {
                (
                    i,
                    score_turn_with_idf(t, &self.messages, &query_tokens, Some(&idf_weights)),
                )
            })
            .collect();
        scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));

        // Phase 5: verbatim budget = 40% of target
        let verbatim_budget = (target_tokens as f32 * 0.4) as usize;
        let mut verbatim_used = 0usize;
        let mut verbatim_turns: std::collections::HashSet<usize> = always_keep.clone();
        for (turn_idx, _) in &scored {
            let turn_tokens: usize = turns[*turn_idx]
                .indices
                .iter()
                .map(|&mi| Self::estimate_message_tokens(&self.messages[mi]))
                .sum();
            if verbatim_used + turn_tokens <= verbatim_budget {
                verbatim_turns.insert(*turn_idx);
                verbatim_used += turn_tokens;
            }
        }

        // Phase 6: build summary from dropped turns
        let drop_turns: Vec<usize> = (0..turns.len())
            .filter(|i| !verbatim_turns.contains(i))
            .collect();
        if drop_turns.is_empty() {
            return;
        }

        let first_drop_msg = turns[drop_turns[0]].indices[0];
        let last_drop_turn_idx = *drop_turns.last().unwrap();
        let last_drop_msg = turns[last_drop_turn_idx]
            .indices
            .last()
            .copied()
            .unwrap_or(first_drop_msg)
            + 1;
        let summary = self.build_structured_summary(first_drop_msg, last_drop_msg);

        // Phase 7: reassemble in original order
        let mut new_messages: Vec<Message> = Vec::new();
        let mut summary_inserted = false;
        for (t_idx, turn) in turns.iter().enumerate() {
            if verbatim_turns.contains(&t_idx) {
                for &mi in &turn.indices {
                    new_messages.push(self.messages[mi].clone());
                }
            } else if !summary_inserted {
                new_messages.push(summary.clone());
                summary_inserted = true;
            }
        }

        if new_messages.len() < self.messages.len().saturating_sub(2) {
            self.messages = new_messages;
            self.cached_msg_tokens = self
                .messages
                .iter()
                .map(Self::estimate_message_tokens)
                .sum();
        }
    }
}

// ---------------------------------------------------------------------------
// Compaction helpers: SimHash, tokenizer, turn grouping, scoring
// ---------------------------------------------------------------------------

/// 64-bit SimHash fingerprint from a token list.
pub(crate) fn simhash(tokens: &[String]) -> u64 {
    use std::collections::hash_map::DefaultHasher;
    use std::hash::{Hash, Hasher};
    let mut v = [0i32; 64];
    for token in tokens {
        let mut h = DefaultHasher::new();
        token.hash(&mut h);
        let bits = h.finish();
        for i in 0..64u32 {
            if (bits >> i) & 1 == 1 {
                v[i as usize] += 1;
            } else {
                v[i as usize] -= 1;
            }
        }
    }
    let mut fp = 0u64;
    for (i, &val) in v.iter().enumerate() {
        if val > 0 {
            fp |= 1u64 << i;
        }
    }
    fp
}

/// Number of differing bits between two SimHash fingerprints.
/// < 8 ≈ near-duplicate (>87.5% similar).
#[inline]
pub(crate) fn hamming_distance(a: u64, b: u64) -> u32 {
    (a ^ b).count_ones()
}

/// Lightweight tokenizer for importance scoring.
/// Splits on non-alphanumeric chars, expands snake_case, removes stop words.
pub(crate) fn tokenize_for_scoring(content: &str) -> Vec<String> {
    const STOP: &[&str] = &[
        "the", "and", "for", "that", "this", "with", "from", "have", "will", "are", "was", "been",
        "has", "its", "but", "not", "you", "can", "use", "get", "set", "run", "let", "all",
    ];
    let mut tokens = Vec::new();
    for word in content.split(|c: char| !c.is_alphanumeric() && c != '_') {
        if word.len() < 3 || word.len() > 50 {
            continue;
        }
        let lower = word.to_lowercase();
        if STOP.contains(&lower.as_str()) {
            continue;
        }
        if lower.contains('_') {
            for part in lower.split('_') {
                if part.len() >= 3 {
                    tokens.push(part.to_string());
                }
            }
        } else {
            tokens.push(lower);
        }
    }
    tokens
}

/// A conversation "turn": starts at a user message and ends just before
/// the next user message. Stores indices into the message array.
pub(crate) struct Turn {
    pub(crate) indices: Vec<usize>,
}

/// Group messages into turns delimited by user messages.
pub(crate) fn group_into_turns(messages: &[Message]) -> Vec<Turn> {
    let mut turns: Vec<Turn> = Vec::new();
    let mut cur: Vec<usize> = Vec::new();
    for (i, msg) in messages.iter().enumerate() {
        if msg.role == "user" && !cur.is_empty() {
            turns.push(Turn { indices: cur });
            cur = Vec::new();
        }
        cur.push(i);
    }
    if !cur.is_empty() {
        turns.push(Turn { indices: cur });
    }
    turns
}

/// Build IDF (inverse document frequency) weights from all messages.
///
/// Rare tokens (specific identifiers) get higher weight than common tokens.
pub(crate) fn build_idf_weights(messages: &[Message]) -> std::collections::HashMap<String, f32> {
    use std::collections::{HashMap, HashSet};
    let mut doc_freq: HashMap<String, u32> = HashMap::new();
    let mut doc_count = 0u32;

    for msg in messages {
        if let Some(ref content) = msg.content {
            let text = content.text_content();
            let unique_tokens: HashSet<String> = tokenize_for_scoring(&text).into_iter().collect();
            for token in &unique_tokens {
                *doc_freq.entry(token.clone()).or_insert(0) += 1;
            }
            doc_count += 1;
        }
    }

    let n = doc_count.max(1) as f32;
    doc_freq
        .into_iter()
        .map(|(token, df)| {
            let idf = (n / df as f32).ln() + 1.0; // smoothed IDF
            (token, idf)
        })
        .collect()
}
/// Score a turn with optional precomputed IDF weights.
pub(crate) fn score_turn_with_idf(
    turn: &Turn,
    messages: &[Message],
    query_tokens: &std::collections::HashSet<String>,
    idf_weights: Option<&std::collections::HashMap<String, f32>>,
) -> f32 {
    let n = messages.len();
    let last_idx = *turn.indices.last().unwrap_or(&0);
    let mut score = 0.0f32;

    for &i in &turn.indices {
        let msg = &messages[i];

        // Structural bonuses
        match msg.role.as_str() {
            "user" => score += 2.0,
            "assistant" => {
                if let Some(ref tcs) = msg.tool_calls {
                    for tc in tcs {
                        match tc.function.name.as_str() {
                            "file_write" | "file_edit" | "git_patch" => score += 3.0,
                            _ => score += 0.3,
                        }
                    }
                }
            }
            "tool" => {
                if let Some(ref c) = msg.content {
                    let text = c.text_content();
                    if text.starts_with("Error:") || text.starts_with("error:") {
                        score += 2.0; // error resolution history is valuable
                    }
                }
            }
            _ => {}
        }

        // IDF-weighted relevance against current query
        if let Some(ref c) = msg.content {
            let msg_tokens: std::collections::HashSet<String> =
                tokenize_for_scoring(&c.text_content())
                    .into_iter()
                    .collect();
            let overlap: Vec<&String> = query_tokens.intersection(&msg_tokens).collect();
            if !query_tokens.is_empty() && !overlap.is_empty() {
                match idf_weights {
                    Some(weights) => {
                        let weighted_sum: f32 = overlap
                            .iter()
                            .map(|t| weights.get(*t).copied().unwrap_or(1.0))
                            .sum();
                        let max_possible: f32 = query_tokens
                            .iter()
                            .map(|t| weights.get(t).copied().unwrap_or(1.0))
                            .sum();
                        score += (weighted_sum / max_possible) * 5.0;
                    }
                    None => {
                        score += (overlap.len() as f32 / query_tokens.len() as f32) * 4.0;
                    }
                }
            }
        }
    }

    // Recency weight: oldest = 0.2×, newest = 1.0× (linear)
    let recency = last_idx as f32 / n.max(1) as f32;
    score * (0.2 + 0.8 * recency)
}

// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------

pub(crate) fn truncate_str(s: &str, max_bytes: usize) -> String {
    if s.len() <= max_bytes {
        s.to_string()
    } else {
        // Find the last valid char boundary at or before max_bytes
        let mut end = max_bytes;
        while end > 0 && !s.is_char_boundary(end) {
            end -= 1;
        }
        format!("{}...", &s[..end])
    }
}

/// Extract file paths from text content (any extension).
pub(crate) fn extract_file_paths(content: &str, modified: &[String], read: &mut Vec<String>) {
    for word in content.split_whitespace() {
        let clean: String = word
            .trim_matches(|c: char| {
                matches!(
                    c,
                    '"' | '\'' | '`' | ',' | ';' | '(' | ')' | '[' | ']' | '{' | '}'
                )
            })
            .to_string();

        if clean.len() < 3 || clean.len() > 200 {
            continue;
        }
        if !clean.contains('/') && !clean.contains('.') {
            continue;
        }
        if let Some(dot_pos) = clean.rfind('.') {
            let ext = &clean[dot_pos + 1..];
            if !ext.is_empty()
                && ext.len() <= 10
                && ext.chars().all(|c| c.is_ascii_alphanumeric())
                && !clean.starts_with("http")
                && clean.contains('/')
                && !modified.contains(&clean)
                && !read.contains(&clean)
            {
                read.push(clean);
            }
        }
    }
}