terraphim_service/

lib.rs

//! Main service layer for Terraphim AI.
//!
//! Provides document search, indexing, and AI-assisted summarisation across
//! multiple haystack backends. Integrates the knowledge graph, thesaurus,
//! and relevance-scoring pipeline into a single async service facade.
use ahash::AHashMap;
use terraphim_automata::builder::{Logseq, ThesaurusBuilder, compute_kg_source_hash};
use terraphim_automata::load_thesaurus;
use terraphim_automata::{LinkType, replace_matches};
use terraphim_config::{ConfigState, Role};
use terraphim_middleware::thesaurus::build_thesaurus_from_haystack;
use terraphim_persistence::Persistable;
use terraphim_rolegraph::{RoleGraph, RoleGraphSync};
use terraphim_types::{
    Document, Index, IndexedDocument, Layer, NormalizedTermValue, RelevanceFunction, RoleName,
    SearchQuery, Thesaurus,
};
mod score;
use crate::score::Query;

pub mod auto_route;
pub use auto_route::{
    AutoRouteContext, AutoRouteReason, AutoRouteResult, JMAP_MISSING_TOKEN_PENALTY,
    auto_select_role,
};

#[cfg(feature = "openrouter")]
pub mod openrouter;

// Generic LLM layer for multiple providers (OpenRouter, Ollama, etc.)
pub mod llm;

// LLM proxy service for unified provider management

// LLM Proxy service\npub mod proxy_client;
// LLM Router configuration integration\n

pub mod llm_proxy;

// LLM Router configuration integration\n

// Centralized HTTP client creation and configuration
pub mod http_client;

// Standardized logging initialization utilities
pub mod logging;

// Summarization queue system for production-ready async processing
pub mod conversation_service;
pub mod rate_limiter;
pub mod summarization_manager;
pub mod summarization_queue;
pub mod summarization_worker;

// Centralized error handling patterns and utilities
pub mod error;

// Context management for LLM conversations
pub mod context;

#[cfg(test)]
mod context_tests;

/// Normalize a filename to be used as a document ID
///
/// This ensures consistent ID generation between server startup and edit API
fn normalize_filename_to_id(filename: &str) -> String {
    let re = regex::Regex::new(r"[^a-zA-Z0-9]+").expect("Failed to create regex");
    re.replace_all(filename, "").to_lowercase()
}

/// Top-level error type for the Terraphim service layer.
#[derive(thiserror::Error, Debug)]
pub enum ServiceError {
    #[error("Middleware error: {0}")]
    Middleware(#[from] terraphim_middleware::Error),

    #[error("OpenDal error: {0}")]
    OpenDal(Box<opendal::Error>),

    #[error("Persistence error: {0}")]
    Persistence(#[from] terraphim_persistence::Error),

    #[error("Config error: {0}")]
    Config(String),

    #[cfg(feature = "openrouter")]
    #[error("OpenRouter error: {0}")]
    OpenRouter(#[from] crate::openrouter::OpenRouterError),

    #[error("Common error: {0}")]
    Common(#[from] crate::error::CommonError),
}

impl From<opendal::Error> for ServiceError {
    fn from(err: opendal::Error) -> Self {
        ServiceError::OpenDal(Box::new(err))
    }
}

impl crate::error::TerraphimError for ServiceError {
    fn category(&self) -> crate::error::ErrorCategory {
        use crate::error::ErrorCategory;
        match self {
            ServiceError::Middleware(_) => ErrorCategory::Integration,
            ServiceError::OpenDal(_) => ErrorCategory::Storage,
            ServiceError::Persistence(_) => ErrorCategory::Storage,
            ServiceError::Config(_) => ErrorCategory::Configuration,
            #[cfg(feature = "openrouter")]
            ServiceError::OpenRouter(_) => ErrorCategory::Integration,
            ServiceError::Common(err) => err.category(),
        }
    }

    fn is_recoverable(&self) -> bool {
        match self {
            ServiceError::Middleware(_) => true,
            ServiceError::OpenDal(_) => false,
            ServiceError::Persistence(_) => false,
            ServiceError::Config(_) => false,
            #[cfg(feature = "openrouter")]
            ServiceError::OpenRouter(_) => true,
            ServiceError::Common(err) => err.is_recoverable(),
        }
    }
}

pub type Result<T> = std::result::Result<T, ServiceError>;

/// Main entry point for search, indexing, and AI operations in Terraphim.
pub struct TerraphimService {
    config_state: ConfigState,
}

impl TerraphimService {
    /// Create a new TerraphimService
    pub fn new(config_state: ConfigState) -> Self {
        Self { config_state }
    }

    /// Build a thesaurus from the haystack and update the knowledge graph automata URL
    async fn build_thesaurus(&mut self, search_query: &SearchQuery) -> Result<()> {
        Ok(build_thesaurus_from_haystack(&mut self.config_state, search_query).await?)
    }
    /// load thesaurus from config object and if absent make sure it's loaded from automata_url
    pub async fn ensure_thesaurus_loaded(&mut self, role_name: &RoleName) -> Result<Thesaurus> {
        async fn load_thesaurus_from_automata_path(
            config_state: &ConfigState,
            role_name: &RoleName,
            rolegraphs: &mut AHashMap<RoleName, RoleGraphSync>,
        ) -> Result<Thesaurus> {
            // CRITICAL: clone the role out, then drop the config lock before
            // doing I/O. Holding the lock across the network/disk operations
            // below blocks every other endpoint that touches /config (e.g.
            // GET /config from `roles select`, `roles list`, etc.) for the
            // duration of the thesaurus load + persistence + RoleGraph build.
            let role = {
                let config = config_state.config.lock().await;
                let Some(role) = config.roles.get(role_name).cloned() else {
                    return Err(ServiceError::Config(format!(
                        "Role '{}' not found in config",
                        role_name
                    )));
                };
                role
            };
            if let Some(kg) = &role.kg {
                if let Some(automata_path) = &kg.automata_path {
                    log::info!("Loading Role `{}` - URL: {:?}", role_name, automata_path);

                    // Try to load from automata path first
                    match load_thesaurus(automata_path).await {
                        Ok(mut thesaurus) => {
                            log::info!("Successfully loaded thesaurus from automata path");

                            // Save thesaurus to persistence to ensure it's available for future loads
                            match thesaurus.save().await {
                                Ok(_) => {
                                    log::info!(
                                        "Thesaurus for role `{}` saved to persistence",
                                        role_name
                                    );
                                    // Reload from persistence to get canonical version
                                    match thesaurus.load().await {
                                        Ok(persisted_thesaurus) => {
                                            thesaurus = persisted_thesaurus;
                                            log::debug!("Reloaded thesaurus from persistence");
                                        }
                                        Err(e) => {
                                            log::warn!(
                                                "Failed to reload thesaurus from persistence, using in-memory version: {:?}",
                                                e
                                            );
                                        }
                                    }
                                }
                                Err(e) => {
                                    log::warn!("Failed to save thesaurus to persistence: {:?}", e);
                                }
                            }

                            let rolegraph =
                                RoleGraph::new(role_name.clone(), thesaurus.clone()).await;
                            match rolegraph {
                                Ok(rolegraph) => {
                                    let rolegraph_value = RoleGraphSync::from(rolegraph);
                                    rolegraphs.insert(role_name.clone(), rolegraph_value);
                                }
                                Err(e) => {
                                    log::error!("Failed to update role and thesaurus: {:?}", e)
                                }
                            }
                            Ok(thesaurus)
                        }
                        Err(e) => {
                            log::warn!("Failed to load thesaurus from automata path: {:?}", e);
                            // Fallback to building from local KG if available
                            if let Some(kg_local) = &kg.knowledge_graph_local {
                                log::info!(
                                    "Fallback: building thesaurus from local KG for role {}",
                                    role_name
                                );
                                let logseq_builder = Logseq::default();
                                match logseq_builder
                                    .build(
                                        role_name.as_lowercase().to_string(),
                                        kg_local.path.clone(),
                                    )
                                    .await
                                {
                                    Ok(mut thesaurus) => {
                                        // Save thesaurus to persistence to ensure it's available for future loads
                                        match thesaurus.save().await {
                                            Ok(_) => {
                                                log::info!(
                                                    "Fallback thesaurus for role `{}` saved to persistence",
                                                    role_name
                                                );
                                                // Reload from persistence to get canonical version
                                                match thesaurus.load().await {
                                                    Ok(persisted_thesaurus) => {
                                                        thesaurus = persisted_thesaurus;
                                                        log::debug!(
                                                            "Reloaded fallback thesaurus from persistence"
                                                        );
                                                    }
                                                    Err(e) => {
                                                        log::warn!(
                                                            "Failed to reload fallback thesaurus from persistence, using in-memory version: {:?}",
                                                            e
                                                        );
                                                    }
                                                }
                                            }
                                            Err(e) => {
                                                log::warn!(
                                                    "Failed to save fallback thesaurus to persistence: {:?}",
                                                    e
                                                );
                                            }
                                        }

                                        let rolegraph =
                                            RoleGraph::new(role_name.clone(), thesaurus.clone())
                                                .await;
                                        match rolegraph {
                                            Ok(rolegraph) => {
                                                let rolegraph_value =
                                                    RoleGraphSync::from(rolegraph);
                                                rolegraphs
                                                    .insert(role_name.clone(), rolegraph_value);
                                            }
                                            Err(e) => log::error!(
                                                "Failed to update role and thesaurus: {:?}",
                                                e
                                            ),
                                        }

                                        Ok(thesaurus)
                                    }
                                    Err(e) => {
                                        // Check if error is "file not found" (expected for optional files)
                                        // and downgrade log level from ERROR to DEBUG
                                        let is_file_not_found =
                                            e.to_string().contains("file not found")
                                                || e.to_string().contains("not found:");

                                        if is_file_not_found {
                                            log::debug!(
                                                "Failed to build thesaurus from local KG (optional file not found) for role {}: {:?}",
                                                role_name,
                                                e
                                            );
                                        } else {
                                            log::error!(
                                                "Failed to build thesaurus from local KG for role {}: {:?}",
                                                role_name,
                                                e
                                            );
                                        }
                                        Err(ServiceError::Config(
                                            "Failed to load or build thesaurus".into(),
                                        ))
                                    }
                                }
                            } else {
                                log::warn!(
                                    "No fallback available for role {}: no local KG path configured, returning empty thesaurus",
                                    role_name
                                );
                                Ok(Thesaurus::new(role_name.as_lowercase().to_string()))
                            }
                        }
                    }
                } else if let Some(kg_local) = &kg.knowledge_graph_local {
                    // Build thesaurus from local KG
                    log::info!(
                        "Role {} has no automata_path, building thesaurus from local KG files at {:?}",
                        role_name,
                        kg_local.path
                    );
                    let logseq_builder = Logseq::default();
                    match logseq_builder
                        .build(role_name.as_lowercase().to_string(), kg_local.path.clone())
                        .await
                    {
                        Ok(mut thesaurus) => {
                            log::info!(
                                "Successfully built thesaurus from local KG for role {}",
                                role_name
                            );

                            // Save thesaurus to persistence to ensure it's available for future loads
                            match thesaurus.save().await {
                                Ok(_) => {
                                    log::info!(
                                        "Local KG thesaurus for role `{}` saved to persistence",
                                        role_name
                                    );
                                    // Reload from persistence to get canonical version
                                    match thesaurus.load().await {
                                        Ok(persisted_thesaurus) => {
                                            log::info!(
                                                "Reloaded local KG thesaurus from persistence: {} entries",
                                                persisted_thesaurus.len()
                                            );
                                            thesaurus = persisted_thesaurus;
                                        }
                                        Err(e) => {
                                            log::warn!(
                                                "Failed to reload local KG thesaurus from persistence, using in-memory version: {:?}",
                                                e
                                            );
                                        }
                                    }
                                }
                                Err(e) => {
                                    log::warn!(
                                        "Failed to save local KG thesaurus to persistence: {:?}",
                                        e
                                    );
                                }
                            }

                            let rolegraph =
                                RoleGraph::new(role_name.clone(), thesaurus.clone()).await;
                            match rolegraph {
                                Ok(rolegraph) => {
                                    let rolegraph_value = RoleGraphSync::from(rolegraph);
                                    rolegraphs.insert(role_name.clone(), rolegraph_value);
                                }
                                Err(e) => {
                                    log::error!("Failed to update role and thesaurus: {:?}", e)
                                }
                            }

                            Ok(thesaurus)
                        }
                        Err(e) => {
                            // Check if error is "file not found" (expected for optional files)
                            // and downgrade log level from ERROR to DEBUG
                            let is_file_not_found = e.to_string().contains("file not found");

                            if is_file_not_found {
                                log::debug!(
                                    "Failed to build thesaurus from local KG (optional file not found) for role {}: {:?}",
                                    role_name,
                                    e
                                );
                            } else {
                                log::error!(
                                    "Failed to build thesaurus from local KG for role {}: {:?}",
                                    role_name,
                                    e
                                );
                            }
                            Err(ServiceError::Config(format!(
                                "Failed to build thesaurus from local KG for role {}: {}",
                                role_name, e
                            )))
                        }
                    }
                } else {
                    log::warn!(
                        "Role {} is configured for TerraphimGraph but has neither automata_path nor knowledge_graph_local defined.",
                        role_name
                    );
                    if let Some(kg_local) = &kg.knowledge_graph_local {
                        // Build thesaurus from local KG files during startup
                        log::info!(
                            "Building thesaurus from local KG files for role {} at {:?}",
                            role_name,
                            kg_local.path
                        );
                        let logseq_builder = Logseq::default();
                        match logseq_builder
                            .build(role_name.as_lowercase().to_string(), kg_local.path.clone())
                            .await
                        {
                            Ok(mut thesaurus) => {
                                log::info!(
                                    "Successfully built thesaurus from local KG for role {}",
                                    role_name
                                );

                                // Save thesaurus to persistence to ensure it's available for future loads
                                match thesaurus.save().await {
                                    Ok(_) => {
                                        log::info!(
                                            "No-automata thesaurus for role `{}` saved to persistence",
                                            role_name
                                        );
                                        // Reload from persistence to get canonical version
                                        match thesaurus.load().await {
                                            Ok(persisted_thesaurus) => {
                                                thesaurus = persisted_thesaurus;
                                                log::debug!(
                                                    "Reloaded no-automata thesaurus from persistence"
                                                );
                                            }
                                            Err(e) => {
                                                log::warn!(
                                                    "Failed to reload no-automata thesaurus from persistence, using in-memory version: {:?}",
                                                    e
                                                );
                                            }
                                        }
                                    }
                                    Err(e) => {
                                        log::warn!(
                                            "Failed to save no-automata thesaurus to persistence: {:?}",
                                            e
                                        );
                                    }
                                }

                                let rolegraph =
                                    RoleGraph::new(role_name.clone(), thesaurus.clone()).await;
                                match rolegraph {
                                    Ok(rolegraph) => {
                                        let rolegraph_value = RoleGraphSync::from(rolegraph);
                                        rolegraphs.insert(role_name.clone(), rolegraph_value);
                                    }
                                    Err(e) => {
                                        // Check if error is "file not found" (expected for optional files)
                                        // and downgrade log level from ERROR to DEBUG
                                        let is_file_not_found =
                                            e.to_string().contains("file not found");

                                        if is_file_not_found {
                                            log::debug!(
                                                "Failed to update role and thesaurus (optional file not found): {:?}",
                                                e
                                            );
                                        } else {
                                            log::error!(
                                                "Failed to update role and thesaurus: {:?}",
                                                e
                                            );
                                        }
                                    }
                                }

                                Ok(thesaurus)
                            }
                            Err(e) => {
                                log::error!(
                                    "Failed to build thesaurus from local KG for role {}: {:?}",
                                    role_name,
                                    e
                                );
                                Err(ServiceError::Config(
                                    "Failed to build thesaurus from local KG".into(),
                                ))
                            }
                        }
                    } else {
                        log::debug!(
                            "Role '{}' has no local KG path, returning empty thesaurus",
                            role_name
                        );
                        Ok(Thesaurus::new(role_name.as_lowercase().to_string()))
                    }
                }
            } else {
                log::debug!("Role '{}' has no knowledge graph configured", role_name);
                Err(ServiceError::Config(format!(
                    "Knowledge graph not configured for role '{}'",
                    role_name
                )))
            }
        }

        log::debug!("Loading thesaurus for role: {}", role_name);
        log::debug!("Role keys {:?}", self.config_state.roles.keys());

        if let Some(rolegraph_value) = self.config_state.roles.get(role_name) {
            let thesaurus_result = rolegraph_value.lock().await.thesaurus.clone().load().await;
            match thesaurus_result {
                Ok(thesaurus) => {
                    log::debug!("Thesaurus loaded: {:?}", thesaurus);
                    log::info!("Rolegraph loaded: for role name {:?}", role_name);

                    // Check if the cached thesaurus is stale by comparing source hashes
                    let is_stale = if let Some(ref cached_hash) = thesaurus.source_hash {
                        let role = {
                            let config = self.config_state.config.lock().await;
                            config.roles.get(role_name).cloned()
                        };
                        if let Some(role) = role {
                            if let Some(ref kg) = role.kg {
                                if let Some(ref kg_local) = kg.knowledge_graph_local {
                                    match compute_kg_source_hash(&kg_local.path) {
                                        Ok(Some(current_hash)) => {
                                            let stale = current_hash != *cached_hash;
                                            if stale {
                                                log::info!(
                                                    "Thesaurus cache stale for role '{}': hash mismatch (cached {} != current {})",
                                                    role_name,
                                                    cached_hash,
                                                    current_hash
                                                );
                                            }
                                            stale
                                        }
                                        Ok(None) => {
                                            log::debug!(
                                                "No markdown files found in KG path {:?}",
                                                kg_local.path
                                            );
                                            false
                                        }
                                        Err(e) => {
                                            log::warn!(
                                                "Failed to compute source hash for role '{}': {}",
                                                role_name,
                                                e
                                            );
                                            false
                                        }
                                    }
                                } else {
                                    false
                                }
                            } else {
                                false
                            }
                        } else {
                            false
                        }
                    } else {
                        log::debug!(
                            "No source_hash in cached thesaurus for role '{}'",
                            role_name
                        );
                        false
                    };

                    if is_stale {
                        let mut rolegraphs = self.config_state.roles.clone();
                        let result = load_thesaurus_from_automata_path(
                            &self.config_state,
                            role_name,
                            &mut rolegraphs,
                        )
                        .await;

                        if result.is_ok() {
                            if let Some(updated_rolegraph) = rolegraphs.get(role_name) {
                                self.config_state
                                    .roles
                                    .insert(role_name.clone(), updated_rolegraph.clone());
                                log::info!(
                                    "Updated config_state with rebuilt rolegraph for role: {}",
                                    role_name
                                );
                            }
                        }
                        result
                    } else {
                        Ok(thesaurus)
                    }
                }
                Err(e) => {
                    // Check if error is "file not found" (expected for optional files)
                    // and downgrade log level from ERROR to DEBUG
                    let is_file_not_found = e.to_string().contains("file not found")
                        || e.to_string().contains("not found:");

                    if is_file_not_found {
                        log::debug!("Thesaurus file not found (optional): {:?}", e);
                    } else {
                        log::error!("Failed to load thesaurus: {:?}", e);
                    }
                    // Try to build thesaurus from KG and update the config_state directly
                    let mut rolegraphs = self.config_state.roles.clone();
                    let result = load_thesaurus_from_automata_path(
                        &self.config_state,
                        role_name,
                        &mut rolegraphs,
                    )
                    .await;

                    // Update the actual config_state with the new rolegraph
                    if result.is_ok() {
                        if let Some(updated_rolegraph) = rolegraphs.get(role_name) {
                            self.config_state
                                .roles
                                .insert(role_name.clone(), updated_rolegraph.clone());
                            log::info!(
                                "Updated config_state with new rolegraph for role: {}",
                                role_name
                            );
                        }
                    }

                    result
                }
            }
        } else {
            // Role not found, try to build from KG
            let mut rolegraphs = self.config_state.roles.clone();
            let result =
                load_thesaurus_from_automata_path(&self.config_state, role_name, &mut rolegraphs)
                    .await;

            // Update the actual config_state with the new rolegraph
            if result.is_ok() {
                if let Some(new_rolegraph) = rolegraphs.get(role_name) {
                    self.config_state
                        .roles
                        .insert(role_name.clone(), new_rolegraph.clone());
                    log::info!(
                        "Added new rolegraph to config_state for role: {}",
                        role_name
                    );
                }
            }

            result
        }
    }

    /// Preprocess document content to create clickable KG links when terraphim_it is enabled
    ///
    /// This function replaces KG terms in the document body with markdown links
    /// in the format `[term](kg:term)` which can be intercepted by the frontend
    /// to display KG documents when clicked.
    pub async fn preprocess_document_content(
        &mut self,
        mut document: Document,
        role: &Role,
    ) -> Result<Document> {
        // Only preprocess if terraphim_it is enabled and role has KG configured
        if !role.terraphim_it {
            log::info!(
                "🔍 terraphim_it disabled for role '{}', skipping KG preprocessing",
                role.name
            );
            return Ok(document);
        }

        let Some(_kg) = &role.kg else {
            log::info!(
                "⚠️ No KG configured for role '{}', skipping KG preprocessing",
                role.name
            );
            return Ok(document);
        };

        log::info!(
            "🧠 Starting KG preprocessing for document '{}' in role '{}' (terraphim_it enabled)",
            document.title,
            role.name
        );
        log::debug!(
            "📄 Document preview: {} characters starting with: {}",
            document.body.len(),
            &document.body.chars().take(100).collect::<String>()
        );

        // Load thesaurus for the role
        let thesaurus = match self.ensure_thesaurus_loaded(&role.name).await {
            Ok(thesaurus) => thesaurus,
            Err(e) => {
                log::warn!("Failed to load thesaurus for role {}: {:?}", role.name, e);
                return Ok(document); // Return original document if thesaurus fails to load
            }
        };

        // Filter thesaurus to only include meaningful terms and avoid over-linking
        let mut kg_thesaurus = Thesaurus::new(format!("kg_links_{}", role.name));

        // Prioritize important KG terms while excluding overly generic ones
        // Key KG concepts should always be included even if they're common
        let important_kg_terms = [
            "graph",
            "haystack",
            "service",
            "terraphim",
            "knowledge",
            "embedding",
            "search",
            "automata",
            "thesaurus",
            "rolegraph",
        ];

        // Exclude only very generic programming/technical terms that don't add value
        let excluded_common_terms = [
            "system",
            "config",
            "configuration",
            "type",
            "method",
            "function",
            "class",
            "component",
            "module",
            "library",
            "framework",
            "interface",
            "api",
            "data",
            "file",
            "path",
            "url",
            "string",
            "number",
            "value",
            "option",
            "parameter",
            "field",
            "property",
            "attribute",
            "element",
            "item",
            "object",
            "array",
            "list",
            "map",
            "set",
            "collection",
            "server",
            "client",
            "request",
            "response",
            "error",
            "result",
            "success",
            "failure",
            "true",
            "false",
            "null",
            "undefined",
            "empty",
            "full",
            "start",
            "end",
            "begin",
            "finish",
            "create",
            "delete",
            "update",
            "read",
            "write",
            "load",
            "save",
            "process",
            "handle",
            "manage",
            "control",
            "execute",
            "run",
            "call",
            "invoke",
            "trigger",
            "event",
            "action",
            "command",
            "query",
            "search",
            "filter",
            "sort",
            "order",
            "group",
            "match",
            "find",
            "replace",
            "insert",
            "remove",
            "add",
            "set",
            "get",
            "put",
            "post",
            "head",
            "patch",
            "delete",
        ];

        let mut sorted_terms: Vec<_> = (&thesaurus)
            .into_iter()
            .filter(|(key, _)| {
                let term = key.as_str();

                // Always exclude empty or very short terms
                if term.is_empty() || term.len() < 3 {
                    return false;
                }

                // Always include important KG terms, even if they're short
                if important_kg_terms.contains(&term) {
                    return true;
                }

                // Exclude generic technical terms
                if excluded_common_terms.contains(&term) {
                    return false;
                }

                // Include terms that are:
                // 1. Moderately long (>5 chars) OR
                // 2. Hyphenated compound terms OR
                // 3. Underscore-separated compound terms OR
                // 4. Capitalized terms (likely proper nouns or important concepts)
                term.len() > 5
                    || term.contains('-')
                    || term.contains('_')
                    || term.chars().next().is_some_and(|c| c.is_uppercase())
            })
            .collect();

        // Sort by relevance, but prioritize important KG terms
        #[allow(clippy::unnecessary_sort_by)]
        sorted_terms.sort_by(|a, b| {
            let a_important = important_kg_terms.contains(&a.0.as_str());
            let b_important = important_kg_terms.contains(&b.0.as_str());

            match (a_important, b_important) {
                (true, false) => std::cmp::Ordering::Less, // a comes first
                (false, true) => std::cmp::Ordering::Greater, // b comes first
                _ => b.1.id.cmp(&a.1.id),                  // Both or neither important, sort by ID
            }
        });

        // Take more terms since we're being more selective about quality
        let max_kg_terms = 8;
        for (key, value) in sorted_terms.into_iter().take(max_kg_terms) {
            let mut kg_value = value.clone();
            // IMPORTANT: Keep the original term (key) as visible text, link to root concept (value.value)
            // This creates links like: [graph embeddings](kg:terraphim-graph)
            // where "graph embeddings" stays visible but links to the root concept "terraphim-graph"
            kg_value.value = key.clone(); // Keep original term as visible text
            kg_value.url = Some(format!("kg:{}", value.value)); // Link to the root concept
            kg_thesaurus.insert(key.clone(), kg_value);
        }

        let kg_terms_count = kg_thesaurus.len();
        log::info!(
            "📋 KG thesaurus filtering: {} → {} terms (prioritizing: {}, filters: len>5, hyphenated, or important KG terms)",
            thesaurus.len(),
            kg_terms_count,
            important_kg_terms.join(", ")
        );

        // Log the actual terms that passed filtering for debugging
        if kg_terms_count > 0 {
            let terms: Vec<String> = (&kg_thesaurus)
                .into_iter()
                .map(|(k, v)| format!("'{}' → kg:{}", k, v.value))
                .collect();
            log::info!("🔍 KG terms selected for linking: {}", terms.join(", "));
        } else {
            log::info!(
                "⚠️ No KG terms passed filtering criteria - document '{}' will have no KG links",
                document.title
            );
        }

        // Apply KG term replacement to document body (only if we have terms to replace)
        if !kg_thesaurus.is_empty() {
            // Debug: log what we're about to pass to replace_matches
            let debug_thesaurus: Vec<String> = (&kg_thesaurus)
                .into_iter()
                .map(|(k, v)| format!("'{}' -> '{}' (url: {:?})", k, v.value, v.url))
                .take(3) // Limit to first 3 entries to avoid spam
                .collect();
            log::info!(
                "🔧 Passing to replace_matches: {} (total terms: {})",
                debug_thesaurus.join(", "),
                kg_thesaurus.len()
            );
            let preview = if document.body.chars().count() > 200 {
                document.body.chars().take(200).collect::<String>() + "..."
            } else {
                document.body.clone()
            };
            log::info!("📝 Document body preview (first 200 chars): {}", preview);

            match replace_matches(&document.body, kg_thesaurus, LinkType::MarkdownLinks) {
                Ok(processed_bytes) => {
                    match String::from_utf8(processed_bytes) {
                        Ok(processed_content) => {
                            log::info!(
                                "✅ Successfully preprocessed document '{}' with {} KG terms → created [term](kg:concept) links",
                                document.title,
                                kg_terms_count
                            );

                            // Debug: Check if content actually changed
                            let content_changed = processed_content != document.body;
                            log::info!(
                                "🔄 Content changed: {} (original: {} chars, processed: {} chars)",
                                content_changed,
                                document.body.len(),
                                processed_content.len()
                            );

                            // Debug: Show actual KG links in the processed content
                            let kg_links: Vec<&str> = processed_content
                                .split("[")
                                .filter_map(|s| s.find("](kg:").map(|closing| &s[..closing]))
                                .collect();

                            if !kg_links.is_empty() {
                                log::info!(
                                    "🔗 Found KG links in processed content: [{}](kg:...)",
                                    kg_links.join("], [")
                                );

                                let snippet = snippet_around(&processed_content, "](kg:", 50, 100);
                                if !snippet.is_empty() {
                                    log::info!(
                                        "📄 Content snippet with KG link: ...{}...",
                                        snippet
                                    );
                                }
                            } else {
                                log::warn!(
                                    "⚠️ No KG links found in processed content despite successful replacement"
                                );
                            }

                            document.body = processed_content;
                        }
                        Err(e) => {
                            log::warn!(
                                "Failed to convert processed content to UTF-8 for document '{}': {:?}",
                                document.title,
                                e
                            );
                        }
                    }
                }
                Err(e) => {
                    log::warn!(
                        "Failed to replace KG terms in document '{}': {:?}",
                        document.title,
                        e
                    );
                }
            }
        } else {
            log::info!(
                "💭 No specific KG terms found for document '{}' (filters excluded generic terms)",
                document.title
            );
        }

        Ok(document)
    }

    /// Preprocess document content with both KG linking and search term highlighting
    pub async fn preprocess_document_content_with_search(
        &mut self,
        document: Document,
        role: &Role,
        search_query: Option<&SearchQuery>,
    ) -> Result<Document> {
        // First apply KG preprocessing if enabled
        let mut processed_doc = self.preprocess_document_content(document, role).await?;

        // Then apply search term highlighting if query is provided
        if let Some(query) = search_query {
            log::debug!(
                "Applying search term highlighting to document '{}'",
                processed_doc.title
            );
            processed_doc.body = Self::highlight_search_terms(&processed_doc.body, query);
        }

        Ok(processed_doc)
    }

    /// Create document
    pub async fn create_document(&mut self, document: Document) -> Result<Document> {
        // Persist the document using the fastest available Operator. The document becomes
        // available on all profiles/devices thanks to the Persistable implementation.
        document.save().await?;

        // Index the freshly-saved document inside all role graphs so it can be discovered via
        // search immediately.
        self.config_state.add_to_roles(&document).await?;

        // 🔄 Persist the updated body back to on-disk Markdown files for every writable
        // ripgrep haystack so that subsequent searches (and external tooling) see the
        // changes instantly.
        use terraphim_config::ServiceType;
        use terraphim_middleware::indexer::RipgrepIndexer;

        let ripgrep = RipgrepIndexer::default();
        let config_snapshot = { self.config_state.config.lock().await.clone() };

        for role in config_snapshot.roles.values() {
            for haystack in &role.haystacks {
                if haystack.service == ServiceType::Ripgrep && !haystack.read_only {
                    if let Err(e) = ripgrep.update_document(&document).await {
                        log::warn!(
                            "Failed to write document {} to haystack {:?}: {:?}",
                            document.id,
                            haystack.location,
                            e
                        );
                    }
                }
            }
        }

        Ok(document)
    }

    /// Get document by ID
    ///
    /// This method supports both normalized IDs (e.g., "haystackmd") and original filenames (e.g., "haystack.md").
    /// It tries to find the document using the provided ID first, then tries with a normalized version,
    /// and finally falls back to searching by title.
    pub async fn get_document_by_id(&mut self, document_id: &str) -> Result<Option<Document>> {
        log::debug!("Getting document by ID: '{}'", document_id);

        // Validate document_id is not empty or whitespace-only
        if document_id.trim().is_empty() {
            log::warn!("Empty or whitespace-only document_id provided");
            return Ok(None);
        }

        // 1️⃣ Try to load the document directly using the provided ID
        let mut placeholder = Document {
            id: document_id.to_string(),
            ..Default::default()
        };
        match placeholder.load().await {
            Ok(doc) => {
                log::debug!("Found document '{}' with direct ID lookup", document_id);
                return self.apply_kg_preprocessing_if_needed(doc).await.map(Some);
            }
            Err(e) => {
                log::debug!(
                    "Document '{}' not found with direct lookup: {:?}",
                    document_id,
                    e
                );
            }
        }

        // 2️⃣ If the provided ID looks like a filename, try with normalized ID
        if document_id.contains('.') || document_id.contains('-') || document_id.contains('_') {
            let normalized_id = normalize_filename_to_id(document_id);
            log::debug!(
                "Trying normalized ID '{}' for filename '{}'",
                normalized_id,
                document_id
            );

            let mut normalized_placeholder = Document {
                id: normalized_id.clone(),
                ..Default::default()
            };
            match normalized_placeholder.load().await {
                Ok(doc) => {
                    log::debug!(
                        "Found document '{}' with normalized ID '{}'",
                        document_id,
                        normalized_id
                    );
                    return self.apply_kg_preprocessing_if_needed(doc).await.map(Some);
                }
                Err(e) => {
                    log::debug!(
                        "Document '{}' not found with normalized ID '{}': {:?}",
                        document_id,
                        normalized_id,
                        e
                    );
                }
            }
        }

        // 3️⃣ Fallback: search by title (for documents where title contains the original filename)
        log::debug!("Falling back to search for document '{}'", document_id);
        let search_query = SearchQuery {
            search_term: NormalizedTermValue::new(document_id.to_string()),
            search_terms: None,
            operator: None,
            limit: Some(5), // Get a few results to check titles
            skip: None,
            role: None,
            layer: Layer::default(),
            include_pinned: false,
            min_quality: None,
        };

        let documents = self.search(&search_query).await?;

        // Look for a document whose title matches the requested ID
        for doc in documents {
            if doc.title == document_id || doc.id == document_id {
                log::debug!("Found document '{}' via search fallback", document_id);
                return self.apply_kg_preprocessing_if_needed(doc).await.map(Some);
            }
        }

        log::debug!("Document '{}' not found anywhere", document_id);
        Ok(None)
    }

    /// Apply KG preprocessing to a document if needed based on the current selected role
    ///
    /// This helper method checks if the selected role has terraphim_it enabled
    /// and applies KG term preprocessing accordingly. It prevents double processing
    /// by checking if KG links already exist in the document.
    async fn apply_kg_preprocessing_if_needed(&mut self, document: Document) -> Result<Document> {
        log::debug!(
            "🔍 [KG-DEBUG] apply_kg_preprocessing_if_needed called for document: '{}'",
            document.title
        );
        log::debug!(
            "🔍 [KG-DEBUG] Document body preview: {}",
            document.body.chars().take(100).collect::<String>()
        );

        let role = {
            let config = self.config_state.config.lock().await;
            let selected_role = &config.selected_role;

            log::debug!("🔍 [KG-DEBUG] Selected role: '{}'", selected_role);

            match config.roles.get(selected_role) {
                Some(role) => {
                    log::debug!(
                        "🔍 [KG-DEBUG] Role found: '{}', terraphim_it: {}",
                        role.name,
                        role.terraphim_it
                    );
                    role.clone() // Clone to avoid borrowing issues
                }
                None => {
                    log::warn!(
                        "❌ [KG-DEBUG] Selected role '{}' not found in config, skipping KG preprocessing",
                        selected_role
                    );
                    return Ok(document);
                }
            }
        }; // Release the lock here

        // Only apply preprocessing if role has terraphim_it enabled
        if !role.terraphim_it {
            log::info!(
                "🔍 [KG-DEBUG] terraphim_it disabled for role '{}', skipping KG preprocessing",
                role.name
            );
            return Ok(document);
        }

        // Check if document already has KG links to prevent double processing
        let has_existing_kg_links = document.body.contains("](kg:");
        log::debug!(
            "🔍 [KG-DEBUG] Document already has KG links: {}",
            has_existing_kg_links
        );
        if has_existing_kg_links {
            log::info!(
                "🔍 [KG-DEBUG] Document '{}' already has KG links, skipping preprocessing to prevent double processing",
                document.title
            );
            return Ok(document);
        }

        log::info!(
            "🧠 [KG-DEBUG] Starting KG preprocessing for document '{}' with role '{}' (terraphim_it enabled)",
            document.title,
            role.name
        );

        // Apply KG preprocessing
        let document_title = document.title.clone(); // Save title before moving document
        let processed_doc = match self.preprocess_document_content(document, &role).await {
            Ok(doc) => {
                let links_added = doc.body.contains("](kg:");
                log::info!(
                    "✅ [KG-DEBUG] KG preprocessing completed for document '{}'. Links added: {}",
                    doc.title,
                    links_added
                );
                if links_added {
                    log::debug!(
                        "🔍 [KG-DEBUG] Processed body preview: {}",
                        doc.body.chars().take(200).collect::<String>()
                    );
                }
                doc
            }
            Err(e) => {
                log::error!(
                    "❌ [KG-DEBUG] KG preprocessing failed for document '{}': {:?}",
                    document_title,
                    e
                );
                return Err(e);
            }
        };

        Ok(processed_doc)
    }

    /// Enhance document descriptions with AI-generated summaries using OpenRouter
    ///
    /// This method uses the OpenRouter service to generate intelligent summaries
    /// of document content, replacing basic text excerpts with AI-powered descriptions.
    #[allow(dead_code)] // Used in 7+ places but compiler can't see due to async/feature boundaries
    async fn enhance_descriptions_with_ai(
        &self,
        mut documents: Vec<Document>,
        role: &Role,
    ) -> Result<Vec<Document>> {
        use crate::llm::{SummarizeOptions, build_llm_from_role};

        eprintln!("🤖 Attempting to build LLM client for role: {}", role.name);
        let llm = match build_llm_from_role(role) {
            Some(client) => {
                eprintln!("✅ LLM client successfully created: {}", client.name());
                client
            }
            None => {
                eprintln!("❌ No LLM client available for role: {}", role.name);
                return Ok(documents);
            }
        };

        log::info!(
            "Enhancing {} document descriptions with LLM provider: {}",
            documents.len(),
            llm.name()
        );

        let mut enhanced_count = 0;
        let mut error_count = 0;

        for document in &mut documents {
            if self.should_generate_ai_summary(document) {
                let summary_length = 250;
                match llm
                    .summarize(
                        &document.body,
                        SummarizeOptions {
                            max_length: summary_length,
                        },
                    )
                    .await
                {
                    Ok(ai_summary) => {
                        log::debug!(
                            "Generated AI summary for '{}': {} characters",
                            document.title,
                            ai_summary.len()
                        );
                        document.description = Some(ai_summary);
                        enhanced_count += 1;
                    }
                    Err(e) => {
                        log::warn!(
                            "Failed to generate AI summary for '{}': {}",
                            document.title,
                            e
                        );
                        error_count += 1;
                    }
                }
            }
        }

        log::info!(
            "LLM enhancement complete: {} enhanced, {} errors, {} skipped",
            enhanced_count,
            error_count,
            documents.len() - enhanced_count - error_count
        );

        Ok(documents)
    }

    /// Determine if a document should receive an AI-generated summary
    ///
    /// This helper method checks various criteria to decide whether a document
    /// would benefit from AI summarization.
    #[allow(dead_code)] // Used by enhance_descriptions_with_ai, compiler can't see due to async boundaries
    fn should_generate_ai_summary(&self, document: &Document) -> bool {
        // Don't enhance if the document body is too short to summarize meaningfully
        if document.body.trim().len() < 200 {
            return false;
        }

        // Don't enhance if we already have a high-quality description
        if let Some(ref description) = document.description {
            // If the description is substantial and doesn't look like a simple excerpt, keep it
            if description.len() > 100 && !description.ends_with("...") {
                return false;
            }
        }

        // Don't enhance very large documents (cost control)
        if document.body.len() > 8000 {
            return false;
        }

        // Good candidates for AI summarization
        true
    }

    /// Get the role for the given search query
    async fn get_search_role(&self, search_query: &SearchQuery) -> Result<Role> {
        let search_role = match &search_query.role {
            Some(role) => role.clone(),
            None => self.config_state.get_default_role().await,
        };

        log::debug!("Searching for role: {:?}", search_role);
        let Some(role) = self.config_state.get_role(&search_role).await else {
            return Err(ServiceError::Config(format!(
                "Role `{}` not found in config",
                search_role
            )));
        };
        Ok(role)
    }

    /// Check if a character is a word boundary (not alphanumeric or underscore).
    /// This provides Unicode-aware word boundary detection.
    fn is_word_boundary_char(c: char) -> bool {
        !c.is_alphanumeric() && c != '_'
    }

    /// Check if a match position is at word boundaries in the text.
    /// Returns true if the character before start (or start of string) and
    /// the character after end (or end of string) are word boundary characters.
    fn is_at_word_boundary(text: &str, start: usize, end: usize) -> bool {
        let before_ok = if start == 0 {
            true
        } else {
            text[..start]
                .chars()
                .last()
                .map(Self::is_word_boundary_char)
                .unwrap_or(true)
        };

        let after_ok = if end >= text.len() {
            true
        } else {
            text[end..]
                .chars()
                .next()
                .map(Self::is_word_boundary_char)
                .unwrap_or(true)
        };

        before_ok && after_ok
    }

    /// Match a term against text using unicode-aware word boundaries.
    /// Returns true if the term appears as a complete word (not as part of another word).
    /// Both inputs should already be lowercase for efficiency.
    fn term_matches_with_word_boundaries(term: &str, text: &str) -> bool {
        // Find all occurrences of the term in the text
        let mut start = 0;
        while let Some(pos) = text[start..].find(term) {
            let abs_start = start + pos;
            let abs_end = abs_start + term.len();

            if Self::is_at_word_boundary(text, abs_start, abs_end) {
                return true;
            }
            start = abs_end;
        }
        false
    }

    /// Apply logical operators (AND/OR) to filter documents based on multiple search terms
    pub async fn apply_logical_operators_to_documents(
        &mut self,
        search_query: &SearchQuery,
        documents: Vec<Document>,
    ) -> Result<Vec<Document>> {
        use terraphim_types::LogicalOperator;

        let all_terms = search_query.get_all_terms();
        let operator = search_query.get_operator();

        let initial_doc_count = documents.len();

        log::debug!(
            "Applying {:?} operator to {} documents with {} search terms",
            operator,
            initial_doc_count,
            all_terms.len()
        );

        // Pre-compute lowercase terms once for efficiency
        let terms_lower: Vec<String> = all_terms
            .iter()
            .map(|t| t.as_str().to_lowercase())
            .collect();

        let filtered_docs: Vec<Document> = documents
            .into_iter()
            .filter(|doc| {
                // Create searchable text from document
                let searchable_text = format!(
                    "{} {} {}",
                    doc.title.to_lowercase(),
                    doc.body.to_lowercase(),
                    doc.description
                        .as_ref()
                        .unwrap_or(&String::new())
                        .to_lowercase()
                );

                match operator {
                    LogicalOperator::And => {
                        // Document must contain ALL terms as whole words
                        terms_lower.iter().all(|term| {
                            Self::term_matches_with_word_boundaries(term, &searchable_text)
                        })
                    }
                    LogicalOperator::Or => {
                        // Document must contain ANY term as a whole word
                        terms_lower.iter().any(|term| {
                            Self::term_matches_with_word_boundaries(term, &searchable_text)
                        })
                    }
                }
            })
            .collect();

        log::debug!(
            "Logical operator filtering: {} -> {} documents",
            initial_doc_count,
            filtered_docs.len()
        );

        // Sort filtered documents by relevance using a combined query
        let combined_query_string = terms_lower.join(" ");
        let query = Query::new(&combined_query_string);
        let sorted_docs = score::sort_documents(&query, filtered_docs);

        Ok(sorted_docs)
    }

    /// search for documents in the haystacks with selected role from the config
    /// and return the documents sorted by relevance
    pub async fn search_documents_selected_role(
        &mut self,
        search_term: &NormalizedTermValue,
    ) -> Result<Vec<Document>> {
        let role = self.config_state.get_selected_role().await;
        let documents = self
            .search(&SearchQuery {
                search_term: search_term.clone(),
                search_terms: None,
                operator: None,
                role: Some(role),
                skip: None,
                limit: None,
                layer: Layer::default(),
                include_pinned: false,
                min_quality: None,
            })
            .await?;
        Ok(documents)
    }

    /// Filter documents by minimum composite quality score.
    ///
    /// Documents without a quality score are excluded when `min_quality` is set.
    /// Out-of-range thresholds are clamped to `[0.0, 1.0]` before comparison.
    fn apply_min_quality_filter(docs: Vec<Document>, min_quality: Option<f64>) -> Vec<Document> {
        let Some(threshold) = min_quality else {
            return docs;
        };
        let threshold = threshold.clamp(0.0, 1.0);
        docs.into_iter()
            .filter(|doc| {
                doc.quality_score
                    .as_ref()
                    .map(|qs| qs.composite() >= threshold)
                    .unwrap_or(false)
            })
            .collect()
    }

    /// Search for documents in the haystacks
    pub async fn search(&mut self, search_query: &SearchQuery) -> Result<Vec<Document>> {
        // Get the role from the config
        log::debug!("Role for searching: {:?}", search_query.role);
        let role = self.get_search_role(search_query).await?;

        log::trace!("Building index for search query: {:?}", search_query);
        let index: Index =
            terraphim_middleware::search_haystacks(self.config_state.clone(), search_query.clone())
                .await?;

        let min_quality = search_query.min_quality;

        let docs_result: Result<Vec<Document>> = match role.relevance_function {
            RelevanceFunction::TitleScorer => {
                log::debug!("Searching haystack with title scorer");

                let documents = index.get_all_documents();

                log::debug!("Sorting documents by relevance");

                let documents = if search_query.is_multi_term_query() {
                    // Handle multi-term queries with logical operators
                    self.apply_logical_operators_to_documents(search_query, documents)
                        .await?
                } else {
                    // Single term query (backward compatibility)
                    let query = Query::new(&search_query.search_term.to_string());
                    score::sort_documents(&query, documents)
                };
                let total_length = documents.len();
                let mut docs_ranked = Vec::new();
                for (idx, doc) in documents.iter().enumerate() {
                    let mut document: terraphim_types::Document = doc.clone();
                    let rank = (total_length - idx).try_into().unwrap();
                    document.rank = Some(rank);

                    // 🔄 Enhanced persistence layer integration for both local and Atomic Data documents
                    if document.id.starts_with("http://") || document.id.starts_with("https://") {
                        // Atomic Data document: Check persistence first, then save for future queries
                        log::debug!(
                            "Processing Atomic Data document '{}' (URL: {})",
                            document.title,
                            document.id
                        );

                        // Try to load from persistence first (for cached Atomic Data documents)
                        let mut placeholder = Document {
                            id: document.id.clone(),
                            ..Default::default()
                        };
                        match placeholder.load().await {
                            Ok(persisted_doc) => {
                                // Found in persistence - use cached version
                                log::debug!(
                                    "Found cached Atomic Data document '{}' in persistence",
                                    document.title
                                );
                                if let Some(better_description) = persisted_doc.description {
                                    document.description = Some(better_description);
                                }
                                // Update body if the persisted version has better content
                                // But DO NOT overwrite if this role uses KG preprocessing (terraphim_it)
                                // because we need to preserve the processed content with KG links
                                if !persisted_doc.body.is_empty() && !role.terraphim_it {
                                    log::debug!(
                                        "Updated body from persistence for Atomic document '{}' (role: '{}', terraphim_it: {})",
                                        document.title,
                                        role.name,
                                        role.terraphim_it
                                    );
                                    document.body = persisted_doc.body;
                                } else if role.terraphim_it {
                                    log::debug!(
                                        "Keeping search result body for Atomic document '{}' because role '{}' uses KG preprocessing (terraphim_it=true)",
                                        document.title,
                                        role.name
                                    );
                                }
                            }
                            Err(_) => {
                                // Not in persistence - save this Atomic Data document for future queries
                                log::debug!(
                                    "Caching Atomic Data document '{}' to persistence for future queries",
                                    document.title
                                );

                                // Save in background to avoid blocking the response
                                let doc_to_save = document.clone();
                                tokio::spawn(async move {
                                    if let Err(e) = doc_to_save.save().await {
                                        log::warn!(
                                            "Failed to cache Atomic Data document '{}': {}",
                                            doc_to_save.title,
                                            e
                                        );
                                    } else {
                                        log::debug!(
                                            "Successfully cached Atomic Data document '{}'",
                                            doc_to_save.title
                                        );
                                    }
                                });
                            }
                        }
                    } else {
                        // Local document: Try direct persistence lookup first
                        let should_lookup_persistence = document
                            .get_source_haystack()
                            .and_then(|source| {
                                role.haystacks
                                    .iter()
                                    .find(|haystack| haystack.location == *source)
                            })
                            .map(|haystack| haystack.fetch_content)
                            .unwrap_or(true);

                        if !should_lookup_persistence {
                            log::trace!(
                                "Skipping persistence lookup for '{}' (haystack fetch_content=false)",
                                document.title
                            );
                        } else {
                            let mut placeholder = Document {
                                id: document.id.clone(),
                                ..Default::default()
                            };
                            if let Ok(persisted_doc) = placeholder.load().await {
                                if let Some(better_description) = persisted_doc.description {
                                    log::debug!(
                                        "Replaced ripgrep description for '{}' with persistence description",
                                        document.title
                                    );
                                    document.description = Some(better_description);
                                }
                            } else {
                                // Try normalized ID based on document title (filename)
                                // For KG files, the title might be "haystack" but persistence ID is "haystackmd"
                                let normalized_id = normalize_filename_to_id(&document.title);

                                let mut normalized_placeholder = Document {
                                    id: normalized_id.clone(),
                                    ..Default::default()
                                };
                                if let Ok(persisted_doc) = normalized_placeholder.load().await {
                                    if let Some(better_description) = persisted_doc.description {
                                        log::debug!(
                                            "Replaced ripgrep description for '{}' with persistence description (normalized from title: {})",
                                            document.title,
                                            normalized_id
                                        );
                                        document.description = Some(better_description);
                                    }
                                } else {
                                    // Try with "md" suffix for KG files (title "haystack" -> ID "haystackmd")
                                    let normalized_id_with_md = format!("{}md", normalized_id);
                                    let mut md_placeholder = Document {
                                        id: normalized_id_with_md.clone(),
                                        ..Default::default()
                                    };
                                    if let Ok(persisted_doc) = md_placeholder.load().await {
                                        if let Some(better_description) = persisted_doc.description
                                        {
                                            log::debug!(
                                                "Replaced ripgrep description for '{}' with persistence description (normalized with md: {})",
                                                document.title,
                                                normalized_id_with_md
                                            );
                                            document.description = Some(better_description);
                                        }
                                    } else {
                                        log::debug!(
                                            "No persistence document found for '{}' (tried ID: '{}', normalized: '{}', with md: '{}')",
                                            document.title,
                                            document.id,
                                            normalized_id,
                                            normalized_id_with_md
                                        );
                                    }
                                }
                            }
                        }
                    }

                    docs_ranked.push(document);
                }

                // Apply OpenRouter AI summarization if enabled for this role and auto-summarize is on
                // Apply AI summarization if enabled via OpenRouter or generic LLM config
                #[cfg(feature = "openrouter")]
                if role.has_llm_config() && role.llm_auto_summarize {
                    log::debug!(
                        "Applying OpenRouter AI summarization to {} search results for role '{}'",
                        docs_ranked.len(),
                        role.name
                    );
                    docs_ranked = self
                        .enhance_descriptions_with_ai(docs_ranked, &role)
                        .await?;
                } else {
                    // Always apply LLM AI summarization if LLM client is available
                    eprintln!(
                        "📋 Entering LLM AI summarization branch for role: {}",
                        role.name
                    );
                    log::debug!(
                        "Applying LLM AI summarization to {} search results for role '{}'",
                        docs_ranked.len(),
                        role.name
                    );
                    docs_ranked = self
                        .enhance_descriptions_with_ai(docs_ranked, &role)
                        .await?;
                }

                // Apply KG preprocessing if enabled for this role (but only once, not in individual document loads)
                if role.terraphim_it {
                    log::info!(
                        "🧠 Applying KG preprocessing to {} TerraphimGraph search results for role '{}'",
                        docs_ranked.len(),
                        role.name
                    );
                    let mut processed_docs = Vec::new();
                    let mut total_kg_terms = 0;
                    let mut docs_with_kg_links = 0;

                    for document in docs_ranked {
                        let original_body_len = document.body.len();
                        let processed_doc =
                            self.preprocess_document_content(document, &role).await?;

                        // Count KG links added (rough estimate by body size increase)
                        let new_body_len = processed_doc.body.len();
                        if new_body_len > original_body_len {
                            docs_with_kg_links += 1;
                            // Rough estimate: each KG link adds ~15-20 chars on average
                            let estimated_links = (new_body_len - original_body_len) / 17;
                            total_kg_terms += estimated_links;
                        }

                        processed_docs.push(processed_doc);
                    }

                    log::info!(
                        "✅ KG preprocessing complete: {} documents processed, {} received KG links (~{} total links)",
                        processed_docs.len(),
                        docs_with_kg_links,
                        total_kg_terms
                    );
                    Ok(processed_docs)
                } else {
                    Ok(docs_ranked)
                }
            }
            RelevanceFunction::BM25 => {
                log::debug!("Searching haystack with BM25 scorer");

                let documents = index.get_all_documents();

                log::debug!("Sorting documents by BM25 relevance");

                let documents = if search_query.is_multi_term_query() {
                    // Handle multi-term queries with logical operators
                    let filtered_docs = self
                        .apply_logical_operators_to_documents(search_query, documents)
                        .await?;
                    // Apply BM25 scoring to filtered documents
                    let combined_query_string = search_query
                        .get_all_terms()
                        .iter()
                        .map(|t| t.as_str())
                        .collect::<Vec<_>>()
                        .join(" ");
                    let query =
                        Query::new(&combined_query_string).name_scorer(score::QueryScorer::BM25);
                    score::sort_documents(&query, filtered_docs)
                } else {
                    // Single term query (backward compatibility)
                    let query = Query::new(&search_query.search_term.to_string())
                        .name_scorer(score::QueryScorer::BM25);
                    score::sort_documents(&query, documents)
                };
                let total_length = documents.len();
                let mut docs_ranked = Vec::new();
                for (idx, doc) in documents.iter().enumerate() {
                    let mut document: terraphim_types::Document = doc.clone();
                    let rank = (total_length - idx).try_into().unwrap();
                    document.rank = Some(rank);
                    docs_ranked.push(document);
                }

                // Apply OpenRouter AI summarization if enabled for this role and auto-summarize is on
                #[cfg(feature = "openrouter")]
                if role.has_llm_config() && role.llm_auto_summarize {
                    log::debug!(
                        "Applying OpenRouter AI summarization to {} BM25 search results for role '{}'",
                        docs_ranked.len(),
                        role.name
                    );
                    docs_ranked = self
                        .enhance_descriptions_with_ai(docs_ranked, &role)
                        .await?;
                } else {
                    // Always apply LLM AI summarization if LLM client is available
                    log::debug!(
                        "Applying LLM AI summarization to {} BM25 search results for role '{}'",
                        docs_ranked.len(),
                        role.name
                    );
                    docs_ranked = self
                        .enhance_descriptions_with_ai(docs_ranked, &role)
                        .await?;
                }

                // Apply KG preprocessing if enabled for this role
                if role.terraphim_it {
                    log::info!(
                        "🧠 Applying KG preprocessing to {} BM25 search results for role '{}'",
                        docs_ranked.len(),
                        role.name
                    );
                    let mut processed_docs = Vec::new();
                    let mut total_kg_terms = 0;
                    let mut docs_with_kg_links = 0;

                    for document in docs_ranked {
                        let original_body_len = document.body.len();
                        let processed_doc =
                            self.preprocess_document_content(document, &role).await?;

                        // Count KG links added (rough estimate by body size increase)
                        let new_body_len = processed_doc.body.len();
                        if new_body_len > original_body_len {
                            docs_with_kg_links += 1;
                            let estimated_links = (new_body_len - original_body_len) / 17;
                            total_kg_terms += estimated_links;
                        }

                        processed_docs.push(processed_doc);
                    }

                    log::info!(
                        "✅ KG preprocessing complete: {} documents processed, {} received KG links (~{} total links)",
                        processed_docs.len(),
                        docs_with_kg_links,
                        total_kg_terms
                    );
                    Ok(processed_docs)
                } else {
                    Ok(docs_ranked)
                }
            }
            RelevanceFunction::BM25F => {
                log::debug!("Searching haystack with BM25F scorer");

                let documents = index.get_all_documents();

                log::debug!("Sorting documents by BM25F relevance");

                let documents = if search_query.is_multi_term_query() {
                    // Handle multi-term queries with logical operators
                    let filtered_docs = self
                        .apply_logical_operators_to_documents(search_query, documents)
                        .await?;
                    // Apply BM25F scoring to filtered documents
                    let combined_query_string = search_query
                        .get_all_terms()
                        .iter()
                        .map(|t| t.as_str())
                        .collect::<Vec<_>>()
                        .join(" ");
                    let query =
                        Query::new(&combined_query_string).name_scorer(score::QueryScorer::BM25F);
                    score::sort_documents(&query, filtered_docs)
                } else {
                    // Single term query (backward compatibility)
                    let query = Query::new(&search_query.search_term.to_string())
                        .name_scorer(score::QueryScorer::BM25F);
                    score::sort_documents(&query, documents)
                };
                let total_length = documents.len();
                let mut docs_ranked = Vec::new();
                for (idx, doc) in documents.iter().enumerate() {
                    let mut document: terraphim_types::Document = doc.clone();
                    let rank = (total_length - idx).try_into().unwrap();
                    document.rank = Some(rank);
                    docs_ranked.push(document);
                }

                // Apply OpenRouter AI summarization if enabled for this role and auto-summarize is on
                #[cfg(feature = "openrouter")]
                if role.has_llm_config() && role.llm_auto_summarize {
                    log::debug!(
                        "Applying OpenRouter AI summarization to {} BM25F search results for role '{}'",
                        docs_ranked.len(),
                        role.name
                    );
                    docs_ranked = self
                        .enhance_descriptions_with_ai(docs_ranked, &role)
                        .await?;
                } else {
                    // Always apply LLM AI summarization if LLM client is available
                    log::debug!(
                        "Applying LLM AI summarization to {} BM25F search results for role '{}'",
                        docs_ranked.len(),
                        role.name
                    );
                    docs_ranked = self
                        .enhance_descriptions_with_ai(docs_ranked, &role)
                        .await?;
                }

                // Apply KG preprocessing if enabled for this role
                if role.terraphim_it {
                    log::info!(
                        "🧠 Applying KG preprocessing to {} BM25F search results for role '{}'",
                        docs_ranked.len(),
                        role.name
                    );
                    let mut processed_docs = Vec::new();
                    let mut total_kg_terms = 0;
                    let mut docs_with_kg_links = 0;

                    for document in docs_ranked {
                        let original_body_len = document.body.len();
                        let processed_doc =
                            self.preprocess_document_content(document, &role).await?;

                        // Count KG links added (rough estimate by body size increase)
                        let new_body_len = processed_doc.body.len();
                        if new_body_len > original_body_len {
                            docs_with_kg_links += 1;
                            let estimated_links = (new_body_len - original_body_len) / 17;
                            total_kg_terms += estimated_links;
                        }

                        processed_docs.push(processed_doc);
                    }

                    log::info!(
                        "✅ KG preprocessing complete: {} documents processed, {} received KG links (~{} total links)",
                        processed_docs.len(),
                        docs_with_kg_links,
                        total_kg_terms
                    );
                    Ok(processed_docs)
                } else {
                    Ok(docs_ranked)
                }
            }
            RelevanceFunction::BM25Plus => {
                log::debug!("Searching haystack with BM25Plus scorer");

                let documents = index.get_all_documents();

                log::debug!("Sorting documents by BM25Plus relevance");

                let documents = if search_query.is_multi_term_query() {
                    // Handle multi-term queries with logical operators
                    let filtered_docs = self
                        .apply_logical_operators_to_documents(search_query, documents)
                        .await?;
                    // Apply BM25Plus scoring to filtered documents
                    let combined_query_string = search_query
                        .get_all_terms()
                        .iter()
                        .map(|t| t.as_str())
                        .collect::<Vec<_>>()
                        .join(" ");
                    let query = Query::new(&combined_query_string)
                        .name_scorer(score::QueryScorer::BM25Plus);
                    score::sort_documents(&query, filtered_docs)
                } else {
                    // Single term query (backward compatibility)
                    let query = Query::new(&search_query.search_term.to_string())
                        .name_scorer(score::QueryScorer::BM25Plus);
                    score::sort_documents(&query, documents)
                };
                let total_length = documents.len();
                let mut docs_ranked = Vec::new();
                for (idx, doc) in documents.iter().enumerate() {
                    let mut document: terraphim_types::Document = doc.clone();
                    let rank = (total_length - idx).try_into().unwrap();
                    document.rank = Some(rank);
                    docs_ranked.push(document);
                }

                // Apply OpenRouter AI summarization if enabled for this role and auto-summarize is on
                #[cfg(feature = "openrouter")]
                if role.has_llm_config() && role.llm_auto_summarize {
                    log::debug!(
                        "Applying OpenRouter AI summarization to {} BM25Plus search results for role '{}'",
                        docs_ranked.len(),
                        role.name
                    );
                    docs_ranked = self
                        .enhance_descriptions_with_ai(docs_ranked, &role)
                        .await?;
                }

                // Apply KG preprocessing if enabled for this role
                if role.terraphim_it {
                    log::info!(
                        "🧠 Applying KG preprocessing to {} BM25Plus search results for role '{}'",
                        docs_ranked.len(),
                        role.name
                    );
                    let mut processed_docs = Vec::new();
                    let mut total_kg_terms = 0;
                    let mut docs_with_kg_links = 0;

                    for document in docs_ranked {
                        let original_body_len = document.body.len();
                        let processed_doc =
                            self.preprocess_document_content(document, &role).await?;

                        // Count KG links added (rough estimate by body size increase)
                        let new_body_len = processed_doc.body.len();
                        if new_body_len > original_body_len {
                            docs_with_kg_links += 1;
                            let estimated_links = (new_body_len - original_body_len) / 17;
                            total_kg_terms += estimated_links;
                        }

                        processed_docs.push(processed_doc);
                    }

                    log::info!(
                        "✅ KG preprocessing complete: {} documents processed, {} received KG links (~{} total links)",
                        processed_docs.len(),
                        docs_with_kg_links,
                        total_kg_terms
                    );
                    Ok(processed_docs)
                } else {
                    Ok(docs_ranked)
                }
            }
            RelevanceFunction::TerraphimGraph => {
                log::debug!("TerraphimGraph search initiated for role: {}", role.name);
                self.build_thesaurus(search_query).await?;
                let _thesaurus = self.ensure_thesaurus_loaded(&role.name).await?;
                let scored_index_docs: Vec<IndexedDocument> = self
                    .config_state
                    .search_indexed_documents(search_query, &role)
                    .await;

                log::debug!(
                    "TerraphimGraph search found {} indexed documents",
                    scored_index_docs.len()
                );

                // Apply to ripgrep vector of document output
                // I.e. use the ranking of thesaurus to rank the documents here
                log::debug!("Ranking documents with thesaurus");
                let mut documents = index.get_documents(scored_index_docs.clone());

                // CRITICAL FIX: Index all haystack documents into rolegraph if not already present
                // This ensures TerraphimGraph search can find documents discovered by haystacks
                let all_haystack_docs = index.get_all_documents();
                log::debug!(
                    "Found {} total documents from haystacks, checking which need indexing",
                    all_haystack_docs.len()
                );
                let mut need_reindexing = false;

                if let Some(rolegraph_sync) = self.config_state.roles.get(&role.name) {
                    let mut rolegraph = rolegraph_sync.lock().await;
                    let mut newly_indexed = 0;

                    for doc in &all_haystack_docs {
                        // Only index documents that aren't already in the rolegraph
                        if !rolegraph.has_document(&doc.id) && !doc.body.is_empty() {
                            log::debug!(
                                "Indexing new document '{}' into rolegraph for TerraphimGraph search",
                                doc.id
                            );
                            rolegraph.insert_document(&doc.id, doc.clone());

                            // Save document to persistence to ensure it's available for kg_search
                            // Drop the rolegraph lock temporarily to avoid deadlocks during async save
                            drop(rolegraph);
                            if let Err(e) = doc.save().await {
                                log::warn!(
                                    "Failed to save document '{}' to persistence: {}",
                                    doc.id,
                                    e
                                );
                            } else {
                                log::debug!(
                                    "Successfully saved document '{}' to persistence",
                                    doc.id
                                );
                            }
                            // Re-acquire the lock
                            rolegraph = rolegraph_sync.lock().await;

                            newly_indexed += 1;
                        }
                    }

                    if newly_indexed > 0 {
                        log::info!(
                            "✅ Indexed {} new documents into rolegraph for role '{}'",
                            newly_indexed,
                            role.name
                        );
                        log::debug!(
                            "RoleGraph now has {} nodes, {} edges, {} documents",
                            rolegraph.get_node_count(),
                            rolegraph.get_edge_count(),
                            rolegraph.get_document_count()
                        );
                        need_reindexing = true; // We'll use the existing re-search logic below
                    }
                }

                // CRITICAL FIX: Ensure documents have body content loaded from persistence
                // If documents don't have body content, they won't contribute to graph nodes properly
                let mut documents_with_content = Vec::new();

                for mut document in documents {
                    // Check if document body is empty or missing
                    if document.body.is_empty() {
                        log::debug!(
                            "Document '{}' has empty body, attempting to load from persistence",
                            document.id
                        );

                        // Try to load full document from persistence with fallback
                        let mut full_doc = Document::new(document.id.clone());
                        match full_doc.load().await {
                            Ok(loaded_doc) => {
                                if !loaded_doc.body.is_empty() {
                                    log::info!(
                                        "✅ Loaded body content for document '{}' from persistence",
                                        document.id
                                    );
                                    document.body = loaded_doc.body.clone();
                                    if loaded_doc.description.is_some() {
                                        document.description = loaded_doc.description.clone();
                                    }

                                    // Re-index document into rolegraph with proper content
                                    if let Some(rolegraph_sync) =
                                        self.config_state.roles.get(&role.name)
                                    {
                                        let mut rolegraph = rolegraph_sync.lock().await;
                                        rolegraph.insert_document(&document.id, loaded_doc);
                                        need_reindexing = true;
                                        log::debug!(
                                            "Re-indexed document '{}' into rolegraph with content",
                                            document.id
                                        );
                                    }
                                } else {
                                    log::warn!(
                                        "Document '{}' still has empty body after loading from persistence",
                                        document.id
                                    );
                                }
                            }
                            Err(e) => {
                                log::warn!(
                                    "Failed to load document '{}' from persistence: {}",
                                    document.id,
                                    e
                                );

                                // Try to read from original file path if it's a local file
                                if document.url.starts_with('/')
                                    || document.url.starts_with("docs/")
                                {
                                    match tokio::fs::read_to_string(&document.url).await {
                                        Ok(content) => {
                                            log::info!(
                                                "✅ Loaded content for '{}' from file: {}",
                                                document.id,
                                                document.url
                                            );
                                            document.body = content.clone();

                                            // Create and save full document
                                            let full_doc = Document {
                                                id: document.id.clone(),
                                                title: document.title.clone(),
                                                body: content,
                                                url: document.url.clone(),
                                                description: document.description.clone(),
                                                summarization: document.summarization.clone(),
                                                stub: None,
                                                tags: document.tags.clone(),
                                                rank: document.rank,
                                                source_haystack: document.source_haystack.clone(),
                                                doc_type: terraphim_types::DocumentType::KgEntry,
                                                synonyms: None,
                                                route: None,
                                                priority: None,
                                                quality_score: None,
                                            };

                                            // Save to persistence for future use
                                            if let Err(e) = full_doc.save().await {
                                                log::warn!(
                                                    "Failed to save document '{}' to persistence: {}",
                                                    document.id,
                                                    e
                                                );
                                            }

                                            // Re-index into rolegraph
                                            if let Some(rolegraph_sync) =
                                                self.config_state.roles.get(&role.name)
                                            {
                                                let mut rolegraph = rolegraph_sync.lock().await;
                                                rolegraph.insert_document(&document.id, full_doc);
                                                need_reindexing = true;
                                                log::debug!(
                                                    "Re-indexed document '{}' into rolegraph from file",
                                                    document.id
                                                );
                                            }
                                        }
                                        Err(file_e) => {
                                            log::warn!(
                                                "Failed to read file '{}' for document '{}': {}",
                                                document.url,
                                                document.id,
                                                file_e
                                            );
                                        }
                                    }
                                }
                            }
                        }
                    }
                    documents_with_content.push(document);
                }

                documents = documents_with_content;

                if need_reindexing {
                    log::info!("🔄 Re-running TerraphimGraph search after indexing new documents");

                    // Re-run the rolegraph search to get updated rankings
                    let updated_scored_docs: Vec<IndexedDocument> = self
                        .config_state
                        .search_indexed_documents(search_query, &role)
                        .await;

                    if !updated_scored_docs.is_empty() {
                        log::debug!(
                            "✅ Updated rolegraph search found {} documents",
                            updated_scored_docs.len()
                        );
                        // Update documents with new ranking from rolegraph
                        let updated_documents = index.get_documents(updated_scored_docs);
                        if !updated_documents.is_empty() {
                            documents = updated_documents;
                        }
                    }
                }

                if documents.is_empty() && !all_haystack_docs.is_empty() {
                    log::info!(
                        "TerraphimGraph returned no results for role '{}'; falling back to lexical haystack ranking",
                        role.name
                    );
                    documents = if search_query.is_multi_term_query() {
                        let filtered_docs = self
                            .apply_logical_operators_to_documents(
                                search_query,
                                all_haystack_docs.clone(),
                            )
                            .await?;
                        let combined_query_string = search_query
                            .get_all_terms()
                            .iter()
                            .map(|t| t.as_str())
                            .collect::<Vec<_>>()
                            .join(" ");
                        let query = Query::new(&combined_query_string);
                        score::sort_documents(&query, filtered_docs)
                    } else {
                        let query = Query::new(&search_query.search_term.to_string());
                        score::sort_documents(&query, all_haystack_docs.clone())
                    };
                }

                // Apply TF-IDF scoring to enhance Terraphim Graph ranking
                if !documents.is_empty() {
                    log::debug!(
                        "Applying TF-IDF scoring to {} documents for enhanced ranking",
                        documents.len()
                    );

                    use crate::score::bm25_additional::TFIDFScorer;
                    let mut tfidf_scorer = TFIDFScorer::new();
                    tfidf_scorer.initialize(&documents);

                    // Re-score documents using TF-IDF
                    let query_text = &search_query.search_term.to_string();
                    for document in &mut documents {
                        let tfidf_score = tfidf_scorer.score(query_text, document);
                        // Combine TF-IDF score with existing rank using a weighted approach
                        if let Some(rank) = document.rank {
                            document.rank = Some(rank + (tfidf_score * 0.3) as u64);
                        // 30% weight for TF-IDF
                        } else {
                            document.rank = Some((tfidf_score * 10.0) as u64); // Scale TF-IDF for ranking
                        }
                    }

                    // Re-sort documents by the new combined rank
                    documents.sort_by_key(|d| std::cmp::Reverse(d.rank.unwrap_or(0)));

                    log::debug!("TF-IDF scoring applied successfully");
                }

                // 🔄 Enhanced persistence layer integration for both local and Atomic Data documents
                for document in &mut documents {
                    if document.id.starts_with("http://") || document.id.starts_with("https://") {
                        // Atomic Data document: Check persistence first, then save for future queries
                        log::debug!(
                            "Processing Atomic Data document '{}' (URL: {})",
                            document.title,
                            document.id
                        );

                        // Try to load from persistence first (for cached Atomic Data documents)
                        let mut placeholder = Document {
                            id: document.id.clone(),
                            ..Default::default()
                        };
                        match placeholder.load().await {
                            Ok(persisted_doc) => {
                                // Found in persistence - use cached version
                                log::debug!(
                                    "Found cached Atomic Data document '{}' in persistence",
                                    document.title
                                );
                                if let Some(better_description) = persisted_doc.description {
                                    document.description = Some(better_description);
                                }
                                // Update body if the persisted version has better content
                                // But DO NOT overwrite if this role uses KG preprocessing (terraphim_it)
                                // because we need to preserve the processed content with KG links
                                if !persisted_doc.body.is_empty() && !role.terraphim_it {
                                    log::debug!(
                                        "Updated body from persistence for Atomic document '{}' (role: '{}', terraphim_it: {})",
                                        document.title,
                                        role.name,
                                        role.terraphim_it
                                    );
                                    document.body = persisted_doc.body;
                                } else if role.terraphim_it {
                                    log::debug!(
                                        "Keeping search result body for Atomic document '{}' because role '{}' uses KG preprocessing (terraphim_it=true)",
                                        document.title,
                                        role.name
                                    );
                                }
                            }
                            Err(_) => {
                                // Not in persistence - save this Atomic Data document for future queries
                                log::debug!(
                                    "Caching Atomic Data document '{}' to persistence for future queries",
                                    document.title
                                );

                                // Save in background to avoid blocking the response
                                let doc_to_save = document.clone();
                                tokio::spawn(async move {
                                    if let Err(e) = doc_to_save.save().await {
                                        log::warn!(
                                            "Failed to cache Atomic Data document '{}': {}",
                                            doc_to_save.title,
                                            e
                                        );
                                    } else {
                                        log::debug!(
                                            "Successfully cached Atomic Data document '{}'",
                                            doc_to_save.title
                                        );
                                    }
                                });
                            }
                        }
                    } else {
                        // Local document: Try direct persistence lookup first
                        let mut placeholder = Document {
                            id: document.id.clone(),
                            ..Default::default()
                        };
                        if let Ok(persisted_doc) = placeholder.load().await {
                            if let Some(better_description) = persisted_doc.description {
                                log::debug!(
                                    "Replaced ripgrep description for '{}' with persistence description",
                                    document.title
                                );
                                document.description = Some(better_description);
                            }
                        } else {
                            // Try normalized ID based on document title (filename)
                            // For KG files, the title might be "haystack" but persistence ID is "haystackmd"
                            let normalized_id = normalize_filename_to_id(&document.title);

                            let mut normalized_placeholder = Document {
                                id: normalized_id.clone(),
                                ..Default::default()
                            };
                            if let Ok(persisted_doc) = normalized_placeholder.load().await {
                                if let Some(better_description) = persisted_doc.description {
                                    log::debug!(
                                        "Replaced ripgrep description for '{}' with persistence description (normalized from title: {})",
                                        document.title,
                                        normalized_id
                                    );
                                    document.description = Some(better_description);
                                }
                            } else {
                                // Try with "md" suffix for KG files (title "haystack" -> ID "haystackmd")
                                let normalized_id_with_md = format!("{}md", normalized_id);
                                let mut md_placeholder = Document {
                                    id: normalized_id_with_md.clone(),
                                    ..Default::default()
                                };
                                if let Ok(persisted_doc) = md_placeholder.load().await {
                                    if let Some(better_description) = persisted_doc.description {
                                        log::debug!(
                                            "Replaced ripgrep description for '{}' with persistence description (normalized with md: {})",
                                            document.title,
                                            normalized_id_with_md
                                        );
                                        document.description = Some(better_description);
                                    }
                                } else {
                                    log::debug!(
                                        "No persistence document found for '{}' (tried ID: '{}', normalized: '{}', with md: '{}')",
                                        document.title,
                                        document.id,
                                        normalized_id,
                                        normalized_id_with_md
                                    );
                                }
                            }
                        }
                    }
                }

                // Apply OpenRouter AI summarization if enabled for this role
                #[cfg(feature = "openrouter")]
                if role.has_llm_config() {
                    log::debug!(
                        "Applying OpenRouter AI summarization to {} search results for role '{}'",
                        documents.len(),
                        role.name
                    );
                    documents = self.enhance_descriptions_with_ai(documents, &role).await?;
                } else {
                    // Always apply LLM AI summarization if LLM client is available
                    log::debug!(
                        "Applying LLM AI summarization to {} search results for role '{}'",
                        documents.len(),
                        role.name
                    );
                    documents = self.enhance_descriptions_with_ai(documents, &role).await?;
                }

                // Apply KG preprocessing if enabled for this role (but only once, not in individual document loads)
                if role.terraphim_it {
                    log::debug!(
                        "Applying KG preprocessing to {} search results for role '{}'",
                        documents.len(),
                        role.name
                    );
                    let mut processed_docs = Vec::new();
                    for document in documents {
                        let processed_doc =
                            self.preprocess_document_content(document, &role).await?;
                        processed_docs.push(processed_doc);
                    }
                    Ok(processed_docs)
                } else {
                    Ok(documents)
                }
            }
        };
        let docs = docs_result?;
        Ok(Self::apply_min_quality_filter(docs, min_quality))
    }

    /// Check if a document ID appears to be hash-based (16 hex characters)
    fn is_hash_based_id(id: &str) -> bool {
        id.len() == 16 && id.chars().all(|c| c.is_ascii_hexdigit())
    }

    /// Find documents that contain a given knowledge graph term
    ///
    /// This method searches for documents that were the source of a knowledge graph term.
    /// For example, given "haystack", it will find documents like "haystack.md" that contain
    /// this term or its synonyms ("datasource", "service", "agent").
    ///
    /// For KG protocol resolution, this method also directly looks for KG definition documents
    /// when the term appears to be a KG concept (like "terraphim-graph" -> "./docs/src/kg/terraphim-graph.md").
    ///
    /// Returns a vector of Documents that contain the term, with KG preprocessing applied if enabled for the role.
    pub async fn find_documents_for_kg_term(
        &mut self,
        role_name: &RoleName,
        term: &str,
    ) -> Result<Vec<Document>> {
        log::debug!(
            "Finding documents for KG term '{}' in role '{}'",
            term,
            role_name
        );

        // Ensure the thesaurus is loaded for this role
        let thesaurus = self.ensure_thesaurus_loaded(role_name).await?;

        // Get the role configuration to check if KG preprocessing should be applied
        let role = self.config_state.get_role(role_name).await.ok_or_else(|| {
            ServiceError::Config(format!("Role '{}' not found in config", role_name))
        })?;

        let mut documents = Vec::new();

        // ENHANCEMENT: First, check if this is a direct KG definition document request
        // This handles KG protocol resolution like kg:terraphim-graph -> ./docs/src/kg/terraphim-graph.md
        // Also handles synonyms like kg:graph -> terraphim-graph -> ./docs/src/kg/terraphim-graph.md
        if let Some(kg_config) = &role.kg {
            log::debug!("Found KG config for role");
            if let Some(kg_local) = &kg_config.knowledge_graph_local {
                let mut potential_concepts = vec![term.to_string()];

                // Use the loaded thesaurus to resolve synonyms to root concepts
                log::debug!("Checking thesaurus for term '{}'", term);

                // Create normalized term to look up in thesaurus
                let normalized_search_term =
                    terraphim_types::NormalizedTermValue::new(term.to_string());

                // Look up the term in the thesaurus - this will find the root concept if term is a synonym
                if let Some(root_concept) = thesaurus.get(&normalized_search_term) {
                    log::debug!("Found root concept for '{}': {:?}", term, root_concept);

                    // The root concept's value contains the canonical concept name
                    let root_concept_name = root_concept.value.as_str();

                    // If we have a URL, extract concept name from it, otherwise use the concept value
                    let concept_name = if let Some(url) = &root_concept.url {
                        url.split('/')
                            .next_back()
                            .and_then(|s| s.strip_suffix(".md"))
                            .unwrap_or(root_concept_name)
                    } else {
                        root_concept_name
                    };

                    if !potential_concepts.contains(&concept_name.to_string()) {
                        potential_concepts.push(concept_name.to_string());
                        log::debug!(
                            "Added concept from thesaurus: {} (root: {})",
                            concept_name,
                            root_concept_name
                        );
                    }
                } else {
                    log::debug!("No direct mapping found for '{}' in thesaurus", term);
                }

                log::debug!(
                    "Trying {} potential concepts: {:?}",
                    potential_concepts.len(),
                    potential_concepts
                );

                // Try to find KG definition documents for all potential concepts
                for concept in potential_concepts {
                    let potential_kg_file = kg_local.path.join(format!("{}.md", concept));
                    log::debug!("Looking for KG definition file: {:?}", potential_kg_file);

                    if potential_kg_file.exists() {
                        log::info!("Found KG definition file: {:?}", potential_kg_file);

                        // Check if we already have this document to avoid duplicates
                        let file_path = potential_kg_file.to_string_lossy().to_string();
                        if documents.iter().any(|d: &Document| d.url == file_path) {
                            log::debug!("Skipping duplicate KG document: {}", file_path);
                            continue;
                        }

                        // Load the KG definition document directly from filesystem
                        // Don't use Document::load() as it relies on persistence layer
                        match std::fs::read_to_string(&potential_kg_file) {
                            Ok(content) => {
                                let mut kg_doc =
                                    Document::new(potential_kg_file.to_string_lossy().to_string());
                                kg_doc.url = potential_kg_file.to_string_lossy().to_string();
                                kg_doc.body = content.clone();

                                // Extract title from markdown content (first # line)
                                let title = content
                                    .lines()
                                    .find(|line| line.starts_with("# "))
                                    .map(|line| line.trim_start_matches("# ").trim())
                                    .unwrap_or(&concept)
                                    .to_string();
                                kg_doc.title = title;

                                log::debug!(
                                    "Successfully loaded KG definition document: {}",
                                    kg_doc.title
                                );
                                documents.push(kg_doc);

                                // Found the definition document, no need to check other concepts
                                break;
                            }
                            Err(e) => {
                                log::warn!(
                                    "Failed to read KG definition file '{}': {}",
                                    potential_kg_file.display(),
                                    e
                                );
                            }
                        }
                    } else {
                        log::debug!("KG definition file not found: {:?}", potential_kg_file);
                    }
                }
            } else {
                log::debug!("No KG local config found");
            }
        } else {
            log::debug!("No KG config found for role");
        }

        // Also search through the rolegraph for any documents that contain this term
        let rolegraph_sync = self
            .config_state
            .roles
            .get(role_name)
            .ok_or_else(|| ServiceError::Config(format!("Role '{}' not found", role_name)))?;

        let rolegraph = rolegraph_sync.lock().await;
        let document_ids = rolegraph.find_document_ids_for_term(term);
        drop(rolegraph); // Release the lock early

        log::debug!(
            "Found {} document IDs from rolegraph for term '{}'",
            document_ids.len(),
            term
        );

        // Load documents found in the rolegraph (if any)
        for doc_id in &document_ids {
            // Skip if we already have this document from the KG definition lookup
            if documents
                .iter()
                .any(|d| d.id == *doc_id || d.url == *doc_id)
            {
                log::debug!("Skipping duplicate document from rolegraph: {}", doc_id);
                continue;
            }

            // Load the actual documents using the persistence layer
            // Handle both local and Atomic Data documents properly
            if doc_id.starts_with("http://") || doc_id.starts_with("https://") {
                // Atomic Data document: Try to load from persistence first
                log::debug!("Loading Atomic Data document '{}' from persistence", doc_id);
                let mut placeholder = Document {
                    id: doc_id.clone(),
                    ..Default::default()
                };
                match placeholder.load().await {
                    Ok(loaded_doc) => {
                        log::debug!(
                            "Found cached Atomic Data document '{}' in persistence",
                            doc_id
                        );
                        documents.push(loaded_doc);
                    }
                    Err(_) => {
                        log::warn!(
                            "Atomic Data document '{}' not found in persistence - this may indicate the document hasn't been cached yet",
                            doc_id
                        );
                        // Skip this document for now - it will be cached when accessed through search
                        // In a production system, you might want to fetch it from the Atomic Server here
                    }
                }
            } else {
                // Local document: Use the standard persistence loading
                let mut doc = Document::new(doc_id.clone());
                match doc.load().await {
                    Ok(loaded_doc) => {
                        documents.push(loaded_doc);
                        log::trace!("Successfully loaded local document: {}", doc_id);
                    }
                    Err(e) => {
                        log::warn!("Failed to load local document '{}': {}", doc_id, e);

                        // Check if this might be a hash-based ID from old ripgrep documents
                        if Self::is_hash_based_id(doc_id) {
                            log::debug!(
                                "Document ID '{}' appears to be hash-based (legacy document), skipping for now",
                                doc_id
                            );
                            log::info!(
                                "💡 Hash-based document IDs are deprecated. This document will be re-indexed with normalized IDs on next haystack search."
                            );
                            // Skip legacy hash-based documents - they will be re-indexed with proper normalized IDs
                            // when the haystack is searched again
                        }

                        // Continue processing other documents even if this one fails
                    }
                }
            }
        }

        // Apply KG preprocessing if enabled for this role
        if role.terraphim_it {
            log::info!(
                "🧠 Applying KG preprocessing to {} KG term documents for role '{}' (terraphim_it enabled)",
                documents.len(),
                role_name
            );
            let mut processed_documents = Vec::new();
            let mut total_kg_terms = 0;
            let mut docs_with_kg_links = 0;

            for document in documents {
                let original_body_len = document.body.len();
                let processed_doc = self.preprocess_document_content(document, &role).await?;

                // Count KG links added (rough estimate by body size increase)
                let new_body_len = processed_doc.body.len();
                if new_body_len > original_body_len {
                    docs_with_kg_links += 1;
                    let estimated_links = (new_body_len - original_body_len) / 17;
                    total_kg_terms += estimated_links;
                }

                processed_documents.push(processed_doc);
            }

            log::info!(
                "✅ KG preprocessing complete: {} documents processed, {} received KG links (~{} total links)",
                processed_documents.len(),
                docs_with_kg_links,
                total_kg_terms
            );
            documents = processed_documents;
        } else {
            log::info!(
                "🔍 terraphim_it disabled for role '{}', skipping KG preprocessing for {} documents",
                role_name,
                documents.len()
            );
        }

        // Assign ranks based on order (same logic as regular search)
        // Higher rank for earlier results to maintain consistency
        let total_length = documents.len();
        for (idx, doc) in documents.iter_mut().enumerate() {
            let rank = (total_length - idx) as u64;
            doc.rank = Some(rank);
            log::trace!("Assigned rank {} to document '{}'", rank, doc.title);
        }

        log::debug!(
            "Successfully loaded and processed {} documents for term '{}', ranks assigned from {} to 1",
            documents.len(),
            term,
            total_length
        );
        Ok(documents)
    }

    /// Generate a summary for a document using OpenRouter
    ///
    /// This method takes a document and generates an AI-powered summary using the OpenRouter service.
    /// The summary is generated based on the document's content and can be customized with different
    /// models and length constraints.
    ///
    /// # Arguments
    ///
    /// * `document` - The document to summarize
    /// * `api_key` - The OpenRouter API key
    /// * `model` - The model to use for summarization (e.g., "openai/gpt-3.5-turbo")
    /// * `max_length` - Maximum length of the summary in characters
    ///
    /// # Returns
    ///
    /// Returns a `Result<String>` containing the generated summary or an error if summarization fails.
    #[cfg(feature = "openrouter")]
    pub async fn generate_document_summary(
        &self,
        document: &Document,
        api_key: &str,
        model: &str,
        max_length: usize,
    ) -> Result<String> {
        use crate::openrouter::OpenRouterService;

        log::debug!(
            "Generating summary for document '{}' using model '{}'",
            document.id,
            model
        );

        // Create the OpenRouter service
        let openrouter_service =
            OpenRouterService::new(api_key, model).map_err(ServiceError::OpenRouter)?;

        // Use the document body for summarization
        let content = &document.body;

        if content.trim().is_empty() {
            return Err(ServiceError::Config(
                "Document body is empty, cannot generate summary".to_string(),
            ));
        }

        // Generate the summary
        let summary = openrouter_service
            .generate_summary(content, max_length)
            .await
            .map_err(ServiceError::OpenRouter)?;

        log::info!(
            "Generated {}-character summary for document '{}' using model '{}'",
            summary.len(),
            document.id,
            model
        );

        Ok(summary)
    }

    /// Generate a summary for a document using OpenRouter (stub when feature is disabled)
    #[cfg(not(feature = "openrouter"))]
    pub async fn generate_document_summary(
        &self,
        _document: &Document,
        _api_key: &str,
        _model: &str,
        _max_length: usize,
    ) -> Result<String> {
        Err(ServiceError::Config(
            "OpenRouter feature not enabled during compilation".to_string(),
        ))
    }

    /// Fetch the current config
    pub async fn fetch_config(&self) -> terraphim_config::Config {
        let current_config = self.config_state.config.lock().await;
        current_config.clone()
    }

    // Test helper methods
    #[cfg(test)]
    pub async fn get_role(&self, role_name: &RoleName) -> Result<Role> {
        let config = self.config_state.config.lock().await;
        config
            .roles
            .get(role_name)
            .cloned()
            .ok_or_else(|| ServiceError::Config(format!("Role '{}' not found", role_name)))
    }

    /// Update the config
    ///
    /// Overwrites the config in the config state and returns the updated
    /// config.
    pub async fn update_config(
        &self,
        config: terraphim_config::Config,
    ) -> Result<terraphim_config::Config> {
        // Lock briefly to swap in the new config, then drop before save so
        // the disk write doesn't block other /config endpoints.
        {
            let mut current_config = self.config_state.config.lock().await;
            *current_config = config.clone();
        }
        config.save().await?;
        log::info!("Config updated");
        Ok(config)
    }

    /// Update only the `selected_role` in the config without mutating the rest of the
    /// configuration. Returns the up-to-date `Config` object.
    pub async fn update_selected_role(
        &self,
        role_name: terraphim_types::RoleName,
    ) -> Result<terraphim_config::Config> {
        // Lock briefly: validate, mutate in-memory state, snapshot. Drop the
        // lock BEFORE the disk save -- holding the config mutex across an
        // async I/O write blocks every other endpoint that touches /config
        // (e.g. concurrent search, get_config) for the duration of the save.
        let snapshot = {
            let mut current_config = self.config_state.config.lock().await;

            if !current_config.roles.contains_key(&role_name) {
                return Err(ServiceError::Config(format!(
                    "Role `{}` not found in config",
                    role_name
                )));
            }

            current_config.selected_role = role_name.clone();
            current_config.clone()
        };
        // Persist asynchronously: in-memory update is the source of truth for
        // subsequent reads; disk save is best-effort and must not delay the
        // HTTP response. save_to_all() can take many seconds depending on the
        // configured persistence profiles (sled WAL flush, S3 PUT, etc.) and
        // should never block role selection.
        let snapshot_for_save = snapshot.clone();
        let role_for_log = role_name.clone();
        tokio::spawn(async move {
            if let Err(e) = snapshot_for_save.save().await {
                log::warn!(
                    "background persist of selected_role={} failed: {}",
                    role_for_log,
                    e
                );
            }
        });
        // Log role selection from the snapshot (no need to re-lock).
        if let Some(role) = snapshot.roles.get(&role_name) {
            if role.terraphim_it {
                log::info!(
                    "🎯 Selected role '{}' → terraphim_it: ENABLED (KG preprocessing will be applied)",
                    role_name
                );
            } else {
                log::info!("🎯 Selected role '{}' → terraphim_it: DISABLED", role_name);
            }
        }

        Ok(snapshot)
    }

    /// Highlight search terms in the given text content
    ///
    /// This method wraps matching search terms with HTML-style highlighting tags
    /// to make them visually distinct in the frontend.
    fn highlight_search_terms(content: &str, search_query: &SearchQuery) -> String {
        let mut highlighted_content = content.to_string();

        // Get all terms from the search query
        let terms = search_query.get_all_terms();

        // Sort terms by length (longest first) to avoid partial replacements
        let mut sorted_terms: Vec<&str> = terms.iter().map(|t| t.as_str()).collect();
        sorted_terms.sort_by_key(|term| std::cmp::Reverse(term.len()));

        for term in sorted_terms {
            if term.trim().is_empty() {
                continue;
            }

            // Create case-insensitive regex for the term
            // Escape special regex characters in the search term
            let escaped_term = regex::escape(term);

            if let Ok(regex) = regex::RegexBuilder::new(&escaped_term)
                .case_insensitive(true)
                .build()
            {
                // Replace all matches with highlighted version
                // Use a unique delimiter to avoid conflicts with existing HTML
                let highlight_open = "<mark class=\"search-highlight\">";
                let highlight_close = "</mark>";

                highlighted_content = regex
                    .replace_all(
                        &highlighted_content,
                        format!("{}{}{}", highlight_open, "$0", highlight_close),
                    )
                    .to_string();
            }
        }

        highlighted_content
    }
}

pub(crate) fn snippet_around(s: &str, marker: &str, before: usize, after: usize) -> String {
    let Some(marker_byte) = s.find(marker) else {
        return String::new();
    };
    let marker_char_index = s[..marker_byte].chars().count();
    let total_chars = s.chars().count();

    let start_char_index = marker_char_index.saturating_sub(before);
    let end_char_index = (marker_char_index + marker.len() + after).min(total_chars);

    if start_char_index >= end_char_index {
        return String::new();
    }

    s.chars()
        .skip(start_char_index)
        .take(end_char_index - start_char_index)
        .collect()
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::path::PathBuf;
    use terraphim_config::ConfigBuilder;
    use terraphim_types::NormalizedTermValue;

    #[tokio::test]
    async fn test_get_config() {
        let mut config = ConfigBuilder::new()
            .build_default_desktop()
            .build()
            .unwrap();
        let config_state = ConfigState::new(&mut config).await.unwrap();
        let service = TerraphimService::new(config_state);
        let fetched_config = service.fetch_config().await;
        assert_eq!(fetched_config.id, terraphim_config::ConfigId::Desktop);
    }

    #[tokio::test]
    async fn test_search_documents_selected_role() {
        // Check if KG directory exists before running test
        let project_root = std::env::current_dir().unwrap_or_else(|_| PathBuf::from("."));
        let kg_path = project_root.join("docs/src/kg");
        if !kg_path.exists() {
            println!("Skipping test: KG directory not found at {:?}", kg_path);
            return;
        }

        let mut config = ConfigBuilder::new()
            .build_default_desktop()
            .build()
            .unwrap();
        let config_state = match ConfigState::new(&mut config).await {
            Ok(state) => state,
            Err(e) => {
                println!("Skipping test: Failed to create config state: {:?}", e);
                return;
            }
        };
        let mut service = TerraphimService::new(config_state);
        let search_term = NormalizedTermValue::new("terraphim".to_string());
        let documents = match service.search_documents_selected_role(&search_term).await {
            Ok(docs) => docs,
            Err(e) => {
                println!(
                    "Skipping test: Search failed (expected in some environments): {:?}",
                    e
                );
                return;
            }
        };
        assert!(documents.is_empty() || !documents.is_empty()); // Either empty or has results
    }

    #[tokio::test]
    async fn test_ensure_thesaurus_loaded_terraphim_engineer() {
        // Create a fresh config with correct KG path for testing
        let project_root = std::env::current_dir().unwrap_or_else(|_| PathBuf::from("."));
        let kg_path = project_root.join("docs/src/kg");

        // Skip test gracefully if KG directory doesn't exist
        if !kg_path.exists() {
            println!("⚠️ KG directory not found at {:?}, skipping test", kg_path);
            return;
        }

        let mut config = ConfigBuilder::new()
            .build_default_desktop()
            .build()
            .unwrap();

        // Update the Terraphim Engineer role to use project KG directory
        if let Some(terr_eng_role) = config.roles.get_mut(&"Terraphim Engineer".into()) {
            if let Some(kg) = &mut terr_eng_role.kg {
                if let Some(kg_local) = &mut kg.knowledge_graph_local {
                    kg_local.path = kg_path;
                }
            }
        }

        let config_state = ConfigState::new(&mut config).await.unwrap();
        let mut service = TerraphimService::new(config_state);

        let role_name = RoleName::new("Terraphim Engineer");
        let thesaurus_result = service.ensure_thesaurus_loaded(&role_name).await;

        match thesaurus_result {
            Ok(thesaurus) => {
                println!(
                    "✅ Successfully loaded thesaurus with {} entries",
                    thesaurus.len()
                );
                // Verify thesaurus contains expected terms
                assert!(!thesaurus.is_empty(), "Thesaurus should not be empty");

                // Check for expected terms from docs/src/kg using &thesaurus for iteration
                let has_terraphim = (&thesaurus)
                    .into_iter()
                    .any(|(term, _)| term.as_str().to_lowercase().contains("terraphim"));
                let has_graph = (&thesaurus)
                    .into_iter()
                    .any(|(term, _)| term.as_str().to_lowercase().contains("graph"));

                println!("   Contains 'terraphim': {}", has_terraphim);
                println!("   Contains 'graph': {}", has_graph);

                // At least one of these should be present
                assert!(
                    has_terraphim || has_graph,
                    "Thesaurus should contain expected terms"
                );
            }
            Err(e) => {
                println!("❌ Failed to load thesaurus: {:?}", e);
                // This might fail if the local KG files don't exist, which is expected in some test environments
                // We'll just log the error but not fail the test
            }
        }
    }

    #[tokio::test]
    #[ignore = "Requires local KG fixtures at ~/.terraphim/kg"]
    async fn test_config_building_with_local_kg() {
        // Test that config building works correctly with local KG files
        let mut config = ConfigBuilder::new()
            .build_default_desktop()
            .build()
            .unwrap();
        let config_state_result = ConfigState::new(&mut config).await;

        match config_state_result {
            Ok(config_state) => {
                println!("✅ Successfully built config state");
                // Verify that roles were created
                assert!(
                    !config_state.roles.is_empty(),
                    "Config state should have roles"
                );

                // Check if Terraphim Engineer role was created
                let terraphim_engineer_role = RoleName::new("Terraphim Engineer");
                let has_terraphim_engineer =
                    config_state.roles.contains_key(&terraphim_engineer_role);
                println!("   Has Terraphim Engineer role: {}", has_terraphim_engineer);

                // The role should exist even if thesaurus building failed
                assert!(
                    has_terraphim_engineer,
                    "Terraphim Engineer role should exist"
                );
            }
            Err(e) => {
                println!("❌ Failed to build config state: {:?}", e);
                // This might fail if the local KG files don't exist, which is expected in some test environments
                // We'll just log the error but not fail the test
            }
        }
    }

    #[tokio::test]
    async fn test_atomic_data_persistence_skip() {
        use ahash::AHashMap;
        use terraphim_config::{Config, Haystack, Role, ServiceType};
        use terraphim_persistence::DeviceStorage;
        use terraphim_types::{NormalizedTermValue, RoleName, SearchQuery};

        // Initialize memory-only persistence for testing
        DeviceStorage::init_memory_only().await.unwrap();

        // Create a test config with a role
        let mut config = Config::default();
        let role_name = RoleName::new("test_role");
        let role = Role {
            shortname: None,
            name: "test_role".into(),
            haystacks: vec![Haystack {
                location: "test".to_string(),
                service: ServiceType::Ripgrep,
                read_only: false,
                atomic_server_secret: None,
                extra_parameters: std::collections::HashMap::new(),
                fetch_content: false,
            }],
            kg: None,
            terraphim_it: false,
            theme: "default".to_string(),
            relevance_function: terraphim_types::RelevanceFunction::TitleScorer,
            llm_enabled: false,
            llm_api_key: None,
            llm_model: None,
            llm_auto_summarize: false,
            llm_chat_enabled: false,
            llm_chat_system_prompt: None,
            llm_chat_model: None,
            llm_context_window: None,
            extra: AHashMap::new(),
            llm_router_enabled: false,
            llm_router_config: None,
        };
        config.roles.insert(role_name.clone(), role);

        let config_state = ConfigState::new(&mut config).await.unwrap();
        let mut service = TerraphimService::new(config_state);

        // Create a test search query
        let search_query = SearchQuery {
            search_term: NormalizedTermValue::new("test".to_string()),
            search_terms: None,
            operator: None,
            limit: Some(10),
            skip: None,
            role: Some(role_name),
            layer: Layer::default(),
            include_pinned: false,
            min_quality: None,
        };

        // Test that Atomic Data URLs are skipped during persistence lookup
        // This test verifies that the debug message is logged instead of trying to load from persistence
        let result = service.search(&search_query).await;

        // The search should complete without errors, even though no documents are found
        // The important thing is that Atomic Data URLs don't cause persistence lookup errors
        assert!(result.is_ok(), "Search should complete without errors");
    }

    #[tokio::test]
    async fn test_atomic_data_caching() {
        use ahash::AHashMap;
        use terraphim_config::{Config, Haystack, Role, ServiceType};
        use terraphim_persistence::DeviceStorage;
        use terraphim_types::{Document, NormalizedTermValue, RoleName, SearchQuery};

        // Initialize memory-only persistence for testing
        DeviceStorage::init_memory_only().await.unwrap();

        // Create a test config with a role
        let mut config = Config::default();
        let role_name = RoleName::new("test_role");
        let role = Role {
            shortname: None,
            name: "test_role".into(),
            haystacks: vec![Haystack {
                location: "test".to_string(),
                service: ServiceType::Ripgrep,
                read_only: false,
                atomic_server_secret: None,
                extra_parameters: std::collections::HashMap::new(),
                fetch_content: false,
            }],
            kg: None,
            terraphim_it: false,
            theme: "default".to_string(),
            relevance_function: terraphim_types::RelevanceFunction::TitleScorer,
            llm_enabled: false,
            llm_api_key: None,
            llm_model: None,
            llm_auto_summarize: false,
            llm_chat_enabled: false,
            llm_chat_system_prompt: None,
            llm_chat_model: None,
            llm_context_window: None,
            extra: AHashMap::new(),
            llm_router_enabled: false,
            llm_router_config: None,
        };
        config.roles.insert(role_name.clone(), role);

        let config_state = ConfigState::new(&mut config).await.unwrap();
        let mut service = TerraphimService::new(config_state);

        // Create a mock Atomic Data document
        let atomic_doc = Document {
            id: "http://localhost:9883/borrower-portal/form-field/requestedLoanAmount".to_string(),
            url: "http://localhost:9883/borrower-portal/form-field/requestedLoanAmount".to_string(),
            title: "Requested Loan Amount ($)".to_string(),
            body: "Form field for Requested Loan Amount ($)".to_string(),
            description: Some("Form field for Requested Loan Amount ($)".to_string()),
            summarization: None,
            stub: None,
            tags: None,
            rank: None,
            source_haystack: None,
            doc_type: terraphim_types::DocumentType::KgEntry,
            synonyms: None,
            route: None,
            priority: None,
            quality_score: None,
        };

        // Test 1: Save Atomic Data document to persistence
        log::info!("Testing Atomic Data document caching...");
        match atomic_doc.save().await {
            Ok(_) => log::info!("✅ Successfully saved Atomic Data document to persistence"),
            Err(e) => {
                log::error!("❌ Failed to save Atomic Data document: {}", e);
                panic!("Atomic Data document save failed");
            }
        }

        // Test 2: Verify the document can be loaded from persistence
        let mut placeholder = Document {
            id: atomic_doc.id.clone(),
            ..Default::default()
        };
        match placeholder.load().await {
            Ok(loaded_doc) => {
                log::info!("✅ Successfully loaded Atomic Data document from persistence");
                assert_eq!(loaded_doc.title, atomic_doc.title);
                assert_eq!(loaded_doc.body, atomic_doc.body);
                assert_eq!(loaded_doc.description, atomic_doc.description);
            }
            Err(e) => {
                log::error!(
                    "❌ Failed to load Atomic Data document from persistence: {}",
                    e
                );
                panic!("Atomic Data document load failed");
            }
        }

        // Test 3: Verify the search logic would find the cached document
        let search_query = SearchQuery {
            search_term: NormalizedTermValue::new("test".to_string()),
            search_terms: None,
            operator: None,
            limit: Some(10),
            skip: None,
            role: Some(role_name),
            layer: Layer::default(),
            include_pinned: false,
            min_quality: None,
        };

        let result = service.search(&search_query).await;
        assert!(result.is_ok(), "Search should complete without errors");

        log::info!("✅ All Atomic Data caching tests passed!");
    }

    #[tokio::test]
    #[ignore = "Requires local KG fixtures at 'test' directory"]
    async fn test_kg_term_search_with_atomic_data() {
        use ahash::AHashMap;
        use std::path::PathBuf;
        use terraphim_config::{
            Config, Haystack, KnowledgeGraph, KnowledgeGraphLocal, Role, ServiceType,
        };
        use terraphim_persistence::DeviceStorage;
        use terraphim_types::{Document, KnowledgeGraphInputType, RoleName};

        // Initialize memory-only persistence for testing
        DeviceStorage::init_memory_only().await.unwrap();

        // Create a test config with a role that has KG enabled
        let mut config = Config::default();
        let role_name = RoleName::new("test_kg_role");
        let role = Role {
            shortname: None,
            name: "test_kg_role".into(),
            haystacks: vec![Haystack {
                location: "test".to_string(),
                service: ServiceType::Ripgrep,
                read_only: false,
                atomic_server_secret: None,
                extra_parameters: std::collections::HashMap::new(),
                fetch_content: false,
            }],
            kg: Some(KnowledgeGraph {
                automata_path: None,
                knowledge_graph_local: Some(KnowledgeGraphLocal {
                    input_type: KnowledgeGraphInputType::Markdown,
                    path: PathBuf::from("test"),
                }),
                public: true,
                publish: true,
            }),
            terraphim_it: true,
            theme: "default".to_string(),
            relevance_function: terraphim_types::RelevanceFunction::TerraphimGraph,
            llm_enabled: false,
            llm_api_key: None,
            llm_model: None,
            llm_auto_summarize: false,
            llm_chat_enabled: false,
            llm_chat_system_prompt: None,
            llm_chat_model: None,
            llm_context_window: None,
            extra: AHashMap::new(),
            llm_router_enabled: false,
            llm_router_config: None,
        };
        config.roles.insert(role_name.clone(), role);

        let config_state = ConfigState::new(&mut config).await.unwrap();
        let mut service = TerraphimService::new(config_state);

        // Create and cache an Atomic Data document
        let atomic_doc = Document {
            id: "http://localhost:9883/borrower-portal/form-field/requestedLoanAmount".to_string(),
            url: "http://localhost:9883/borrower-portal/form-field/requestedLoanAmount".to_string(),
            title: "Requested Loan Amount ($)".to_string(),
            body: "Form field for Requested Loan Amount ($)".to_string(),
            description: Some("Form field for Requested Loan Amount ($)".to_string()),
            summarization: None,
            stub: None,
            tags: None,
            rank: None,
            source_haystack: None,
            doc_type: terraphim_types::DocumentType::KgEntry,
            synonyms: None,
            route: None,
            priority: None,
            quality_score: None,
        };

        // Save the Atomic Data document to persistence
        log::info!("Testing KG term search with Atomic Data documents...");
        match atomic_doc.save().await {
            Ok(_) => log::info!("✅ Successfully saved Atomic Data document to persistence"),
            Err(e) => {
                log::error!("❌ Failed to save Atomic Data document: {}", e);
                panic!("Atomic Data document save failed");
            }
        }

        // Test that find_documents_for_kg_term can handle Atomic Data document IDs
        // Note: In a real scenario, the rolegraph would contain the Atomic Data document ID
        // For this test, we're verifying that the function can handle Atomic Data URLs properly
        let result = service.find_documents_for_kg_term(&role_name, "test").await;

        // The function should complete without errors, even if no documents are found
        // The important thing is that it doesn't crash when encountering Atomic Data URLs
        assert!(
            result.is_ok(),
            "find_documents_for_kg_term should complete without errors"
        );

        let documents = result.unwrap();
        log::info!(
            "✅ KG term search completed successfully, found {} documents",
            documents.len()
        );

        // Verify that the function can handle Atomic Data document loading
        // by manually testing the document loading logic
        let atomic_doc_id = "http://localhost:9883/borrower-portal/form-field/requestedLoanAmount";
        let mut placeholder = Document {
            id: atomic_doc_id.to_string(),
            ..Default::default()
        };

        match placeholder.load().await {
            Ok(loaded_doc) => {
                log::info!(
                    "✅ Successfully loaded Atomic Data document from persistence in KG term search context"
                );
                assert_eq!(loaded_doc.title, atomic_doc.title);
                assert_eq!(loaded_doc.body, atomic_doc.body);
            }
            Err(e) => {
                log::error!(
                    "❌ Failed to load Atomic Data document in KG term search context: {}",
                    e
                );
                panic!("Atomic Data document load failed in KG term search context");
            }
        }

        log::info!("✅ All KG term search with Atomic Data tests passed!");
    }

    #[tokio::test]
    async fn test_kg_term_search_rank_assignment() -> Result<()> {
        use ahash::AHashMap;
        use terraphim_config::{Config, Haystack, Role, ServiceType};
        use terraphim_persistence::DeviceStorage;
        use terraphim_types::{Document, RoleName};

        // Initialize memory-only persistence for testing
        DeviceStorage::init_memory_only().await.unwrap();

        // Create a test config with a role that has KG capabilities
        let mut config = Config::default();
        let role_name = RoleName::new("Test KG Role");
        let role = Role {
            shortname: Some("test-kg".to_string()),
            name: role_name.clone(),
            haystacks: vec![Haystack {
                location: "test".to_string(),
                service: ServiceType::Ripgrep,
                read_only: false,
                atomic_server_secret: None,
                extra_parameters: std::collections::HashMap::new(),
                fetch_content: false,
            }],
            kg: Some(terraphim_config::KnowledgeGraph {
                automata_path: Some(terraphim_automata::AutomataPath::local_example()),
                knowledge_graph_local: None,
                public: false,
                publish: false,
            }),
            terraphim_it: false,
            theme: "default".to_string(),
            relevance_function: terraphim_types::RelevanceFunction::TitleScorer,
            llm_enabled: false,
            llm_api_key: None,
            llm_model: None,
            llm_auto_summarize: false,
            llm_chat_enabled: false,
            llm_chat_system_prompt: None,
            llm_chat_model: None,
            llm_context_window: None,
            extra: AHashMap::new(),
            llm_router_enabled: false,
            llm_router_config: None,
        };
        config.roles.insert(role_name.clone(), role);

        let config_state = ConfigState::new(&mut config).await.unwrap();
        let _service = TerraphimService::new(config_state);

        // Create test documents and save them to persistence
        let test_documents = vec![
            Document {
                id: "test-doc-1".to_string(),
                title: "First Test Document".to_string(),
                body: "This is the first test document body".to_string(),
                url: "test://doc1".to_string(),
                description: Some("First document description".to_string()),
                summarization: None,
                stub: None,
                tags: Some(vec!["test".to_string(), "first".to_string()]),
                rank: None, // Should be assigned by the function
                source_haystack: None,
                doc_type: terraphim_types::DocumentType::KgEntry,
                synonyms: None,
                route: None,
                priority: None,
                quality_score: None,
            },
            Document {
                id: "test-doc-2".to_string(),
                title: "Second Test Document".to_string(),
                body: "This is the second test document body".to_string(),
                url: "test://doc2".to_string(),
                description: Some("Second document description".to_string()),
                summarization: None,
                stub: None,
                tags: Some(vec!["test".to_string(), "second".to_string()]),
                rank: None, // Should be assigned by the function
                source_haystack: None,
                doc_type: terraphim_types::DocumentType::KgEntry,
                synonyms: None,
                route: None,
                priority: None,
                quality_score: None,
            },
            Document {
                id: "test-doc-3".to_string(),
                title: "Third Test Document".to_string(),
                body: "This is the third test document body".to_string(),
                url: "test://doc3".to_string(),
                description: Some("Third document description".to_string()),
                summarization: None,
                stub: None,
                tags: Some(vec!["test".to_string(), "third".to_string()]),
                rank: None, // Should be assigned by the function
                source_haystack: None,
                doc_type: terraphim_types::DocumentType::KgEntry,
                synonyms: None,
                route: None,
                priority: None,
                quality_score: None,
            },
        ];

        // Save test documents to persistence
        for doc in &test_documents {
            doc.save().await.expect("Failed to save test document");
        }

        // The rolegraph will be created automatically by ensure_thesaurus_loaded
        // We don't need to manually create it for this test

        // Test the rank assignment logic directly
        // This validates the core functionality we implemented in find_documents_for_kg_term
        let mut simulated_documents = test_documents.clone();

        // Apply the same rank assignment logic as in find_documents_for_kg_term
        let total_length = simulated_documents.len();
        for (idx, doc) in simulated_documents.iter_mut().enumerate() {
            let rank = (total_length - idx) as u64;
            doc.rank = Some(rank);
        }

        // Verify rank assignment
        assert_eq!(simulated_documents.len(), 3, "Should have 3 test documents");

        // Check that all documents have ranks assigned
        for doc in &simulated_documents {
            assert!(
                doc.rank.is_some(),
                "Document '{}' should have a rank assigned",
                doc.title
            );
            assert!(
                doc.rank.unwrap() > 0,
                "Document '{}' should have a positive rank",
                doc.title
            );
        }

        // Check that ranks are in descending order (first document has highest rank)
        assert_eq!(
            simulated_documents[0].rank,
            Some(3),
            "First document should have highest rank (3)"
        );
        assert_eq!(
            simulated_documents[1].rank,
            Some(2),
            "Second document should have rank 2"
        );
        assert_eq!(
            simulated_documents[2].rank,
            Some(1),
            "Third document should have rank 1"
        );

        // Verify ranks are unique and properly ordered
        let mut ranks: Vec<u64> = simulated_documents
            .iter()
            .map(|doc| doc.rank.unwrap())
            .collect();
        ranks.sort_by_key(|r| std::cmp::Reverse(*r));
        assert_eq!(
            ranks,
            vec![3, 2, 1],
            "Ranks should be unique and in descending order"
        );

        log::info!("✅ KG term search rank assignment test completed successfully!");
        Ok(())
    }

    // Helper to build a Document with a given composite quality score.
    fn doc_with_quality(id: &str, knowledge: f64, logic: f64, structure: f64) -> Document {
        Document {
            id: id.to_string(),
            url: format!("https://example.com/{id}"),
            title: id.to_string(),
            body: String::new(),
            quality_score: Some(terraphim_types::QualityScore {
                knowledge: Some(knowledge),
                logic: Some(logic),
                structure: Some(structure),
                last_evaluated: None,
            }),
            ..Default::default()
        }
    }

    fn doc_without_quality(id: &str) -> Document {
        Document {
            id: id.to_string(),
            url: format!("https://example.com/{id}"),
            title: id.to_string(),
            body: String::new(),
            quality_score: None,
            ..Default::default()
        }
    }

    #[test]
    fn test_min_quality_none_returns_all_documents() {
        // When min_quality is None, all documents are returned unchanged.
        let docs = vec![
            doc_with_quality("a", 0.9, 0.9, 0.9),
            doc_with_quality("b", 0.1, 0.1, 0.1),
            doc_without_quality("c"),
        ];
        let result = TerraphimService::apply_min_quality_filter(docs, None);
        assert_eq!(result.len(), 3);
    }

    #[test]
    fn test_min_quality_keeps_documents_at_or_above_threshold() {
        // composite = (0.8 + 0.6 + 0.7) / 3 = 0.7
        let high = doc_with_quality("high", 0.8, 0.6, 0.7);
        // composite = (0.3 + 0.2 + 0.1) / 3 ≈ 0.2
        let low = doc_with_quality("low", 0.3, 0.2, 0.1);
        let docs = vec![high, low];

        let result = TerraphimService::apply_min_quality_filter(docs, Some(0.5));
        assert_eq!(result.len(), 1);
        assert_eq!(result[0].id, "high");
    }

    #[test]
    fn test_min_quality_excludes_documents_below_threshold() {
        // composite = 0.4
        let doc = doc_with_quality("below", 0.4, 0.4, 0.4);
        let result = TerraphimService::apply_min_quality_filter(vec![doc], Some(0.5));
        assert!(result.is_empty());
    }

    #[test]
    fn test_min_quality_excludes_documents_without_quality_score() {
        // Documents with no quality_score must be excluded when a threshold is set.
        let no_score = doc_without_quality("no-score");
        let result = TerraphimService::apply_min_quality_filter(vec![no_score], Some(0.0));
        assert!(result.is_empty());
    }

    #[test]
    fn test_min_quality_exact_threshold_is_included() {
        // composite = 0.5 exactly — must satisfy >= threshold
        let doc = doc_with_quality("exact", 0.5, 0.5, 0.5);
        let result = TerraphimService::apply_min_quality_filter(vec![doc], Some(0.5));
        assert_eq!(result.len(), 1);
    }

    #[test]
    fn test_min_quality_threshold_zero_excludes_no_score_docs() {
        // Threshold 0.0 passes any document that has a score, but not scoreless ones.
        let with_score = doc_with_quality("scored", 0.0, 0.0, 0.0);
        let no_score = doc_without_quality("unscored");
        let result =
            TerraphimService::apply_min_quality_filter(vec![with_score, no_score], Some(0.0));
        assert_eq!(result.len(), 1);
        assert_eq!(result[0].id, "scored");
    }

    #[test]
    fn test_min_quality_empty_input_returns_empty() {
        let result = TerraphimService::apply_min_quality_filter(vec![], Some(0.5));
        assert!(result.is_empty());
    }

    #[test]
    fn test_min_quality_preserves_document_order() {
        // Verify that documents passing the filter are returned in original order.
        let a = doc_with_quality("a", 0.9, 0.9, 0.9);
        let b = doc_with_quality("b", 0.8, 0.8, 0.8);
        let c = doc_with_quality("c", 0.7, 0.7, 0.7);
        let result = TerraphimService::apply_min_quality_filter(vec![a, b, c], Some(0.5));
        assert_eq!(result.len(), 3);
        assert_eq!(result[0].id, "a");
        assert_eq!(result[1].id, "b");
        assert_eq!(result[2].id, "c");
    }

    #[test]
    fn test_min_quality_negative_threshold_clamped_to_zero() {
        // A negative threshold is clamped to 0.0: documents with any score pass,
        // documents without a score are still excluded.
        let with_score = doc_with_quality("scored", 0.1, 0.1, 0.1);
        let no_score = doc_without_quality("unscored");
        let result =
            TerraphimService::apply_min_quality_filter(vec![with_score, no_score], Some(-0.1));
        assert_eq!(result.len(), 1, "only scored document should pass");
        assert_eq!(result[0].id, "scored");
    }

    #[test]
    fn test_snippet_around_ascii_simple() {
        let s = "Hello World foo](kg:bar Baz";
        let result = snippet_around(s, "](kg:", 10, 10);
        assert_eq!(result, " World foo](kg:bar Baz");
    }

    #[test]
    fn test_snippet_around_ascii_truncation_left() {
        let s = "xyz Hello World foo](kg:bar";
        let result = snippet_around(s, "](kg:", 10, 10);
        assert_eq!(result, " World foo](kg:bar");
    }

    #[test]
    fn test_snippet_around_ascii_truncation_right() {
        let s = "Hello World foo](kg:bar xyz";
        let result = snippet_around(s, "](kg:", 10, 10);
        assert_eq!(result, " World foo](kg:bar xyz");
    }

    #[test]
    fn test_snippet_around_multibyte_cjk() {
        let s = "日本語 Hello](kg:bar 日本語";
        let result = snippet_around(s, "](kg:", 5, 5);
        assert!(!result.is_empty());
        assert!(result.contains("Hello"));
        assert!(result.contains("](kg:"));
    }

    #[test]
    fn test_snippet_around_multibyte_emoji() {
        let s = "Hello 😂 World](kg:bar";
        let result = snippet_around(s, "](kg:", 10, 10);
        assert!(!result.is_empty());
        assert!(result.contains("😂"));
        assert!(result.contains("](kg:"));
    }

    #[test]
    fn test_snippet_around_marker_not_found() {
        let s = "Hello World";
        let result = snippet_around(s, "](kg:", 10, 10);
        assert_eq!(result, "");
    }

    #[test]
    fn test_snippet_around_empty_string() {
        let s = "";
        let result = snippet_around(s, "](kg:", 10, 10);
        assert_eq!(result, "");
    }

    #[test]
    fn test_snippet_around_marker_at_start() {
        let s = "](kg:bar Hello";
        let result = snippet_around(s, "](kg:", 10, 10);
        assert_eq!(result, "](kg:bar Hello");
    }

    #[test]
    fn test_snippet_around_marker_at_end() {
        let s = "Hello ](kg:bar";
        let result = snippet_around(s, "](kg:", 10, 10);
        assert_eq!(result, "Hello ](kg:bar");
    }
}