rag-module 0.6.7

//! Search service for collection-specific search per business architecture
//! Chat History: Scroll API for filter-only search (1D dummy vectors)
//! AWS Estate: Vector search for semantic + filters (1024D BGE-M3 vectors)

use anyhow::{Result, anyhow};
use async_trait::async_trait;
use std::sync::Arc;
use std::collections::{HashMap, HashSet};
use serde_json::{json, Value as JsonValue};
use regex::Regex;
use tracing::{info, debug, warn, error};

use crate::types::{SearchOptions, SearchResult, Document, SearchFilter, FilterCondition, MatchCondition};
use crate::db::VectorStore;
use crate::services::{EmbeddingService, SecurityService, EncryptionService};

/// Collection configuration for business architecture
#[derive(Debug, Clone)]
pub struct CollectionConfig {
    pub use_vector_search: bool,
    pub requires_embedding: bool,
    pub default_limit: usize,
    pub max_limit: usize,
    pub default_filters: Vec<String>,
}

/// Chat search options
#[derive(Debug, Clone)]
pub struct ChatSearchOptions {
    pub context_id: Option<String>,
    pub role: Option<String>,
    pub from_timestamp: Option<i64>,
    pub to_timestamp: Option<i64>,
    pub from_message_index: Option<i32>,
    pub to_message_index: Option<i32>,
    pub limit: Option<usize>,
    pub include_metadata: bool,
    pub user_id: Option<String>,
}

/// Estate search options
#[derive(Debug, Clone)]
pub struct EstateSearchOptions {
    pub resource_types: Option<Vec<String>>,
    pub account_ids: Option<Vec<String>>,
    pub regions: Option<Vec<String>>,
    pub services: Option<Vec<String>>,
    pub states: Option<Vec<String>>,
    pub environment: Option<String>,
    pub application: Option<String>,
    pub synced_after: Option<i64>,
    pub limit: Option<usize>,
    pub score_threshold: Option<f32>,
    pub include_metadata: bool,
    pub use_anonymous_ids: bool,
    pub parameters: Option<HashMap<String, serde_json::Value>>, // Supports any JSON value (string, array, number, boolean, etc.)
}

/// IAM context for permission filtering
#[derive(Debug, Clone)]
pub struct IAMContext {
    pub user_id: String,
    pub roles: Vec<String>,
    pub permissions: Vec<String>,
}

pub struct SearchService {
    vector_store: Arc<dyn VectorStore + Send + Sync>,
    embedding_service: Arc<EmbeddingService>,
    security_service: Option<Arc<SecurityService>>,
    encryption_service: Option<Arc<EncryptionService>>,
    mapping_service: Option<Arc<dyn MappingService + Send + Sync>>,
    iam_service: Option<Arc<dyn IAMService + Send + Sync>>,
    collection_configs: HashMap<String, CollectionConfig>,
    spell_corrections: HashMap<String, String>,
    base_path: std::path::PathBuf,
}

/// Trait for anonymous ID mapping service
#[async_trait::async_trait]
pub trait MappingService {
    async fn get_anonymous_id(&self, original_id: &str) -> Result<Option<String>>;
}

/// Trait for IAM permission service
#[async_trait::async_trait]
pub trait IAMService {
    async fn check_resource_access(&self, context: &IAMContext, resource_id: &str, action: &str) -> Result<bool>;
}

impl SearchService {
    pub async fn new(
        vector_store: Arc<dyn VectorStore + Send + Sync>,
        embedding_service: Arc<EmbeddingService>,
    ) -> Result<Self> {
        let mut collection_configs = HashMap::new();
        
        // Chat collection configuration per business architecture
        collection_configs.insert("chat_history".to_string(), CollectionConfig {
            use_vector_search: false, // Scroll API only for chat history
            requires_embedding: false,
            default_limit: 50,
            max_limit: 200,
            default_filters: vec!["context_id".to_string(), "role".to_string(), "timestamp".to_string()],
        });
        
        // Estate collection configuration per business architecture
        // This serves as the default/template config for all estate collections
        // (aws_estate, azure_estate, gcp_estate, core_estate, etc.)
        let estate_config = CollectionConfig {
            use_vector_search: true, // Vector search for estate resources
            requires_embedding: true,
            default_limit: 100,
            max_limit: 400,
            default_filters: vec!["resource_type".to_string(), "account_id".to_string(), "region".to_string(), "service".to_string()],
        };

        // Register default estate collections
        collection_configs.insert("aws_estate".to_string(), estate_config.clone());
        collection_configs.insert("core_estate".to_string(), estate_config.clone());
        collection_configs.insert("azure_estate".to_string(), estate_config.clone());
        collection_configs.insert("gcp_estate".to_string(), estate_config.clone());
        
        let spell_corrections = Self::create_spell_corrections();
        
        Ok(Self {
            vector_store,
            embedding_service,
            security_service: None,
            encryption_service: None,
            mapping_service: None,
            iam_service: None,
            collection_configs,
            spell_corrections,
            base_path: std::path::PathBuf::from("."),
        })
    }

    /// Set the base path for local file storage
    pub fn set_base_path(&mut self, path: std::path::PathBuf) {
        self.base_path = path;
    }
    
    /// Create a new search service with optional services
    pub async fn new_with_services(
        vector_store: Arc<dyn VectorStore + Send + Sync>,
        embedding_service: Arc<EmbeddingService>,
        security_service: Option<Arc<SecurityService>>,
        encryption_service: Option<Arc<EncryptionService>>,
        mapping_service: Option<Arc<dyn MappingService + Send + Sync>>,
        iam_service: Option<Arc<dyn IAMService + Send + Sync>>,
    ) -> Result<Self> {
        let mut service = Self::new(vector_store, embedding_service).await?;
        service.security_service = security_service;
        service.encryption_service = encryption_service;
        service.mapping_service = mapping_service;
        service.iam_service = iam_service;
        Ok(service)
    }
    
    /// Create spell corrections map for multi-cloud estate queries
    fn create_spell_corrections() -> HashMap<String, String> {
        let mut corrections = HashMap::new();
        
        // AWS corrections
        corrections.insert("lamda".to_string(), "lambda".to_string());
        corrections.insert("lambada".to_string(), "lambda".to_string());
        corrections.insert("lamabda".to_string(), "lambda".to_string());
        corrections.insert("lamnda".to_string(), "lambda".to_string());
        corrections.insert("dynamo".to_string(), "dynamodb".to_string());
        corrections.insert("api-gateway".to_string(), "api gateway".to_string());
        corrections.insert("apigateway".to_string(), "api gateway".to_string());
        corrections.insert("secrets".to_string(), "secrets manager".to_string());
        corrections.insert("parameter".to_string(), "parameter store".to_string());
        corrections.insert("systems".to_string(), "systems manager".to_string());
        corrections.insert("certificate".to_string(), "certificate manager".to_string());
        corrections.insert("directory".to_string(), "directory service".to_string());
        corrections.insert("control-tower".to_string(), "control tower".to_string());
        corrections.insert("security-hub".to_string(), "security hub".to_string());
        corrections.insert("disaster".to_string(), "disaster recovery".to_string());
        corrections.insert("running instances".to_string(), "running instance ec2".to_string());
        corrections.insert("stopped instances".to_string(), "stopped instance ec2".to_string());
        corrections.insert("database instances".to_string(), "database instance rds".to_string());
        corrections.insert("storage buckets".to_string(), "storage bucket s3".to_string());
        
        // Azure corrections
        corrections.insert("virtual machines".to_string(), "virtual machine vm".to_string());
        corrections.insert("storage accounts".to_string(), "storage account".to_string());
        corrections.insert("app services".to_string(), "app service".to_string());
        corrections.insert("key vaults".to_string(), "key vault".to_string());
        corrections.insert("sql databases".to_string(), "sql database".to_string());
        corrections.insert("service bus".to_string(), "servicebus".to_string());
        corrections.insert("cosmos db".to_string(), "cosmosdb".to_string());
        corrections.insert("azure ad".to_string(), "active directory".to_string());
        
        // GCP corrections
        corrections.insert("compute engine".to_string(), "compute-engine vm".to_string());
        corrections.insert("cloud storage".to_string(), "cloud-storage bucket".to_string());
        corrections.insert("cloud functions".to_string(), "cloud-functions".to_string());
        corrections.insert("cloud sql".to_string(), "cloud-sql database".to_string());
        corrections.insert("pub sub".to_string(), "pub-sub".to_string());
        corrections.insert("cloud run".to_string(), "cloud-run".to_string());
        corrections.insert("kubernetes engine".to_string(), "gke cluster".to_string());
        
        corrections
    }
    
    // ============ BUSINESS ARCHITECTURE SEARCH METHODS ============

    /// Load chat documents from local encrypted files
    async fn load_local_chat_documents(&self, user_id: &str) -> Result<Vec<Document>> {
        use tokio::fs;
        use serde::{Deserialize};

        #[derive(Debug, Deserialize)]
        struct EncryptedDoc {
            id: String,
            content: String,
            #[serde(default)]
            metadata: std::collections::HashMap<String, serde_json::Value>,
        }

        let user_dir = self.base_path.join("qdrant-data").join(user_id);
        let docs_file = user_dir.join("chat_history-documents.json");

        println!("🔍 DEBUG: Looking for chat documents at: {}", docs_file.display());

        if !docs_file.exists() {
            println!("⚠️  DEBUG: File not found at {}", docs_file.display());
            return Ok(Vec::new());
        }

        let content = fs::read_to_string(&docs_file).await?;

        // Parse JSON - it might be wrapped in a "documents" field or be a direct array
        let encrypted_docs: Vec<EncryptedDoc> = if let Ok(wrapper) = serde_json::from_str::<serde_json::Value>(&content) {
            if let Some(docs_array) = wrapper.get("documents") {
                // Wrapped format: { "documents": [...] }
                serde_json::from_value(docs_array.clone())?
            } else {
                // Direct array format: [...]
                serde_json::from_str(&content)?
            }
        } else {
            serde_json::from_str(&content)?
        };

        println!("🔍 DEBUG: Loaded {} encrypted documents from file", encrypted_docs.len());

        // Convert to Document format
        let documents: Vec<Document> = encrypted_docs.into_iter().map(|doc| {
            let metadata_map: indexmap::IndexMap<String, serde_json::Value> =
                doc.metadata.into_iter().collect();

            Document {
                id: doc.id.clone(),
                vector_id: doc.id,
                content: doc.content,
                embedding: None,
                metadata: metadata_map,
                created_at: chrono::Utc::now(),
                updated_at: chrono::Utc::now(),
            }
        }).collect();

        Ok(documents)
    }

    /// Search chat history using local encrypted files (Business Architecture API)
    pub async fn search_chat_history(&self, options: ChatSearchOptions) -> Result<Vec<JsonValue>> {
        let config = self.collection_configs.get("chat_history")
            .ok_or_else(|| anyhow!("Chat collection configuration not found"))?;

        let limit = std::cmp::min(
            options.limit.unwrap_or(config.default_limit),
            config.max_limit
        );

        // Get user_id from options, if not provided we can't search local files
        let user_id = options.user_id.as_ref()
            .ok_or_else(|| anyhow!("user_id is required for local chat search"))?;

        // Load from local encrypted files instead of Qdrant
        let all_results = self.load_local_chat_documents(user_id).await?;

        println!("🔐 DEBUG: Retrieved {} chat documents from local storage for user {}", all_results.len(), user_id);

        // If context_id is provided, collect all messages and format as conversation
        if let Some(ref context_id) = options.context_id {
            return self.format_chat_history_by_context(all_results, context_id, &options).await;
        }

        // Default behavior: return individual messages
        let chat_filter = self.create_chat_filter_function(&options);
        let mut filtered_results = Vec::new();

        for document in all_results {
            // Decrypt document content and metadata
            let decrypted_content = if document.content.len() > 100 {
                if let Some(ref enc_service) = self.encryption_service {
                    match enc_service.decrypt_content(&document.content).await {
                        Ok(content) => content,
                        Err(_) => document.content.clone(),
                    }
                } else {
                    document.content.clone()
                }
            } else {
                document.content.clone()
            };

            // Decrypt metadata
            let decrypted_metadata = self.decrypt_document_metadata(&document.metadata).await;

            // Create JSON object for filtering
            let mut json_obj = serde_json::Map::new();
            json_obj.insert("id".to_string(), JsonValue::String(document.id.clone()));

            // Parse decrypted content as JSON to extract fields like context_id, role, etc.
            if let Ok(content_json) = serde_json::from_str::<JsonValue>(&decrypted_content) {
                if let Some(content_obj) = content_json.as_object() {
                    // Extract fields from content JSON for filtering
                    for (key, value) in content_obj {
                        json_obj.insert(key.clone(), value.clone());
                    }
                }
            } else {
                // If content isn't JSON, store as plain content
                json_obj.insert("content".to_string(), JsonValue::String(decrypted_content.clone()));
            }

            // Add metadata fields (these override content fields if there are conflicts)
            for (key, value) in &decrypted_metadata {
                json_obj.insert(key.clone(), value.clone());
            }

            let json_value = JsonValue::Object(json_obj);

            // Apply chat filters
            if chat_filter(&json_value) {
                let mut final_json = serde_json::Map::new();
                final_json.insert("id".to_string(), JsonValue::String(document.id));
                final_json.insert("content".to_string(), JsonValue::String(decrypted_content));

                if options.include_metadata {
                    for (key, value) in decrypted_metadata {
                        // Skip content field from metadata to avoid overwriting decrypted content
                        if key != "content" && key != "_encrypted_content" {
                            final_json.insert(key, value);
                        }
                    }
                }

                filtered_results.push(JsonValue::Object(final_json));
            }
        }

        // Sort by timestamp if available (for context_id queries)
        filtered_results.sort_by(|a, b| {
            let a_timestamp = a.get("t").and_then(|v| v.as_i64()).unwrap_or(0);
            let b_timestamp = b.get("t").and_then(|v| v.as_i64()).unwrap_or(0);
            a_timestamp.cmp(&b_timestamp)
        });

        // Limit results
        filtered_results.truncate(limit);

        println!("🔍 DEBUG: Final chat search results: {}", filtered_results.len());
        Ok(filtered_results)
    }

    /// Format chat history by context_id - collects all messages and formats as conversation
    async fn format_chat_history_by_context(
        &self,
        documents: Vec<Document>,
        context_id: &str,
        options: &ChatSearchOptions,
    ) -> Result<Vec<JsonValue>> {
        // Helper function to normalize role for sorting
        fn normalize_role_for_sorting(role: &JsonValue) -> i32 {
            match role {
                JsonValue::String(s) => {
                    let s_lower = s.to_lowercase();
                    if s_lower == "0" || s_lower == "user" {
                        0
                    } else if s_lower == "1" || s_lower == "assistant" {
                        1
                    } else {
                        999 // Unknown roles go last
                    }
                },
                JsonValue::Number(n) => {
                    n.as_i64().unwrap_or(999) as i32
                },
                _ => 999,
            }
        }

        #[derive(Debug, Clone)]
        struct MessageEntry {
            index: i64,
            role: JsonValue,
            content: JsonValue,
            timestamp: Option<String>,
        }

        // Collect all messages for this context_id
        let mut messages: Vec<MessageEntry> = Vec::new();
        let mut chat_metadata: Option<serde_json::Map<String, JsonValue>> = None;
        let mut seen_document_ids: std::collections::HashSet<String> = std::collections::HashSet::new();

        for document in documents {
            // Skip duplicate documents (same document ID)
            if !seen_document_ids.insert(document.id.clone()) {
                println!("🔍 DEBUG: Skipping duplicate document ID: {}", document.id);
                continue;
            }
            // Decrypt document content
            let decrypted_content = if document.content.len() > 100 {
                if let Some(ref enc_service) = self.encryption_service {
                    match enc_service.decrypt_content(&document.content).await {
                        Ok(content) => content,
                        Err(_) => document.content.clone(),
                    }
                } else {
                    document.content.clone()
                }
            } else {
                document.content.clone()
            };

            // Parse the content as JSON
            if let Ok(content_json) = serde_json::from_str::<JsonValue>(&decrypted_content) {
                // Check if this matches our context_id
                let doc_context_id = content_json.get("i")
                    .or_else(|| content_json.get("context_id"))
                    .and_then(|v| v.as_str());

                if doc_context_id == Some(context_id) {
                    // Get message index - try both "m" and "message_index"
                    let message_index = content_json.get("m")
                        .or_else(|| content_json.get("message_index"))
                        .and_then(|v| v.as_i64());

                    // Get role - try both "r" and "role"
                    let role = content_json.get("r")
                        .or_else(|| content_json.get("role"));

                    // Get content - try both "c" and "content"
                    let message_content = content_json.get("c")
                        .or_else(|| content_json.get("content"));

                    // Get timestamp for deduplication
                    let timestamp = content_json.get("t")
                        .or_else(|| content_json.get("timestamp"))
                        .and_then(|v| v.as_str())
                        .map(|s| s.to_string());

                    if let (Some(idx), Some(role_val), Some(content_val)) = (message_index, role, message_content) {
                        messages.push(MessageEntry {
                            index: idx,
                            role: role_val.clone(),
                            content: content_val.clone(),
                            timestamp,
                        });
                    }

                    // Capture chat metadata from first matching document
                    if chat_metadata.is_none() {
                        let mut metadata = serde_json::Map::new();

                        if let Some(i) = content_json.get("i") {
                            metadata.insert("i".to_string(), i.clone());
                        }
                        if let Some(a) = content_json.get("a") {
                            metadata.insert("a".to_string(), a.clone());
                        }
                        if let Some(ct) = content_json.get("ct").or_else(|| content_json.get("chat_title")) {
                            metadata.insert("ct".to_string(), ct.clone());
                        }
                        if let Some(t) = content_json.get("t").or_else(|| content_json.get("timestamp")) {
                            metadata.insert("t".to_string(), t.clone());
                        }
                        if let Some(l) = content_json.get("l") {
                            metadata.insert("l".to_string(), l.clone());
                        }

                        chat_metadata = Some(metadata);
                    }
                }
            }
        }

        // If no messages found, return empty result
        if messages.is_empty() {
            println!("🔍 DEBUG: No messages found for context_id: {}", context_id);
            return Ok(Vec::new());
        }

        // Sort messages by index first, then by role (to ensure prompts come before responses)
        messages.sort_by(|a, b| {
            match a.index.cmp(&b.index) {
                std::cmp::Ordering::Equal => {
                    // When indices are equal, sort by role (0/user before 1/assistant)
                    let role_a = normalize_role_for_sorting(&a.role);
                    let role_b = normalize_role_for_sorting(&b.role);
                    match role_a.cmp(&role_b) {
                        std::cmp::Ordering::Equal => a.timestamp.cmp(&b.timestamp),
                        other => other,
                    }
                },
                other => other,
            }
        });

        println!("🔍 DEBUG: Total messages for context_id {}: {}", context_id, messages.len());

        let deduplicated_messages = messages;

        // Get metadata or create default
        let metadata = chat_metadata.unwrap_or_else(|| {
            let mut default = serde_json::Map::new();
            default.insert("i".to_string(), JsonValue::String(context_id.to_string()));
            default
        });

        // Find the message with the highest index to get the latest timestamp
        let latest_timestamp = deduplicated_messages.iter()
            .max_by_key(|msg| msg.index)
            .and_then(|msg| msg.timestamp.as_ref())
            .map(|s| JsonValue::String(s.clone()));

        // Get field values
        let i_val = metadata.get("i").cloned().unwrap_or_else(|| JsonValue::String(context_id.to_string()));
        let a_val = metadata.get("a").cloned().unwrap_or_else(|| JsonValue::Number(serde_json::Number::from(deduplicated_messages.len())));
        let ct_val = metadata.get("ct").cloned();
        let t_val = latest_timestamp.or_else(|| metadata.get("t").cloned()); // Use latest timestamp, fallback to metadata
        let l_val = metadata.get("l").cloned();

        // Convert messages to the final format with r first, then c using json! macro for order
        let formatted_messages: Vec<JsonValue> = deduplicated_messages
            .into_iter()
            .map(|msg| {
                // Normalize role to always be "0" or "1"
                let normalized_role = match &msg.role {
                    JsonValue::String(s) => {
                        let s_lower = s.to_lowercase();
                        if s_lower == "0" || s_lower == "user" {
                            "0"
                        } else if s_lower == "1" || s_lower == "assistant" {
                            "1"
                        } else {
                            s.as_str()
                        }
                    },
                    JsonValue::Number(n) => {
                        if n.as_i64() == Some(0) {
                            "0"
                        } else if n.as_i64() == Some(1) {
                            "1"
                        } else {
                            "1"
                        }
                    },
                    _ => "1",
                };

                // Use json! macro to ensure field order: r, then c
                serde_json::json!({
                    "r": normalized_role,
                    "c": msg.content
                })
            })
            .collect();

        // Build the final JSON with proper field order using json! macro
        let mut formatted_chat = if let (Some(ct), Some(t), Some(l)) = (&ct_val, &t_val, &l_val) {
            serde_json::json!({
                "i": i_val,
                "a": a_val,
                "ct": ct,
                "m": formatted_messages,
                "t": t,
                "l": l
            })
        } else if let (Some(ct), Some(t)) = (&ct_val, &t_val) {
            serde_json::json!({
                "i": i_val,
                "a": a_val,
                "ct": ct,
                "m": formatted_messages,
                "t": t
            })
        } else if let Some(ct) = &ct_val {
            serde_json::json!({
                "i": i_val,
                "a": a_val,
                "ct": ct,
                "m": formatted_messages
            })
        } else {
            serde_json::json!({
                "i": i_val,
                "a": a_val,
                "m": formatted_messages
            })
        };

        println!("🔍 DEBUG: Formatted chat history with {} messages for context_id: {}",
                 formatted_chat.get("m").and_then(|v| v.as_array()).map(|a| a.len()).unwrap_or(0),
                 context_id);

        Ok(vec![formatted_chat])
    }
    
    /// Search estate resources using vector search (Business Architecture API)
    /// Supports dynamic collection names (aws_estate, azure_estate, gcp_estate, core_estate, etc.)
    pub async fn search_estate_resources(
        &self,
        collection_name: &str,
        query: &str,
        options: EstateSearchOptions,
        iam_context: Option<IAMContext>,
        user_id: &str,
    ) -> Result<Vec<JsonValue>> {
        // Clear document cache before search to ensure fresh data after profile switches
        info!("🗑️  Clearing document cache before search to get latest data from disk");
        if let Err(e) = self.vector_store.clear_document_cache().await {
            warn!("⚠️  Failed to clear document cache: {}", e);
            // Continue anyway - search will use cached data
        }

        // Default to "core_estate" if empty string is passed
        let effective_collection_name = if collection_name.is_empty() {
            "core_estate"
        } else {
            collection_name
        };

        // Try to get collection-specific config, fall back to aws_estate config as template
        let config = self.collection_configs.get(effective_collection_name)
            .or_else(|| self.collection_configs.get("aws_estate"))
            .ok_or_else(|| anyhow!("Estate collection configuration not found for '{}'", effective_collection_name))?;
        
        // Handle None limit more carefully for estate search  
        let limit = match options.limit {
            Some(user_limit) => std::cmp::min(user_limit, config.max_limit),
            None => config.max_limit // Use max limit when no specific limit requested - find all documents
        };
        
        // Preprocess query for estate search
        info!("🔄 Preprocessing query...");
        let preprocessed_query = self.preprocess_estate_query(query);
        info!("   📝 Original query: '{}'", query);
        info!("   🔄 Preprocessed query: '{}'", preprocessed_query);
        
        // Generate query embedding for semantic search with detailed error handling
        info!("🧠 Generating query embedding...");
        let embedding_start = std::time::Instant::now();
        let (query_embedding, embedding_duration) = match self.embedding_service.generate_embedding(&preprocessed_query).await {
            Ok(embedding) => {
                let duration = embedding_start.elapsed();
                info!("   ✅ Query embedding generated: {} dimensions in {:?}", embedding.len(), duration);
                (embedding, duration)
            },
            Err(e) => {
                let duration = embedding_start.elapsed();
                error!("❌ EMBEDDING GENERATION FAILED after {:?}", duration);
                error!("   🔍 Query: '{}'", preprocessed_query);
                error!("   💥 Error: {}", e);
                error!("   🛠️ Troubleshooting:");
                error!("      1. Check if BGE-M3 model files are properly downloaded");
                error!("      2. Verify sufficient memory (2GB+ RAM available)");
                error!("      3. Try restarting the application to reset model state");
                error!("      4. Check device compatibility (Metal vs CPU issues)");
                
                // Check for specific error types
                let error_str = format!("{}", e);
                if error_str.contains("Metal") || error_str.contains("pipeline") {
                    error!("   🔧 Metal GPU Error Detected!");
                    error!("      - This is a known issue with Metal backend on macOS");
                    error!("      - The model should automatically fall back to CPU");
                    error!("      - If this persists, restart the application");
                } else if error_str.contains("memory") || error_str.contains("allocation") {
                    error!("   🔧 Memory Error Detected!");
                    error!("      - Close other applications to free memory");
                    error!("      - BGE-M3 model requires ~2GB RAM");
                } else if error_str.contains("file") || error_str.contains("No such file") {
                    error!("   🔧 Model File Error Detected!");
                    error!("      - Model files may be corrupted or missing");
                    error!("      - Try deleting model cache and redownloading");
                }
                
                return Err(anyhow!("BGE-M3 model inference failed - check model files and memory: {}", e));
            }
        };
        
        // Parse parameters to separate filter criteria from extraction fields
        let (filter_params, extract_fields) = if let Some(ref params) = options.parameters {
            // FILTERING: Only use 'profile' and 'services' fields (hardcoded for Qdrant pre-filtering)
            // Map "services" parameter to "service" document field for compatibility
            // Supports both strings (exact match) and arrays (OR condition)
            let filters: HashMap<String, serde_json::Value> = params.iter()
                .filter_map(|(k, v)| {
                    let key = k.as_str();

                    // Only profile and services are used for filtering
                    // Map "services" parameter to "service" field (document field is singular)
                    let filter_key = match key {
                        "profile" => "profile",
                        "services" => "service",  // Map plural → singular for document field
                        _ => return None,  // Skip other fields
                    };

                    // Check if value is non-empty (string or non-empty array)
                    let is_valid = match v {
                        serde_json::Value::String(s) => !s.is_empty(),
                        serde_json::Value::Array(arr) => !arr.is_empty(),
                        _ => false,
                    };

                    if is_valid {
                        Some((filter_key.to_string(), v.clone()))
                    } else {
                        None
                    }
                })
                .collect();

            // EXTRACTION: Use ALL parameter keys (extract all fields from _encrypted_metadata)
            let fields: Vec<String> = params.keys()
                .map(|k| k.clone())
                .collect();

            info!("🔍 Parsed parameters: {} filter criteria (profile/services only), {} extraction fields (all keys)",
                  filters.len(), fields.len());

            if !filters.is_empty() {
                for (key, value) in &filters {
                    match value {
                        serde_json::Value::String(s) => {
                            info!("   🎯 Filter: {} = \"{}\" (exact match)", key, s);
                        }
                        serde_json::Value::Array(arr) => {
                            let values: Vec<String> = arr.iter()
                                .filter_map(|v| v.as_str().map(|s| s.to_string()))
                                .collect();
                            info!("   🎯 Filter: {} IN {:?} (OR condition)", key, values);
                        }
                        _ => {}
                    }
                }
            }

            (Some(filters), Some(fields))
        } else {
            (None, None)
        };

        // Use decrypt-first approach for filtering on encrypted metadata
        info!("🔐 Initiating decrypt-first vector search...");
        let estate_filter = self.create_estate_filter_function(&options, filter_params.as_ref());
        let search_start = std::time::Instant::now();

        // Only use dynamic filtering if user didn't provide explicit score threshold
        let use_dynamic_filtering = options.score_threshold.is_none();
        info!("   🎚️ Dynamic filtering: {} (score_threshold: {:?})",
              if use_dynamic_filtering { "enabled" } else { "disabled" },
              options.score_threshold);

        let search_results = self.search_with_decrypt_first(
            effective_collection_name,
            &preprocessed_query,
            user_id,
            limit,
            options.score_threshold,
            Some(estate_filter),
            use_dynamic_filtering,
            filter_params.as_ref() // Pass filter_params for Qdrant pre-filtering
        ).await?;
        
        let search_duration = search_start.elapsed();
        info!("   ✅ Vector search completed: {} results in {:?}", search_results.len(), search_duration);
        
        // Apply IAM permission filtering if context provided
        let permission_filtered_results = if let Some(context) = iam_context {
            info!("🔐 Applying IAM permission filtering...");
            let iam_start = std::time::Instant::now();
            let search_results_count = search_results.len(); // Store count before move
            let iam_filtered = self.apply_iam_filtering(search_results, &context).await?;
            let iam_duration = iam_start.elapsed();
            info!("   ✅ IAM filtering completed: {} → {} results in {:?}", search_results_count, iam_filtered.len(), iam_duration);
            iam_filtered
        } else {
            info!("   ⏭️ No IAM context provided, skipping IAM filtering");
            search_results
        };

        // Convert SearchResult to JsonValue for consistency
        info!("🔄 Converting results to JSON format...");
        let conversion_start = std::time::Instant::now();
        let mut json_results = self.process_estate_results(permission_filtered_results, &options, effective_collection_name).await?;
        let conversion_duration = conversion_start.elapsed();
        info!("   ✅ JSON conversion completed: {} results in {:?}", json_results.len(), conversion_duration);

        // Parse and extract fields from Qdrant's _encrypted_metadata field (v0.5.9 format)
        info!("🔍 Processing metadata from Qdrant for {} results...", json_results.len());
        for result in &mut json_results {
            if let JsonValue::Object(ref mut map) = result {
                // Get _encrypted_metadata field from Qdrant payload
                if let Some(encrypted_metadata_str) = map.get("_encrypted_metadata").and_then(|v| v.as_str()) {
                    // Parse the JSON string from Qdrant
                    match serde_json::from_str::<JsonValue>(encrypted_metadata_str) {
                        Ok(metadata_json) => {
                            if let Some(metadata_obj) = metadata_json.as_object() {
                                // Extract fields based on parameters (extract ALL keys, not just empty ones)
                                if let Some(ref field_names) = extract_fields {
                                    // Only extract requested fields (all parameter keys)
                                    info!("📋 Extracting {} requested fields from Qdrant metadata", field_names.len());
                                    for field_name in field_names {
                                        if let Some(field_value) = metadata_obj.get(field_name) {
                                            map.insert(field_name.clone(), field_value.clone());
                                            debug!("  ✅ Extracted field '{}' from Qdrant", field_name);
                                        } else {
                                            debug!("  ⚠️ Field '{}' not found in Qdrant metadata", field_name);
                                        }
                                    }
                                } else {
                                    // No parameters specified - extract all fields from Qdrant
                                    info!("📋 Extracting all {} fields from Qdrant metadata", metadata_obj.len());
                                    for (key, value) in metadata_obj {
                                        map.insert(key.clone(), value.clone());
                                    }
                                }
                                debug!("✅ Extracted metadata from Qdrant");
                            } else {
                                warn!("⚠️ Qdrant metadata is not a JSON object");
                            }
                        },
                        Err(e) => {
                            warn!("⚠️ Failed to parse Qdrant metadata: {}", e);
                        }
                    }
                } else {
                    debug!("⚠️ No _encrypted_metadata field found in Qdrant payload");
                }
            }
        }

        // Dynamic filtering already handles deduplication and relevance filtering
        info!("🎯 Dynamic filtering completed all quality control");
        let final_results = json_results;

        let total_duration = std::time::Instant::now() - embedding_start + embedding_duration;
        info!("🎯 ESTATE SEARCH SUMMARY:");
        info!("   📊 Final Results: {} estate resources found", final_results.len());
        info!("   ⏱️ Total Duration: {:?}", total_duration);
        info!("   🧠 Embedding Generation: {:?}", embedding_duration);
        info!("   🔍 Vector Search: {:?}", search_duration);
        info!("   🔄 Result Conversion: {:?}", conversion_duration);
        if final_results.is_empty() {
            warn!("⚠️ No results found - check if AWS estate data is loaded for user: {}", user_id);
        }
        
        Ok(final_results)
    }
    
    pub async fn initialize(&self) -> Result<()> {
        Ok(())
    }
    
    pub async fn shutdown(&self) -> Result<()> {
        Ok(())
    }
    
    // ============ COLLECTION-SPECIFIC FILTER BUILDERS ============
    
    /// Build chat history filters per business architecture
    fn build_chat_filters(&self, options: &ChatSearchOptions) -> Result<SearchFilter> {
        let mut must_conditions = Vec::new();
        
        // Required context_id filter for chat history
        if let Some(context_id) = &options.context_id {
            must_conditions.push(FilterCondition {
                key: "i".to_string(),
                r#match: MatchCondition::Value { value: json!(context_id) }
            });
        }
        
        // Optional role filter
        if let Some(role) = &options.role {
            must_conditions.push(FilterCondition {
                key: "r".to_string(),
                r#match: MatchCondition::Value { value: json!(role) }
            });
        }
        
        // Timestamp range filter
        if options.from_timestamp.is_some() || options.to_timestamp.is_some() {
            must_conditions.push(FilterCondition {
                key: "t".to_string(),
                r#match: MatchCondition::Range {
                    gte: options.from_timestamp.map(|ts| ts as f64),
                    lte: options.to_timestamp.map(|ts| ts as f64),
                }
            });
        }
        
        // Message index range (for pagination)
        if options.from_message_index.is_some() || options.to_message_index.is_some() {
            must_conditions.push(FilterCondition {
                key: "m".to_string(),
                r#match: MatchCondition::Range {
                    gte: options.from_message_index.map(|idx| idx as f64),
                    lte: options.to_message_index.map(|idx| idx as f64),
                }
            });
        }
        
        Ok(SearchFilter {
            must: if !must_conditions.is_empty() { Some(must_conditions) } else { None },
            must_not: None,
            should: None,
        })
    }
    
    /// Build estate resource filters per business architecture
    fn build_estate_filters(&self, options: &EstateSearchOptions) -> Result<SearchFilter> {
        let mut must_conditions = Vec::new();
        
        // Helper function to add filter condition
        let add_filter = |conditions: &mut Vec<FilterCondition>, key: &str, values: &Option<Vec<String>>| {
            if let Some(vals) = values {
                if vals.len() == 1 {
                    conditions.push(FilterCondition {
                        key: key.to_string(),
                        r#match: MatchCondition::Value { value: json!(vals[0]) }
                    });
                } else if vals.len() > 1 {
                    let json_vals = vals.iter().map(|v| json!(v)).collect();
                    conditions.push(FilterCondition {
                        key: key.to_string(),
                        r#match: MatchCondition::Any { any: json_vals }
                    });
                }
            }
        };
        
        // Apply all filters
        add_filter(&mut must_conditions, "resource_type", &options.resource_types);
        add_filter(&mut must_conditions, "account_id", &options.account_ids);
        add_filter(&mut must_conditions, "region", &options.regions);
        add_filter(&mut must_conditions, "service", &options.services);
        add_filter(&mut must_conditions, "state", &options.states);
        
        // Single value filters
        if let Some(env) = &options.environment {
            must_conditions.push(FilterCondition {
                key: "tags.env".to_string(),
                r#match: MatchCondition::Value { value: json!(env) }
            });
        }
        
        if let Some(app) = &options.application {
            must_conditions.push(FilterCondition {
                key: "tags.app".to_string(),
                r#match: MatchCondition::Value { value: json!(app) }
            });
        }
        
        // Last synced filter (for freshness)
        if let Some(synced_after) = options.synced_after {
            must_conditions.push(FilterCondition {
                key: "last_synced".to_string(),
                r#match: MatchCondition::Range {
                    gte: Some(synced_after as f64),
                    lte: None,
                }
            });
        }
        
        Ok(SearchFilter {
            must: if !must_conditions.is_empty() { Some(must_conditions) } else { None },
            must_not: None,
            should: None,
        })
    }
    
    // ============ QUERY PREPROCESSING ============
    
    /// Preprocess estate search queries with AWS/Azure/GCP spell correction
    fn preprocess_estate_query(&self, query: &str) -> String {
        if query.is_empty() {
            return String::new();
        }
        
        let mut query = query.to_lowercase().trim().to_string();
        
        // Apply word-level corrections
        let words: Vec<String> = query.split_whitespace()
            .map(|word| {
                let clean_word = word.trim_matches(|c: char| ".,!?;:".contains(c));
                self.spell_corrections.get(clean_word)
                    .cloned()
                    .unwrap_or_else(|| word.to_string())
            })
            .collect();
        
        // Apply phrase-level corrections
        let mut corrected_query = words.join(" ");
        for (mistake, correction) in &self.spell_corrections {
            if corrected_query.contains(mistake) {
                corrected_query = corrected_query.replace(mistake, correction);
            }
        }
        
        corrected_query
    }
    
    /// Calculate optimal search limit (from traditional RAG)
    fn calculate_optimal_limit(&self, requested_limit: usize) -> usize {
        // For estate search, ensure we get all documents by being very generous
        // Don't clamp the limit - use what's requested or a reasonable default
        let base_limit = if requested_limit == 0 { 20 } else { requested_limit };
        
        // Use generous over-fetch to ensure we find all relevant documents
        std::cmp::max(base_limit * 3, 50) // At least 50, or 3x the requested limit
    }
    
    /// Check if content is likely encrypted
    fn is_content_encrypted(&self, content: &str) -> bool {
        // Check for common encrypted data patterns
        if content.is_empty() {
            return false;
        }
        
        // 1. Check for base64 pattern (common encryption output)
        let is_base64_like = content.len() > 100 && 
            content.chars().all(|c| c.is_alphanumeric() || c == '+' || c == '/' || c == '=') &&
            content.contains('='); // Base64 padding
        
        // 2. Check for JSON with encrypted metadata markers
        let has_encrypted_markers = content.contains("_encrypted_metadata") || 
            content.contains("_encrypted_content") || 
            content.contains("\"data\":") || // Encrypted JSON structure
            content.contains("\"nonce\":");
        
        // 3. Check for very long single-line content (likely encrypted)
        let is_long_single_line = content.len() > 500 && !content.contains('\n');
        
        // 4. Check entropy - encrypted content has high entropy
        let entropy = self.calculate_entropy(content);
        let has_high_entropy = entropy > 4.5; // Threshold for likely encrypted content
        
        is_base64_like || has_encrypted_markers || (is_long_single_line && has_high_entropy)
    }
    
    /// Calculate Shannon entropy of a string (measure of randomness)
    fn calculate_entropy(&self, text: &str) -> f64 {
        use std::collections::HashMap;
        
        if text.is_empty() {
            return 0.0;
        }
        
        let mut frequencies = HashMap::new();
        for byte in text.bytes() {
            *frequencies.entry(byte).or_insert(0) += 1;
        }
        
        let length = text.len() as f64;
        let mut entropy = 0.0;
        
        for count in frequencies.values() {
            let probability = *count as f64 / length;
            entropy -= probability * probability.log2();
        }
        
        entropy
    }
    
    /// Apply dynamic relevance filtering with intelligent thresholding
    fn apply_dynamic_relevance_filtering(&self, results: Vec<SearchResult>, requested_limit: usize) -> Vec<SearchResult> {
        if results.is_empty() {
            return results;
        }

        // Use the requested limit directly without capping to 20
        let max_results = if requested_limit == 0 { results.len() } else { requested_limit };
        let mut filtered = Vec::new();
        let mut seen_ids = HashSet::new();
        let best_score = results.first().map(|r| r.score).unwrap_or(0.0);

        println!("🔍 DEBUG: Best score: {:.4}", best_score);


        // More permissive dynamic threshold calculation for better results with parsing logic
        let dynamic_threshold = if best_score > 0.8 {
            // Very high similarity case - use 30% of best score (more permissive)
            best_score * 0.3
        } else if best_score > 0.5 {
            // High similarity case - use 20% of best score, minimum 0.03
            f32::max(best_score * 0.2, 0.03)
        } else if best_score > 0.2 {
            // Medium similarity - use 15% of best score, minimum 0.02
            f32::max(best_score * 0.15, 0.02)
        } else if best_score > 0.1 {
            // Low-medium similarity - use 10% of best score, minimum 0.01
            f32::max(best_score * 0.1, 0.01)
        } else {
            // Very low similarity case - extremely permissive threshold
            f32::max(best_score * 0.05, 0.005)
        };

        println!("🔍 DEBUG: Dynamic threshold: {:.4}", dynamic_threshold);

        for result in results {
            if filtered.len() >= max_results {
                break;
            }

            // Skip duplicates by document ID
            if seen_ids.contains(&result.id) {
                continue;
            }

            // Apply dynamic relevance threshold
            if result.score < dynamic_threshold {
                println!("🔍 DEBUG: Filtered out result with score {:.4} (below {:.4})", result.score, dynamic_threshold);
                continue;
            }

            // Apply score drop threshold - very permissive for better recall
            const MAX_SCORE_DROP: f32 = 0.8; // Allow up to 0.8 score drop (very permissive)
            if best_score - result.score > MAX_SCORE_DROP {
                println!("🔍 DEBUG: Filtered out result due to score drop: {:.4} vs best {:.4}", result.score, best_score);
                continue;
            }

            println!("🔍 DEBUG: Accepting result with score {:.4}", result.score);

            // Add to results and mark as seen
            seen_ids.insert(result.id.clone());
            filtered.push(result);
        }

        println!("🔍 DEBUG: Dynamic filtering kept {} out of original results", filtered.len());
        filtered
    }
    
    // ============ COLLECTION-SPECIFIC RESULT PROCESSORS ============
    
    /// Process chat history search results
    async fn process_chat_results(&self, results: Vec<Document>, options: &ChatSearchOptions) -> Result<Vec<JsonValue>> {
        if results.is_empty() {
            return Ok(Vec::new());
        }
        
        let mut processed_results = Vec::new();
        
        for result in results {
            let mut processed_result = json!({
                "id": result.id,
                "i": result.metadata.get("i").or(result.metadata.get("i")),
                "m": result.metadata.get("m"),
                "r": result.metadata.get("r"),
                "t": result.metadata.get("t"),
                "c": result.content
            });
            
            // Include metadata if requested
            if options.include_metadata {
                processed_result["metadata"] = json!({
                    "created_at": result.metadata.get("t"),
                    "collection_type": "chat"
                });
            }
            
            processed_results.push(processed_result);
        }
        
        // Sort by message_index for proper conversation order
        processed_results.sort_by(|a, b| {
            let a_idx = a.get("m").and_then(|v| v.as_i64()).unwrap_or(0);
            let b_idx = b.get("m").and_then(|v| v.as_i64()).unwrap_or(0);
            a_idx.cmp(&b_idx)
        });
        
        Ok(processed_results)
    }
    
    /// Process estate resource search results
    async fn process_estate_results(&self, results: Vec<SearchResult>, options: &EstateSearchOptions, collection_name: &str) -> Result<Vec<JsonValue>> {
        if results.is_empty() {
            return Ok(Vec::new());
        }
        
        let mut processed_results = Vec::new();
        
        for result in results {
            // Helper function to extract from decrypted document metadata (nested structure)
            let get_field = |field: &str| -> Option<serde_json::Value> {
                result.document.as_ref()
                    .and_then(|doc| {
                        // Try nested metadata first (metadata.metadata.field)
                        doc.metadata.get("metadata")
                            .and_then(|m| m.get(field))
                            .cloned()
                            .or_else(|| {
                                // Then try top level (metadata.field)  
                                doc.metadata.get(field).cloned()
                            })
                    })
            };
            
            // Start with base fields
            let mut processed_result = json!({
                "id": result.id,
                "score": result.score,
                "updated_at": result.document.as_ref().map(|d| d.updated_at),
                "created_at": result.document.as_ref().map(|d| d.created_at),

            });

            // Add payload fields (includes doc_ref and small filterable fields from Qdrant)
            // Skip _encrypted_metadata - we'll extract individual fields from it instead
            if let Some(ref payload) = result.payload {
                for (key, value) in payload.iter() {
                    // Don't copy _encrypted_metadata to output - only extracted fields should appear
                    if key != "_encrypted_metadata" {
                        processed_result[key.clone()] = value.clone();
                    }
                }
            }

            // Dynamically extract all metadata fields from the document
            if let Some(doc) = result.document.as_ref() {
                // Define system fields that should be excluded from dynamic extraction
                let system_fields = vec![
                    "embedding", "_vectors", "vector", "embeddings",
                    "document_type", "cloud_provider", "collection_type"
                ];

                // Helper to get all metadata fields (nested or top-level)
                let get_all_metadata = || -> serde_json::Map<String, serde_json::Value> {
                    let mut all_fields = serde_json::Map::new();

                    // First, try to get nested metadata (metadata.metadata)
                    if let Some(nested_metadata) = doc.metadata.get("metadata").and_then(|m| m.as_object()) {
                        for (key, value) in nested_metadata {
                            if !system_fields.contains(&key.as_str()) {
                                all_fields.insert(key.clone(), value.clone());
                            }
                        }
                    }

                    // Then add/override with top-level metadata fields (IndexMap iteration)
                    for (key, value) in doc.metadata.iter() {
                        // Skip the nested "metadata" object itself and system fields
                        if key != "metadata" && !system_fields.contains(&key.as_str()) {
                            all_fields.insert(key.clone(), value.clone());
                        }
                    }

                    all_fields
                };

                // Extract metadata fields based on parameters option
                if let Some(ref parameters) = options.parameters {
                    // Extract ALL keys from parameters HashMap (regardless of empty/non-empty values)
                    let all_metadata = get_all_metadata();
                    for field_name in parameters.keys() {
                        // Handle nested field access using dot notation (e.g., "endpoint.port")
                        if field_name.contains('.') {
                            let parts: Vec<&str> = field_name.split('.').collect();
                            let mut current_value = None;

                            // Start from the root level
                            if let Some(root_value) = all_metadata.get(parts[0]) {
                                current_value = Some(root_value.clone());

                                // Navigate through the nested structure
                                for part in &parts[1..] {
                                    if let Some(obj) = current_value.as_ref().and_then(|v| v.as_object()) {
                                        if let Some(nested_value) = obj.get(*part) {
                                            current_value = Some(nested_value.clone());
                                        } else {
                                            current_value = None;
                                            break;
                                        }
                                    } else {
                                        current_value = None;
                                        break;
                                    }
                                }

                                // If we found the nested value, add it to results
                                if let Some(final_value) = current_value {
                                    processed_result[field_name.clone()] = final_value;
                                }
                            }
                        } else {
                            // Handle regular top-level fields
                            if let Some(value) = all_metadata.get(field_name) {
                                processed_result[field_name.clone()] = value.clone();
                            }
                        }
                    }
                } else {
                    // Extract all metadata fields dynamically (existing behavior)
                    let all_metadata = get_all_metadata();
                    for (key, value) in all_metadata {
                        processed_result[key] = value;
                    }
                }
            }
            
            // Handle tags - convert AWS array format to object format if needed
            if let Some(tags_value) = processed_result.get("tags").cloned() {
                // If tags is an array (AWS format: [{"key": "Name", "value": "..."}]), convert to object
                if let Some(tags_array) = tags_value.as_array() {
                    let mut tags_obj = serde_json::Map::new();
                    for tag in tags_array {
                        if let (Some(key), Some(value)) = (
                            tag.get("key").and_then(|k| k.as_str()),
                            tag.get("value")
                        ) {
                            tags_obj.insert(key.to_string(), value.clone());
                        }
                    }
                    processed_result["tags"] = json!(tags_obj);
                }
                // If tags is already an object, keep it as is
            } else {
                // Fallback: try to get tags from payload if not in metadata
                let mut tags = serde_json::Map::new();
                if let Some(env) = result.payload.as_ref().and_then(|p| p.get("tags.env")) {
                    tags.insert("env".to_string(), env.clone());
                }
                if let Some(app) = result.payload.as_ref().and_then(|p| p.get("tags.app")) {
                    tags.insert("app".to_string(), app.clone());
                }
                if let Some(name) = result.payload.as_ref().and_then(|p| p.get("tags.name")) {
                    tags.insert("name".to_string(), name.clone());
                }
                // Only set tags if we found some in payload
                if !tags.is_empty() {
                    processed_result["tags"] = json!(tags);
                }
            }
            
            // Include metadata if requested
            if options.include_metadata {
                processed_result["metadata"] = json!({
                    "collection_type": collection_name,
                    "search_score": result.score
                });
            }
            
            // Apply anonymous ID mapping if enabled
            if options.use_anonymous_ids {
                if let Some(mapping_service) = &self.mapping_service {
                    if let Ok(Some(anonymous_id)) = mapping_service.get_anonymous_id(&result.id).await {
                        processed_result["anonymous_id"] = json!(anonymous_id);
                    }
                }
            }
            
            processed_results.push(processed_result);
        }
        
        // Results are already sorted by score from vector search
        Ok(processed_results)
    }
    
    /// Search playbook collections (escher_library and tenant collections)
    pub async fn search_playbooks(
        &self,
        collection_name: &str,
        query: &str,
        options: &SearchOptions,
    ) -> Result<Vec<SearchResult>> {
        println!("🔍 Searching playbook collection '{}' with query: '{}'", collection_name, query);
        
        // Generate query embedding using BGE-M3 model
        let query_texts = vec![query];
        let query_embedding = self.embedding_service.generate_embeddings(&query_texts).await?;
        if query_embedding.is_empty() || query_embedding[0].is_empty() {
            eprintln!("❌ Failed to generate query embedding");
            return Ok(Vec::new());
        }
        
        println!("✅ Generated query embedding: {} dimensions", query_embedding[0].len());
        
        // Create proper SearchOptions for vector store
        let search_options = SearchOptions {
            limit: options.limit,
            score_threshold: options.score_threshold,
            filter: options.filter.clone(),
            collection_name: Some(collection_name.to_string()),
            privacy_level: None,
            with_payload: Some(true),
            parameters:None
        };
        
        // Perform vector search in Qdrant
        let search_results = self.vector_store
            .search(
                collection_name,
                query_embedding[0].clone(),
                search_options,
            )
            .await?;
        
        println!("🔍 Vector search returned {} results", search_results.len());
        
        // Convert VectorSearchResult to SearchResult format
        let mut results = Vec::new();
        for search_result in search_results {
            // Create proper Document from search result if available
            let document = if let Some(doc) = &search_result.document {
                Some(doc.clone())
            } else {
                None
            };
            
            let mut payload = HashMap::new();
            payload.insert("id".to_string(), JsonValue::String(search_result.id.clone()));
            payload.insert("score".to_string(), JsonValue::from(search_result.score as f64));
            
            // Add payload from search result if available
            if let Some(search_payload) = &search_result.payload {
                for (key, value) in search_payload {
                    payload.insert(key.clone(), value.clone());
                }
            }
            
            let result = SearchResult {
                id: search_result.id,
                score: search_result.score,
                document,
                payload: Some(payload),
            };
            
            results.push(result);
        }
        
        println!("🔍 Converted {} search results for playbook collection", results.len());
        Ok(results)
    }
    
    /// Apply IAM permission filtering to estate results
    async fn apply_iam_filtering(&self, results: Vec<SearchResult>, iam_context: &IAMContext) -> Result<Vec<SearchResult>> {
        if let Some(iam_service) = &self.iam_service {
            let mut filtered_results = Vec::new();
            
            for result in results {
                match iam_service.check_resource_access(iam_context, &result.id, "read").await {
                    Ok(true) => filtered_results.push(result),
                    Ok(false) => {}, // Skip unauthorized results
                    Err(e) => {
                        eprintln!("❌ IAM filtering failed for resource {}: {}", result.id, e);
                        // On IAM errors, return result (fail open for availability)
                        filtered_results.push(result);
                    }
                }
            }
            
            Ok(filtered_results)
        } else {
            Ok(results) // No IAM filtering if service not available
        }
    }

    /// Universal decrypt-first search approach for all collections
    /// This method handles encrypted metadata by decrypting first, then applying filters
    pub async fn search_with_decrypt_first(
        &self,
        collection_name: &str,
        query: &str,
        user_id: &str,
        limit: usize,
        score_threshold: Option<f32>,
        post_filters: Option<Box<dyn Fn(&JsonValue) -> bool + Send>>,
        use_dynamic_filtering: bool,
        filter_params: Option<&HashMap<String, serde_json::Value>>, // Pre-filter params (string or array)
    ) -> Result<Vec<SearchResult>> {
        
        // Set user context for user-specific document access (embedded, server, and dual modes)
        if let Some(embedded_store) = self.vector_store.as_any().downcast_ref::<crate::db::EmbeddedQdrantVectorStore>() {
            embedded_store.set_user_context(user_id).await;
        } else if let Some(server_store) = self.vector_store.as_any().downcast_ref::<crate::db::QdrantServerVectorStore>() {
            server_store.set_user_context(user_id).await;
        } else if let Some(dual_store) = self.vector_store.as_any().downcast_ref::<crate::db::DualVectorStore>() {
            dual_store.set_user_context(user_id).await;
        }
        
        // Step 1: Build Qdrant pre-filter from filter_params (ONLY profile and service fields)
        let qdrant_filter = if let Some(filters) = filter_params {
            if !filters.is_empty() {
                let mut conditions = Vec::new();

                // Build filter conditions - filters only contains profile/service at this point
                for (key, value) in filters {
                    match value {
                        serde_json::Value::String(s) => {
                            // Single value - exact match
                            info!("   🔍 Qdrant filter: {} = \"{}\" (exact match)", key, s);
                            conditions.push(FilterCondition {
                                key: key.clone(),
                                r#match: MatchCondition::Value {
                                    value: serde_json::Value::String(s.clone()),
                                },
                            });
                        }
                        serde_json::Value::Array(arr) => {
                            // Array of values - OR condition (match ANY)
                            let values: Vec<String> = arr.iter()
                                .filter_map(|v| v.as_str().map(|s| s.to_string()))
                                .collect();
                            info!("   🔍 Qdrant filter: {} IN {:?} (OR condition)", key, values);

                            let any_values: Vec<serde_json::Value> = arr.iter()
                                .filter_map(|v| {
                                    if let Some(s) = v.as_str() {
                                        Some(serde_json::Value::String(s.to_string()))
                                    } else {
                                        None
                                    }
                                })
                                .collect();

                            if !any_values.is_empty() {
                                conditions.push(FilterCondition {
                                    key: key.clone(),
                                    r#match: MatchCondition::Any {
                                        any: any_values,
                                    },
                                });
                            }
                        }
                        _ => {
                            info!("   ⚠️ Skipping unsupported filter type for key: {}", key);
                        }
                    }
                }

                info!("✅ Built Qdrant pre-filter with {} conditions", conditions.len());
                Some(SearchFilter {
                    must: Some(conditions),
                    must_not: None,
                    should: None,
                })
            } else {
                info!("⏭️ No filter parameters provided, skipping Qdrant pre-filtering");
                None
            }
        } else {
            info!("⏭️ No filter parameters provided, skipping Qdrant pre-filtering");
            None
        };

        // Step 2: Perform vector search WITH Qdrant pre-filters
        let query_embedding = self.embedding_service.generate_embedding(query).await?;
        let search_limit = self.calculate_optimal_limit(limit); // Use limit directly, calculate_optimal_limit already does over-fetch

        let search_options = SearchOptions {
            limit: Some(search_limit),
            score_threshold: score_threshold, // Use exact threshold provided (None means no threshold)
            filter: qdrant_filter, // Pre-filter using profile and service fields in Qdrant
            with_payload: Some(true),
            ..Default::default()
        };
        
        let vector_results = self.vector_store.search(collection_name, query_embedding, search_options).await?;
        
        // Step 2: Decrypt all results
        let mut decrypted_results = Vec::new();
        for result in vector_results {
            if let Some(document) = &result.document {
                // Better encryption detection and content decryption
                let decrypted_content = if let Some(ref enc_service) = self.encryption_service {
                    // Check if content looks like it's encrypted (base64 pattern or JSON with encrypted fields)
                    let is_likely_encrypted = self.is_content_encrypted(&document.content);
                    
                    if is_likely_encrypted {
                        match enc_service.decrypt_content(&document.content).await {
                            Ok(content) => content,
                            Err(_) => document.content.clone() // Fallback to original
                        }
                    } else {
                        document.content.clone()
                    }
                } else {
                    document.content.clone()
                };
                
                // Decrypt metadata
                let decrypted_metadata = self.decrypt_document_metadata(&document.metadata).await;
                
                // Create decrypted search result
                let mut decrypted_result = result.clone();
                if let Some(ref mut doc) = decrypted_result.document {
                    doc.content = decrypted_content;
                    doc.metadata = decrypted_metadata;
                }
                
                decrypted_results.push(decrypted_result);
            }
        }
        
        // Step 3: Apply post-processing filters on decrypted data
        let mut filtered_results = if let Some(filter_fn) = post_filters {
            let mut filtered = Vec::new();
            for result in decrypted_results {
                // Convert result to JsonValue for filtering
                if let Some(doc) = &result.document {
                    let mut json_obj = serde_json::Map::new();
                    json_obj.insert("id".to_string(), JsonValue::String(result.id.clone()));
                    json_obj.insert("score".to_string(), JsonValue::Number(serde_json::Number::from_f64(result.score as f64).unwrap()));
                    json_obj.insert("content".to_string(), JsonValue::String(doc.content.clone()));
                    
                    // Add ALL decrypted metadata fields (this was the bug - metadata is now decrypted)
                    for (key, value) in &doc.metadata {
                        json_obj.insert(key.clone(), value.clone());
                    }
                    
                    let json_value = JsonValue::Object(json_obj);
                    
                    if filter_fn(&json_value) {
                        filtered.push(result);
                    }
                }
            }
            filtered
        } else {
            decrypted_results
        };
        
        // Step 4: Deduplicate results by resource identifier to avoid duplicate resources
        let mut seen_resources = std::collections::HashSet::new();
        let mut deduplicated_results = Vec::new();
        
        for result in filtered_results {
            // Create a unique key based on the actual resource (not document ID)
            let resource_key = if let Some(doc) = &result.document {
                // Helper function to look in nested metadata first, then top level
                let get_nested_field = |field: &str| -> Option<&str> {
                    doc.metadata.get("metadata")
                        .and_then(|m| m.get(field))
                        .and_then(|v| v.as_str())
                        .or_else(|| doc.metadata.get(field).and_then(|v| v.as_str()))
                };
                
                // Try to get a unique resource identifier from nested metadata
                let instance_id = get_nested_field("instance_id")
                    .or_else(|| get_nested_field("InstanceId"));
                let db_id = get_nested_field("db_instance_identifier")
                    .or_else(|| get_nested_field("DBInstanceIdentifier"));
                let bucket_name = get_nested_field("bucket_name")
                    .or_else(|| get_nested_field("Name"));
                let user_name = get_nested_field("user_name")
                    .or_else(|| get_nested_field("UserName"))
                    .or_else(|| get_nested_field("role_name"))
                    .or_else(|| get_nested_field("RoleName"));
                let function_name = get_nested_field("function_name")
                    .or_else(|| get_nested_field("FunctionName"));
                    
                // Use the first available resource identifier
                instance_id
                    .or(db_id)
                    .or(bucket_name)  
                    .or(user_name)
                    .or(function_name)
                    .map(|s| s.to_string())
                    .unwrap_or_else(|| result.id.clone()) // Fallback to document ID
            } else {
                result.id.clone()
            };
            
            if seen_resources.insert(resource_key) {
                deduplicated_results.push(result);
            }
        }
        
        // Step 5: Apply intelligent dynamic relevance filtering (replaces crude truncation)
        deduplicated_results.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap_or(std::cmp::Ordering::Equal));
        // let intelligently_filtered_results = self.apply_dynamic_relevance_filtering(deduplicated_results, limit);
        let intelligently_filtered_results = if use_dynamic_filtering {
            info!("🎯 Applying dynamic relevance filtering");
            self.apply_dynamic_relevance_filtering(deduplicated_results, limit)
        } else {
            info!("⏭️ Skipping dynamic filtering (explicit score threshold provided or disabled)");
            // Just apply limit without additional filtering
            deduplicated_results.into_iter().take(limit).collect()
        };
        Ok(intelligently_filtered_results)
    }

    /// Helper method to decrypt document metadata
    async fn decrypt_document_metadata(&self, metadata: &indexmap::IndexMap<String, JsonValue>) -> indexmap::IndexMap<String, JsonValue> {
        let mut decrypted_metadata = indexmap::IndexMap::new();
        
        for (key, value) in metadata {
            if key == "_encrypted_metadata" {
                // Decrypt the encrypted metadata blob
                if let Some(encrypted_str) = value.as_str() {
                    if let Some(ref enc_service) = self.encryption_service {
                        match enc_service.decrypt_content(encrypted_str).await {
                            Ok(decrypted_str) => {
                                // Try to parse as JSON first
                                match serde_json::from_str::<JsonValue>(&decrypted_str) {
                                    Ok(decrypted_json) => {
                                        if let Some(obj) = decrypted_json.as_object() {
                                            for (k, v) in obj {
                                                decrypted_metadata.insert(k.clone(), v.clone());
                                            }
                                        } else if let Some(nested_encrypted_str) = decrypted_json.as_str() {
                                            // Handle double encryption - try to decrypt again
                                            match enc_service.decrypt_content(nested_encrypted_str).await {
                                                Ok(double_decrypted_str) => {
                                                    match serde_json::from_str::<JsonValue>(&double_decrypted_str) {
                                                        Ok(final_json) => {
                                                            if let Some(obj) = final_json.as_object() {
                                                                for (k, v) in obj {
                                                                    decrypted_metadata.insert(k.clone(), v.clone());
                                                                }
                                                            }
                                                        },
                                                        Err(_) => {
                                                            decrypted_metadata.insert(key.clone(), value.clone());
                                                        }
                                                    }
                                                },
                                                Err(_) => {
                                                    decrypted_metadata.insert(key.clone(), value.clone());
                                                }
                                            }
                                        } else {
                                            decrypted_metadata.insert(key.clone(), value.clone());
                                        }
                                    },
                                    Err(_) => {
                                        // Keep original if JSON parsing fails
                                        decrypted_metadata.insert(key.clone(), value.clone());
                                    }
                                }
                            },
                            Err(_) => {
                                // Keep original if decryption fails
                                decrypted_metadata.insert(key.clone(), value.clone());
                            }
                        }
                    } else {
                        // No encryption service, keep original
                        decrypted_metadata.insert(key.clone(), value.clone());
                    }
                }
            } else {
                // Keep non-encrypted metadata as-is
                decrypted_metadata.insert(key.clone(), value.clone());
            }
        }
        
        decrypted_metadata
    }

    /// Create estate filter function for post-processing
    fn create_estate_filter_function(&self, options: &EstateSearchOptions, param_filters: Option<&HashMap<String, serde_json::Value>>) -> Box<dyn Fn(&JsonValue) -> bool + Send> {
        let resource_types = options.resource_types.clone();
        let account_ids = options.account_ids.clone();
        let regions = options.regions.clone();
        let services = options.services.clone();
        let states = options.states.clone();
        let environment = options.environment.clone();
        let application = options.application.clone();
        let param_filters = param_filters.cloned();
        
        Box::new(move |json: &JsonValue| {
            // Helper function to get field value from multiple nesting levels
            let get_field_value = |field_name: &str| -> Option<String> {
                // Try top-level field first
                if let Some(value) = json.get(field_name).and_then(|v| v.as_str()) {
                    return Some(value.to_string());
                }

                // Try metadata.metadata.field (triple nested - common after decryption)
                if let Some(metadata) = json.get("metadata") {
                    if let Some(nested_metadata) = metadata.get("metadata") {
                        if let Some(value) = nested_metadata.get(field_name).and_then(|v| v.as_str()) {
                            return Some(value.to_string());
                        }
                    }

                    // Try metadata.field (double nested)
                    if let Some(value) = metadata.get(field_name).and_then(|v| v.as_str()) {
                        return Some(value.to_string());
                    }
                }

                None
            };

            // POST-FILTER: Check parameter-based filters (from parameters with non-empty values)
            // Note: Pre-filtering already done in Qdrant, this is a secondary check after decryption
            if let Some(ref filters) = param_filters {
                for (field_name, expected_value) in filters {
                    if let Some(actual_value) = get_field_value(field_name) {
                        // Handle both string (exact match) and array (OR condition)
                        let matches = match expected_value {
                            serde_json::Value::String(expected_str) => {
                                &actual_value == expected_str
                            }
                            serde_json::Value::Array(expected_arr) => {
                                // Check if actual_value is in the array (OR condition)
                                expected_arr.iter().any(|v| {
                                    if let Some(s) = v.as_str() {
                                        s == actual_value
                                    } else {
                                        false
                                    }
                                })
                            }
                            _ => false,
                        };

                        if !matches {
                            return false; // Value doesn't match - filter out
                        }
                    } else {
                        return false; // Field not found - filter out
                    }
                }
            }

            // Check resource_types filter
            if let Some(ref types) = resource_types {
                if let Some(resource_type) = get_field_value("resource_type") {
                    if !types.contains(&resource_type) {
                        return false;
                    }
                } else {
                    return false;
                }
            }
            
            // Check account_ids filter
            if let Some(ref accounts) = account_ids {
                if let Some(account_id) = get_field_value("account_id") {
                    if !accounts.contains(&account_id) {
                        return false;
                    }
                } else {
                    return false;
                }
            }
            
            // Check regions filter
            if let Some(ref regions_list) = regions {
                if let Some(region) = get_field_value("region") {
                    if !regions_list.contains(&region) {
                        return false;
                    }
                } else {
                    return false;
                }
            }
            
            // Check services filter
            if let Some(ref services_list) = services {
                if let Some(service) = get_field_value("service") {
                    if !services_list.contains(&service) {
                        return false;
                    }
                } else {
                    return false;
                }
            }
            
            // Check states filter
            if let Some(ref states_list) = states {
                if let Some(state) = get_field_value("state") {
                    if !states_list.contains(&state) {
                        return false;
                    }
                } else {
                    return false;
                }
            }
            
            // Check environment filter (tags.env)
            if let Some(ref env) = environment {
                if let Some(tags) = json.get("tags") {
                    if let Some(tag_env) = tags.get("env").or_else(|| tags.get("Environment")).and_then(|v| v.as_str()) {
                        if tag_env != env {
                            return false;
                        }
                    } else {
                        return false;
                    }
                }
            }
            
            // Check application filter (tags.app)
            if let Some(ref app) = application {
                if let Some(tags) = json.get("tags") {
                    if let Some(tag_app) = tags.get("app").or_else(|| tags.get("Application")).and_then(|v| v.as_str()) {
                        if tag_app != app {
                            return false;
                        }
                    } else {
                        return false;
                    }
                }
            }
            
            true // Passed all filters
        })
    }

    /// Convert SearchResult to JsonValue for estate resources
    async fn convert_search_results_to_json(&self, results: Vec<SearchResult>, options: &EstateSearchOptions) -> Result<Vec<JsonValue>> {
        let mut json_results = Vec::new();
        
        for result in results {
            if let Some(document) = result.document {
                let mut json_obj = serde_json::Map::new();
                
                // Basic fields
                json_obj.insert("id".to_string(), JsonValue::String(result.id));
                json_obj.insert("score".to_string(), JsonValue::Number(serde_json::Number::from_f64(result.score as f64).unwrap()));
                
                if options.include_metadata {
                    json_obj.insert("content".to_string(), JsonValue::String(document.content));
                }
                
                // Add all metadata fields (now decrypted)
                for (key, value) in document.metadata {
                    json_obj.insert(key, value);
                }
                
                json_results.push(JsonValue::Object(json_obj));
            }
        }
        
        Ok(json_results)
    }

    /// Create chat filter function for post-processing
    fn create_chat_filter_function(&self, options: &ChatSearchOptions) -> Box<dyn Fn(&JsonValue) -> bool + Send> {
        let context_id = options.context_id.clone();
        let role = options.role.clone();
        let from_timestamp = options.from_timestamp;
        let to_timestamp = options.to_timestamp;
        let from_message_index = options.from_message_index;
        let to_message_index = options.to_message_index;

        Box::new(move |json: &JsonValue| {
            // Check context_id filter - use "i" field (matches JSON format)
            if let Some(ref ctx_id) = context_id {
                // Try both "i" (new format) and "context_id" (legacy format)
                let msg_ctx_id = json.get("i").and_then(|v| v.as_str())
                    .or_else(|| json.get("context_id").and_then(|v| v.as_str()));

                if let Some(msg_ctx_id) = msg_ctx_id {
                    if msg_ctx_id != ctx_id {
                        return false;
                    }
                } else {
                    return false;
                }
            }

            // Check role filter - use "r" field for message-level filtering
            if let Some(ref msg_role) = role {
                // This would apply to individual messages, not top-level documents
                if let Some(current_role) = json.get("r").and_then(|v| v.as_i64()) {
                    let role_str = if current_role == 0 { "user" } else { "assistant" };
                    if role_str != msg_role {
                        return false;
                    }
                } else if let Some(current_role) = json.get("role").and_then(|v| v.as_str()) {
                    if current_role != msg_role {
                        return false;
                    }
                } else {
                    return false;
                }
            }

            // Check timestamp filters - use "t" field
            if let (Some(from_ts), Some(msg_timestamp)) = (from_timestamp, json.get("t").and_then(|v| v.as_i64()).or_else(|| json.get("timestamp").and_then(|v| v.as_i64()))) {
                if msg_timestamp < from_ts {
                    return false;
                }
            }

            if let (Some(to_ts), Some(msg_timestamp)) = (to_timestamp, json.get("t").and_then(|v| v.as_i64()).or_else(|| json.get("timestamp").and_then(|v| v.as_i64()))) {
                if msg_timestamp > to_ts {
                    return false;
                }
            }

            // Check message index filters - use "m" field for array of messages
            if let (Some(from_idx), Some(msg_idx)) = (from_message_index, json.get("m").and_then(|v| v.as_i64()).or_else(|| json.get("message_index").and_then(|v| v.as_i64()))) {
                if msg_idx < from_idx as i64 {
                    return false;
                }
            }

            if let (Some(to_idx), Some(msg_idx)) = (to_message_index, json.get("m").and_then(|v| v.as_i64()).or_else(|| json.get("message_index").and_then(|v| v.as_i64()))) {
                if msg_idx > to_idx as i64 {
                    return false;
                }
            }

            true // Passed all filters
        })
    }

    /// Check if filters have active conditions
    fn has_active_filters(&self, filters: &SearchFilter) -> bool {
        filters.must.as_ref().map_or(false, |v| !v.is_empty()) || 
        filters.should.as_ref().map_or(false, |v| !v.is_empty()) || 
        filters.must_not.as_ref().map_or(false, |v| !v.is_empty())
    }
}