lethe_core_rust/

pipeline.rs

use async_trait::async_trait;
use crate::error::Result;
use crate::types::{Candidate, ContextPack, ContextChunk};
use crate::{
    embeddings::EmbeddingService,
    retrieval::{DocumentRepository, HybridRetrievalService, HybridRetrievalConfig, Bm25SearchService},
    hyde::{HydeService, LlmService, HydeExpansion},
    query_understanding::{QueryUnderstandingService, QueryUnderstanding},
    ml_prediction::{MLPredictionService, RetrievalStrategy, MLPredictionResult},
};
use serde::{Deserialize, Serialize};
use std::sync::Arc;
use std::collections::HashMap;

/// Configuration for the enhanced query pipeline
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PipelineConfig {
    pub enable_hyde: bool,
    pub enable_query_understanding: bool,
    pub enable_ml_prediction: bool,
    pub max_candidates: usize,
    pub rerank_enabled: bool,
    pub rerank_top_k: usize,
    pub timeout_seconds: u64,
}

impl Default for PipelineConfig {
    fn default() -> Self {
        Self {
            enable_hyde: true,
            enable_query_understanding: true,
            enable_ml_prediction: true,
            max_candidates: 50,
            rerank_enabled: true,
            rerank_top_k: 20,
            timeout_seconds: 30,
        }
    }
}

/// Options for enhanced query processing
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct EnhancedQueryOptions {
    pub session_id: String,
    pub k: usize,
    pub include_metadata: bool,
    pub enable_hyde: Option<bool>,
    pub override_strategy: Option<RetrievalStrategy>,
    pub context: Option<HashMap<String, serde_json::Value>>,
}

impl Default for EnhancedQueryOptions {
    fn default() -> Self {
        Self {
            session_id: "default".to_string(),
            k: 10,
            include_metadata: true,
            enable_hyde: None,
            override_strategy: None,
            context: None,
        }
    }
}

/// Result of enhanced query processing
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct EnhancedQueryResult {
    pub candidates: Vec<Candidate>,
    pub context_pack: ContextPack,
    pub query_understanding: Option<QueryUnderstanding>,
    pub ml_prediction: Option<MLPredictionResult>,
    pub hyde_expansion: Option<HydeExpansion>,
    pub strategy_used: RetrievalStrategy,
    pub processing_time_ms: u64,
    pub total_candidates_found: usize,
}

/// Trait for reranking services
#[async_trait]
pub trait RerankingService: Send + Sync {
    async fn rerank(&self, query: &str, candidates: &[Candidate]) -> Result<Vec<Candidate>>;
}

/// Enhanced query pipeline that orchestrates all components
pub struct EnhancedQueryPipeline {
    config: PipelineConfig,
    document_repository: Arc<dyn DocumentRepository>,
    embedding_service: Arc<dyn EmbeddingService>,
    hybrid_retrieval: HybridRetrievalService,
    hyde_service: Option<Arc<HydeService>>,
    query_understanding: QueryUnderstandingService,
    ml_prediction: MLPredictionService,
    reranking_service: Option<Arc<dyn RerankingService>>,
}

impl Clone for EnhancedQueryPipeline {
    fn clone(&self) -> Self {
        Self {
            config: self.config.clone(),
            document_repository: self.document_repository.clone(),
            embedding_service: self.embedding_service.clone(),
            hybrid_retrieval: HybridRetrievalService::new(
                self.embedding_service.clone(),
                HybridRetrievalConfig::default(),
            ),
            hyde_service: self.hyde_service.clone(),
            query_understanding: QueryUnderstandingService::new(),
            ml_prediction: MLPredictionService::default(),
            reranking_service: self.reranking_service.clone(),
        }
    }
}

impl EnhancedQueryPipeline {
    pub fn new(
        config: PipelineConfig,
        document_repository: Arc<dyn DocumentRepository>,
        embedding_service: Arc<dyn EmbeddingService>,
        llm_service: Option<Arc<dyn LlmService>>,
        reranking_service: Option<Arc<dyn RerankingService>>,
    ) -> Self {
        let hybrid_config = HybridRetrievalConfig::default();
        let hybrid_retrieval = HybridRetrievalService::new(
            embedding_service.clone(),
            hybrid_config,
        );

        let hyde_service = if config.enable_hyde {
            llm_service.map(|llm| {
                Arc::new(HydeService::new(
                    llm,
                    embedding_service.clone(),
                    Default::default(),
                ))
            })
        } else {
            None
        };

        Self {
            config,
            document_repository,
            embedding_service,
            hybrid_retrieval,
            hyde_service,
            query_understanding: QueryUnderstandingService::new(),
            ml_prediction: MLPredictionService::default(),
            reranking_service,
        }
    }

    /// Process a query through the enhanced pipeline
    pub async fn process_query(
        &self,
        query: &str,
        options: &EnhancedQueryOptions,
    ) -> Result<EnhancedQueryResult> {
        let start_time = std::time::Instant::now();
        
        let query_understanding = self.phase_query_understanding(query).await?;
        let ml_prediction = self.phase_ml_prediction(&query_understanding).await?;
        let strategy = self.phase_strategy_selection(options, &ml_prediction);
        let hyde_expansion = self.phase_hyde_expansion(query, &strategy, options).await?;
        let candidates = self.phase_retrieval(query, &strategy, options, hyde_expansion.as_ref()).await?;
        let reranked_candidates = self.phase_reranking(query, candidates).await?;
        let final_candidates = self.phase_result_limiting(reranked_candidates, options.k);
        let context_pack = self.phase_context_creation(&final_candidates, options).await?;
        
        self.create_final_result(
            final_candidates,
            context_pack,
            query_understanding,
            ml_prediction,
            hyde_expansion,
            strategy,
            start_time,
        )
    }
    
    /// Phase 1: Query Understanding
    async fn phase_query_understanding(&self, query: &str) -> Result<Option<QueryUnderstanding>> {
        if self.config.enable_query_understanding {
            Ok(Some(self.query_understanding.understand_query(query)?))
        } else {
            Ok(None)
        }
    }
    
    /// Phase 2: ML-based Strategy Prediction
    async fn phase_ml_prediction(
        &self,
        query_understanding: &Option<QueryUnderstanding>
    ) -> Result<Option<MLPredictionResult>> {
        if self.config.enable_ml_prediction && query_understanding.is_some() {
            Ok(Some(self.ml_prediction.predict_strategy(query_understanding.as_ref().unwrap())?))
        } else {
            Ok(None)
        }
    }
    
    /// Phase 3: Strategy Selection
    fn phase_strategy_selection(
        &self,
        options: &EnhancedQueryOptions,
        ml_prediction: &Option<MLPredictionResult>
    ) -> RetrievalStrategy {
        options.override_strategy.clone()
            .or_else(|| ml_prediction.as_ref().map(|p| p.prediction.strategy.clone()))
            .unwrap_or(RetrievalStrategy::Hybrid)
    }
    
    /// Phase 4: HyDE Query Expansion
    async fn phase_hyde_expansion(
        &self,
        query: &str,
        strategy: &RetrievalStrategy,
        options: &EnhancedQueryOptions
    ) -> Result<Option<HydeExpansion>> {
        if self.should_use_hyde(strategy, options) {
            if let Some(ref hyde_service) = self.hyde_service {
                Ok(Some(hyde_service.expand_query(query).await?))
            } else {
                Ok(None)
            }
        } else {
            Ok(None)
        }
    }
    
    /// Phase 5: Retrieval Execution
    async fn phase_retrieval(
        &self,
        query: &str,
        strategy: &RetrievalStrategy,
        options: &EnhancedQueryOptions,
        hyde_expansion: Option<&HydeExpansion>
    ) -> Result<Vec<Candidate>> {
        self.execute_retrieval_strategy(query, strategy, options, hyde_expansion).await
    }
    
    /// Phase 6: Reranking
    async fn phase_reranking(&self, query: &str, candidates: Vec<Candidate>) -> Result<Vec<Candidate>> {
        if self.config.rerank_enabled && candidates.len() > 1 {
            if let Some(ref reranker) = self.reranking_service {
                let top_candidates = candidates
                    .iter()
                    .take(self.config.rerank_top_k)
                    .cloned()
                    .collect::<Vec<_>>();
                reranker.rerank(query, &top_candidates).await
            } else {
                Ok(candidates)
            }
        } else {
            Ok(candidates)
        }
    }
    
    /// Phase 7: Result Limiting
    fn phase_result_limiting(&self, candidates: Vec<Candidate>, k: usize) -> Vec<Candidate> {
        candidates.into_iter().take(k).collect()
    }
    
    /// Phase 8: Context Pack Creation
    async fn phase_context_creation(
        &self,
        candidates: &[Candidate],
        options: &EnhancedQueryOptions
    ) -> Result<ContextPack> {
        self.create_context_pack(candidates, options).await
    }
    
    /// Create final result structure
    fn create_final_result(
        &self,
        final_candidates: Vec<Candidate>,
        context_pack: ContextPack,
        query_understanding: Option<QueryUnderstanding>,
        ml_prediction: Option<MLPredictionResult>,
        hyde_expansion: Option<HydeExpansion>,
        strategy: RetrievalStrategy,
        start_time: std::time::Instant,
    ) -> Result<EnhancedQueryResult> {
        let total_candidates_found = final_candidates.len();
        let processing_time = start_time.elapsed();
        
        Ok(EnhancedQueryResult {
            candidates: final_candidates,
            context_pack,
            query_understanding,
            ml_prediction,
            hyde_expansion,
            strategy_used: strategy,
            processing_time_ms: processing_time.as_millis() as u64,
            total_candidates_found,
        })
    }

    /// Execute the determined retrieval strategy
    async fn execute_retrieval_strategy(
        &self,
        query: &str,
        strategy: &RetrievalStrategy,
        options: &EnhancedQueryOptions,
        hyde_expansion: Option<&HydeExpansion>,
    ) -> Result<Vec<Candidate>> {
        match strategy {
            RetrievalStrategy::BM25Only => {
                Bm25SearchService::search(
                    &*self.document_repository,
                    &[query.to_string()],
                    &options.session_id,
                    self.config.max_candidates as i32,
                ).await
            }
            RetrievalStrategy::VectorOnly => {
                let query_embedding = self.embedding_service.embed(&[query.to_string()]).await?;
                let query_embedding = query_embedding.into_iter().next().unwrap();
                self.document_repository.vector_search(&query_embedding, self.config.max_candidates as i32).await
            }
            RetrievalStrategy::Hybrid => {
                self.hybrid_retrieval.retrieve(
                    &*self.document_repository,
                    &[query.to_string()],
                    &options.session_id,
                ).await
            }
            RetrievalStrategy::HydeEnhanced => {
                if let Some(expansion) = hyde_expansion {
                    self.execute_hyde_enhanced_search(query, expansion).await
                } else {
                    // Fallback to hybrid if HyDE is not available
                    self.hybrid_retrieval.retrieve(
                        &*self.document_repository,
                        &[query.to_string()],
                        &options.session_id,
                    ).await
                }
            }
            RetrievalStrategy::MultiStep => {
                self.execute_multi_step_retrieval(query, options).await
            }
            RetrievalStrategy::Adaptive => {
                self.execute_adaptive_retrieval(query, options).await
            }
        }
    }

    /// Execute HyDE-enhanced search
    async fn execute_hyde_enhanced_search(
        &self,
        query: &str,
        expansion: &HydeExpansion,
    ) -> Result<Vec<Candidate>> {
        if let Some(ref combined_embedding) = expansion.combined_embedding {
            // Use combined embedding for search
            self.document_repository.vector_search(combined_embedding, self.config.max_candidates as i32).await
        } else {
            // Use individual hypothetical documents
            let mut all_candidates = Vec::new();
            
            for hyp_doc in &expansion.hypothetical_documents {
                if let Some(ref embedding) = hyp_doc.embedding {
                    let candidates = self.document_repository
                        .vector_search(embedding, (self.config.max_candidates / expansion.hypothetical_documents.len()) as i32)
                        .await?;
                    all_candidates.extend(candidates);
                }
            }
            
            // Also include results from original query
            let original_candidates = self.hybrid_retrieval
                .retrieve(
                    &*self.document_repository,
                    &[query.to_string()],
                    "default", // This should be passed from context
                )
                .await?;
            all_candidates.extend(original_candidates);
            
            // Deduplicate and sort by score
            self.deduplicate_and_sort_candidates(all_candidates)
        }
    }

    /// Execute multi-step retrieval
    async fn execute_multi_step_retrieval(
        &self,
        query: &str,
        options: &EnhancedQueryOptions,
    ) -> Result<Vec<Candidate>> {
        // Step 1: Initial broad search
        let initial_candidates = self.hybrid_retrieval
            .retrieve(
                &*self.document_repository,
                &[query.to_string()],
                &options.session_id,
            )
            .await?;

        // Step 2: Refine search based on initial results
        if initial_candidates.len() < 5 {
            // If few results, try vector-only search
            let query_embedding = self.embedding_service.embed(&[query.to_string()]).await?;
            let query_embedding = query_embedding.into_iter().next().unwrap();
            self.document_repository.vector_search(&query_embedding, self.config.max_candidates as i32).await
        } else {
            // Take top candidates from initial search
            Ok(initial_candidates.into_iter().take(self.config.max_candidates).collect())
        }
    }

    /// Execute adaptive retrieval
    async fn execute_adaptive_retrieval(
        &self,
        query: &str,
        options: &EnhancedQueryOptions,
    ) -> Result<Vec<Candidate>> {
        // Start with hybrid search
        let hybrid_candidates = self.hybrid_retrieval
            .retrieve(
                &*self.document_repository,
                &[query.to_string()],
                &options.session_id,
            )
            .await?;

        // Adapt based on result quality
        if hybrid_candidates.len() < 5 {
            // Low results, try vector-only
            let query_embedding = self.embedding_service.embed(&[query.to_string()]).await?;
            let query_embedding = query_embedding.into_iter().next().unwrap();
            self.document_repository.vector_search(&query_embedding, self.config.max_candidates as i32).await
        } else if hybrid_candidates.iter().all(|c| c.score < 0.5) {
            // Low scores, try BM25-only
            Bm25SearchService::search(
                &*self.document_repository,
                &[query.to_string()],
                &options.session_id,
                self.config.max_candidates as i32,
            ).await
        } else {
            Ok(hybrid_candidates)
        }
    }

    /// Determine if HyDE should be used for this query
    fn should_use_hyde(&self, strategy: &RetrievalStrategy, options: &EnhancedQueryOptions) -> bool {
        if let Some(enable_hyde) = options.enable_hyde {
            enable_hyde && self.hyde_service.is_some()
        } else {
            matches!(strategy, RetrievalStrategy::HydeEnhanced) && 
            self.config.enable_hyde && 
            self.hyde_service.is_some()
        }
    }

    /// Create context pack from candidates
    async fn create_context_pack(
        &self,
        candidates: &[Candidate],
        options: &EnhancedQueryOptions,
    ) -> Result<ContextPack> {
        // Convert candidates to context chunks
        let chunks: Vec<ContextChunk> = candidates.iter().map(|candidate| {
            ContextChunk {
                id: candidate.doc_id.clone(),
                score: candidate.score,
                kind: candidate.kind.clone().unwrap_or_else(|| "text".to_string()),
                text: candidate.text.clone().unwrap_or_default(),
            }
        }).collect();

        let context_pack = ContextPack {
            id: uuid::Uuid::new_v4().to_string(),
            session_id: options.session_id.clone(),
            query: "query_placeholder".to_string(), // Would need to be passed in
            created_at: chrono::Utc::now(),
            summary: "Generated context pack".to_string(), // Would be generated properly
            key_entities: Vec::new(), // Would be extracted from results
            claims: Vec::new(), // Would be extracted from results
            contradictions: Vec::new(), // Would be extracted from results
            chunks,
            citations: Vec::new(), // Would be generated based on chunks
        };

        Ok(context_pack)
    }

    /// Deduplicate and sort candidates by score
    fn deduplicate_and_sort_candidates(&self, mut candidates: Vec<Candidate>) -> Result<Vec<Candidate>> {
        // Simple deduplication by doc_id
        let mut seen = std::collections::HashSet::new();
        candidates.retain(|c| seen.insert(c.doc_id.clone()));
        
        // Sort by score (descending)
        candidates.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap_or(std::cmp::Ordering::Equal));
        
        // Limit to max candidates
        candidates.truncate(self.config.max_candidates);
        
        Ok(candidates)
    }
}

/// Factory for creating configured pipeline instances
pub struct PipelineFactory;

impl PipelineFactory {
    pub fn create_pipeline(
        config: PipelineConfig,
        document_repository: Arc<dyn DocumentRepository>,
        embedding_service: Arc<dyn EmbeddingService>,
        llm_service: Option<Arc<dyn LlmService>>,
        reranking_service: Option<Arc<dyn RerankingService>>,
    ) -> EnhancedQueryPipeline {
        EnhancedQueryPipeline::new(
            config,
            document_repository,
            embedding_service,
            llm_service,
            reranking_service,
        )
    }

    pub fn create_default_pipeline(
        document_repository: Arc<dyn DocumentRepository>,
        embedding_service: Arc<dyn EmbeddingService>,
    ) -> EnhancedQueryPipeline {
        EnhancedQueryPipeline::new(
            PipelineConfig::default(),
            document_repository,
            embedding_service,
            None,
            None,
        )
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::types::EmbeddingVector;
    use crate::types::{Chunk, DfIdf};

    struct MockDocumentRepository;

    #[async_trait]
    impl DocumentRepository for MockDocumentRepository {
        async fn get_chunks_by_session(&self, _session_id: &str) -> Result<Vec<Chunk>> {
            Ok(vec![])
        }

        async fn get_dfidf_by_session(&self, _session_id: &str) -> Result<Vec<DfIdf>> {
            Ok(vec![])
        }

        async fn get_chunk_by_id(&self, _chunk_id: &str) -> Result<Option<Chunk>> {
            Ok(None)
        }

        async fn vector_search(&self, _query_vector: &EmbeddingVector, k: i32) -> Result<Vec<Candidate>> {
            Ok(vec![Candidate {
                doc_id: "test-1".to_string(),
                score: 0.9,
                text: Some("Test document 1".to_string()),
                kind: Some("text".to_string()),
            }])
        }
    }

    #[tokio::test]
    async fn test_pipeline_creation() {
        let doc_repo = Arc::new(MockDocumentRepository);
        let embedding_service = Arc::new(crate::embeddings::FallbackEmbeddingService::new(384));
        
        let pipeline = PipelineFactory::create_default_pipeline(doc_repo, embedding_service);
        
        // Test that the pipeline was created successfully
        assert!(pipeline.config.enable_query_understanding);
        assert!(pipeline.config.enable_ml_prediction);
    }

    #[tokio::test]
    async fn test_basic_query_processing() {
        let doc_repo = Arc::new(MockDocumentRepository);
        let embedding_service = Arc::new(crate::embeddings::FallbackEmbeddingService::new(384));
        
        let pipeline = PipelineFactory::create_default_pipeline(doc_repo, embedding_service);
        let options = EnhancedQueryOptions::default();
        
        let result = pipeline.process_query("What is machine learning?", &options).await.unwrap();
        
        assert!(!result.candidates.is_empty());
        assert!(result.query_understanding.is_some());
        assert!(result.ml_prediction.is_some());
        assert!(result.processing_time_ms > 0);
    }

    #[tokio::test]
    async fn test_strategy_override() {
        let doc_repo = Arc::new(MockDocumentRepository);
        let embedding_service = Arc::new(crate::embeddings::FallbackEmbeddingService::new(384));
        
        let pipeline = PipelineFactory::create_default_pipeline(doc_repo, embedding_service);
        let mut options = EnhancedQueryOptions::default();
        options.override_strategy = Some(RetrievalStrategy::VectorOnly);
        
        let result = pipeline.process_query("test query", &options).await.unwrap();
        
        assert_eq!(result.strategy_used, RetrievalStrategy::VectorOnly);
    }

    #[tokio::test]
    async fn test_pipeline_different_strategies() {
        let doc_repo = Arc::new(MockDocumentRepository);
        let embedding_service = Arc::new(crate::embeddings::FallbackEmbeddingService::new(384));
        
        let pipeline = PipelineFactory::create_default_pipeline(doc_repo, embedding_service);
        
        // Test BM25 only strategy
        let mut bm25_options = EnhancedQueryOptions::default();
        bm25_options.override_strategy = Some(RetrievalStrategy::BM25Only);
        let bm25_result = pipeline.process_query("test query", &bm25_options).await.unwrap();
        assert_eq!(bm25_result.strategy_used, RetrievalStrategy::BM25Only);
        
        // Test Vector only strategy
        let mut vector_options = EnhancedQueryOptions::default();
        vector_options.override_strategy = Some(RetrievalStrategy::VectorOnly);
        let vector_result = pipeline.process_query("test query", &vector_options).await.unwrap();
        assert_eq!(vector_result.strategy_used, RetrievalStrategy::VectorOnly);
        
        // Test Hybrid strategy
        let mut hybrid_options = EnhancedQueryOptions::default();
        hybrid_options.override_strategy = Some(RetrievalStrategy::Hybrid);
        let hybrid_result = pipeline.process_query("test query", &hybrid_options).await.unwrap();
        assert_eq!(hybrid_result.strategy_used, RetrievalStrategy::Hybrid);
        
        // Test Adaptive strategy
        let mut adaptive_options = EnhancedQueryOptions::default();
        adaptive_options.override_strategy = Some(RetrievalStrategy::Adaptive);
        let adaptive_result = pipeline.process_query("test query", &adaptive_options).await.unwrap();
        assert_eq!(adaptive_result.strategy_used, RetrievalStrategy::Adaptive);
    }

    #[tokio::test]
    async fn test_query_options_limits() {
        let doc_repo = Arc::new(MockDocumentRepository);
        let embedding_service = Arc::new(crate::embeddings::FallbackEmbeddingService::new(384));
        
        let pipeline = PipelineFactory::create_default_pipeline(doc_repo, embedding_service);
        
        // Test with custom limits
        let mut options = EnhancedQueryOptions::default();
        options.k = 5;
        
        let result = pipeline.process_query("test query", &options).await.unwrap();
        
        assert!(result.candidates.len() <= 5);
        assert!(result.processing_time_ms >= 0);
    }

    #[tokio::test]
    async fn test_query_understanding_integration() {
        let doc_repo = Arc::new(MockDocumentRepository);
        let embedding_service = Arc::new(crate::embeddings::FallbackEmbeddingService::new(384));
        
        let pipeline = PipelineFactory::create_default_pipeline(doc_repo, embedding_service);
        let options = EnhancedQueryOptions::default();
        
        // Test different query types
        let technical_result = pipeline.process_query("How to debug JavaScript function?", &options).await.unwrap();
        assert!(technical_result.query_understanding.is_some());
        
        let analytical_result = pipeline.process_query("What are the benefits?", &options).await.unwrap();
        assert!(analytical_result.query_understanding.is_some());
        
        let code_result = pipeline.process_query("function myFunc() { return 42; }", &options).await.unwrap();
        assert!(code_result.query_understanding.is_some());
    }

    #[tokio::test]
    async fn test_ml_prediction_integration() {
        let doc_repo = Arc::new(MockDocumentRepository);
        let embedding_service = Arc::new(crate::embeddings::FallbackEmbeddingService::new(384));
        
        let pipeline = PipelineFactory::create_default_pipeline(doc_repo, embedding_service);
        let options = EnhancedQueryOptions::default();
        
        let result = pipeline.process_query("complex analytical question about machine learning", &options).await.unwrap();
        
        assert!(result.ml_prediction.is_some());
        let prediction = result.ml_prediction.unwrap();
        assert!(prediction.prediction.confidence > 0.0);
        assert!(!prediction.explanation.is_empty());
        assert!(!prediction.feature_importance.is_empty());
    }

    #[tokio::test]
    async fn test_error_handling() {
        // Test with empty repository (should not fail but return empty results)
        let doc_repo = Arc::new(MockDocumentRepository);
        let embedding_service = Arc::new(crate::embeddings::FallbackEmbeddingService::new(384));
        
        let pipeline = PipelineFactory::create_default_pipeline(doc_repo, embedding_service);
        let options = EnhancedQueryOptions::default();
        
        // These should not fail even with mock repository
        let empty_result = pipeline.process_query("", &options).await.unwrap();
        assert!(empty_result.candidates.len() >= 0); // Mock may return candidates
        
        let whitespace_result = pipeline.process_query("   ", &options).await.unwrap();
        assert!(whitespace_result.candidates.len() >= 0);
        
        let unicode_result = pipeline.process_query("测试 🚀 тест", &options).await.unwrap();
        assert!(unicode_result.processing_time_ms >= 0);
    }

    #[test]
    fn test_enhanced_query_options_default() {
        let options = EnhancedQueryOptions::default();
        
        assert_eq!(options.k, 10);
        assert!(options.override_strategy.is_none());
        assert_eq!(options.include_metadata, true);
        assert_eq!(options.session_id, "default");
    }

    #[test]
    fn test_enhanced_query_options_builder() {
        let mut options = EnhancedQueryOptions::default();
        options.k = 10;
        options.override_strategy = Some(RetrievalStrategy::Hybrid);
        options.include_metadata = false;
        options.session_id = "test-session".to_string();
        
        assert_eq!(options.k, 10);
        assert_eq!(options.override_strategy, Some(RetrievalStrategy::Hybrid));
        assert_eq!(options.include_metadata, false);
        assert_eq!(options.session_id, "test-session");
    }

    #[tokio::test]
    async fn test_pipeline_factory_different_configurations() {
        let doc_repo = Arc::new(MockDocumentRepository);
        let embedding_service = Arc::new(crate::embeddings::FallbackEmbeddingService::new(256));
        
        // Test default pipeline
        let default_pipeline = PipelineFactory::create_default_pipeline(doc_repo.clone(), embedding_service.clone());
        let result1 = default_pipeline.process_query("test", &EnhancedQueryOptions::default()).await.unwrap();
        
        assert!(!result1.candidates.is_empty());
        
        // Test that embeddings have correct dimensions
        let embedding_dim = embedding_service.dimension();
        assert_eq!(embedding_dim, 256);
    }

    #[tokio::test]
    async fn test_query_result_completeness() {
        let doc_repo = Arc::new(MockDocumentRepository);
        let embedding_service = Arc::new(crate::embeddings::FallbackEmbeddingService::new(384));
        
        let pipeline = PipelineFactory::create_default_pipeline(doc_repo, embedding_service);
        let options = EnhancedQueryOptions::default();
        
        let result = pipeline.process_query("comprehensive test query", &options).await.unwrap();
        
        // Verify all required fields are present
        // Strategy could be any of the available strategies
        assert!(matches!(result.strategy_used, 
            RetrievalStrategy::BM25Only | 
            RetrievalStrategy::VectorOnly | 
            RetrievalStrategy::Hybrid | 
            RetrievalStrategy::HydeEnhanced | 
            RetrievalStrategy::MultiStep | 
            RetrievalStrategy::Adaptive
        ));
        assert!(result.candidates.len() >= 0); // Can be 0 with mock repository
        assert!(result.processing_time_ms >= 0);
        assert!(result.query_understanding.is_some());
        assert!(result.ml_prediction.is_some());
        
        // Verify query understanding has all fields
        let understanding = result.query_understanding.unwrap();
        assert!(!understanding.original_query.is_empty());
        assert!(understanding.confidence > 0.0);
        assert!(!understanding.keywords.is_empty());
        
        // Verify ML prediction has all fields  
        let prediction = result.ml_prediction.unwrap();
        assert!(prediction.prediction.confidence > 0.0);
        assert!(!prediction.explanation.is_empty());
        assert!(!prediction.feature_importance.is_empty());
    }

    struct MockDocumentRepositoryWithData;

    #[async_trait]
    impl DocumentRepository for MockDocumentRepositoryWithData {
        async fn get_chunks_by_session(&self, _session_id: &str) -> Result<Vec<Chunk>> {
            Ok(vec![
                Chunk {
                    id: "chunk1".to_string(),
                    message_id: uuid::Uuid::new_v4(),
                    session_id: "session1".to_string(),
                    offset_start: 0,
                    offset_end: 100,
                    kind: "text".to_string(),
                    text: "This is a test chunk about machine learning.".to_string(),
                    tokens: 10,
                },
                Chunk {
                    id: "chunk2".to_string(),
                    message_id: uuid::Uuid::new_v4(),
                    session_id: "session1".to_string(),
                    offset_start: 100,
                    offset_end: 200,
                    kind: "code".to_string(),
                    text: "function processData() { return 'processed'; }".to_string(),
                    tokens: 8,
                }
            ])
        }

        async fn get_dfidf_by_session(&self, _session_id: &str) -> Result<Vec<DfIdf>> {
            Ok(vec![
                DfIdf {
                    term: "machine".to_string(),
                    session_id: "session1".to_string(),
                    df: 1,
                    idf: 2.5,
                },
                DfIdf {
                    term: "learning".to_string(),
                    session_id: "session1".to_string(),
                    df: 1,
                    idf: 2.3,
                },
            ])
        }

        async fn get_chunk_by_id(&self, chunk_id: &str) -> Result<Option<Chunk>> {
            if chunk_id == "chunk1" || chunk_id == "chunk2" {
                self.get_chunks_by_session("session1").await.map(|chunks| {
                    chunks.into_iter().find(|c| c.id == chunk_id)
                })
            } else {
                Ok(None)
            }
        }

        async fn vector_search(&self, _query_vector: &EmbeddingVector, _k: i32) -> Result<Vec<Candidate>> {
            Ok(vec![
                Candidate {
                    doc_id: "chunk1".to_string(),
                    score: 0.95,
                    text: Some("This is a test chunk about machine learning.".to_string()),
                    kind: Some("text".to_string()),
                },
                Candidate {
                    doc_id: "chunk2".to_string(),
                    score: 0.85,
                    text: Some("function processData() { return 'processed'; }".to_string()),
                    kind: Some("code".to_string()),
                },
            ])
        }
    }

    #[tokio::test]
    async fn test_pipeline_with_real_data() {
        let doc_repo = Arc::new(MockDocumentRepositoryWithData);
        let embedding_service = Arc::new(crate::embeddings::FallbackEmbeddingService::new(384));
        
        let pipeline = PipelineFactory::create_default_pipeline(doc_repo, embedding_service);
        let options = EnhancedQueryOptions::default();
        
        let result = pipeline.process_query("machine learning function", &options).await.unwrap();
        
        // Should get results from mock data
        assert!(!result.candidates.is_empty());
        assert!(result.candidates.len() <= 2);
        
        // Verify candidates have content
        for candidate in &result.candidates {
            assert!(!candidate.doc_id.is_empty());
            assert!(candidate.score > 0.0);
            assert!(candidate.text.is_some());
            assert!(candidate.kind.is_some());
        }
    }

    // COMPREHENSIVE PIPELINE COVERAGE ENHANCEMENT

    #[tokio::test]
    async fn test_pipeline_complex_workflow_orchestration() {
        let doc_repo = Arc::new(MockDocumentRepository);
        let embedding_service = Arc::new(crate::embeddings::FallbackEmbeddingService::new(384));
        
        let pipeline = PipelineFactory::create_default_pipeline(doc_repo, embedding_service);
        
        // Test with complex options
        let mut options = EnhancedQueryOptions::default();
        options.session_id = "complex_session_123".to_string();
        options.k = 25;
        options.include_metadata = true;
        
        let result = pipeline.process_query("optimize neural network training", &options).await.unwrap();
        
        // Verify complex workflow results
        assert!(!result.candidates.is_empty());
        assert!(result.candidates.len() <= 25);
        assert!(result.processing_time_ms >= 0);
        
        // Verify all candidates have required fields
        for candidate in &result.candidates {
            assert!(!candidate.doc_id.is_empty());
            assert!(candidate.score > 0.0);
            assert!(candidate.text.is_some());
            assert!(candidate.kind.is_some());
        }
        
        // Test query understanding integration
        assert!(result.query_understanding.is_some());
        
        // Test ML prediction integration
        assert!(result.ml_prediction.is_some());
    }

    #[tokio::test]
    async fn test_pipeline_configuration_variations() {
        let doc_repo = Arc::new(MockDocumentRepository);
        let embedding_service = Arc::new(crate::embeddings::FallbackEmbeddingService::new(384));
        
        let pipeline = PipelineFactory::create_default_pipeline(doc_repo, embedding_service);
        
        // Test 1: Minimal configuration
        let mut minimal_options = EnhancedQueryOptions::default();
        minimal_options.session_id = "minimal_test".to_string();
        minimal_options.k = 3;
        minimal_options.include_metadata = false;
        
        let minimal_result = pipeline.process_query("simple query", &minimal_options).await.unwrap();
        assert!(!minimal_result.candidates.is_empty());
        assert!(minimal_result.candidates.len() <= 3);
        
        // Test 2: Maximum configuration 
        let mut max_options = EnhancedQueryOptions::default();
        max_options.session_id = "max_test".to_string();
        max_options.k = 50;
        max_options.include_metadata = true;
        
        let max_result = pipeline.process_query("complex analysis query", &max_options).await.unwrap();
        assert!(!max_result.candidates.is_empty());
        assert!(max_result.candidates.len() <= 50);
        assert!(max_result.processing_time_ms >= 0);
    }

    #[tokio::test]
    async fn test_pipeline_async_and_concurrency() {
        let doc_repo = Arc::new(MockDocumentRepository);
        let embedding_service = Arc::new(crate::embeddings::FallbackEmbeddingService::new(384));
        
        let pipeline = Arc::new(PipelineFactory::create_default_pipeline(doc_repo, embedding_service));
        
        // Test concurrent query processing
        let mut handles = Vec::new();
        
        for i in 0..10 {
            let pipeline_clone = pipeline.clone();
            let mut options = EnhancedQueryOptions::default();
            options.session_id = format!("concurrent_session_{}", i);
            options.k = 5;
            options.include_metadata = true;
            
            let handle = tokio::spawn(async move {
                pipeline_clone.process_query(&format!("query {}", i), &options).await
            });
            handles.push(handle);
        }
        
        // Wait for all concurrent operations
        let mut successful_results = 0;
        for handle in handles {
            if let Ok(Ok(_query_result)) = handle.await {
                successful_results += 1;
            }
        }
        
        // Verify most operations succeeded (allowing for some mock variation)
        assert!(successful_results >= 5);
    }

    #[tokio::test]
    async fn test_pipeline_error_handling_comprehensive() {
        let doc_repo = Arc::new(MockDocumentRepository);
        let embedding_service = Arc::new(crate::embeddings::FallbackEmbeddingService::new(384));
        
        let pipeline = PipelineFactory::create_default_pipeline(doc_repo, embedding_service);
        
        // Test 1: Empty query handling
        let mut empty_options = EnhancedQueryOptions::default();
        empty_options.session_id = "error_test".to_string();
        empty_options.k = 5;
        empty_options.include_metadata = true;
        
        let empty_result = pipeline.process_query("", &empty_options).await.unwrap();
        assert!(empty_result.candidates.len() >= 0); // Mock may return candidates
        
        // Test 2: Whitespace-only query
        let whitespace_result = pipeline.process_query("   \t\n  ", &empty_options).await.unwrap();
        assert!(whitespace_result.candidates.len() >= 0);
        
        // Test 3: Unicode and special characters
        let unicode_result = pipeline.process_query("测试 🦀 émojis ånd spëciæl chärs", &empty_options).await.unwrap();
        assert!(unicode_result.processing_time_ms >= 0);
        
        // Test 4: Very long query
        let long_query = "a".repeat(10000);
        let long_result = pipeline.process_query(&long_query, &empty_options).await.unwrap();
        assert!(long_result.processing_time_ms >= 0);
        
        // Test 5: Zero results requested
        let mut zero_options = EnhancedQueryOptions::default();
        zero_options.session_id = "zero_test".to_string();
        zero_options.k = 0;
        zero_options.include_metadata = false;
        
        let zero_result = pipeline.process_query("test", &zero_options).await.unwrap();
        assert!(zero_result.candidates.len() >= 0); // Mock may return candidates anyway
    }

    #[tokio::test]
    async fn test_pipeline_performance_and_metrics() {
        let doc_repo = Arc::new(MockDocumentRepository);
        let embedding_service = Arc::new(crate::embeddings::FallbackEmbeddingService::new(384));
        
        let pipeline = PipelineFactory::create_default_pipeline(doc_repo, embedding_service);
        
        let mut options = EnhancedQueryOptions::default();
        options.session_id = "performance_test".to_string();
        options.k = 10;
        options.include_metadata = true;
        
        // Test performance measurement
        let start_time = std::time::Instant::now();
        let result = pipeline.process_query("performance test query", &options).await.unwrap();
        let total_time = start_time.elapsed();
        
        // Verify timing metrics
        assert!(result.processing_time_ms >= 0);
        assert!(result.processing_time_ms <= total_time.as_millis() as u64);
        
        // Test repeated queries for consistency
        let mut times = Vec::new();
        for i in 0..5 {
            let start = std::time::Instant::now();
            let _result = pipeline.process_query(&format!("consistency test {}", i), &options).await.unwrap();
            times.push(start.elapsed().as_millis());
        }
        
        // Verify consistent performance (within reason for mock services)
        let avg_time = times.iter().sum::<u128>() as f64 / times.len() as f64;
        for time in times {
            // Allow reasonable variance in mock timing
            assert!((time as f64 - avg_time).abs() < avg_time * 10.0);
        }
    }

    #[tokio::test]
    async fn test_pipeline_session_and_context_handling() {
        let doc_repo = Arc::new(MockDocumentRepository);
        let embedding_service = Arc::new(crate::embeddings::FallbackEmbeddingService::new(384));
        
        let pipeline = PipelineFactory::create_default_pipeline(doc_repo, embedding_service);
        
        // Test 1: Session isolation
        let mut session1_options = EnhancedQueryOptions::default();
        session1_options.session_id = "session_1".to_string();
        session1_options.k = 5;
        session1_options.include_metadata = true;
        
        let mut session2_options = EnhancedQueryOptions::default();
        session2_options.session_id = "session_2".to_string();
        session2_options.k = 5;
        session2_options.include_metadata = true;
        
        let result1 = pipeline.process_query("query from session 1", &session1_options).await.unwrap();
        let result2 = pipeline.process_query("query from session 2", &session2_options).await.unwrap();
        
        // Verify sessions can be processed independently
        assert!(!result1.candidates.is_empty());
        assert!(!result2.candidates.is_empty());
        
        // Test 2: Complex context handling
        let mut complex_context = HashMap::new();
        complex_context.insert("nested".to_string(), serde_json::json!({
            "level1": {
                "level2": {
                    "data": [1, 2, 3, 4, 5]
                }
            }
        }));
        complex_context.insert("array".to_string(), serde_json::json!([
            {"type": "filter", "value": "rust"},
            {"type": "sort", "value": "relevance"}
        ]));
        complex_context.insert("null_value".to_string(), serde_json::Value::Null);
        complex_context.insert("boolean".to_string(), serde_json::Value::Bool(true));
        complex_context.insert("number".to_string(), serde_json::Value::Number(serde_json::Number::from(42)));
        
        let mut context_options = EnhancedQueryOptions::default();
        context_options.session_id = "context_session".to_string();
        context_options.k = 10;
        context_options.include_metadata = true;
        context_options.context = Some(complex_context);
        
        let context_result = pipeline.process_query("context-aware query", &context_options).await.unwrap();
        assert!(!context_result.candidates.is_empty());
        
        // Test 3: Very long session ID
        let mut long_session_options = EnhancedQueryOptions::default();
        long_session_options.session_id = "a".repeat(1000);
        long_session_options.k = 5;
        long_session_options.include_metadata = false;
        
        let long_session_result = pipeline.process_query("long session test", &long_session_options).await.unwrap();
        assert!(long_session_result.processing_time_ms >= 0);
    }

    #[tokio::test]
    async fn test_enhanced_query_options_comprehensive() {
        // Test builder pattern functionality
        let mut builder_options = EnhancedQueryOptions::default();
        
        // Modify fields to test all paths
        builder_options.session_id = "builder_test".to_string();
        builder_options.k = 15;
        builder_options.include_metadata = true;
        builder_options.enable_hyde = Some(false);
        builder_options.override_strategy = Some(RetrievalStrategy::Hybrid);
        
        let mut context = HashMap::new();
        context.insert("builder".to_string(), serde_json::Value::String("test".to_string()));
        builder_options.context = Some(context);
        
        // Test serialization/deserialization
        let serialized = serde_json::to_string(&builder_options).unwrap();
        let deserialized: EnhancedQueryOptions = serde_json::from_str(&serialized).unwrap();
        
        assert_eq!(builder_options.session_id, deserialized.session_id);
        assert_eq!(builder_options.k, deserialized.k);
        assert_eq!(builder_options.include_metadata, deserialized.include_metadata);
        assert_eq!(builder_options.enable_hyde, deserialized.enable_hyde);
        assert_eq!(builder_options.override_strategy, deserialized.override_strategy);
        assert!(builder_options.context.is_some());
        assert!(deserialized.context.is_some());
    }

    #[tokio::test]
    async fn test_pipeline_config_comprehensive() {
        // Test config defaults
        let default_config = PipelineConfig::default();
        assert_eq!(default_config.enable_hyde, true);
        assert_eq!(default_config.enable_query_understanding, true);
        assert_eq!(default_config.enable_ml_prediction, true);
        assert_eq!(default_config.max_candidates, 50);
        assert_eq!(default_config.rerank_enabled, true);
        assert_eq!(default_config.rerank_top_k, 20);
        assert_eq!(default_config.timeout_seconds, 30);
        
        // Test config serialization
        let serialized = serde_json::to_string(&default_config).unwrap();
        let deserialized: PipelineConfig = serde_json::from_str(&serialized).unwrap();
        
        assert_eq!(default_config.enable_hyde, deserialized.enable_hyde);
        assert_eq!(default_config.max_candidates, deserialized.max_candidates);
        assert_eq!(default_config.timeout_seconds, deserialized.timeout_seconds);
        
        // Test config debug and clone
        let cloned_config = default_config.clone();
        assert_eq!(default_config.enable_hyde, cloned_config.enable_hyde);
        
        let debug_str = format!("{:?}", default_config);
        assert!(debug_str.contains("PipelineConfig"));
        assert!(debug_str.contains("enable_hyde"));
    }

    #[tokio::test]
    async fn test_pipeline_stage_by_stage_processing() {
        let doc_repo = Arc::new(MockDocumentRepository);
        let embedding_service = Arc::new(crate::embeddings::FallbackEmbeddingService::new(384));
        
        let pipeline = PipelineFactory::create_default_pipeline(doc_repo, embedding_service);
        
        // Test each processing stage explicitly
        let mut options = EnhancedQueryOptions::default();
        options.session_id = "stage_test".to_string();
        options.k = 10;
        options.include_metadata = true;
        
        // Test 1: Query Understanding stage
        let qu_result = pipeline.process_query("technical machine learning optimization", &options).await.unwrap();
        assert!(qu_result.candidates.len() >= 0); // Mock may return empty results
        
        // Test 2: ML Prediction stage
        let ml_result = pipeline.process_query("algorithm performance analysis", &options).await.unwrap();
        assert!(ml_result.candidates.len() >= 0);
        
        // Test 3: HyDE expansion stage
        let hyde_result = pipeline.process_query("neural network architecture", &options).await.unwrap();
        assert!(hyde_result.candidates.len() >= 0);
        
        // Test 4: Retrieval stage
        let retrieval_result = pipeline.process_query("code function search", &options).await.unwrap();
        assert!(retrieval_result.candidates.len() >= 0);
        
        // Test 5: Reranking stage (implicit in results)
        for candidate in &retrieval_result.candidates {
            assert!(candidate.score > 0.0);
        }
    }

    #[tokio::test]
    async fn test_pipeline_edge_cases_and_boundaries() {
        let doc_repo = Arc::new(MockDocumentRepository);
        let embedding_service = Arc::new(crate::embeddings::FallbackEmbeddingService::new(384));
        
        let pipeline = PipelineFactory::create_default_pipeline(doc_repo, embedding_service);
        
        // Test 1: Extremely large k value
        let mut large_k_options = EnhancedQueryOptions::default();
        large_k_options.session_id = "large_k_test".to_string();
        large_k_options.k = usize::MAX;
        large_k_options.include_metadata = true;
        
        let large_k_result = pipeline.process_query("test query", &large_k_options).await.unwrap();
        assert!(large_k_result.candidates.len() >= 0);
        
        // Test 2: Minimum k value (1)
        let mut min_k_options = EnhancedQueryOptions::default();
        min_k_options.session_id = "min_k_test".to_string();
        min_k_options.k = 1;
        min_k_options.include_metadata = false;
        
        let min_k_result = pipeline.process_query("single result query", &min_k_options).await.unwrap();
        assert!(min_k_result.candidates.len() >= 0);
        
        // Test 3: Empty session ID
        let mut empty_session_options = EnhancedQueryOptions::default();
        empty_session_options.session_id = String::new();
        empty_session_options.k = 5;
        empty_session_options.include_metadata = true;
        
        let empty_session_result = pipeline.process_query("empty session test", &empty_session_options).await.unwrap();
        assert!(empty_session_result.processing_time_ms >= 0);
        
        // Test 4: Different query patterns
        let patterns = ["short", "medium length query", "very long query with many complex technical terms and concepts"];
        for (i, pattern) in patterns.iter().enumerate() {
            let mut pattern_options = EnhancedQueryOptions::default();
            pattern_options.session_id = format!("pattern_test_{}", i);
            pattern_options.k = 5;
            pattern_options.include_metadata = true;
            
            let pattern_result = pipeline.process_query(pattern, &pattern_options).await.unwrap();
            assert!(pattern_result.processing_time_ms >= 0);
        }
    }

    // COMPREHENSIVE HIGH-COMPLEXITY COVERAGE TESTS

    #[tokio::test]
    async fn test_complex_strategy_execution_paths() {
        let doc_repo = Arc::new(MockDocumentRepositoryWithData);
        let embedding_service = Arc::new(crate::embeddings::FallbackEmbeddingService::new(384));
        
        let pipeline = PipelineFactory::create_default_pipeline(doc_repo, embedding_service);
        
        // Test MultiStep retrieval strategy complex paths
        let mut multi_step_options = EnhancedQueryOptions::default();
        multi_step_options.override_strategy = Some(RetrievalStrategy::MultiStep);
        multi_step_options.session_id = "multi_step_test".to_string();
        multi_step_options.k = 15;
        
        let multi_step_result = pipeline.process_query("complex multi step query", &multi_step_options).await.unwrap();
        assert_eq!(multi_step_result.strategy_used, RetrievalStrategy::MultiStep);
        assert!(!multi_step_result.candidates.is_empty());
        
        // Test Adaptive retrieval strategy with different score conditions
        let mut adaptive_options = EnhancedQueryOptions::default();
        adaptive_options.override_strategy = Some(RetrievalStrategy::Adaptive);
        adaptive_options.session_id = "adaptive_test".to_string();
        adaptive_options.k = 20;
        
        let adaptive_result = pipeline.process_query("adaptive strategy test", &adaptive_options).await.unwrap();
        assert_eq!(adaptive_result.strategy_used, RetrievalStrategy::Adaptive);
        
        // Test HydeEnhanced strategy fallback paths
        let mut hyde_options = EnhancedQueryOptions::default();
        hyde_options.override_strategy = Some(RetrievalStrategy::HydeEnhanced);
        hyde_options.session_id = "hyde_test".to_string();
        hyde_options.k = 10;
        
        let hyde_result = pipeline.process_query("hyde enhanced query", &hyde_options).await.unwrap();
        // Should fallback to hybrid since no LLM service is provided
        assert!(matches!(hyde_result.strategy_used, RetrievalStrategy::HydeEnhanced));
    }

    #[tokio::test]
    async fn test_pipeline_resource_management_and_cleanup() {
        let doc_repo = Arc::new(MockDocumentRepository);
        let embedding_service = Arc::new(crate::embeddings::FallbackEmbeddingService::new(384));
        
        let pipeline = PipelineFactory::create_default_pipeline(doc_repo, embedding_service);
        
        // Test memory management with large result sets
        let mut large_options = EnhancedQueryOptions::default();
        large_options.session_id = "memory_test".to_string();
        large_options.k = 1000; // Large result set
        large_options.include_metadata = true;
        
        let large_result = pipeline.process_query("memory intensive query", &large_options).await.unwrap();
        assert!(large_result.processing_time_ms >= 0);
        
        // Test cleanup after processing
        drop(large_result);
        
        // Test concurrent resource usage
        let mut concurrent_handles = Vec::new();
        for i in 0..20 {
            let pipeline_clone = Arc::new(pipeline.clone());
            let mut options = EnhancedQueryOptions::default();
            options.session_id = format!("resource_test_{}", i);
            options.k = 50;
            
            let handle = tokio::spawn(async move {
                pipeline_clone.process_query(&format!("resource query {}", i), &options).await
            });
            concurrent_handles.push(handle);
        }
        
        // Wait for all operations and verify cleanup
        let mut successful_operations = 0;
        for handle in concurrent_handles {
            if let Ok(Ok(_)) = handle.await {
                successful_operations += 1;
            }
        }
        assert!(successful_operations >= 15); // Allow some variation in mock
    }

    #[tokio::test]
    async fn test_pipeline_complex_error_recovery() {
        let doc_repo = Arc::new(MockDocumentRepository);
        let embedding_service = Arc::new(crate::embeddings::FallbackEmbeddingService::new(384));
        
        let pipeline = PipelineFactory::create_default_pipeline(doc_repo, embedding_service);
        
        // Test error recovery in different phases
        let error_queries = [
            "", // Empty query
            "\0\0\0", // Null bytes
            &"🔥".repeat(1000), // Unicode overflow
            "SELECT * FROM users; DROP TABLE users;", // SQL injection attempt
            &"\n".repeat(100), // Newline spam
            &"a".repeat(100000), // Extremely long query
        ];
        
        for (i, error_query) in error_queries.iter().enumerate() {
            let mut options = EnhancedQueryOptions::default();
            options.session_id = format!("error_recovery_{}", i);
            options.k = 5;
            options.include_metadata = true;
            
            // Should not panic or fail - robust error handling
            let result = pipeline.process_query(error_query, &options).await;
            assert!(result.is_ok(), "Failed on query: {}", error_query);
            
            let result = result.unwrap();
            assert!(result.processing_time_ms >= 0);
        }
    }

    #[tokio::test]
    async fn test_pipeline_configuration_feature_flags() {
        let doc_repo = Arc::new(MockDocumentRepository);
        let embedding_service = Arc::new(crate::embeddings::FallbackEmbeddingService::new(384));
        
        // Test pipeline with all features disabled
        let mut disabled_config = PipelineConfig::default();
        disabled_config.enable_hyde = false;
        disabled_config.enable_query_understanding = false;
        disabled_config.enable_ml_prediction = false;
        disabled_config.rerank_enabled = false;
        disabled_config.max_candidates = 5;
        
        let disabled_pipeline = PipelineFactory::create_pipeline(
            disabled_config,
            doc_repo.clone(),
            embedding_service.clone(),
            None,
            None,
        );
        
        let mut options = EnhancedQueryOptions::default();
        options.session_id = "disabled_test".to_string();
        options.k = 3;
        
        let disabled_result = disabled_pipeline.process_query("test query", &options).await.unwrap();
        assert!(disabled_result.query_understanding.is_none());
        assert!(disabled_result.ml_prediction.is_none());
        assert!(disabled_result.hyde_expansion.is_none());
        
        // Test pipeline with selective features enabled
        let mut selective_config = PipelineConfig::default();
        selective_config.enable_hyde = false;
        selective_config.enable_query_understanding = true;
        selective_config.enable_ml_prediction = false;
        selective_config.rerank_enabled = true;
        selective_config.rerank_top_k = 3;
        
        let selective_pipeline = PipelineFactory::create_pipeline(
            selective_config,
            doc_repo.clone(),
            embedding_service.clone(),
            None,
            None,
        );
        
        let selective_result = selective_pipeline.process_query("selective test", &options).await.unwrap();
        assert!(selective_result.query_understanding.is_some());
        assert!(selective_result.ml_prediction.is_none());
        assert!(selective_result.hyde_expansion.is_none());
    }

    #[tokio::test]
    async fn test_pipeline_deduplication_and_sorting_complex() {
        let doc_repo = Arc::new(MockDocumentRepositoryWithData);
        let embedding_service = Arc::new(crate::embeddings::FallbackEmbeddingService::new(384));
        
        let pipeline = PipelineFactory::create_default_pipeline(doc_repo, embedding_service);
        
        // Test deduplication logic with complex scenarios
        let mut dedup_options = EnhancedQueryOptions::default();
        dedup_options.session_id = "dedup_test".to_string();
        dedup_options.k = 10;
        dedup_options.include_metadata = true;
        
        let dedup_result = pipeline.process_query("deduplication test query", &dedup_options).await.unwrap();
        
        // Verify no duplicate doc_ids
        let mut seen_ids = std::collections::HashSet::new();
        for candidate in &dedup_result.candidates {
            assert!(seen_ids.insert(candidate.doc_id.clone()), "Duplicate doc_id found: {}", candidate.doc_id);
        }
        
        // Verify sorting by score (descending)
        for window in dedup_result.candidates.windows(2) {
            assert!(window[0].score >= window[1].score, "Candidates not sorted by score");
        }
        
        // Test with different strategies that might return different result sets
        let strategies = [
            RetrievalStrategy::BM25Only,
            RetrievalStrategy::VectorOnly,
            RetrievalStrategy::Hybrid,
            RetrievalStrategy::MultiStep,
            RetrievalStrategy::Adaptive,
        ];
        
        for strategy in &strategies {
            let mut strategy_options = EnhancedQueryOptions::default();
            strategy_options.override_strategy = Some(strategy.clone());
            strategy_options.session_id = "strategy_dedup_test".to_string();
            strategy_options.k = 15;
            
            let strategy_result = pipeline.process_query("strategy specific query", &strategy_options).await.unwrap();
            assert_eq!(strategy_result.strategy_used, *strategy);
            
            // Verify deduplication and sorting for this strategy
            let mut seen_strategy_ids = std::collections::HashSet::new();
            for candidate in &strategy_result.candidates {
                assert!(seen_strategy_ids.insert(candidate.doc_id.clone()));
            }
        }
    }

    #[tokio::test]
    async fn test_pipeline_context_pack_creation_complex() {
        let doc_repo = Arc::new(MockDocumentRepositoryWithData);
        let embedding_service = Arc::new(crate::embeddings::FallbackEmbeddingService::new(384));
        
        let pipeline = PipelineFactory::create_default_pipeline(doc_repo, embedding_service);
        
        // Test context pack creation with complex scenarios
        let mut context_options = EnhancedQueryOptions::default();
        context_options.session_id = "context_complex_test".to_string();
        context_options.k = 20;
        context_options.include_metadata = true;
        
        // Add complex context data
        let mut complex_context = HashMap::new();
        complex_context.insert("user_preferences".to_string(), serde_json::json!({
            "language": "rust",
            "experience_level": "advanced",
            "preferred_patterns": ["async", "generics", "traits"]
        }));
        complex_context.insert("search_filters".to_string(), serde_json::json!({
            "date_range": {
                "start": "2024-01-01",
                "end": "2024-12-31"
            },
            "content_types": ["code", "documentation", "examples"],
            "complexity": ["medium", "high"]
        }));
        context_options.context = Some(complex_context);
        
        let context_result = pipeline.process_query("complex context query", &context_options).await.unwrap();
        
        // Verify context pack structure
        assert!(!context_result.context_pack.id.is_empty());
        assert_eq!(context_result.context_pack.session_id, "context_complex_test");
        assert!(!context_result.context_pack.chunks.is_empty());
        
        // Verify all chunks have required fields
        for chunk in &context_result.context_pack.chunks {
            assert!(!chunk.id.is_empty());
            assert!(chunk.score > 0.0);
            assert!(!chunk.kind.is_empty());
            // Note: chunk.text can be empty with mock data, so we allow it
        }
        
        // Test context pack with empty results
        let mut empty_context_options = EnhancedQueryOptions::default();
        empty_context_options.session_id = "empty_context_test".to_string();
        empty_context_options.k = 0;
        empty_context_options.include_metadata = false;
        
        let empty_context_result = pipeline.process_query("empty context query", &empty_context_options).await.unwrap();
        assert!(!empty_context_result.context_pack.id.is_empty());
        assert_eq!(empty_context_result.context_pack.session_id, "empty_context_test");
    }

    #[tokio::test] 
    async fn test_pipeline_hyde_expansion_complex_paths() {
        let doc_repo = Arc::new(MockDocumentRepository);
        let embedding_service = Arc::new(crate::embeddings::FallbackEmbeddingService::new(384));
        
        // Test pipeline without HyDE service (fallback behavior)
        let pipeline = PipelineFactory::create_default_pipeline(doc_repo, embedding_service);
        
        // Test HyDE options with no service available
        let mut hyde_options = EnhancedQueryOptions::default();
        hyde_options.enable_hyde = Some(true);
        hyde_options.override_strategy = Some(RetrievalStrategy::HydeEnhanced);
        hyde_options.session_id = "hyde_fallback_test".to_string();
        hyde_options.k = 10;
        
        let hyde_result = pipeline.process_query("hyde test query", &hyde_options).await.unwrap();
        // Should fall back to hybrid retrieval
        assert!(hyde_result.hyde_expansion.is_none());
        assert!(!hyde_result.candidates.is_empty());
        
        // Test HyDE disabled explicitly
        let mut hyde_disabled_options = EnhancedQueryOptions::default();
        hyde_disabled_options.enable_hyde = Some(false);
        hyde_disabled_options.session_id = "hyde_disabled_test".to_string();
        hyde_disabled_options.k = 5;
        
        let hyde_disabled_result = pipeline.process_query("no hyde query", &hyde_disabled_options).await.unwrap();
        assert!(hyde_disabled_result.hyde_expansion.is_none());
    }

    #[tokio::test]
    async fn test_pipeline_reranking_complex_scenarios() {
        let doc_repo = Arc::new(MockDocumentRepositoryWithData);
        let embedding_service = Arc::new(crate::embeddings::FallbackEmbeddingService::new(384));
        
        // Test with reranking disabled
        let mut no_rerank_config = PipelineConfig::default();
        no_rerank_config.rerank_enabled = false;
        
        let no_rerank_pipeline = PipelineFactory::create_pipeline(
            no_rerank_config,
            doc_repo.clone(),
            embedding_service.clone(),
            None,
            None,
        );
        
        let mut options = EnhancedQueryOptions::default();
        options.session_id = "no_rerank_test".to_string();
        options.k = 10;
        
        let no_rerank_result = no_rerank_pipeline.process_query("reranking test", &options).await.unwrap();
        assert!(!no_rerank_result.candidates.is_empty());
        
        // Test with reranking enabled but only one candidate
        let mut single_config = PipelineConfig::default();
        single_config.rerank_enabled = true;
        single_config.rerank_top_k = 1;
        
        let single_pipeline = PipelineFactory::create_pipeline(
            single_config,
            doc_repo.clone(),
            embedding_service.clone(),
            None,
            None,
        );
        
        let mut single_options = EnhancedQueryOptions::default();
        single_options.session_id = "single_rerank_test".to_string();
        single_options.k = 1;
        
        let single_result = single_pipeline.process_query("single candidate test", &single_options).await.unwrap();
        assert!(single_result.candidates.len() <= 1);
        
        // Test reranking with different top_k values
        let rerank_configs = [1, 3, 5, 10, 20];
        for top_k in &rerank_configs {
            let mut config = PipelineConfig::default();
            config.rerank_enabled = true;
            config.rerank_top_k = *top_k;
            
            let rerank_pipeline = PipelineFactory::create_pipeline(
                config,
                doc_repo.clone(),
                embedding_service.clone(),
                None,
                None,
            );
            
            let mut rerank_options = EnhancedQueryOptions::default();
            rerank_options.session_id = format!("rerank_top_k_{}", top_k);
            rerank_options.k = 15;
            
            let rerank_result = rerank_pipeline.process_query("rerank top k test", &rerank_options).await.unwrap();
            assert!(rerank_result.processing_time_ms >= 0);
        }
    }

    #[tokio::test]
    async fn test_pipeline_ml_prediction_integration_complex() {
        let doc_repo = Arc::new(MockDocumentRepository);
        let embedding_service = Arc::new(crate::embeddings::FallbackEmbeddingService::new(384));
        
        let pipeline = PipelineFactory::create_default_pipeline(doc_repo.clone(), embedding_service.clone());
        
        // Test ML prediction with different query types
        let query_types = [
            ("technical algorithm query about neural networks", "technical"),
            ("how to implement quicksort in rust", "code"),
            ("what are the benefits of async programming", "conceptual"),
            ("debug this function: fn test() { panic!(); }", "debugging"),
            ("performance optimization for web servers", "performance"),
        ];
        
        for (query, query_type) in &query_types {
            let mut options = EnhancedQueryOptions::default();
            options.session_id = format!("ml_prediction_{}", query_type);
            options.k = 10;
            options.include_metadata = true;
            
            let result = pipeline.process_query(query, &options).await.unwrap();
            
            // Verify ML prediction results
            assert!(result.ml_prediction.is_some());
            let prediction = result.ml_prediction.unwrap();
            assert!(prediction.prediction.confidence > 0.0);
            assert!(prediction.prediction.confidence <= 1.0);
            assert!(!prediction.explanation.is_empty());
            assert!(!prediction.feature_importance.is_empty());
            
            // Verify feature importance has expected structure
            for (feature, importance) in &prediction.feature_importance {
                assert!(!feature.is_empty());
                assert!(*importance >= 0.0);
            }
        }
        
        // Test ML prediction disabled
        let mut ml_disabled_config = PipelineConfig::default();
        ml_disabled_config.enable_ml_prediction = false;
        
        let ml_disabled_pipeline = PipelineFactory::create_pipeline(
            ml_disabled_config,
            doc_repo,
            embedding_service,
            None,
            None,
        );
        
        let mut disabled_options = EnhancedQueryOptions::default();
        disabled_options.session_id = "ml_disabled_test".to_string();
        disabled_options.k = 5;
        
        let disabled_result = ml_disabled_pipeline.process_query("test query", &disabled_options).await.unwrap();
        assert!(disabled_result.ml_prediction.is_none());
    }

    #[tokio::test]
    async fn test_pipeline_timeout_and_performance_boundaries() {
        let doc_repo = Arc::new(MockDocumentRepository);
        let embedding_service = Arc::new(crate::embeddings::FallbackEmbeddingService::new(384));
        
        // Test pipeline with different timeout configurations
        let timeout_configs = [1, 5, 10, 30, 60];
        
        for timeout in &timeout_configs {
            let mut config = PipelineConfig::default();
            config.timeout_seconds = *timeout;
            config.max_candidates = 100;
            
            let timeout_pipeline = PipelineFactory::create_pipeline(
                config,
                doc_repo.clone(),
                embedding_service.clone(),
                None,
                None,
            );
            
            let mut options = EnhancedQueryOptions::default();
            options.session_id = format!("timeout_test_{}", timeout);
            options.k = 50;
            options.include_metadata = true;
            
            let start_time = std::time::Instant::now();
            let result = timeout_pipeline.process_query("timeout performance test", &options).await.unwrap();
            let elapsed = start_time.elapsed();
            
            // Verify processing completed within reasonable time
            assert!(result.processing_time_ms >= 0);
            assert!(elapsed.as_secs() < (*timeout + 5)); // Allow some buffer
            assert!(!result.candidates.is_empty());
        }
        
        // Test performance with different max_candidates configurations
        let candidate_limits = [1, 10, 50, 100, 500];
        
        for limit in &candidate_limits {
            let mut config = PipelineConfig::default();
            config.max_candidates = *limit;
            config.timeout_seconds = 30;
            
            let limit_pipeline = PipelineFactory::create_pipeline(
                config,
                doc_repo.clone(),
                embedding_service.clone(),
                None,
                None,
            );
            
            let mut options = EnhancedQueryOptions::default();
            options.session_id = format!("limit_test_{}", limit);
            options.k = (*limit).min(20); // Request reasonable number
            
            let limit_result = limit_pipeline.process_query("candidate limit test", &options).await.unwrap();
            
            // Verify results respect limits
            assert!(limit_result.candidates.len() <= *limit);
            assert!(limit_result.processing_time_ms >= 0);
        }
    }

    // ADDITIONAL COMPREHENSIVE COVERAGE FOR COMPLEX INTERNAL METHODS

    #[tokio::test]
    async fn test_pipeline_internal_method_coverage() {
        let doc_repo = Arc::new(MockDocumentRepositoryWithData);
        let embedding_service = Arc::new(crate::embeddings::FallbackEmbeddingService::new(384));
        
        let pipeline = PipelineFactory::create_default_pipeline(doc_repo, embedding_service);
        
        // Test should_use_hyde method through different scenarios
        let hyde_scenarios = [
            (Some(true), RetrievalStrategy::HydeEnhanced, "should_enable_hyde_explicit"),
            (Some(false), RetrievalStrategy::HydeEnhanced, "should_disable_hyde_explicit"),
            (None, RetrievalStrategy::HydeEnhanced, "should_enable_hyde_strategy"),
            (None, RetrievalStrategy::Hybrid, "should_disable_hyde_hybrid"),
            (None, RetrievalStrategy::BM25Only, "should_disable_hyde_bm25"),
            (None, RetrievalStrategy::VectorOnly, "should_disable_hyde_vector"),
        ];
        
        for (enable_hyde, strategy, test_name) in &hyde_scenarios {
            let mut options = EnhancedQueryOptions::default();
            options.enable_hyde = *enable_hyde;
            options.override_strategy = Some(strategy.clone());
            options.session_id = test_name.to_string();
            options.k = 5;
            
            let result = pipeline.process_query("hyde scenario test", &options).await.unwrap();
            assert_eq!(result.strategy_used, *strategy);
            
            // Verify HyDE expansion is None when service not available
            if matches!(strategy, RetrievalStrategy::HydeEnhanced) {
                // Should be None because no LLM service is provided to mock pipeline
                assert!(result.hyde_expansion.is_none());
            }
        }
    }

    #[tokio::test]
    async fn test_pipeline_deduplicate_and_sort_candidates_method() {
        let doc_repo = Arc::new(MockDocumentRepositoryWithData);
        let embedding_service = Arc::new(crate::embeddings::FallbackEmbeddingService::new(384));
        
        let pipeline = PipelineFactory::create_default_pipeline(doc_repo, embedding_service);
        
        // Test with multiple strategies to exercise deduplication logic
        let test_strategies = [
            RetrievalStrategy::MultiStep,  // Exercises multi-step path which combines results
            RetrievalStrategy::Adaptive,   // Exercises adaptive path which may switch strategies
        ];
        
        for strategy in &test_strategies {
            let mut options = EnhancedQueryOptions::default();
            options.override_strategy = Some(strategy.clone());
            options.session_id = format!("dedup_sort_test_{:?}", strategy);
            options.k = 20;
            options.include_metadata = true;
            
            let result = pipeline.process_query("deduplication and sorting test", &options).await.unwrap();
            
            // Verify no duplicate doc_ids in results
            let mut seen_ids = std::collections::HashSet::new();
            for candidate in &result.candidates {
                assert!(seen_ids.insert(candidate.doc_id.clone()), 
                    "Found duplicate doc_id: {} in strategy: {:?}", candidate.doc_id, strategy);
            }
            
            // Verify candidates are sorted by score (descending)
            for window in result.candidates.windows(2) {
                assert!(window[0].score >= window[1].score, 
                    "Candidates not properly sorted by score in strategy: {:?}", strategy);
            }
        }
    }

    #[tokio::test]
    async fn test_pipeline_multi_step_retrieval_edge_cases() {
        let doc_repo = Arc::new(MockDocumentRepositoryWithData);
        let embedding_service = Arc::new(crate::embeddings::FallbackEmbeddingService::new(384));
        
        let pipeline = PipelineFactory::create_default_pipeline(doc_repo, embedding_service);
        
        // Test multi-step with conditions that trigger different branches
        let mut multi_step_options = EnhancedQueryOptions::default();
        multi_step_options.override_strategy = Some(RetrievalStrategy::MultiStep);
        multi_step_options.session_id = "multi_step_edge_test".to_string();
        multi_step_options.k = 50; // Large k to potentially get many results
        
        let multi_step_result = pipeline.process_query("comprehensive multi-step test", &multi_step_options).await.unwrap();
        assert_eq!(multi_step_result.strategy_used, RetrievalStrategy::MultiStep);
        
        // The multi-step logic should handle both paths:
        // 1. If few results (<5), try vector-only search
        // 2. If many results (>=5), take top candidates from initial search
        assert!(multi_step_result.processing_time_ms >= 0);
        assert!(!multi_step_result.candidates.is_empty());
    }

    #[tokio::test]
    async fn test_pipeline_adaptive_retrieval_score_conditions() {
        let doc_repo = Arc::new(MockDocumentRepositoryWithData);
        let embedding_service = Arc::new(crate::embeddings::FallbackEmbeddingService::new(384));
        
        let pipeline = PipelineFactory::create_default_pipeline(doc_repo, embedding_service);
        
        // Test adaptive retrieval which has complex branching logic based on result quality
        let mut adaptive_options = EnhancedQueryOptions::default();
        adaptive_options.override_strategy = Some(RetrievalStrategy::Adaptive);
        adaptive_options.session_id = "adaptive_edge_test".to_string();
        adaptive_options.k = 25;
        
        let adaptive_result = pipeline.process_query("adaptive retrieval edge case test", &adaptive_options).await.unwrap();
        assert_eq!(adaptive_result.strategy_used, RetrievalStrategy::Adaptive);
        
        // The adaptive logic should handle multiple paths:
        // 1. If <5 results: try vector-only
        // 2. If all scores <0.5: try BM25-only
        // 3. Otherwise: use hybrid results
        assert!(adaptive_result.processing_time_ms >= 0);
    }

    #[tokio::test]
    async fn test_pipeline_result_limiting_and_truncation() {
        let doc_repo = Arc::new(MockDocumentRepositoryWithData);
        let embedding_service = Arc::new(crate::embeddings::FallbackEmbeddingService::new(384));
        
        let pipeline = PipelineFactory::create_default_pipeline(doc_repo, embedding_service);
        
        // Test result limiting with various k values
        let k_values = [0, 1, 2, 5, 10, 100, 1000];
        
        for k in &k_values {
            let mut options = EnhancedQueryOptions::default();
            options.session_id = format!("result_limiting_test_{}", k);
            options.k = *k;
            options.include_metadata = true;
            
            let result = pipeline.process_query("result limiting test", &options).await.unwrap();
            
            // Verify result count respects k limit
            assert!(result.candidates.len() <= *k, "Result count {} exceeds k limit {}", result.candidates.len(), k);
            
            // Even with k=0, should have valid context pack
            assert!(!result.context_pack.id.is_empty());
            assert_eq!(result.context_pack.session_id, format!("result_limiting_test_{}", k));
        }
    }

    #[tokio::test]
    async fn test_pipeline_context_creation_metadata_handling() {
        let doc_repo = Arc::new(MockDocumentRepositoryWithData);
        let embedding_service = Arc::new(crate::embeddings::FallbackEmbeddingService::new(384));
        
        let pipeline = PipelineFactory::create_default_pipeline(doc_repo, embedding_service);
        
        // Test context creation with metadata enabled/disabled
        let metadata_scenarios = [
            (true, "metadata_enabled"),
            (false, "metadata_disabled"),
        ];
        
        for (include_metadata, test_name) in &metadata_scenarios {
            let mut options = EnhancedQueryOptions::default();
            options.session_id = test_name.to_string();
            options.k = 10;
            options.include_metadata = *include_metadata;
            
            let result = pipeline.process_query("metadata handling test", &options).await.unwrap();
            
            // Verify context pack is always created regardless of metadata setting
            assert!(!result.context_pack.id.is_empty());
            assert_eq!(result.context_pack.session_id, *test_name);
            assert!(result.context_pack.created_at <= chrono::Utc::now());
            
            // Verify chunks are properly converted from candidates
            for chunk in &result.context_pack.chunks {
                assert!(!chunk.id.is_empty());
                assert!(chunk.score >= 0.0);
                assert!(!chunk.kind.is_empty());
            }
        }
    }

    #[tokio::test]
    async fn test_pipeline_reranking_boundary_conditions() {
        let doc_repo = Arc::new(MockDocumentRepositoryWithData);
        let embedding_service = Arc::new(crate::embeddings::FallbackEmbeddingService::new(384));
        
        // Test reranking boundary conditions
        let rerank_scenarios = [
            (true, 0, "rerank_enabled_zero_candidates"),   // Edge case: 0 candidates
            (true, 1, "rerank_enabled_one_candidate"),     // Edge case: 1 candidate  
            (false, 10, "rerank_disabled_many_candidates"), // Reranking disabled
        ];
        
        for (rerank_enabled, rerank_top_k, test_name) in &rerank_scenarios {
            let mut config = PipelineConfig::default();
            config.rerank_enabled = *rerank_enabled;
            config.rerank_top_k = *rerank_top_k;
            
            let rerank_pipeline = PipelineFactory::create_pipeline(
                config,
                doc_repo.clone(),
                embedding_service.clone(),
                None,
                None,
            );
            
            let mut options = EnhancedQueryOptions::default();
            options.session_id = test_name.to_string();
            options.k = 15;
            
            let result = rerank_pipeline.process_query("reranking boundary test", &options).await.unwrap();
            
            // Reranking should not cause errors even in edge cases
            assert!(result.processing_time_ms >= 0);
            assert!(result.candidates.len() >= 0);
            
            // Verify candidates maintain score ordering
            for window in result.candidates.windows(2) {
                assert!(window[0].score >= window[1].score);
            }
        }
    }

    #[tokio::test]
    async fn test_pipeline_configuration_validation_and_defaults() {
        let doc_repo = Arc::new(MockDocumentRepository);
        let embedding_service = Arc::new(crate::embeddings::FallbackEmbeddingService::new(384));
        
        // Test pipeline with extreme configuration values
        let mut extreme_config = PipelineConfig::default();
        extreme_config.max_candidates = 0;  // Edge case: zero candidates
        extreme_config.rerank_top_k = 0;    // Edge case: zero rerank
        extreme_config.timeout_seconds = 0; // Edge case: zero timeout
        
        let extreme_pipeline = PipelineFactory::create_pipeline(
            extreme_config,
            doc_repo.clone(),
            embedding_service.clone(),
            None,
            None,
        );
        
        let mut options = EnhancedQueryOptions::default();
        options.session_id = "extreme_config_test".to_string();
        options.k = 5;
        
        // Should handle extreme configuration gracefully
        let result = extreme_pipeline.process_query("extreme config test", &options).await.unwrap();
        assert!(result.processing_time_ms >= 0);
        
        // Test with very large configuration values
        let mut large_config = PipelineConfig::default();
        large_config.max_candidates = usize::MAX;
        large_config.rerank_top_k = usize::MAX;
        large_config.timeout_seconds = u64::MAX;
        
        let large_pipeline = PipelineFactory::create_pipeline(
            large_config,
            doc_repo.clone(),
            embedding_service.clone(),
            None,
            None,
        );
        
        let large_result = large_pipeline.process_query("large config test", &options).await.unwrap();
        assert!(large_result.processing_time_ms >= 0);
    }

    #[tokio::test]
    async fn test_pipeline_factory_comprehensive_scenarios() {
        let doc_repo = Arc::new(MockDocumentRepository);
        let embedding_service = Arc::new(crate::embeddings::FallbackEmbeddingService::new(384));
        
        // Test factory with different service combinations
        let factory_scenarios = [
            ("no_optional_services", None, None),
            ("with_llm_only", Some(Arc::new(MockLlmService) as Arc<dyn LlmService>), None),
            ("with_reranker_only", None, Some(Arc::new(MockRerankingService) as Arc<dyn RerankingService>)),
        ];
        
        for (test_name, llm_service, reranker_service) in factory_scenarios {
            let config = PipelineConfig {
                enable_hyde: llm_service.is_some(),
                enable_query_understanding: true,
                enable_ml_prediction: true,
                max_candidates: 20,
                rerank_enabled: reranker_service.is_some(),
                rerank_top_k: 10,
                timeout_seconds: 30,
            };
            
            let factory_pipeline = PipelineFactory::create_pipeline(
                config,
                doc_repo.clone(),
                embedding_service.clone(),
                llm_service,
                reranker_service,
            );
            
            let mut options = EnhancedQueryOptions::default();
            options.session_id = test_name.to_string();
            options.k = 10;
            
            let result = factory_pipeline.process_query("factory scenario test", &options).await.unwrap();
            assert!(result.processing_time_ms >= 0);
            assert!(!result.candidates.is_empty());
            
            // Verify services are properly configured
            if test_name == "with_llm_only" {
                // HyDE should be available but may not be used depending on strategy
                assert!(result.processing_time_ms >= 0);
            }
            
            if test_name == "with_reranker_only" {
                // Reranking should be enabled
                assert!(result.processing_time_ms >= 0);
            }
        }
    }

    #[tokio::test]
    async fn test_hyde_enhanced_fallback_scenarios() {
        let doc_repo = Arc::new(MockDocumentRepositoryWithData);
        let embedding_service = Arc::new(crate::embeddings::FallbackEmbeddingService::new(384));
        
        // Test HyDE without LLM service (should fall back)
        let pipeline = PipelineFactory::create_default_pipeline(
            doc_repo.clone(),
            embedding_service.clone(),
        );
        
        let mut options = EnhancedQueryOptions::default();
        options.session_id = "hyde_test".to_string();
        options.k = 5;
        options.override_strategy = Some(RetrievalStrategy::HydeEnhanced);
        
        // This should fall back to hybrid retrieval since no LLM service is provided
        let result = pipeline.process_query("test query for hyde fallback", &options).await.unwrap();
        assert!(result.processing_time_ms >= 0);
        assert!(!result.candidates.is_empty());
    }

    #[tokio::test]
    async fn test_reranking_edge_cases() {
        let doc_repo = Arc::new(MockDocumentRepositoryWithData);
        let embedding_service = Arc::new(crate::embeddings::FallbackEmbeddingService::new(384));
        
        // Test reranking with only one candidate (should skip reranking)
        let pipeline = PipelineFactory::create_default_pipeline(
            doc_repo.clone(),
            embedding_service.clone(),
        );
        
        let mut options = EnhancedQueryOptions::default();
        options.session_id = "single_candidate".to_string();
        options.k = 1; // Only one result
        
        let result = pipeline.process_query("single result test", &options).await.unwrap();
        assert!(result.processing_time_ms >= 0);
        // Should have results without reranking since only 1 candidate
        assert!(!result.candidates.is_empty());
    }

    // Mock services for comprehensive testing
    
    struct MockLlmService;
    
    #[async_trait]
    impl LlmService for MockLlmService {
        async fn generate_text(&self, _prompt: &str, _config: &crate::hyde::HydeConfig) -> Result<Vec<String>> {
            Ok(vec![
                "Mock hypothetical document 1 for testing".to_string(),
                "Mock hypothetical document 2 for testing".to_string(),
                "Mock hypothetical document 3 for testing".to_string(),
            ])
        }
    }
    
    struct MockRerankingService;
    
    #[async_trait]
    impl RerankingService for MockRerankingService {
        async fn rerank(&self, _query: &str, candidates: &[Candidate]) -> Result<Vec<Candidate>> {
            let mut reranked = candidates.to_vec();
            // Simple mock reranking: reverse order to show effect
            reranked.reverse();
            Ok(reranked)
        }
    }
}