lethe_core_rust/

lib.rs

//! # lethe-core-rust
//! 
//! A high-performance hybrid retrieval engine that combines BM25 lexical search with vector similarity 
//! using z-score fusion. Lethe Core provides state-of-the-art context selection for conversational AI 
//! and retrieval-augmented generation (RAG) systems.
//!
//! ## Features
//!
//! - **Hybrid Retrieval**: Combines BM25 lexical search with vector similarity for optimal relevance
//! - **Z-Score Fusion**: Normalizes and fuses scores using statistical z-score transformation (α=0.5, β=0.5)  
//! - **Hero Configuration**: Pre-tuned parameters achieving parity with splade baseline performance
//! - **Gamma Boosting**: Context-aware score boosting for code, errors, and technical content
//! - **Chunking Pipeline**: Intelligent text segmentation with sentence-level granularity
//! - **Async-First**: Built on Tokio for high-performance concurrent operations
//!
//! ## Quick Start
//!
//! ```rust
//! use lethe_core_rust::{get_hero_config, apply_zscore_fusion, Candidate};
//!
//! # #[tokio::main]
//! # async fn main() -> Result<(), Box<dyn std::error::Error>> {
//! // Get the hero configuration (optimal for splade parity)
//! let config = get_hero_config();
//! println!("Hero config: α={}, β={}", config.alpha, config.beta);
//! 
//! // Example candidates from BM25 and vector search
//! let bm25_candidates = vec![
//!     Candidate {
//!         doc_id: "doc1".to_string(),
//!         score: 0.8,
//!         text: Some("Rust async programming".to_string()),
//!         kind: Some("bm25".to_string()),
//!     },
//! ];
//! 
//! let vector_candidates = vec![
//!     Candidate {
//!         doc_id: "doc1".to_string(),
//!         score: 0.9,
//!         text: Some("Rust async programming".to_string()),
//!         kind: Some("vector".to_string()),
//!     },
//! ];
//! 
//! // Apply z-score fusion with hero configuration (α=0.5)
//! let results = apply_zscore_fusion(bm25_candidates, vector_candidates, 0.5);
//! println!("Fused {} candidates", results.len());
//! # Ok(())
//! # }
//! ```

// Re-export all shared types and utilities
pub mod types;
pub mod error;
pub mod config;
pub mod utils;

// Domain services and business logic
pub mod chunker;
pub mod retrieval;
pub mod embeddings;
pub mod hyde;
pub mod query_understanding;
pub mod ml_prediction;
pub mod pipeline;

// Infrastructure services (optional with database feature)
#[cfg(feature = "database")]
pub mod database;
#[cfg(feature = "database")]
pub mod repositories;

// Re-export everything for convenience
pub use types::*;
pub use error::*;
pub use config::*;
pub use utils::*;

pub use chunker::*;
pub use retrieval::*;
pub use embeddings::*;
pub use hyde::*;
pub use query_understanding::*;
pub use ml_prediction::*;
pub use pipeline::*;

#[cfg(feature = "database")]
pub use database::*;
#[cfg(feature = "database")]
pub use repositories::*;

/// Get a hero configuration for testing and benchmarks
/// 
/// The hero configuration provides optimal parameters validated against splade baseline performance:
/// - α = 0.5, β = 0.5: Equal weighting of lexical and semantic signals
/// - k_initial = 200: Large candidate pool for comprehensive coverage  
/// - k_final = 5: Focused results optimized for Recall@5 metrics
/// - Diversification = "splade": Advanced diversification matching baseline method
/// - Gamma boosting disabled: Clean z-score fusion without latent multipliers
/// 
/// # Examples
///
/// ```rust
/// use lethe_core_rust::get_hero_config;
/// 
/// let config = get_hero_config();
/// assert_eq!(config.alpha, 0.5);
/// assert_eq!(config.beta, 0.5);
/// assert_eq!(config.k_final, 5);
/// ```
pub fn get_hero_config() -> HybridRetrievalConfig {
    HybridRetrievalConfig::hero()
}

/// Process candidates using z-score fusion
/// 
/// This function demonstrates the core z-score fusion algorithm by:
/// 1. Converting raw scores to z-scores (mean=0, std=1)
/// 2. Combining using weighted fusion: `α * z_bm25 + β * z_vector`
/// 3. Returning unified candidates sorted by hybrid score
///
/// # Arguments
///
/// * `bm25_candidates` - Candidates from BM25 lexical search
/// * `vector_candidates` - Candidates from vector semantic search
/// * `alpha` - Weight for BM25 z-scores (typically 0.5 for equal weighting)
///
/// # Returns
///
/// Vector of candidates with hybrid scores, sorted descending by relevance
///
/// # Examples
///
/// ```rust
/// use lethe_core_rust::{apply_zscore_fusion, Candidate};
/// 
/// let bm25_candidates = vec![
///     Candidate {
///         doc_id: "doc1".to_string(),
///         score: 0.8,
///         text: Some("Programming tutorial".to_string()),
///         kind: Some("bm25".to_string()),
///     },
/// ];
/// 
/// let vector_candidates = vec![
///     Candidate {
///         doc_id: "doc1".to_string(),
///         score: 0.9,
///         text: Some("Programming tutorial".to_string()),
///         kind: Some("vector".to_string()),
///     },
/// ];
/// 
/// let results = apply_zscore_fusion(bm25_candidates, vector_candidates, 0.5);
/// assert!(!results.is_empty());
/// assert_eq!(results[0].kind, Some("hybrid".to_string()));
/// ```
pub fn apply_zscore_fusion(
    bm25_candidates: Vec<Candidate>, 
    vector_candidates: Vec<Candidate>, 
    alpha: f64
) -> Vec<Candidate> {
    let service = HybridRetrievalService::mock_for_testing();
    
    // Calculate z-scores for each set
    let zscore_bm25 = service.calculate_zscores(&bm25_candidates);
    let zscore_vector = service.calculate_zscores(&vector_candidates);
    
    // Combine with weighted fusion
    let mut combined = Vec::new();
    let mut doc_scores = std::collections::HashMap::new();
    
    for candidate in zscore_bm25 {
        doc_scores.insert(candidate.doc_id.clone(), alpha * candidate.score);
    }
    
    for candidate in zscore_vector {
        let entry = doc_scores.entry(candidate.doc_id.clone()).or_insert(0.0);
        *entry += (1.0 - alpha) * candidate.score;
    }
    
    // Convert back to candidates
    for (doc_id, score) in doc_scores {
        combined.push(Candidate {
            doc_id,
            score,
            text: None,
            kind: Some("hybrid".to_string()),
        });
    }
    
    // Sort by score descending
    combined.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap());
    combined
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_hero_config() {
        let config = get_hero_config();
        
        // Verify hero configuration parameters
        assert_eq!(config.alpha, 0.5);
        assert_eq!(config.beta, 0.5);
        assert_eq!(config.k_final, 5);
        assert_eq!(config.k_initial, 200);
        assert_eq!(config.diversify_method, "splade");
        
        // Hero config has gamma boosting disabled by default (no latent multipliers)
        assert!(config.gamma_kind_boost.is_empty(), "Gamma boosting should be disabled by default");
    }

    #[test]
    fn test_zscore_fusion() {
        // Create test candidates
        let bm25_candidates = vec![
            Candidate {
                doc_id: "doc1".to_string(),
                score: 0.8,
                text: Some("Rust programming".to_string()),
                kind: Some("bm25".to_string()),
            },
            Candidate {
                doc_id: "doc2".to_string(),
                score: 0.6,
                text: Some("Python data".to_string()), 
                kind: Some("bm25".to_string()),
            },
        ];
        
        let vector_candidates = vec![
            Candidate {
                doc_id: "doc1".to_string(),
                score: 0.9,
                text: Some("Rust programming".to_string()),
                kind: Some("vector".to_string()),
            },
            Candidate {
                doc_id: "doc3".to_string(),
                score: 0.7,
                text: Some("Machine learning".to_string()),
                kind: Some("vector".to_string()),
            },
        ];
        
        // Test z-score fusion with alpha=0.5 (hero config)
        let results = apply_zscore_fusion(bm25_candidates, vector_candidates, 0.5);
        
        // Should have 3 unique documents (doc1, doc2, doc3)
        assert_eq!(results.len(), 3);
        
        // Results should be sorted by score descending
        for i in 1..results.len() {
            assert!(results[i-1].score >= results[i].score);
        }
        
        // All results should be marked as hybrid
        for result in &results {
            assert_eq!(result.kind, Some("hybrid".to_string()));
        }
    }

    #[test]
    fn test_library_integration() {
        // Test that all main components can be imported and used together
        let _config = get_hero_config();
        let _chunking_config = ChunkingConfig::default();
        let _service = HybridRetrievalService::mock_for_testing();
        
        // Test that error types are available
        let _error = LetheError::InvalidQuery("test".to_string());
        
        // Test that all types are available
        let _candidate = Candidate {
            doc_id: "test".to_string(),
            score: 1.0,
            text: None,
            kind: None,
        };
    }
}