lethe_core_rust/

lib.rs

//! # lethe-core-rust
//! 
//! A high-performance hybrid retrieval engine that combines BM25 lexical search with vector similarity 
//! using z-score fusion. Lethe Core provides state-of-the-art context selection for conversational AI 
//! and retrieval-augmented generation (RAG) systems.
//!
//! ## Features
//!
//! - **Hybrid Retrieval**: Combines BM25 lexical search with vector similarity for optimal relevance
//! - **Z-Score Fusion**: Normalizes and fuses scores using statistical z-score transformation (α=0.5, β=0.5)  
//! - **Hero Configuration**: Pre-tuned parameters achieving parity with splade baseline performance
//! - **Gamma Boosting**: Context-aware score boosting for code, errors, and technical content
//! - **Chunking Pipeline**: Intelligent text segmentation with sentence-level granularity
//! - **Async-First**: Built on Tokio for high-performance concurrent operations
//!
//! ## Quick Start
//!
//! ```rust
//! use lethe_core_rust::{get_hero_config, apply_zscore_fusion, Candidate};
//!
//! # #[tokio::main]
//! # async fn main() -> Result<(), Box<dyn std::error::Error>> {
//! // Get the hero configuration (optimal for splade parity)
//! let config = get_hero_config();
//! println!("Hero config: α={}, β={}", config.alpha, config.beta);
//! 
//! // Example candidates from BM25 and vector search
//! let bm25_candidates = vec![
//!     Candidate {
//!         doc_id: "doc1".to_string(),
//!         score: 0.8,
//!         text: Some("Rust async programming".to_string()),
//!         kind: Some("bm25".to_string()),
//!     },
//! ];
//! 
//! let vector_candidates = vec![
//!     Candidate {
//!         doc_id: "doc1".to_string(),
//!         score: 0.9,
//!         text: Some("Rust async programming".to_string()),
//!         kind: Some("vector".to_string()),
//!     },
//! ];
//! 
//! // Apply z-score fusion with hero configuration (α=0.5)
//! let results = apply_zscore_fusion(bm25_candidates, vector_candidates, 0.5);
//! println!("Fused {} candidates", results.len());
//! # Ok(())
//! # }
//! ```

// Re-export all shared types and utilities
pub mod types;
pub mod error;
pub mod config;
pub mod utils;

// Domain services and business logic
pub mod chunker;
pub mod retrieval;
pub mod embeddings;
pub mod hyde;
pub mod query_understanding;
pub mod ml_prediction;
pub mod pipeline;

// Infrastructure services (optional with database feature)
#[cfg(feature = "database")]
pub mod database;
#[cfg(feature = "database")]
pub mod repositories;

// Re-export everything for convenience
pub use types::*;
pub use error::*;
pub use config::*;
pub use utils::*;

pub use chunker::*;
pub use retrieval::*;
pub use embeddings::*;
pub use hyde::*;
pub use query_understanding::*;
pub use ml_prediction::*;
pub use pipeline::*;

#[cfg(feature = "database")]
pub use database::*;
#[cfg(feature = "database")]
pub use repositories::*;

/// Get a hero configuration for testing and benchmarks
/// 
/// The hero configuration provides optimal parameters validated against splade baseline performance:
/// - α = 0.5, β = 0.5: Equal weighting of lexical and semantic signals
/// - k_initial = 200: Large candidate pool for comprehensive coverage  
/// - k_final = 5: Focused results optimized for Recall@5 metrics
/// - Diversification = "splade": Advanced diversification matching baseline method
/// - Gamma boosting disabled: Clean z-score fusion without latent multipliers
/// 
/// # Examples
///
/// ```rust
/// use lethe_core_rust::get_hero_config;
/// 
/// let config = get_hero_config();
/// assert_eq!(config.alpha, 0.5);
/// assert_eq!(config.beta, 0.5);
/// assert_eq!(config.k_final, 5);
/// ```
pub fn get_hero_config() -> HybridRetrievalConfig {
    HybridRetrievalConfig::hero()
}

/// Get a validated hero configuration against canonical hash
/// 
/// This function creates a hero configuration and validates it against the expected
/// hash from the canonical audit manifest. It will refuse to run unless the 
/// configuration matches exactly, providing integrity verification.
/// 
/// # Arguments
/// 
/// * `expected_hash` - The expected SHA-256 hash of the canonical hero configuration
/// * `allow_override` - Whether to continue with mismatched config (logs warning)
/// 
/// # Returns
/// 
/// A validated `HybridRetrievalConfig` or an error if validation fails
/// 
/// # Examples
/// 
/// ```rust
/// use lethe_core_rust::{get_hero_config_validated, LetheError};
/// 
/// // With canonical hash (this would be from your manifest)
/// let canonical_hash = "8dd2de7e89ed4af1aede4cc89d8c9d8435d03a340d918f45e640b4b84959a80f";
/// 
/// // Strict validation - fails on mismatch
/// match get_hero_config_validated(canonical_hash, false) {
///     Ok(config) => {
///         // Use validated config
///         assert_eq!(config.alpha, 0.5);
///     },
///     Err(e) => {
///         // Handle configuration validation failure
///         eprintln!("Hero config validation failed: {}", e);
///     }
/// }
/// 
/// // Permissive validation - warns on mismatch but continues
/// let config = get_hero_config_validated(canonical_hash, true)
///     .expect("Should not fail with override");
/// ```
pub fn get_hero_config_validated(expected_hash: &str, allow_override: bool) -> Result<HybridRetrievalConfig> {
    HybridRetrievalConfig::hero_with_validation(expected_hash, allow_override)
}

/// Process candidates using z-score fusion
/// 
/// This function demonstrates the core z-score fusion algorithm by:
/// 1. Converting raw scores to z-scores (mean=0, std=1)
/// 2. Combining using weighted fusion: `α * z_bm25 + β * z_vector`
/// 3. Returning unified candidates sorted by hybrid score
///
/// # Arguments
///
/// * `bm25_candidates` - Candidates from BM25 lexical search
/// * `vector_candidates` - Candidates from vector semantic search
/// * `alpha` - Weight for BM25 z-scores (typically 0.5 for equal weighting)
///
/// # Returns
///
/// Vector of candidates with hybrid scores, sorted descending by relevance
///
/// # Examples
///
/// ```rust
/// use lethe_core_rust::{apply_zscore_fusion, Candidate};
/// 
/// let bm25_candidates = vec![
///     Candidate {
///         doc_id: "doc1".to_string(),
///         score: 0.8,
///         text: Some("Programming tutorial".to_string()),
///         kind: Some("bm25".to_string()),
///     },
/// ];
/// 
/// let vector_candidates = vec![
///     Candidate {
///         doc_id: "doc1".to_string(),
///         score: 0.9,
///         text: Some("Programming tutorial".to_string()),
///         kind: Some("vector".to_string()),
///     },
/// ];
/// 
/// let results = apply_zscore_fusion(bm25_candidates, vector_candidates, 0.5);
/// assert!(!results.is_empty());
/// assert_eq!(results[0].kind, Some("hybrid".to_string()));
/// ```
pub fn apply_zscore_fusion(
    bm25_candidates: Vec<Candidate>, 
    vector_candidates: Vec<Candidate>, 
    alpha: f64
) -> Vec<Candidate> {
    let service = HybridRetrievalService::mock_for_testing();
    
    // Calculate z-scores for each set
    let zscore_bm25 = service.calculate_zscores(&bm25_candidates);
    let zscore_vector = service.calculate_zscores(&vector_candidates);
    
    // Combine with weighted fusion
    let mut combined = Vec::new();
    let mut doc_scores = std::collections::HashMap::new();
    
    for candidate in zscore_bm25 {
        doc_scores.insert(candidate.doc_id.clone(), alpha * candidate.score);
    }
    
    for candidate in zscore_vector {
        let entry = doc_scores.entry(candidate.doc_id.clone()).or_insert(0.0);
        *entry += (1.0 - alpha) * candidate.score;
    }
    
    // Convert back to candidates
    for (doc_id, score) in doc_scores {
        combined.push(Candidate {
            doc_id,
            score,
            text: None,
            kind: Some("hybrid".to_string()),
        });
    }
    
    // Sort by score descending
    combined.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap());
    combined
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_hero_config() {
        let config = get_hero_config();
        
        // Verify hero configuration parameters
        assert_eq!(config.alpha, 0.5);
        assert_eq!(config.beta, 0.5);
        assert_eq!(config.k_final, 5);
        assert_eq!(config.k_initial, 200);
        assert_eq!(config.diversify_method, "splade");
        
        // Hero config has gamma boosting disabled by default (no latent multipliers)
        assert!(config.gamma_kind_boost.is_empty(), "Gamma boosting should be disabled by default");
    }

    #[test]
    fn test_zscore_fusion() {
        // Create test candidates
        let bm25_candidates = vec![
            Candidate {
                doc_id: "doc1".to_string(),
                score: 0.8,
                text: Some("Rust programming".to_string()),
                kind: Some("bm25".to_string()),
            },
            Candidate {
                doc_id: "doc2".to_string(),
                score: 0.6,
                text: Some("Python data".to_string()), 
                kind: Some("bm25".to_string()),
            },
        ];
        
        let vector_candidates = vec![
            Candidate {
                doc_id: "doc1".to_string(),
                score: 0.9,
                text: Some("Rust programming".to_string()),
                kind: Some("vector".to_string()),
            },
            Candidate {
                doc_id: "doc3".to_string(),
                score: 0.7,
                text: Some("Machine learning".to_string()),
                kind: Some("vector".to_string()),
            },
        ];
        
        // Test z-score fusion with alpha=0.5 (hero config)
        let results = apply_zscore_fusion(bm25_candidates, vector_candidates, 0.5);
        
        // Should have 3 unique documents (doc1, doc2, doc3)
        assert_eq!(results.len(), 3);
        
        // Results should be sorted by score descending
        for i in 1..results.len() {
            assert!(results[i-1].score >= results[i].score);
        }
        
        // All results should be marked as hybrid
        for result in &results {
            assert_eq!(result.kind, Some("hybrid".to_string()));
        }
    }

    #[test]
    fn test_library_integration() {
        // Test that all main components can be imported and used together
        let _config = get_hero_config();
        let _chunking_config = ChunkingConfig::default();
        let _service = HybridRetrievalService::mock_for_testing();
        
        // Test that error types are available
        let _error = LetheError::config("test");
        
        // Test that all types are available
        let _candidate = Candidate {
            doc_id: "test".to_string(),
            score: 1.0,
            text: None,
            kind: None,
        };
    }

    #[test]
    fn test_hero_config_hash_computation() {
        let config = get_hero_config();
        let hash = config.compute_hash();
        
        // Print the hash so we can see it in test output
        println!("Hero configuration hash: {}", hash);
        
        // Print the JSON for verification
        let json = serde_json::to_string_pretty(&config).expect("Failed to serialize");
        println!("Configuration JSON:\n{}", json);
        
        // The hash should be consistent for the same configuration
        let hash2 = config.compute_hash();
        assert_eq!(hash, hash2);
        
        // Hash should be a valid hex string of correct length (SHA-256 = 64 chars)
        assert_eq!(hash.len(), 64);
        assert!(hash.chars().all(|c| c.is_ascii_hexdigit()));
        
        // Verify it matches the expected canonical hash
        assert_eq!(hash, "91ad48c46bfb83257f69b329bf4153a7862765840fbbcfb6fb28ed2408ffe759");
    }

    #[test]
    fn test_hero_config_validation_success() {
        let canonical_hash = "91ad48c46bfb83257f69b329bf4153a7862765840fbbcfb6fb28ed2408ffe759";
        
        // Test strict validation with correct hash - should succeed
        let config = get_hero_config_validated(canonical_hash, false).unwrap();
        assert_eq!(config.alpha, 0.5);
        assert_eq!(config.beta, 0.5);
        assert_eq!(config.k_final, 5);
        
        // Test permissive validation with correct hash - should also succeed
        let config2 = get_hero_config_validated(canonical_hash, true).unwrap();
        assert_eq!(config2.alpha, 0.5);
    }

    #[test]
    fn test_hero_config_validation_failure() {
        let wrong_hash = "0000000000000000000000000000000000000000000000000000000000000000";
        
        // Test strict validation with wrong hash - should fail
        let result = get_hero_config_validated(wrong_hash, false);
        assert!(result.is_err());
        
        // Test permissive validation with wrong hash - should succeed but warn
        let config = get_hero_config_validated(wrong_hash, true).unwrap();
        assert_eq!(config.alpha, 0.5); // Config should still be valid
    }
}