rag-module 0.6.7

use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};

fn main() {
    println!("🧪 Demonstrating Content-Based Deduplication Logic");
    println!("==================================================");

    // Simulate the content hashing logic we implemented
    let content1 = "This is an EC2 instance with standard configuration";
    let content2 = "This is an EC2 instance with standard configuration"; // Same content
    let content3 = "This is a completely different RDS database instance"; // Different content

    // Generate hashes like our implementation does
    let hash1 = generate_content_hash(content1);
    let hash2 = generate_content_hash(content2);
    let hash3 = generate_content_hash(content3);

    let doc_id1 = format!("aws_estate-{:x}", hash1);
    let doc_id2 = format!("aws_estate-{:x}", hash2);
    let doc_id3 = format!("aws_estate-{:x}", hash3);

    println!("\n📄 Content 1: \"{}\"", content1);
    println!("🆔 Document ID 1: {}", doc_id1);
    
    println!("\n📄 Content 2: \"{}\"", content2);
    println!("🆔 Document ID 2: {}", doc_id2);
    
    println!("\n📄 Content 3: \"{}\"", content3);
    println!("🆔 Document ID 3: {}", doc_id3);

    println!("\n✅ **Deduplication Results:**");
    
    if doc_id1 == doc_id2 {
        println!("✅ Same content produces SAME document ID → Will be deduplicated!");
        println!("   Content 1 and 2 will share the same document (upsert behavior)");
    } else {
        println!("❌ Same content produces DIFFERENT document IDs → Would create duplicates!");
    }

    if doc_id1 != doc_id3 {
        println!("✅ Different content produces DIFFERENT document ID → Will create new document!");
        println!("   Content 1 and 3 will be stored as separate documents");
    } else {
        println!("❌ Different content produces SAME document ID → Would cause collisions!");
    }

    println!("\n📊 **Hash Distribution:**");
    println!("Hash 1: {:016x}", hash1);
    println!("Hash 2: {:016x}", hash2);  
    println!("Hash 3: {:016x}", hash3);
    
    println!("\n🎉 **How This Solves Your Problem:**");
    println!("1. 🔄 Same content → Same hash → Same doc ID → Upsert (delete old + add new)");
    println!("2. 📝 Same content + different metadata → Same hash → Updates existing document");
    println!("3. 🆕 Different content → Different hash → Different doc ID → Creates new document");
    println!("4. 📈 Scales to 2M+ documents with O(1) hash lookup");
}

fn generate_content_hash(content: &str) -> u64 {
    let mut hasher = DefaultHasher::new();
    content.hash(&mut hasher);
    hasher.finish()
}