uri-register 0.3.0

// Copyright TELICENT LTD
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

/// Example: Simple web crawler using URI Register for deduplication
///
/// This example demonstrates using uri-register to assign unique IDs to discovered URLs
/// in a web crawler, preventing duplicate processing.
///
/// Run with: cargo run --example web_crawler
///
/// Prerequisites:
/// - PostgreSQL running locally
/// - Database initialized with schema.sql
/// - DATABASE_URL environment variable set (or use default: postgres://localhost/access)
use uri_register::{PostgresUriRegister, UriService};

#[tokio::main]
async fn main() -> uri_register::Result<()> {
    println!("🕷️  Web Crawler URI Deduplication Example\n");

    // Connect to PostgreSQL
    let db_url =
        std::env::var("DATABASE_URL").unwrap_or_else(|_| "postgres://localhost/access".to_string());

    let register = PostgresUriRegister::new(&db_url, "uri_register", 20, 10_000).await?;
    println!("✓ Connected to database\n");

    // Simulate crawler discovering URLs from multiple sources
    println!("📥 Crawler discovered URLs from Page 1:");
    let page1_urls = vec![
        "https://example.com/page1".to_string(),
        "https://example.com/page2".to_string(),
        "https://example.com/page3".to_string(),
    ];

    // Register batch of URLs
    let page1_ids = register.register_uri_batch(&page1_urls).await?;

    for (url, id) in page1_urls.iter().zip(page1_ids.iter()) {
        println!("  {} -> ID {}", url, id);
    }

    // Simulate second page with some duplicate URLs
    println!("\n📥 Crawler discovered URLs from Page 2:");
    let page2_urls = vec![
        "https://example.com/page2".to_string(), // Duplicate from page 1
        "https://example.com/page4".to_string(), // New URL
        "https://example.com/page5".to_string(), // New URL
    ];

    let page2_ids = register.register_uri_batch(&page2_urls).await?;

    for (url, id) in page2_urls.iter().zip(page2_ids.iter()) {
        let status = if page1_ids.contains(id) {
            "(already crawled)"
        } else {
            "(new)"
        };
        println!("  {} -> ID {} {}", url, id, status);
    }

    // Use HashMap variant when you don't need order
    println!("\n📥 Batch processing discovered links (using HashMap):");
    let batch_urls = vec![
        "https://example.com/page1".to_string(), // Duplicate
        "https://example.com/page6".to_string(), // New
        "https://example.com/page7".to_string(), // New
        "https://example.com/page1".to_string(), // Duplicate within batch
    ];

    let batch_map = register.register_uri_batch_hashmap(&batch_urls).await?;

    println!(
        "  Processed {} URLs into {} unique entries:",
        batch_urls.len(),
        batch_map.len()
    );
    for (url, id) in &batch_map {
        println!("    {} -> ID {}", url, id);
    }

    // Get statistics
    println!("\n📊 URI Register Statistics:");
    let stats = register.stats().await?;
    println!("  Total unique URLs registered: {}", stats.total_uris);
    println!("  Storage size: {} bytes", stats.size_bytes);

    // Demonstrate deduplication benefit
    println!("\n✅ Deduplication Summary:");
    println!(
        "  Total URLs processed: {}",
        page1_urls.len() + page2_urls.len() + batch_urls.len()
    );
    println!("  Unique URLs in database: {}", stats.total_uris);
    println!(
        "  Duplicates avoided: {}",
        (page1_urls.len() + page2_urls.len() + batch_urls.len()) - stats.total_uris as usize
    );

    Ok(())
}