bgustcrawler 0.2.0

Distributed web crawler and scheduler integrated with Kafka, SurrealDB, and S3 storage
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

use bgustcrawler::{start_background_consumer, scheduler::SchedulerManager};
use anyhow::Result;

#[tokio::main]
async fn main() -> Result<()> {
    println!("👾 [CRAWLER] Starting bgustcrawler Daemon...");

    // 1. Initialize and start the Scheduled Tasks Manager
    let sched_manager = SchedulerManager::new().await?;
    let scheduler_handle = tokio::spawn(async move {
        if let Err(e) = sched_manager.start().await {
            eprintln!("⚠️ [CRAWLER] Scheduler error: {:?}", e);
        }
    });

    // 2. Initialize and start the Redpanda Background Task Consumer
    let brokers = "localhost:9094";
    let consumer_handle = tokio::spawn(async move {
        if let Err(e) = start_background_consumer(brokers).await {
            eprintln!("⚠️ [CRAWLER] Consumer error: {:?}", e);
        }
    });

    println!("👾 [CRAWLER] Daemon components initialized. Running...");
    let _ = tokio::join!(scheduler_handle, consumer_handle);

    Ok(())
}