medina 0.0.1

An asynchronous web crawling engine
Documentation
use super::task_runner::TaskRunner;
use super::Crawler;
use async_mutex::Mutex;
use event_listener::Event;
use std::time::SystemTime;

#[cfg(feature = "tokio")]
use tokio::task;

#[cfg(feature = "async-std")]
use async_std::task;

pub struct CrawlerRunner {
    internal: (Mutex<Crawler>, Event),
    task_runner: (Mutex<TaskRunner>, Event),
}

impl CrawlerRunner {
    pub fn new(crawler: Crawler) -> Self {
        Self {
            internal: (Mutex::new(crawler), Event::new()),
            task_runner: (Mutex::new(TaskRunner::new()), Event::new()),
        }
    }

    pub async fn run_crawler(&self) {
        let start = SystemTime::now();
        Crawler::run(&self.internal, &self.task_runner, start).await;
    }

    pub async fn run_task_runner(&self) {
        TaskRunner::run(&self.task_runner).await;
    }

    #[cfg(any(feature = "tokio", feature = "async-std"))]
    pub async fn run(crawler: Crawler, num_crawler_tasks: usize, num_processing_tasks: usize) {
        let runner = std::sync::Arc::new(CrawlerRunner::new(crawler));

        let processing_tasks = {
            let mut vec = Vec::new();
            for _ in 0..num_crawler_tasks {
                let crawler_runner = runner.clone();
                vec.push(task::spawn(async move {
                    crawler_runner.run_task_runner().await;
                }));
            }
            vec
        };

        let crawling_tasks = {
            let mut vec = Vec::new();
            for _ in 0..num_processing_tasks {
                let crawler_runner = runner.clone();
                vec.push(task::spawn(async move {
                    crawler_runner.run_crawler().await;
                }));
            }
            vec
        };

        #[allow(unused_must_use)]
        {
            for t in crawling_tasks {
                t.await;
            }
            for t in processing_tasks {
                #[allow(unused_must_use)]
                t.await;
            }
        }
    }
}