use super::task_runner::TaskRunner;
use super::Crawler;
use async_mutex::Mutex;
use event_listener::Event;
use std::time::SystemTime;
#[cfg(feature = "tokio")]
use tokio::task;
#[cfg(feature = "async-std")]
use async_std::task;
pub struct CrawlerRunner {
internal: (Mutex<Crawler>, Event),
task_runner: (Mutex<TaskRunner>, Event),
}
impl CrawlerRunner {
pub fn new(crawler: Crawler) -> Self {
Self {
internal: (Mutex::new(crawler), Event::new()),
task_runner: (Mutex::new(TaskRunner::new()), Event::new()),
}
}
pub async fn run_crawler(&self) {
let start = SystemTime::now();
Crawler::run(&self.internal, &self.task_runner, start).await;
}
pub async fn run_task_runner(&self) {
TaskRunner::run(&self.task_runner).await;
}
#[cfg(any(feature = "tokio", feature = "async-std"))]
pub async fn run(crawler: Crawler, num_crawler_tasks: usize, num_processing_tasks: usize) {
let runner = std::sync::Arc::new(CrawlerRunner::new(crawler));
let processing_tasks = {
let mut vec = Vec::new();
for _ in 0..num_crawler_tasks {
let crawler_runner = runner.clone();
vec.push(task::spawn(async move {
crawler_runner.run_task_runner().await;
}));
}
vec
};
let crawling_tasks = {
let mut vec = Vec::new();
for _ in 0..num_processing_tasks {
let crawler_runner = runner.clone();
vec.push(task::spawn(async move {
crawler_runner.run_crawler().await;
}));
}
vec
};
#[allow(unused_must_use)]
{
for t in crawling_tasks {
t.await;
}
for t in processing_tasks {
#[allow(unused_must_use)]
t.await;
}
}
}
}