qrawl 0.6.0

Rust toolkit to crawl web data for AI agents
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20

//! Batch Tools

mod tests;

use futures_util::stream::{self, StreamExt};

/// Batch execute async operations with bounded concurrency.
pub async fn batch<T, F, Fut, R>(items: Vec<T>, concurrency: usize, operation: F) -> Vec<R>
where
    T: Send + 'static,
    F: Fn(T) -> Fut + Send + Sync + 'static,
    Fut: std::future::Future<Output = R> + Send + 'static,
    R: Send + 'static,
{
    stream::iter(items)
        .map(operation)
        .buffer_unordered(concurrency)
        .collect()
        .await
}