simple-concurrent-get 0.3.1

Simply make multiple concurrent HTTP GET requests
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48

use reqwest::{ Client, Response };
use futures_util::stream::{ StreamExt as _ };

use std::sync::Arc;
use std::sync::mpsc::{
    IntoIter,
    channel,
};

use crate::{
    client,
};



pub async fn concurrent_get_foreach<I,S,F,R>(fetch_urls: I, concurrent: usize, run_for_each: F) -> IntoIter<R>
where
    S: reqwest::IntoUrl,
    I: IntoIterator<Item=S>,
    F: Copy + FnOnce(reqwest::Result<Response>) -> R,
{
    let client: Arc<Client> = Arc::new(client::build_preset());

    let (sender, receiver) = channel();

    let bodies = futures_util::stream::iter(fetch_urls)
        .map(|url| {
            let client = client.clone();
            async move {
                run_for_each(
                    client.get(url).send().await
                )
            }
        })
        .buffer_unordered(concurrent);

    bodies
        .for_each(|resp| {
            let sender = sender.clone();
            async move {
                sender.send(resp).unwrap();
            }
        })
        .await;

    drop(sender);
    receiver.into_iter()
}