simple-concurrent-get 0.3.1

Simply make multiple concurrent HTTP GET requests
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45

use reqwest::{ Client, Response };
use futures_util::stream::{ StreamExt as _ };

use std::sync::Arc;
use std::sync::mpsc::{
    IntoIter,
    channel,
};

use crate::{
    client,
};



pub async fn concurrent_get<I,S>(fetch_urls: I, concurrent: usize) -> IntoIter<reqwest::Result<Response>>
where
    I: IntoIterator<Item=S>,
    S: reqwest::IntoUrl,
{
    let client: Arc<Client> = Arc::new(client::build_preset());

    let (sender, receiver) = channel();

    let results = futures_util::stream::iter(fetch_urls)
        .map(|url| {
            let client = client.clone();
            async move {
                client.get(url).send().await
            }
        })
        .buffer_unordered(concurrent);

    results
        .for_each(|resp| {
            let sender = sender.clone();
            async move {
                sender.send(resp).unwrap();
            }
        })
        .await;

    drop(sender);
    receiver.into_iter()
}