Skip to main content

simple_concurrent_get/ordered/
get.rs

1use reqwest::{ Client, Response };
2use futures_util::stream::{ StreamExt as _ };
3
4use std::sync::Arc;
5use std::sync::mpsc::{
6    IntoIter,
7    channel,
8};
9
10use crate::{
11    client,
12};
13
14
15
16pub async fn concurrent_get<I,S>(fetch_urls: I, concurrent: usize) -> IntoIter<reqwest::Result<Response>>
17where
18    I: IntoIterator<Item=S>,
19    S: reqwest::IntoUrl,
20{
21    let client: Arc<Client> = Arc::new(client::build_preset());
22
23    let (sender, receiver) = channel();
24
25    let bodies = futures_util::stream::iter(fetch_urls)
26        .map(|url| {
27            let client = client.clone();
28            async move {
29                client.get(url).send().await
30            }
31        })
32        .buffered(concurrent);
33
34    bodies
35        .for_each(|resp| {
36            let sender = sender.clone();
37            async move {
38                sender.send(resp).unwrap();
39            }
40        })
41        .await;
42
43    drop(sender);
44    receiver.into_iter()
45}