moirai-iter 0.4.0

Parallel and async iterator combinators for Moirai concurrency library
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115

//! Parallel async operations with controlled concurrency.

use futures::stream::{self, StreamExt};
use std::future::Future;

use super::traits::AsyncIterator;

/// Parallel async map with concurrency control
pub struct ParAsyncMap<I, F> {
    iter: I,
    concurrency: usize,
    map_fn: F,
}

impl<I, F> ParAsyncMap<I, F> {
    pub(super) fn new(iter: I, concurrency: usize, map_fn: F) -> Self {
        Self {
            iter,
            concurrency,
            map_fn,
        }
    }
}

impl<I, F, Fut, R> AsyncIterator for ParAsyncMap<I, F>
where
    I: AsyncIterator,
    F: Fn(I::Item) -> Fut + Send + Sync,
    Fut: Future<Output = R> + Send,
    R: Send,
{
    type Item = R;

    fn into_vec(self) -> Vec<Self::Item> {
        let concurrency = self.concurrency.max(1);
        let map_fn = self.map_fn;
        let items = self.iter.into_vec();
        futures::executor::block_on(async move {
            stream::iter(items)
                .map(|item| {
                    let map_fn = &map_fn;
                    async move { map_fn(item).await }
                })
                .buffered(concurrency)
                .collect()
                .await
        })
    }
}

/// Parallel async filter with concurrency control
pub struct ParAsyncFilter<I, F> {
    iter: I,
    concurrency: usize,
    filter_fn: F,
}

impl<I, F> ParAsyncFilter<I, F> {
    pub(super) fn new(iter: I, concurrency: usize, filter_fn: F) -> Self {
        Self {
            iter,
            concurrency,
            filter_fn,
        }
    }
}

impl<I, F, Fut> AsyncIterator for ParAsyncFilter<I, F>
where
    I: AsyncIterator,
    F: Fn(&I::Item) -> Fut + Send + Sync,
    Fut: Future<Output = bool> + Send,
{
    type Item = I::Item;

    fn into_vec(self) -> Vec<Self::Item> {
        let concurrency = self.concurrency.max(1);
        let filter_fn = self.filter_fn;
        let items = self.iter.into_vec();
        futures::executor::block_on(async move {
            stream::iter(items)
                .map(|item| {
                    let filter_fn = &filter_fn;
                    async move {
                        let keep = filter_fn(&item).await;
                        (item, keep)
                    }
                })
                .buffered(concurrency)
                .filter_map(|(item, keep)| async move { keep.then_some(item) })
                .collect()
                .await
        })
    }
}

/// Parallel async for_each with concurrency control.
///
/// Returns a real `Future` driven by the caller's runtime — it never blocks the
/// executor. `for_each` is order-independent, so it uses `buffer_unordered`
/// (no head-of-line blocking) while still keeping at most `concurrency` item
/// futures in flight.
pub(super) async fn for_each<I, F, Fut>(iter: I, concurrency: usize, func: F)
where
    I: AsyncIterator,
    F: Fn(I::Item) -> Fut + Send + Sync,
    Fut: Future<Output = ()> + Send,
{
    let items = iter.into_vec();
    stream::iter(items)
        .map(func)
        .buffer_unordered(concurrency.max(1))
        .for_each(|()| async {})
        .await;
}