stream_shared 0.8.6

A library for creating shareable streams that can be cloned and consumed by multiple consumers
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151

use criterion::{criterion_group, criterion_main, Criterion};
use futures_util::{future::join_all, stream::StreamExt};
use std::time::Duration;
use stream_shared::{SharedStream, SharedStreamExt};
use tokio::runtime::Runtime;

mod utils;
use utils::{
    generate_in_memory_stream, generate_simulated_io_stream, run_channel_fan_out, ArcData,
    NUM_CONSUMERS,
};

// --- Benchmark Runner Function ---

// Function to run the consumption logic for N consumers
async fn consume_streams<S>(streams: Vec<S>)
where
    S: futures_util::stream::Stream<Item = ArcData> + Unpin,
{
    let futures: Vec<_> = streams
        .into_iter()
        .map(|mut stream| async move {
            let mut consumed_count = 0;
            while stream.next().await.is_some() {
                consumed_count += 1;
            }
            consumed_count
        })
        .collect();

    // Wait for all consumers to finish
    let _results = join_all(futures).await;
}

// --- Criterion Benchmarks ---

fn benchmark_group(c: &mut Criterion) {
    let item_counts = [1_000, 10_000, 100_000];

    let rt = Runtime::new().expect("Failed to create Tokio runtime");

    // --- 1. Contention Benchmarks (5 Consumers) ---
    let mut contention_group = c.benchmark_group("Contention (N=5)");

    contention_group.measurement_time(Duration::from_secs(10));
    contention_group.sample_size(50);

    for &count in item_counts.iter() {
        // --- SharedStream (Contended Synchronization) ---
        contention_group.bench_function(format!("SharedStream_MEM__{}", count), |b| {
            b.iter(|| {
                rt.block_on(async {
                    let stream = generate_in_memory_stream(count);
                    let shared_stream = stream.into_shared();
                    let consumers: Vec<_> =
                        (0..NUM_CONSUMERS).map(|_| shared_stream.clone()).collect();
                    consume_streams(consumers).await;
                })
            })
        });

        contention_group.bench_function(format!("SharedStream_IO__{}", count), |b| {
            b.iter(|| {
                rt.block_on(async {
                    let stream = generate_simulated_io_stream(count);
                    let shared_stream = SharedStream::new(stream);
                    let consumers: Vec<_> =
                        (0..NUM_CONSUMERS).map(|_| shared_stream.clone()).collect();
                    consume_streams(consumers).await;
                })
            })
        });

        // --- Channel Fan-Out (Centralized I/O) ---
        contention_group.bench_function(format!("ChannelFanOut_MEM__{}", count), |b| {
            b.iter(|| {
                rt.block_on(async {
                    let stream = generate_in_memory_stream(count);
                    let consumers = run_channel_fan_out(stream);
                    consume_streams(consumers).await;
                })
            })
        });

        contention_group.bench_function(format!("ChannelFanOut_IO__{}", count), |b| {
            b.iter(|| {
                rt.block_on(async {
                    let stream = generate_simulated_io_stream(count);
                    let consumers = run_channel_fan_out(stream);
                    consume_streams(consumers).await;
                })
            })
        });
    }
    contention_group.finish();

    // --- 2. Wrapper Overhead Benchmarks (Single Consumer N=1) ---
    let mut overhead_group = c.benchmark_group("Wrapper Overhead (N=1)");
    overhead_group.measurement_time(Duration::from_secs(10));
    overhead_group.sample_size(50);

    for &count in item_counts.iter() {
        // A. Raw Stream Consumption (Baseline, MEM)
        overhead_group.bench_function(format!("RawStream_MEM__{}", count), |b| {
            b.iter(|| {
                rt.block_on(async {
                    let stream = generate_in_memory_stream(count);
                    consume_streams(vec![stream]).await;
                })
            })
        });

        // B. Shared Wrapper Consumption (Wrapper Overhead, MEM)
        overhead_group.bench_function(format!("SharedWrapper_MEM__{}", count), |b| {
            b.iter(|| {
                rt.block_on(async {
                    let stream = generate_in_memory_stream(count);
                    let shared_stream = SharedStream::new(stream);
                    // Single consumer using the wrapper
                    consume_streams(vec![shared_stream]).await;
                })
            })
        });

        // C. Raw Stream IO Consumption (Baseline, IO)
        overhead_group.bench_function(format!("RawStream_IO__{}", count), |b| {
            b.iter(|| {
                rt.block_on(async {
                    let stream = generate_simulated_io_stream(count);
                    consume_streams(vec![stream]).await;
                })
            })
        });

        // D. Shared Wrapper IO Consumption (Wrapper Overhead, IO)
        overhead_group.bench_function(format!("SharedWrapper_IO__{}", count), |b| {
            b.iter(|| {
                rt.block_on(async {
                    let stream = generate_simulated_io_stream(count);
                    let shared_stream = SharedStream::new(stream);
                    // Single consumer using the wrapper
                    consume_streams(vec![shared_stream]).await;
                })
            })
        });
    }
    overhead_group.finish();
}

criterion_group!(benches, benchmark_group);
criterion_main!(benches);