Skip to main content

bench_throughput/
bench_throughput.rs

1//! Standalone throughput benchmark for the rypipe-core engine.
2//!
3//! This example uses a tiny inline TSV-like adapter so the benchmark measures
4//! the engine itself, not any particular external parser. Run with:
5//!
6//!     cargo run --release -p rypipe-core --example bench_throughput
7
8use std::fs::File;
9use std::io::Write;
10use std::time::Instant;
11
12use rypipe_core::{
13    bounded::MemoryBudget, decoder::ColumnarSink, decoder::RecordParser, decoder::Splitter,
14    pipeline::Pipeline, plan::FieldType, value::Value, Result,
15};
16
17const ROWS: usize = 5_000_000;
18
19fn generate_data(rows: usize) -> Vec<u8> {
20    let mut buf = Vec::with_capacity(rows * 48);
21    for i in 0..rows {
22        writeln!(
23            &mut buf,
24            "id={i}\tstatus=active\tregion=north\tamount=123.45\tcount=7",
25        )
26        .unwrap();
27    }
28    buf
29}
30
31#[derive(Clone, Debug, Default)]
32struct TsvSplitter;
33
34impl Splitter for TsvSplitter {
35    fn find_split_points(&self, bytes: &[u8], max_chunks: usize) -> Vec<usize> {
36        if max_chunks <= 1 || bytes.is_empty() {
37            return vec![0, bytes.len()];
38        }
39        let mut points = vec![0usize];
40        let mut last = 0;
41        for (i, &b) in bytes.iter().enumerate() {
42            if b == b'\n' {
43                let next = i + 1;
44                if next > last && points.len() < max_chunks {
45                    points.push(next);
46                    last = next;
47                }
48            }
49        }
50        if *points.last().unwrap() != bytes.len() {
51            points.push(bytes.len());
52        }
53        points
54    }
55
56    fn estimate_bytes_per_row(&self, sample: &[u8]) -> usize {
57        let newline_count = sample.iter().filter(|&&b| b == b'\n').count().max(1);
58        (sample.len() / newline_count).max(1)
59    }
60}
61
62#[derive(Clone, Debug, Default)]
63struct TsvParser;
64
65impl RecordParser for TsvParser {
66    fn validate(&self, bytes: &[u8]) -> Result<()> {
67        simdutf8::basic::from_utf8(bytes)?;
68        Ok(())
69    }
70
71    fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()> {
72        let text =
73            std::str::from_utf8(bytes).map_err(|e| rypipe_core::Error::Plan(e.to_string()))?;
74        for line in text.lines() {
75            if line.is_empty() {
76                continue;
77            }
78            sink.begin_row();
79            for token in line.split('\t') {
80                if let Some((k, v)) = token.split_once('=') {
81                    if sink.wants(k) {
82                        sink.put_field(k, Value::Str(v));
83                    }
84                }
85            }
86            sink.end_row();
87        }
88        Ok(())
89    }
90}
91
92fn run(name: &str, f: impl FnOnce() -> Result<usize>) {
93    let start = Instant::now();
94    let rows = f().expect("benchmark failed");
95    let elapsed = start.elapsed();
96    let secs = elapsed.as_secs_f64();
97    println!(
98        "{name:24} {secs:8.3}s  {rows:10} rows  {:8.0} rows/s",
99        rows as f64 / secs
100    );
101}
102
103fn main() -> Result<()> {
104    let data = generate_data(ROWS);
105    let bytes = data.len();
106    let mb = bytes as f64 / 1_000_000.0;
107    println!("Generated {ROWS} rows ({mb:.1} MB)");
108
109    let mut path = std::env::temp_dir();
110    path.push("rypipe_bench_throughput.tsv");
111    File::create(&path)?.write_all(&data)?;
112
113    let pipeline = Pipeline::new(TsvSplitter, TsvParser).with_plan(
114        rypipe_core::ExecutionPlan::new()
115            .type_as("id", FieldType::Int64)
116            .type_as("amount", FieldType::Float64)
117            .type_as("count", FieldType::Int64),
118    );
119
120    run("single-thread", || {
121        let batch = pipeline.read_path(&path, false, false)?;
122        Ok(batch.num_rows())
123    });
124
125    run("parallel (4 chunks)", || {
126        let batches = pipeline.read_path_par(&path, 4, false, false)?;
127        Ok(batches.iter().map(|b| b.num_rows()).sum::<usize>())
128    });
129
130    run("parallel (8 chunks)", || {
131        let batches = pipeline.read_path_par(&path, 8, false, false)?;
132        Ok(batches.iter().map(|b| b.num_rows()).sum::<usize>())
133    });
134
135    run("bounded (64 MiB)", || {
136        let batches =
137            pipeline.read_path_stream(&path, MemoryBudget::new(64 * 1024 * 1024), false)?;
138        Ok(batches.iter().map(|b| b.num_rows()).sum::<usize>())
139    });
140
141    std::fs::remove_file(&path)?;
142    Ok(())
143}