bench_throughput/
bench_throughput.rs1use std::fs::File;
9use std::io::Write;
10use std::time::Instant;
11
12use rypipe_core::{
13 bounded::MemoryBudget, decoder::ColumnarSink, decoder::RecordParser, decoder::Splitter,
14 pipeline::Pipeline, plan::FieldType, value::Value, Result,
15};
16
17const ROWS: usize = 5_000_000;
18
19fn generate_data(rows: usize) -> Vec<u8> {
20 let mut buf = Vec::with_capacity(rows * 48);
21 for i in 0..rows {
22 writeln!(
23 &mut buf,
24 "id={i}\tstatus=active\tregion=north\tamount=123.45\tcount=7",
25 )
26 .unwrap();
27 }
28 buf
29}
30
31#[derive(Clone, Debug, Default)]
32struct TsvSplitter;
33
34impl Splitter for TsvSplitter {
35 fn find_split_points(&self, bytes: &[u8], max_chunks: usize) -> Vec<usize> {
36 if max_chunks <= 1 || bytes.is_empty() {
37 return vec![0, bytes.len()];
38 }
39 let mut points = vec![0usize];
40 let mut last = 0;
41 for (i, &b) in bytes.iter().enumerate() {
42 if b == b'\n' {
43 let next = i + 1;
44 if next > last && points.len() < max_chunks {
45 points.push(next);
46 last = next;
47 }
48 }
49 }
50 if *points.last().unwrap() != bytes.len() {
51 points.push(bytes.len());
52 }
53 points
54 }
55
56 fn estimate_bytes_per_row(&self, sample: &[u8]) -> usize {
57 let newline_count = sample.iter().filter(|&&b| b == b'\n').count().max(1);
58 (sample.len() / newline_count).max(1)
59 }
60}
61
62#[derive(Clone, Debug, Default)]
63struct TsvParser;
64
65impl RecordParser for TsvParser {
66 fn validate(&self, bytes: &[u8]) -> Result<()> {
67 simdutf8::basic::from_utf8(bytes)?;
68 Ok(())
69 }
70
71 fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()> {
72 let text =
73 std::str::from_utf8(bytes).map_err(|e| rypipe_core::Error::Plan(e.to_string()))?;
74 for line in text.lines() {
75 if line.is_empty() {
76 continue;
77 }
78 sink.begin_row();
79 for token in line.split('\t') {
80 if let Some((k, v)) = token.split_once('=') {
81 if sink.wants(k) {
82 sink.put_field(k, Value::Str(v));
83 }
84 }
85 }
86 sink.end_row();
87 }
88 Ok(())
89 }
90}
91
92fn run(name: &str, f: impl FnOnce() -> Result<usize>) {
93 let start = Instant::now();
94 let rows = f().expect("benchmark failed");
95 let elapsed = start.elapsed();
96 let secs = elapsed.as_secs_f64();
97 println!(
98 "{name:24} {secs:8.3}s {rows:10} rows {:8.0} rows/s",
99 rows as f64 / secs
100 );
101}
102
103fn main() -> Result<()> {
104 let data = generate_data(ROWS);
105 let bytes = data.len();
106 let mb = bytes as f64 / 1_000_000.0;
107 println!("Generated {ROWS} rows ({mb:.1} MB)");
108
109 let mut path = std::env::temp_dir();
110 path.push("rypipe_bench_throughput.tsv");
111 File::create(&path)?.write_all(&data)?;
112
113 let pipeline = Pipeline::new(TsvSplitter, TsvParser).with_plan(
114 rypipe_core::ExecutionPlan::new()
115 .type_as("id", FieldType::Int64)
116 .type_as("amount", FieldType::Float64)
117 .type_as("count", FieldType::Int64),
118 );
119
120 run("single-thread", || {
121 let batch = pipeline.read_path(&path, false, false)?;
122 Ok(batch.num_rows())
123 });
124
125 run("parallel (4 chunks)", || {
126 let batches = pipeline.read_path_par(&path, 4, false, false)?;
127 Ok(batches.iter().map(|b| b.num_rows()).sum::<usize>())
128 });
129
130 run("parallel (8 chunks)", || {
131 let batches = pipeline.read_path_par(&path, 8, false, false)?;
132 Ok(batches.iter().map(|b| b.num_rows()).sum::<usize>())
133 });
134
135 run("bounded (64 MiB)", || {
136 let batches =
137 pipeline.read_path_stream(&path, MemoryBudget::new(64 * 1024 * 1024), false)?;
138 Ok(batches.iter().map(|b| b.num_rows()).sum::<usize>())
139 });
140
141 std::fs::remove_file(&path)?;
142 Ok(())
143}