use runsync_transfer::codec::compress::{self, Algorithm, FileHint};
use runsync_transfer::config::{CompressionConfig, CompressionMode};
use runsync_transfer::{crypto, Cipher, Secrecy};
use std::collections::BTreeMap;
use std::io::Write;
use std::path::{Path, PathBuf};
use std::time::Instant;
type Any = Box<dyn std::error::Error + Send + Sync>;
fn main() -> Result<(), Any> {
let args: Vec<String> = std::env::args().skip(1).collect();
let cmd = args.first().map(String::as_str).unwrap_or("help");
let o = parse(&args[1.min(args.len())..]);
match cmd {
"corpus" => corpus(&o),
"run" => run(&o),
"stages" => stages(&o),
"crypto" => cryptobench(&o),
"pcm" => pcmbench(&o),
"pcmfile" => pcmfile(&o),
_ => {
eprintln!(
"codecbench corpus --out DIR [--size 64M]\n\
codecbench run --corpus DIR [--level 3] [--chunk 1M] [--threads N]\n\
codecbench stages --corpus DIR [--chunk 1M]\n\
codecbench crypto [--chunk 1M] [--iters 40]"
);
Ok(())
}
}
}
fn parse(args: &[String]) -> BTreeMap<String, String> {
let mut m = BTreeMap::new();
let mut i = 0;
while i < args.len() {
if let Some(k) = args[i].strip_prefix("--") {
match args.get(i + 1) {
Some(v) if !v.starts_with("--") => {
m.insert(k.into(), v.clone());
i += 2;
}
_ => {
m.insert(k.into(), "1".into());
i += 1;
}
}
} else {
i += 1;
}
}
m
}
fn size_of(o: &BTreeMap<String, String>, k: &str, d: u64) -> u64 {
o.get(k).and_then(|s| parse_size(s)).unwrap_or(d)
}
fn parse_size(s: &str) -> Option<u64> {
let (n, m) = match s.chars().last()? {
'k' | 'K' => (&s[..s.len() - 1], 1u64 << 10),
'm' | 'M' => (&s[..s.len() - 1], 1 << 20),
'g' | 'G' => (&s[..s.len() - 1], 1 << 30),
_ => (s, 1),
};
n.parse::<u64>().ok().map(|v| v * m)
}
fn xorshift(seed: u64, out: &mut [u8]) {
let mut s = seed | 1;
for c in out.chunks_mut(8) {
s ^= s << 13;
s ^= s >> 7;
s ^= s << 17;
c.copy_from_slice(&s.to_le_bytes()[..c.len()]);
}
}
fn corpus(o: &BTreeMap<String, String>) -> Result<(), Any> {
let out = PathBuf::from(o.get("out").ok_or("need --out")?);
let size = size_of(o, "size", 64 << 20) as usize;
std::fs::create_dir_all(&out)?;
let mut v = Vec::with_capacity(size);
let mut i = 0u64;
while v.len() < size {
v.extend_from_slice(
format!(
"2026-08-01T22:{:02}:{:02}Z INFO transfer chunk={i:07} offset={} status=ok peer=10.0.{}.{}\n",
(i / 60) % 60, i % 60, i * 1048576, (i / 256) % 256, i % 256
)
.as_bytes(),
);
i += 1;
}
v.truncate(size);
write(&out.join("logs.txt"), &v)?;
let mut src = Vec::new();
collect_files(Path::new("src"), &mut src);
collect_files(Path::new("examples"), &mut src);
collect_files(Path::new("tests"), &mut src);
let mut v = Vec::with_capacity(size);
while v.len() < size && !src.is_empty() {
for f in &src {
if v.len() >= size {
break;
}
if let Ok(b) = std::fs::read(f) {
v.extend_from_slice(&b);
}
}
}
if v.is_empty() {
v = vec![b'x'; size];
}
v.truncate(size);
write(&out.join("source.txt"), &v)?;
let mut v = Vec::with_capacity(size);
let mut n = 0u64;
let mut noise = 0x1234_5678_9ABC_DEF0u64;
while v.len() < size {
let t = n as f64 / 44_100.0;
noise ^= noise << 13;
noise ^= noise >> 7;
noise ^= noise << 17;
let d = ((noise >> 56) as i16) / 64;
let l = ((t * 440.0 * std::f64::consts::TAU).sin() * 11_000.0) as i16 + d;
let r = ((t * 659.25 * std::f64::consts::TAU).sin() * 8_500.0) as i16 - d;
v.extend_from_slice(&l.to_le_bytes());
v.extend_from_slice(&r.to_le_bytes());
n += 1;
}
v.truncate(size);
write(&out.join("audio_pcm.wav"), &v)?;
let mut v = vec![0u8; size];
xorshift(0xC0FFEE, &mut v);
write(&out.join("media.flac"), &v)?;
let mut exe = std::fs::read("target/release/examples/rst")
.or_else(|_| std::fs::read("/bin/bash"))
.unwrap_or_else(|_| vec![0x90; 1 << 20]);
if exe.is_empty() {
exe = vec![0x90; 1 << 20];
}
let mut v = Vec::with_capacity(size);
while v.len() < size {
v.extend_from_slice(&exe);
}
v.truncate(size);
write(&out.join("binary.bin"), &v)?;
write(&out.join("zeros.bin"), &vec![0u8; size])?;
let mut v = Vec::with_capacity(size);
for name in ["logs.txt", "audio_pcm.wav", "media.flac", "binary.bin"] {
let b = std::fs::read(out.join(name))?;
v.extend_from_slice(&b[..(size / 4).min(b.len())]);
}
v.truncate(size);
write(&out.join("mixed.bin"), &v)?;
println!("corpora written to {}", out.display());
Ok(())
}
fn collect_files(dir: &Path, out: &mut Vec<PathBuf>) {
let Ok(rd) = std::fs::read_dir(dir) else {
return;
};
for e in rd.flatten() {
let p = e.path();
if p.is_dir() {
collect_files(&p, out);
} else if p
.extension()
.is_some_and(|x| x == "rs" || x == "toml" || x == "md")
{
out.push(p);
}
}
}
fn write(p: &Path, b: &[u8]) -> Result<(), Any> {
let mut f = std::io::BufWriter::new(std::fs::File::create(p)?);
f.write_all(b)?;
f.flush()?;
Ok(())
}
struct Row {
name: String,
raw: u64,
encoded: u64,
comp_s: f64,
decomp_s: f64,
chunks_compressed: u64,
chunks_total: u64,
}
impl Row {
fn ratio(&self) -> f64 {
if self.encoded == 0 {
return f64::INFINITY;
}
self.raw as f64 / self.encoded as f64
}
fn cmb(&self) -> f64 {
self.raw as f64 / self.comp_s / 1e6
}
fn dmb(&self) -> f64 {
self.raw as f64 / self.decomp_s / 1e6
}
}
fn bench_chunked(
name: &str,
data: &[u8],
cfg: &CompressionConfig,
chunk: usize,
hint: FileHint,
passes: usize,
) -> Row {
let n_chunks = data.len().div_ceil(chunk);
let mut encoded: Vec<(Algorithm, usize, Vec<u8>)> = (0..n_chunks)
.map(|_| (Algorithm::None, 0, Vec::with_capacity(chunk + chunk / 8)))
.collect();
let mut total_enc = 0u64;
let mut compressed_chunks = 0u64;
let mut best_comp = f64::MAX;
let mut best_decomp = f64::MAX;
let mut sink = Vec::with_capacity(chunk + chunk / 8);
for _ in 0..passes.max(1) {
total_enc = 0;
compressed_chunks = 0;
let t0 = Instant::now();
for (slot, c) in encoded.iter_mut().zip(data.chunks(chunk)) {
slot.2.clear();
let e = compress::compress_into(cfg, hint, c, &mut slot.2).expect("compress");
slot.0 = e.algorithm;
slot.1 = e.raw_len;
total_enc += slot.2.len() as u64;
if e.algorithm != Algorithm::None {
compressed_chunks += 1;
}
}
best_comp = best_comp.min(t0.elapsed().as_secs_f64());
let t1 = Instant::now();
for (algo, raw_len, buf) in &encoded {
sink.clear();
compress::decompress_into(*algo, *raw_len, buf, &mut sink).expect("decompress");
std::hint::black_box(&sink);
}
best_decomp = best_decomp.min(t1.elapsed().as_secs_f64());
}
Row {
name: name.to_string(),
raw: data.len() as u64,
encoded: total_enc,
comp_s: best_comp,
decomp_s: best_decomp,
chunks_compressed: compressed_chunks,
chunks_total: encoded.len() as u64,
}
}
fn run(o: &BTreeMap<String, String>) -> Result<(), Any> {
let dir = PathBuf::from(o.get("corpus").ok_or("need --corpus")?);
let level: i32 = o.get("level").and_then(|s| s.parse().ok()).unwrap_or(3);
let chunk = size_of(o, "chunk", 1 << 20) as usize;
let passes: usize = o.get("passes").and_then(|s| s.parse().ok()).unwrap_or(5);
let files = corpus_files(&dir);
println!(
"our chunked pipeline — zstd level {level}, {} chunks, single thread, best of {passes}\n",
runsync_transfer::human_bytes(chunk as u64)
);
println!(
"{:<16} {:>10} {:>8} {:>11} {:>12} {:>10}",
"corpus", "size", "ratio", "comp MB/s", "decomp MB/s", "cmp/total"
);
println!("{}", "-".repeat(72));
for f in &files {
let data = std::fs::read(f)?;
let name = f.file_name().unwrap().to_string_lossy().into_owned();
let cfg = CompressionConfig {
mode: CompressionMode::Adaptive,
algorithm: Algorithm::Zstd,
level,
..Default::default()
};
let hint = FileHint {
known_incompressible: compress::is_incompressible_extension(&cfg, &name),
..Default::default()
};
let r = bench_chunked(&name, &data, &cfg, chunk, hint, passes);
print_row(&r);
}
println!("\nsame corpora, compression forced on every chunk (no adaptive skip),");
println!("which is the apples-to-apples setting against `zstd -b{level} -B{chunk}`\n");
println!(
"{:<16} {:>10} {:>8} {:>11} {:>12} {:>10}",
"corpus", "size", "ratio", "comp MB/s", "decomp MB/s", "cmp/total"
);
println!("{}", "-".repeat(72));
for f in &files {
let data = std::fs::read(f)?;
let name = f.file_name().unwrap().to_string_lossy().into_owned();
let cfg = CompressionConfig {
mode: CompressionMode::Always,
algorithm: Algorithm::Zstd,
level,
min_gain: 0.0,
..Default::default()
};
let r = bench_chunked(&name, &data, &cfg, chunk, FileHint::default(), passes);
print_row(&r);
}
println!("\nLZ4 (same chunking, forced)\n");
println!(
"{:<16} {:>10} {:>8} {:>11} {:>12} {:>10}",
"corpus", "size", "ratio", "comp MB/s", "decomp MB/s", "cmp/total"
);
println!("{}", "-".repeat(72));
for f in &files {
let data = std::fs::read(f)?;
let name = f.file_name().unwrap().to_string_lossy().into_owned();
let cfg = CompressionConfig {
mode: CompressionMode::Always,
algorithm: Algorithm::Lz4,
min_gain: 0.0,
..Default::default()
};
let r = bench_chunked(&name, &data, &cfg, chunk, FileHint::default(), passes);
print_row(&r);
}
Ok(())
}
fn print_row(r: &Row) {
let ratio = if r.ratio().is_finite() {
format!("{:.2}x", r.ratio())
} else {
"inf".into()
};
println!(
"{:<16} {:>10} {:>8} {:>11.1} {:>12.1} {:>10}",
r.name,
runsync_transfer::human_bytes(r.raw),
ratio,
r.cmb(),
r.dmb(),
format!("{}/{}", r.chunks_compressed, r.chunks_total),
);
}
fn corpus_files(dir: &Path) -> Vec<PathBuf> {
let mut v: Vec<_> = std::fs::read_dir(dir)
.map(|rd| {
rd.flatten()
.map(|e| e.path())
.filter(|p| p.is_file())
.collect()
})
.unwrap_or_else(|_| Vec::new());
v.sort();
v
}
fn stages(o: &BTreeMap<String, String>) -> Result<(), Any> {
let dir = PathBuf::from(o.get("corpus").ok_or("need --corpus")?);
let chunk = size_of(o, "chunk", 1 << 20) as usize;
let psk = crypto::random_key();
let a = crypto::Handshake::new(crypto::Role::Initiator, &Secrecy::Psk(psk), Cipher::Auto);
let b = crypto::Handshake::new(crypto::Role::Responder, &Secrecy::Psk(psk), Cipher::Auto);
let (am, bm) = (*a.message(), *b.message());
let sc = a.finish(&bm)?;
let _ = b.finish(&am)?;
let mut sealer = sc.sealer();
println!(
"per-stage cost, {} chunks, MB/s (higher is better)\n",
runsync_transfer::human_bytes(chunk as u64)
);
println!(
"{:<16} {:>11} {:>11} {:>11} {:>11} {:>11}",
"corpus", "read/copy", "zero-scan", "entropy", "blake3", "AEAD"
);
println!("{}", "-".repeat(76));
for f in corpus_files(&dir) {
let data = std::fs::read(&f)?;
let name = f.file_name().unwrap().to_string_lossy().into_owned();
let cfg = CompressionConfig::default();
let t = Instant::now();
let mut dst = Vec::with_capacity(chunk);
for c in data.chunks(chunk) {
dst.clear();
dst.extend_from_slice(c);
std::hint::black_box(&dst);
}
let copy = mbps(data.len(), t.elapsed().as_secs_f64());
let t = Instant::now();
for c in data.chunks(chunk) {
std::hint::black_box(runsync_transfer::send::is_all_zero(c));
}
let zero = mbps(data.len(), t.elapsed().as_secs_f64());
let t = Instant::now();
for c in data.chunks(chunk) {
std::hint::black_box(probe(c, cfg.probe_bytes));
}
let ent = mbps(data.len(), t.elapsed().as_secs_f64());
let t = Instant::now();
for c in data.chunks(chunk) {
std::hint::black_box(blake3::hash(c));
}
let hash = mbps(data.len(), t.elapsed().as_secs_f64());
let mut buf = vec![0u8; chunk];
let t = Instant::now();
for (idx, c) in data.chunks(chunk).enumerate() {
buf[..c.len()].copy_from_slice(c);
let _ = sealer.seal(
0,
idx as u64,
0,
b"header-28-bytes-placeholder!",
&mut buf[..c.len()],
);
}
let aead = mbps(data.len(), t.elapsed().as_secs_f64());
println!(
"{:<16} {:>11.0} {:>11.0} {:>11.0} {:>11.0} {:>11.0}",
name, copy, zero, ent, hash, aead
);
}
println!("\n(AEAD row includes the copy into the working buffer.)");
Ok(())
}
fn cryptobench(o: &BTreeMap<String, String>) -> Result<(), Any> {
let chunk = size_of(o, "chunk", 1 << 20) as usize;
let iters: usize = o.get("iters").and_then(|s| s.parse().ok()).unwrap_or(40);
let mut data = vec![0u8; chunk];
xorshift(0xABCDEF, &mut data);
let aad = [7u8; 28];
println!(
"AEAD and hash throughput, {} blocks, best of {iters}\n",
runsync_transfer::human_bytes(chunk as u64)
);
println!("{:<28} {:>12}", "primitive", "MB/s");
println!("{}", "-".repeat(42));
for (label, cipher) in [
("AES-256-GCM", Cipher::Aes256Gcm),
("ChaCha20-Poly1305", Cipher::ChaCha20Poly1305),
("Auto (what we pick)", Cipher::Auto),
] {
let psk = crypto::random_key();
let a = crypto::Handshake::new(crypto::Role::Initiator, &Secrecy::Psk(psk), cipher);
let b = crypto::Handshake::new(crypto::Role::Responder, &Secrecy::Psk(psk), cipher);
let (am, bm) = (*a.message(), *b.message());
let sc = a.finish(&bm)?;
let _ = b.finish(&am)?;
let mut sealer = sc.sealer();
let mut buf = data.clone();
let mut best = 0f64;
for i in 0..iters {
let t = Instant::now();
let _ = sealer.seal(0, i as u64, 0, &aad, &mut buf);
let v = mbps(chunk, t.elapsed().as_secs_f64());
if v > best {
best = v;
}
buf.copy_from_slice(&data);
}
println!("{:<28} {:>12.0}", format!("seal {label}"), best);
}
let mut best = 0f64;
for _ in 0..iters {
let t = Instant::now();
std::hint::black_box(blake3::hash(&data));
let v = mbps(chunk, t.elapsed().as_secs_f64());
if v > best {
best = v;
}
}
println!("{:<28} {:>12.0}", "blake3 (1 thread)", best);
println!("\nbuild-time feature detection:");
println!(" target_arch : {}", std::env::consts::ARCH);
println!(" target_feature aes : {}", cfg!(target_feature = "aes"));
println!(" target_feature neon : {}", cfg!(target_feature = "neon"));
println!(" target_feature sha3 : {}", cfg!(target_feature = "sha3"));
Ok(())
}
fn mbps(bytes: usize, secs: f64) -> f64 {
bytes as f64 / secs / 1e6
}
fn probe(input: &[u8], probe_bytes: usize) -> bool {
let n = probe_bytes.min(input.len());
if n < 256 {
return false;
}
let mut hist = [0u32; 256];
for &b in &input[..n] {
hist[b as usize] += 1;
}
let len = n as f32;
let mut e = 0.0f32;
for &c in hist.iter() {
if c != 0 {
let p = c as f32 / len;
e -= p * p.log2();
}
}
e > 7.8
}
fn pcmbench(o: &BTreeMap<String, String>) -> Result<(), Any> {
use runsync_transfer::codec::pcm::{self, AudioFormat};
let chunk = size_of(o, "chunk", 1 << 20) as usize;
let passes: usize = o.get("passes").and_then(|s| s.parse().ok()).unwrap_or(5);
let fmt = AudioFormat {
bits_per_sample: 16,
channels: 2,
data_start: 0,
block_align: 4,
sample_format: runsync_transfer::codec::pcm::SampleFormat::SignedInt,
};
let mut cases: Vec<(&str, Vec<u8>)> = Vec::new();
let frames = 8 << 20;
let mut tonal = Vec::with_capacity(frames * 4);
let mut dithered = Vec::with_capacity(frames * 4);
let mut loud = Vec::with_capacity(frames * 4);
let mut s = 0x1234_5678_9ABC_DEF0u64;
for i in 0..frames {
let t = i as f64 / 44_100.0;
s ^= s << 13;
s ^= s >> 7;
s ^= s << 17;
let l = (t * 440.0 * std::f64::consts::TAU).sin();
let r = (t * 659.25 * std::f64::consts::TAU).sin();
tonal.extend_from_slice(&((l * 11_000.0) as i16).to_le_bytes());
tonal.extend_from_slice(&((r * 8_500.0) as i16).to_le_bytes());
let d = ((s >> 56) as i16) / 4;
dithered.extend_from_slice(&(((l * 11_000.0) as i16).wrapping_add(d)).to_le_bytes());
dithered.extend_from_slice(&(((r * 8_500.0) as i16).wrapping_add(d)).to_le_bytes());
let d2 = ((s >> 48) as i16) / 2;
let ml = (l * 0.6 + r * 0.4) * 30_000.0;
let mr = (r * 0.6 + l * 0.4) * 30_000.0;
loud.extend_from_slice(&((ml as i16).wrapping_add(d2)).to_le_bytes());
loud.extend_from_slice(&((mr as i16).wrapping_add(d2)).to_le_bytes());
}
cases.push(("tonal", tonal));
cases.push(("dithered", dithered));
cases.push(("loud+noisy", loud));
println!(
"PCM codec vs zstd-3, 16-bit stereo, {} chunks, best of {passes}\n",
runsync_transfer::human_bytes(chunk as u64)
);
println!(
"{:<12} {:>9} {:>11} {:>12} {:>9} {:>11} {:>12}",
"signal", "pcm ratio", "pcm enc MB/s", "pcm dec MB/s", "zstd", "zstd enc", "zstd dec"
);
println!("{}", "-".repeat(82));
for (name, data) in &cases {
let mut enc: Vec<Vec<u8>> = Vec::new();
let mut best_e = f64::MAX;
let mut total = 0usize;
for _ in 0..passes {
enc.clear();
total = 0;
let t = Instant::now();
for (i, c) in data.chunks(chunk).enumerate() {
let mut o = Vec::with_capacity(c.len());
match pcm::encode(&fmt, (i * chunk) as u64, c, &mut o) {
Some(_) => {}
None => o.extend_from_slice(c),
}
total += o.len();
enc.push(o);
}
best_e = best_e.min(t.elapsed().as_secs_f64());
}
let mut best_d = f64::MAX;
for _ in 0..passes {
let t = Instant::now();
let mut sink = Vec::with_capacity(chunk);
for e in &enc {
sink.clear();
let _ = pcm::decode(e, &mut sink);
std::hint::black_box(&sink);
}
best_d = best_d.min(t.elapsed().as_secs_f64());
}
let zcfg = CompressionConfig {
mode: CompressionMode::Always,
algorithm: Algorithm::Zstd,
level: 3,
min_gain: 0.0,
..Default::default()
};
let z = bench_chunked(name, data, &zcfg, chunk, FileHint::default(), passes);
println!(
"{:<12} {:>8.2}x {:>11.0} {:>12.0} {:>8.2}x {:>11.0} {:>12.0}",
name,
data.len() as f64 / total as f64,
data.len() as f64 / best_e / 1e6,
data.len() as f64 / best_d / 1e6,
z.ratio(),
z.cmb(),
z.dmb(),
);
}
Ok(())
}
fn pcmfile(o: &BTreeMap<String, String>) -> Result<(), Any> {
use runsync_transfer::codec::pcm;
let chunk = size_of(o, "chunk", 1 << 20) as usize;
let paths: Vec<PathBuf> = o
.get("dir")
.map(|d| corpus_files(Path::new(d)))
.unwrap_or_else(|| {
o.get("path")
.map(|p| vec![PathBuf::from(p)])
.unwrap_or_default()
});
if paths.is_empty() {
return Err("need --path FILE or --dir DIR".into());
}
println!(
"{:<34} {:>10} {:>8} {:>9} {:>8} {:>9}",
"file", "size", "ours", "MB/s", "zstd-3", "MB/s"
);
println!("{}", "-".repeat(84));
let (mut tot_raw, mut tot_ours, mut tot_zstd) = (0u64, 0u64, 0u64);
for p in &paths {
let data = match std::fs::read(p) {
Ok(d) => d,
Err(_) => continue,
};
let Some(fmt) = pcm::parse_wav_header(&data[..data.len().min(8192)]) else {
continue;
};
let t = Instant::now();
let mut ours = 0usize;
for (i, c) in data.chunks(chunk).enumerate() {
let mut out = Vec::with_capacity(c.len());
match pcm::encode(&fmt, (i * chunk) as u64, c, &mut out) {
Some(n) => ours += n,
None => ours += c.len(),
}
}
let ours_s = t.elapsed().as_secs_f64();
let zcfg = CompressionConfig {
mode: CompressionMode::Always,
algorithm: Algorithm::Zstd,
level: 3,
min_gain: 0.0,
..Default::default()
};
let z = bench_chunked("", &data, &zcfg, chunk, FileHint::default(), 1);
let name: String = p
.file_name()
.unwrap()
.to_string_lossy()
.chars()
.take(33)
.collect();
println!(
"{:<34} {:>10} {:>7.2}x {:>9.0} {:>7.2}x {:>9.0}",
name,
runsync_transfer::human_bytes(data.len() as u64),
data.len() as f64 / ours as f64,
data.len() as f64 / ours_s / 1e6,
z.ratio(),
z.cmb(),
);
tot_raw += data.len() as u64;
tot_ours += ours as u64;
tot_zstd += z.encoded;
}
if tot_raw > 0 {
println!("{}", "-".repeat(84));
println!(
"{:<34} {:>10} {:>7.2}x {:>9} {:>7.2}x",
"TOTAL",
runsync_transfer::human_bytes(tot_raw),
tot_raw as f64 / tot_ours as f64,
"",
tot_raw as f64 / tot_zstd as f64
);
}
Ok(())
}