urldecoder 2.0.0

High performance URL batch decoder
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136

#![cfg(feature = "bin")]
#[cfg(feature = "verbose-log")]
use std::sync::atomic::AtomicUsize;
use std::{
    fs::File,
    io::{BufWriter, Write},
    path::PathBuf,
};

use criterion::{Criterion, Throughput, criterion_group, criterion_main};
use rayon::iter::{IntoParallelIterator as _, IntoParallelRefIterator, ParallelIterator};
use tempfile::TempDir;
use urldecoder::decode_file;

pub type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>;

fn generate_mixed_data() -> Vec<u8> {
    let url = "https://2.com/1?q=%E5%A4%A9%E6%B0%94";
    let url_len = url.len();

    // Total = URL / 0.1
    let total_len = url_len * 10;
    let text_len = total_len - url_len;

    let mut pattern = String::with_capacity(total_len);
    let raw_text = "This is a standard chunk of text used to simulate the payload which acts as the ninety percent of the content stream. It contains spaces and normal sentences. ";

    while pattern.len() < text_len {
        let needed = text_len - pattern.len();
        if needed >= raw_text.len() {
            pattern.push_str(raw_text);
        } else {
            pattern.push_str(&raw_text[..needed]);
        }
    }
    pattern.push_str(url);
    pattern.into_bytes()
}

fn prepare_test_env() -> (TempDir, Vec<PathBuf>, u64) {
    let dir = tempfile::tempdir().expect("failed to create temp dir");
    let file_count = 32;
    // let min_file_size = 4 * 1024 * 1024; // 4 MB
    let min_file_size = 32 * 1024; // 32 KB

    let pattern = generate_mixed_data();

    let mut file_content = Vec::with_capacity(min_file_size + pattern.len());
    while file_content.len() < min_file_size {
        file_content.extend_from_slice(&pattern);
    }
    let single_file_size = file_content.len() as u64;

    let paths: Vec<PathBuf> = (0..file_count)
        .into_par_iter()
        .map(|i| {
            let file_path = dir.path().join(format!("test_file_{}.txt", i));
            let mut f = BufWriter::new(File::create(&file_path).unwrap());
            f.write_all(&file_content).unwrap();
            f.flush().unwrap();
            file_path
        })
        .collect();

    let total_bytes = single_file_size * file_count as u64;
    (dir, paths, total_bytes)
}

fn bench_decode_throughput(c: &mut Criterion) {
    let (temp_dir, paths, total_bytes) = prepare_test_env();

    let mut group = c.benchmark_group("decode_throughput");

    group.throughput(Throughput::Bytes(total_bytes));

    // 4 MB
    // unsafe: 27.133 GiB/s
    group.bench_function("rayon_decode_dry_run", |b| {
        b.iter(|| {
            let escape_space = false;
            let dry_run = true;
            #[cfg(feature = "verbose-log")]
            let processed_count = AtomicUsize::new(0);
            #[cfg(feature = "verbose-log")]
            let changed_count = AtomicUsize::new(0);
            paths.par_iter().for_each(|path| {
                decode_file(
                    path,
                    escape_space,
                    dry_run,
                    #[cfg(feature = "verbose-log")]
                    false,
                    #[cfg(feature = "verbose-log")]
                    &processed_count,
                    #[cfg(feature = "verbose-log")]
                    &changed_count,
                )
                .unwrap()
            })
        })
    });

    // 4 MB
    // unsafe: 21.860 GiB/s
    // safe: 11.954 GiB/s
    group.bench_function("rayon_decode", |b| {
        b.iter(|| {
            let escape_space = false;
            let dry_run = false;
            #[cfg(feature = "verbose-log")]
            let processed_count = AtomicUsize::new(0);
            #[cfg(feature = "verbose-log")]
            let changed_count = AtomicUsize::new(0);
            paths.par_iter().for_each(|path| {
                decode_file(
                    path,
                    escape_space,
                    dry_run,
                    #[cfg(feature = "verbose-log")]
                    false,
                    #[cfg(feature = "verbose-log")]
                    &processed_count,
                    #[cfg(feature = "verbose-log")]
                    &changed_count,
                )
                .unwrap()
            })
        })
    });

    group.finish();
    drop(temp_dir);
}

criterion_group!(benches, bench_decode_throughput);
criterion_main!(benches);