kodept-parse 0.3.0

Simple compiler with dependent types support in mind
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41

use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput};
use kodept_parse::lexer::{PegLexer, PestLexer};
use kodept_parse::tokenizer::{EagerTokenizer, LazyTokenizer, ParallelTokenizer, Tokenizer};

const FILENAME: &'static str = "benches/benchmarking_file1.kd";

fn get_contents_with_factor(filename: &str, factor: usize) -> String {
    let contents = std::fs::read_to_string(filename).unwrap();
    contents.repeat(factor)
}

fn bench_impls(c: &mut Criterion) {
    let mut group = c.benchmark_group("tokenizer");
    for factor in (5..=10).map(|it| 2usize.pow(it)) {
        let contents = get_contents_with_factor(FILENAME, factor);
        group.throughput(Throughput::Bytes(contents.as_bytes().len() as u64));
        
        group.bench_with_input(BenchmarkId::new("peg", factor), &contents, |b, i| {
            b.iter(|| EagerTokenizer::new(i, PegLexer::<false>::new()).into_vec())
        });
        group.bench_with_input(BenchmarkId::new("pest", factor), &contents, |b, i| {
            b.iter(|| EagerTokenizer::new(i, PestLexer::new()).into_vec())
        });
        group.bench_with_input(BenchmarkId::new("lazy-pest", factor), &contents, |b, i| {
            b.iter(|| LazyTokenizer::new(i, PestLexer::new()).into_vec())
        });
        group.bench_with_input(
            BenchmarkId::new("parallel-peg", factor),
            &contents,
            |b, i| b.iter(|| ParallelTokenizer::new(i, PegLexer::<false>::new()).into_vec()),
        );
        group.bench_with_input(
            BenchmarkId::new("parallel-pest", factor),
            &contents,
            |b, i| b.iter(|| ParallelTokenizer::new(i, PestLexer::new()).into_vec()),
        );
    }
}

criterion_group!(benches, bench_impls);
criterion_main!(benches);