dsi-bitstream 0.7.0

A Rust implementation of read/write bit streams supporting several types of instantaneous codes
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125

#!/usr/bin/env python3

#
# SPDX-FileCopyrightText: 2023 Tommaso Fontana
# SPDX-FileCopyrightText: 2023 Inria
# SPDX-FileCopyrightText: 2023 Sebastiano Vigna
#
# SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later
#

"""Benchmarks codes with different number of bits for the codes tables
and writes results on standard output in CSV format.
"""

import os
import sys
import subprocess
from gen_code_tables import *

if not os.path.exists("benchmarks") or not os.path.exists("python"):
    sys.exit("You must run this script in the main project directory.")

if len(sys.argv) < 2 or len(sys.argv) > 3 or sys.argv[1] not in {"u16", "u32", "u64"}:
    sys.exit("Usage: %s [u16 | u32 | u64] [implied | univ]" % sys.argv[0])

write_word = sys.argv[1]
dist = sys.argv[2] if len(sys.argv) == 3 else "univ"

if dist not in {"implied", "univ"}:
    sys.exit("Distribution must be 'implied' or 'univ'")

first_time = True

for bits in range(1, 17):
    value_max = 2**bits - 1
    print(
        "\nBenchmarking with write word = %s, table bits = %d\n" % (write_word, bits),
        file=sys.stderr,
    )
    for tables_num in [1, 2]:
        # Clean the target to force the recreation of the tables
        subprocess.check_call(
            "cargo clean",
            shell=True,
            cwd="benchmarks",
        )

        merged_table = tables_num == 1
        gen_gamma(
            read_bits=11,  # unused
            write_max_val=value_max,
            merged_table=merged_table,
        )
        gen_delta(
            read_bits=11,  # unused
            write_max_val=value_max,
            merged_table=merged_table,
        )
        gen_zeta(
            read_bits=12,  # unused
            write_max_val=value_max,
            k=3,
            merged_table=merged_table,
        )
        # Kludge: this will leave the original tables intact,
        # but avoids failing the static assert for
        if value_max >= 62:
            gen_omega(
                read_bits=11,  # unused
                write_max_val=value_max,
                merged_table=merged_table,
            )

        # Run the benchmark with native cpu optimizations
        features = write_word
        if dist == "univ":
            features = "univ," + features
        stdout = subprocess.check_output(
            "cargo run --release --no-default-features --features %s" % features,
            shell=True,
            env={
                **os.environ,
            },
            cwd="benchmarks",
        ).decode()

        for i in range(4, 5):
            gamma_bits = 2 * i + 1
            gamma_value_max = 2**gamma_bits - 1
            gen_gamma(
                read_bits=gamma_bits,  # unused
                write_max_val=gamma_value_max,
                merged_table=merged_table,
            )

            # Run the benchmark with native cpu optimizations
            features = "delta_gamma,%s" % write_word
            if dist == "univ":
                features = "univ," + features
            stdout += subprocess.check_output(
                "cargo run --release --no-default-features --features %s" % features,
                shell=True,
                env={
                    **os.environ,
                    "RUSTFLAGS": "-C target-cpu=native",
                },
                cwd="benchmarks",
            ).decode()

        # Dump the header only the first time
        if first_time:
            print("max,tables_num," + stdout.split("\n")[0])
            first_time = False
        # Dump all lines and add the `max` column
        for line in stdout.split("\n")[1:]:
            if value_max < 62 and "omega" in line:
                # ω write tables must represent 62
                continue
            if len(line.strip()) != 0:
                print("{},{},{}".format(value_max + 1, tables_num, line))

        sys.stdout.flush()

# Reset the tables to the original state
generate_default_tables()