opus-codec 0.2.0

Safe Rust bindings for the Opus audio codec
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144

#!/usr/bin/env python3
"""Generate a DNN blob and test runtime-loaded DRED weights end to end."""

import argparse
import hashlib
import os
import shlex
import shutil
import tempfile
from pathlib import Path

import ci_utils


ROOT = Path(__file__).resolve().parent.parent
TARGET_DIR = os.environ.get("CARGO_TARGET_DIR", "target/ci-external-weights")
MODEL_ARCHIVE = "opus_data-735117b.tar.gz"
MODEL_SHA256 = "8f34305a299183509d22c7ba66790f67916a0fc56028ebd4c8f7b938458f2801"
MODEL_URL = f"https://media.xiph.org/opus/models/{MODEL_ARCHIVE}"


def sha256(path: Path) -> str:
    digest = hashlib.sha256()
    with path.open("rb") as source:
        for chunk in iter(lambda: source.read(1024 * 1024), b""):
            digest.update(chunk)
    return digest.hexdigest()


def download_model_archive(archive: Path) -> None:
    failures = []
    wget = shutil.which("wget")
    if wget:
        result = ci_utils.run([wget, "-O", str(archive), MODEL_URL], check=False)
        if result.returncode == 0:
            return
        failures.append(f"wget exited with {result.returncode}")
    else:
        failures.append("wget was not found")

    curl = shutil.which("curl")
    if curl:
        result = ci_utils.run(
            [curl, "--fail", "--location", "--output", str(archive), MODEL_URL],
            check=False,
        )
        if result.returncode == 0:
            return
        failures.append(f"curl exited with {result.returncode}")
    else:
        failures.append("curl was not found")

    ci_utils.fail(
        "failed to download the DRED model archive with wget or curl: "
        + "; ".join(failures)
    )


def ensure_model_archive() -> Path:
    archive = ROOT / "opus" / MODEL_ARCHIVE
    if not archive.exists():
        download_model_archive(archive)

    actual = sha256(archive)
    if actual != MODEL_SHA256:
        ci_utils.fail(
            f"DRED model archive checksum mismatch: expected {MODEL_SHA256}, got {actual}"
        )
    return archive


def generate_weights_blob(archive: Path, temp: Path) -> Path:
    ci_utils.run(["tar", "xzf", str(archive), "-C", str(temp)])

    compiler = shutil.which(os.environ.get("CC", "cc"))
    if not compiler:
        ci_utils.fail(f"C compiler {os.environ.get('CC', 'cc')!r} was not found")

    cflags = shlex.split(os.environ.get("CFLAGS", ""))
    dnn = ROOT / "opus" / "dnn"
    command = [
        compiler,
        *cflags,
        "-O2",
        f"-I{ROOT / 'opus'}",
        f"-I{ROOT / 'opus' / 'include'}",
        f"-I{ROOT / 'opus' / 'celt'}",
        f"-I{dnn}",
        f"-I{temp / 'dnn'}",
        str(dnn / "write_lpcnet_weights.c"),
        str(dnn / "parse_lpcnet_weights.c"),
        "-lm",
        "-o",
        str(temp / "dump_weights_blob"),
    ]
    ci_utils.run(command)
    ci_utils.run([str(temp / "dump_weights_blob")], cwd=temp)

    blob = temp / "weights_blob.bin"
    if not blob.is_file() or blob.stat().st_size == 0:
        ci_utils.fail("weight blob generator produced no data")
    print(f"Generated external DNN blob: {blob.stat().st_size} bytes")
    return blob


def run_test(blob: Path, toolchain: str | None, target: str | None) -> None:
    command = ["cargo"]
    if toolchain:
        command.append(f"+{toolchain}")
    command += ["test"]
    if target:
        command += ["--target", target]
    command += [
        "--features",
        "external-weights",
        "--test",
        "external_weights",
        "--",
        "--ignored",
        "--exact",
        "external_weight_dred_round_trip",
    ]

    env = {
        "CARGO_TARGET_DIR": TARGET_DIR,
        "OPUS_CODEC_DNN_BLOB": str(blob),
    }
    ci_utils.run(command, cwd=ROOT, env=env)


def main() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument("--toolchain")
    parser.add_argument("--target")
    args = parser.parse_args()

    archive = ensure_model_archive()
    with tempfile.TemporaryDirectory(prefix="opus-external-weights-") as temp_dir:
        blob = generate_weights_blob(archive, Path(temp_dir))
        run_test(blob, args.toolchain, args.target)


if __name__ == "__main__":
    main()