1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
name: "Benchmarks: Python"
on:
push:
branches:
paths-ignore:
- '**.md'
- 'docs/**'
- 'mkdocs.yml'
- 'LICENSE'
- '.gitignore'
- '.beads/**'
pull_request:
branches:
paths-ignore:
- '**.md'
- 'docs/**'
- 'mkdocs.yml'
- 'LICENSE'
- '.gitignore'
- '.beads/**'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
codspeed:
name: Codspeed Performance Regression Detection
runs-on: ubuntu-22.04 # CodSpeed only supports Ubuntu 20.04/22.04
timeout-minutes: 30
# Dependency bumps don't need the CodSpeed perf gate; skip for dependabot.
if: github.actor != 'dependabot[bot]'
steps:
- uses: actions/checkout@v6.0.3
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: "3.12"
- name: Install Rust
uses: dtolnay/rust-toolchain@stable
- name: Install uv
uses: astral-sh/setup-uv@v8.1.0
- name: Build mrrc wheel
uses: PyO3/maturin-action@v1
with:
manylinux: off
args: --release --out dist
- name: Install mrrc and dependencies
run: |
uv pip install --system --no-index --find-links dist mrrc
uv pip install --system pytest pytest-benchmark pytest-codspeed
- name: Run benchmarks with Codspeed (1k/10k only)
uses: CodSpeedHQ/action@v4.17.0
with:
# NOTE: Only enabled if CODSPEED_TOKEN is set as a repo secret
token: ${{ secrets.CODSPEED_TOKEN }}
# Simulation mode measures instruction counts under Valgrind:
# deterministic on standard runners, unlike walltime, which cannot
# distinguish real regressions from runner-speed variance.
mode: simulation
# Parallel-throughput benchmarks are excluded: Valgrind serializes
# threads, so multi-threaded speedup cannot be measured under
# simulation. Measure parallel throughput with real walltime runs
# on controlled hardware instead.
# Note: Don't use --benchmark-only here; CodSpeed handles benchmarks differently
run: >-
pytest tests/python/ -m "benchmark and not slow" -v
--ignore=tests/python/test_benchmark_parallel.py
--ignore=tests/python/test_benchmark_pipeline_parallel.py