name: Build and Release
on:
push:
tags:
- 'v*'
workflow_dispatch:
inputs:
tag:
description: 'Existing Git tag to build and publish as a release (e.g., v1.0.0)'
required: true
type: string
concurrency:
group: release-${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref_name }}
cancel-in-progress: false
env:
RUSTFLAGS: "-C target-cpu=generic"
APEXBASE_CI_CPU: "generic"
REGISTRY_USER_AGENT: "apexbase-release-workflow/1.0 (https://github.com/BirchKwok/ApexBase)"
jobs:
resolve-release:
runs-on: ubuntu-latest
outputs:
tag_name: ${{ steps.inspect.outputs.tag_name }}
package_kind: ${{ steps.inspect.outputs.package_kind }}
rust_features: ${{ steps.inspect.outputs.rust_features }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Inspect release tag
id: inspect
env:
RELEASE_TAG: ${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref_name }}
shell: bash
run: |
if [[ ! "$RELEASE_TAG" =~ ^v[0-9]+(\.[0-9]+)*([-.][0-9A-Za-z]+)*$ ]]; then
echo "Release tag must look like v0.0.1; got '$RELEASE_TAG'."
exit 1
fi
git rev-parse "refs/tags/$RELEASE_TAG^{commit}" >/dev/null
git checkout --detach "refs/tags/$RELEASE_TAG"
python - <<'PY'
import json
import os
import sys
import tomllib
from pathlib import Path
tag = os.environ["RELEASE_TAG"]
pyproject = Path("pyproject.toml")
if not pyproject.exists():
sys.exit("The release tag has no pyproject.toml.")
project = tomllib.loads(pyproject.read_text(encoding="utf-8"))
version = project.get("project", {}).get("version")
backend = project.get("build-system", {}).get("build-backend", "")
if version != tag.removeprefix("v"):
sys.exit(f"Package version {version!r} does not match tag {tag!r}.")
package_kind = "maturin" if "maturin" in backend else "legacy"
rust_features = [""]
cargo_file = Path("Cargo.toml")
if package_kind == "maturin" and cargo_file.exists():
cargo = tomllib.loads(cargo_file.read_text(encoding="utf-8"))
cargo_version = cargo.get("package", {}).get("version")
if cargo_version != version:
sys.exit(
f"Cargo package version {cargo_version!r} does not match "
f"Python package version {version!r}."
)
available_features = cargo.get("features", {})
rust_features.extend(
feature for feature in ("server", "flight")
if feature in available_features
)
with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output:
print(f"tag_name={tag}", file=output)
print(f"package_kind={package_kind}", file=output)
print(f"rust_features={json.dumps(rust_features)}", file=output)
PY
rust-test:
needs: resolve-release
if: needs.resolve-release.outputs.package_kind == 'maturin'
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-latest, windows-latest, macos-latest]
features: ${{ fromJSON(needs.resolve-release.outputs.rust_features) }}
rust: [stable]
fail-fast: false
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
ref: ${{ github.event.inputs.tag || github.ref }}
- name: Install Rust ${{ matrix.rust }}
uses: dtolnay/rust-toolchain@master
with:
toolchain: ${{ matrix.rust }}
- name: Rust cache
uses: Swatinem/rust-cache@v2
with:
key: ${{ matrix.os }}-${{ matrix.features }}-${{ env.APEXBASE_CI_CPU }}
- name: Install system dependencies (Linux)
if: runner.os == 'Linux'
run: |
sudo apt-get update
sudo apt-get install -y protobuf-compiler
- name: Install system dependencies (macOS)
if: runner.os == 'macOS'
run: |
brew install protobuf
- name: Install system dependencies (Windows)
if: runner.os == 'Windows'
run: |
choco install protoc -y
- name: Validate Rust (${{ matrix.features && format('{0} feature', matrix.features) || 'core only' }})
shell: bash
run: |
if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
# Historical release backfills must validate the publishable library,
# not recompile old test targets with today's Rust/dependency graph.
cargo check --lib --no-default-features ${{ matrix.features && format('--features {0}', matrix.features) || '' }} --release
else
cargo test --no-default-features ${{ matrix.features && format('--features {0}', matrix.features) || '' }} --release
fi
env:
RUST_BACKTRACE: 1
test:
needs: resolve-release
if: needs.resolve-release.outputs.package_kind == 'maturin'
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-latest, windows-latest, macos-latest]
python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
fail-fast: false
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
ref: ${{ github.event.inputs.tag || github.ref }}
- name: Install Rust stable
uses: dtolnay/rust-toolchain@stable
- name: Rust cache
uses: Swatinem/rust-cache@v2
with:
key: python-${{ matrix.os }}-${{ matrix.python-version }}-${{ env.APEXBASE_CI_CPU }}
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install system dependencies (Linux)
if: runner.os == 'Linux'
run: |
sudo apt-get update
sudo apt-get install -y protobuf-compiler
- name: Install system dependencies (macOS)
if: runner.os == 'macOS'
run: |
brew install protobuf
- name: Install system dependencies (Windows)
if: runner.os == 'Windows'
run: |
choco install protoc -y
- name: Install dependencies
shell: bash
run: |
python -m venv .venv
python - <<'PY'
import os
import pathlib
import subprocess
import sys
venv = pathlib.Path(".venv").resolve()
bin_dir = venv / ("Scripts" if os.name == "nt" else "bin")
python = bin_dir / ("python.exe" if os.name == "nt" else "python")
env = os.environ.copy()
env["VIRTUAL_ENV"] = str(venv)
env["PATH"] = str(bin_dir) + os.pathsep + env.get("PATH", "")
subprocess.check_call([str(python), "-m", "pip", "install", "--upgrade", "pip"])
# Keep the native-extension test stack pinned. The pytest suite loads
# ApexBase, NumPy, PyArrow, pandas, and Polars in the same process; using
# floating latest wheels has produced macOS native hangs during pytest.
# Newer NumPy/PyArrow/pandas/Polars releases have dropped older Python
# versions at different points, so pin by interpreter instead of using
# one native stack for the whole 3.9-3.13 matrix.
test_deps = [
"pytest==8.4.2",
"pytest-timeout==2.4.0",
"maturin==1.9.1",
]
if sys.version_info < (3, 10):
test_deps.extend([
"numpy==2.0.2",
"pyarrow==17.0.0",
"pandas==2.3.2",
"polars==1.36.1",
])
elif sys.version_info < (3, 11):
test_deps.extend([
"numpy==2.1.3",
"pyarrow==17.0.0",
"pandas==2.3.2",
"polars==1.39.3",
])
else:
test_deps.extend([
"numpy==2.1.3",
"pyarrow==23.0.1",
"pandas==3.0.1",
"polars==1.39.3",
])
print("Installing test dependencies:")
for dep in test_deps:
print(f" {dep}")
subprocess.check_call([str(python), "-m", "pip", "install", *test_deps])
subprocess.check_call([str(python), "-m", "maturin", "develop", "--release", "--skip-install"], env=env)
smoke = """
import sys
import tempfile
sys.path.insert(0, 'apexbase/python')
import apexbase
import apexbase._core as core
print(apexbase.__file__)
print(core.__file__)
with tempfile.TemporaryDirectory() as tmpdir:
client = apexbase.ApexClient(dirpath=tmpdir)
client.create_table('smoke')
client.close()
"""
subprocess.check_call([
str(python),
"-c",
smoke,
], env=env)
with open(os.environ["GITHUB_PATH"], "a", encoding="utf-8") as path_file:
path_file.write(str(bin_dir) + os.linesep)
with open(os.environ["GITHUB_ENV"], "a", encoding="utf-8") as env_file:
env_file.write(f"VIRTUAL_ENV={venv}{os.linesep}")
PY
- name: Run tests
timeout-minutes: 10
shell: bash
run: |
python - <<'PY'
import sys
from pathlib import Path
sys.path.insert(0, str(Path("apexbase/python").resolve()))
import apexbase
import apexbase._core as core
import numpy
import pandas
import polars
import pyarrow
import pytest
print(f"python={sys.version}")
print(f"apexbase={apexbase.__file__}")
print(f"apexbase_core={core.__file__}")
print(f"pytest={pytest.__version__}")
print(f"numpy={numpy.__version__}")
print(f"pyarrow={pyarrow.__version__}")
print(f"pandas={pandas.__version__}")
print(f"polars={polars.__version__}")
PY
python -X faulthandler -m pytest \
--ignore=test/test_memory_vs_duckdb.py \
--timeout=30 \
--durations=40
env:
PYTHONFAULTHANDLER: "1"
RUST_BACKTRACE: "1"
build-wheels-linux:
needs: [resolve-release, test, rust-test]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
ref: ${{ github.event.inputs.tag || github.ref }}
- name: Build wheels (manylinux)
uses: PyO3/maturin-action@v1
env:
RUSTFLAGS: ${{ env.RUSTFLAGS }}
with:
maturin-version: v1.9.1
command: build
args: --release --out dist --interpreter /opt/python/cp39-cp39/bin/python /opt/python/cp310-cp310/bin/python /opt/python/cp311-cp311/bin/python /opt/python/cp312-cp312/bin/python /opt/python/cp313-cp313/bin/python
manylinux: 2014
- name: Upload wheel artifacts
uses: actions/upload-artifact@v4
with:
name: wheels-manylinux
path: dist/*.whl
build-wheels:
needs: [resolve-release, test, rust-test]
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [windows-latest, macos-latest]
python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
fail-fast: false
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
ref: ${{ github.event.inputs.tag || github.ref }}
- name: Set up Python ${{ matrix.python-version }}
id: py
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Show Python
run: |
python -c "import sys; print(sys.executable); print(sys.version)"
- name: Build wheels
uses: PyO3/maturin-action@v1
env:
RUSTFLAGS: ${{ env.RUSTFLAGS }}
PYO3_PYTHON: ${{ steps.py.outputs.python-path }}
PYTHON_SYS_EXECUTABLE: ${{ steps.py.outputs.python-path }}
with:
maturin-version: v1.9.1
command: build
args: --release --out dist --interpreter ${{ steps.py.outputs.python-path }}
- name: Smoke test wheel (ApexStorage init)
if: matrix.os == 'macos-latest' && matrix.python-version == '3.12'
shell: bash
run: |
python -m pip install --upgrade pip
python -m pip install "$(ls dist/*.whl | head -1)"
python - <<'PY'
import sys
import tempfile
import os
# Import path used by repro scripts / minimal integrations.
from apexbase._core import ApexStorage
db_dir = tempfile.mkdtemp(prefix="apexbase_wheel_smoke_")
ApexStorage(os.path.join(db_dir, "apexbase.apex"))
print("wheel smoke test OK", sys.version.split()[0])
PY
- name: Upload wheel artifacts
uses: actions/upload-artifact@v4
with:
name: wheels-${{ matrix.os }}-${{ matrix.python-version }}
path: dist/*.whl
build-sdist:
needs: [resolve-release, test, rust-test]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
ref: ${{ github.event.inputs.tag || github.ref }}
- name: Build sdist
uses: PyO3/maturin-action@v1
with:
maturin-version: v1.9.1
command: sdist
args: --out dist
- name: Upload sdist artifact
uses: actions/upload-artifact@v4
with:
name: sdist
path: dist/*.tar.gz
publish-packages:
needs: [resolve-release, test, rust-test, build-wheels-linux, build-wheels, build-sdist]
if: needs.resolve-release.outputs.package_kind == 'maturin'
runs-on: ubuntu-latest
env:
RELEASE_TAG: ${{ needs.resolve-release.outputs.tag_name }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
ref: ${{ needs.resolve-release.outputs.tag_name }}
- name: Install Rust stable
uses: dtolnay/rust-toolchain@stable
- name: Rust cache
uses: Swatinem/rust-cache@v2
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: Install twine
run: |
python -m pip install --upgrade pip
python -m pip install twine
- name: Download artifacts
uses: actions/download-artifact@v4
with:
path: dist
- name: Validate Python distributions
shell: bash
run: |
set -euo pipefail
mapfile -d '' distributions < <(
find dist -type f \( -name '*.whl' -o -name '*.tar.gz' \) -print0
)
if [[ "${#distributions[@]}" -eq 0 ]]; then
echo "No Python distributions were downloaded."
exit 1
fi
python -m twine check "${distributions[@]}"
- name: Check registry state
id: registry-state
shell: bash
run: |
set -euo pipefail
VERSION="${RELEASE_TAG#v}"
registry_status() {
local registry="$1"
local url="$2"
local status
status="$(
curl --silent --show-error \
--user-agent "$REGISTRY_USER_AGENT" \
--output /dev/null \
--write-out '%{http_code}' \
"$url"
)"
case "$status" in
200)
echo "${registry}_exists=true" >> "$GITHUB_OUTPUT"
;;
404)
echo "${registry}_exists=false" >> "$GITHUB_OUTPUT"
;;
*)
echo "Unexpected HTTP ${status} while checking ${registry}."
exit 1
;;
esac
}
registry_status crate "https://crates.io/api/v1/crates/apexbase/${VERSION}"
registry_status pypi "https://pypi.org/pypi/apexbase/${VERSION}/json"
- name: Validate crate package
if: steps.registry-state.outputs.crate_exists != 'true'
run: cargo publish --dry-run --no-default-features --allow-dirty
- name: Publish missing packages together
shell: bash
env:
TWINE_USERNAME: __token__
TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
CRATE_EXISTS: ${{ steps.registry-state.outputs.crate_exists }}
PYPI_EXISTS: ${{ steps.registry-state.outputs.pypi_exists }}
run: |
set -euo pipefail
cargo_pid=""
pypi_pid=""
if [[ "$CRATE_EXISTS" != "true" ]]; then
cargo publish --no-default-features --allow-dirty --no-verify &
cargo_pid="$!"
else
echo "The crate version is already published; no crates.io upload is needed."
fi
if [[ "$PYPI_EXISTS" != "true" ]]; then
(
mapfile -d '' distributions < <(
find dist -type f \( -name '*.whl' -o -name '*.tar.gz' \) -print0
)
python -m twine upload --verbose "${distributions[@]}"
) &
pypi_pid="$!"
else
echo "The Python package version is already published; no PyPI upload is needed."
fi
cargo_status=0
pypi_status=0
if [[ -n "$cargo_pid" ]]; then
if wait "$cargo_pid"; then
echo "crates.io upload completed."
else
cargo_status="$?"
fi
fi
if [[ -n "$pypi_pid" ]]; then
if wait "$pypi_pid"; then
echo "PyPI upload completed."
else
pypi_status="$?"
fi
fi
if [[ "$cargo_status" -ne 0 || "$pypi_status" -ne 0 ]]; then
echo "Package publication failed: crates.io=${cargo_status}, PyPI=${pypi_status}."
echo "Rerun this workflow for the same tag to publish whichever side is missing."
exit 1
fi
- name: Confirm both registries contain the version
shell: bash
run: |
set -euo pipefail
VERSION="${RELEASE_TAG#v}"
for attempt in {1..12}; do
crate_status="$(
curl --silent --show-error \
--user-agent "$REGISTRY_USER_AGENT" \
--output /dev/null \
--write-out '%{http_code}' \
"https://crates.io/api/v1/crates/apexbase/${VERSION}" || true
)"
pypi_status="$(
curl --silent --show-error \
--user-agent "$REGISTRY_USER_AGENT" \
--output /dev/null \
--write-out '%{http_code}' \
"https://pypi.org/pypi/apexbase/${VERSION}/json" || true
)"
if [[ "$crate_status" == "200" && "$pypi_status" == "200" ]]; then
echo "apexbase ${VERSION} is available on both crates.io and PyPI."
exit 0
fi
echo "Registry confirmation ${attempt}/12: crates.io=${crate_status}, PyPI=${pypi_status}."
sleep 10
done
echo "The version was not confirmed on both registries."
exit 1
build-legacy:
needs: resolve-release
if: needs.resolve-release.outputs.package_kind == 'legacy'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
ref: ${{ needs.resolve-release.outputs.tag_name }}
- uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: Build pure-Python wheel and source distribution
run: |
python -m pip install --upgrade pip build
python -m build --wheel --sdist --outdir dist
- name: Upload legacy distributions
uses: actions/upload-artifact@v4
with:
name: legacy-distributions
path: |
dist/*.whl
dist/*.tar.gz
if-no-files-found: error
publish-legacy:
needs: [resolve-release, build-legacy]
if: needs.resolve-release.outputs.package_kind == 'legacy'
runs-on: ubuntu-latest
steps:
- uses: actions/download-artifact@v4
with:
name: legacy-distributions
path: dist
- name: Publish legacy package to PyPI
uses: pypa/gh-action-pypi-publish@release/v1
with:
packages-dir: dist
password: ${{ secrets.PYPI_API_TOKEN }}
skip-existing: true
create-release:
needs: [resolve-release, publish-packages, publish-legacy]
if: >-
always() &&
needs.resolve-release.result == 'success' &&
(
(
needs.resolve-release.outputs.package_kind == 'maturin' &&
needs.publish-packages.result == 'success' &&
needs.publish-legacy.result == 'skipped'
) ||
(
needs.resolve-release.outputs.package_kind == 'legacy' &&
needs.publish-packages.result == 'skipped' &&
needs.publish-legacy.result == 'success'
)
)
runs-on: ubuntu-latest
permissions:
contents: write
env:
RELEASE_TAG: ${{ needs.resolve-release.outputs.tag_name }}
steps:
- name: Checkout release notes from the default branch
uses: actions/checkout@v4
with:
ref: ${{ github.event.repository.default_branch }}
- name: Download all artifacts
uses: actions/download-artifact@v4
with:
path: all-artifacts
- name: Prepare release assets
run: |
mkdir -p release-assets
find all-artifacts -type f \( -name "*.whl" -o -name "*.tar.gz" \) -exec cp {} release-assets/ \;
echo "Release assets prepared:"
ls -lh release-assets/
- name: Extract release notes for tag
env:
RELEASE_NOTES_FILE: docs/release-notes.md
run: |
python - <<'PY'
import os
import re
import sys
from pathlib import Path
tag = os.environ["RELEASE_TAG"]
source = Path(os.environ["RELEASE_NOTES_FILE"])
if not source.exists():
sys.exit(f"Release notes file not found: {source}")
text = source.read_text(encoding="utf-8")
heading = re.compile(
rf"^##\s+(?:\[{re.escape(tag)}\]\([^)]+\)|{re.escape(tag)})\s*$",
re.MULTILINE,
)
match = heading.search(text)
if not match:
sys.exit(f"No release notes section found for {tag} in {source}.")
next_heading = re.search(r"^##\s+", text[match.end():], re.MULTILINE)
end = match.end() + next_heading.start() if next_heading else len(text)
body = text[match.end():end].strip()
body = re.sub(r"\n---\s*$", "", body).strip()
if not body:
sys.exit(f"Release notes section for {tag} is empty.")
Path("release-body.md").write_text(body + "\n", encoding="utf-8")
print(f"Extracted release notes for {tag} from {source}")
PY
- name: Create GitHub Release with assets
uses: softprops/action-gh-release@v2
with:
tag_name: ${{ env.RELEASE_TAG }}
name: ApexBase ${{ env.RELEASE_TAG }}
body_path: release-body.md
files: release-assets/*
fail_on_unmatched_files: true
draft: false
prerelease: ${{ contains(env.RELEASE_TAG, 'alpha') || contains(env.RELEASE_TAG, 'beta') || contains(env.RELEASE_TAG, 'rc') }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}