name: Nightly
on:
schedule:
- cron: '0 3 * * *' workflow_dispatch:
inputs:
suite:
description: 'Which suite to run'
required: false
default: 'auto'
type: choice
options:
- auto - all - mutation - miri - stress - sanitizers
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
env:
CARGO_TERM_COLOR: always
RUST_BACKTRACE: 1
CARGO_INCREMENTAL: 0
jobs:
heavy_plan:
name: Select Heavy Suite
runs-on: ubuntu-latest
outputs:
run_mutation: ${{ steps.select.outputs.run_mutation }}
run_miri: ${{ steps.select.outputs.run_miri }}
mutation_shard: ${{ steps.select.outputs.mutation_shard }}
miri_group: ${{ steps.select.outputs.miri_group }}
steps:
- id: select
env:
EVENT_NAME: ${{ github.event_name }}
SUITE_INPUT: ${{ github.event.inputs.suite }}
run: |
suite="${SUITE_INPUT:-auto}"
run_mutation=false
run_miri=false
epoch_day=$(( $(date -u +%s) / 86400 ))
mutation_shard=$(( epoch_day % 24 ))
miri_group=$(( epoch_day % 5 ))
if [ "$EVENT_NAME" = "schedule" ] || [ "$suite" = "auto" ]; then
# Both run every night with rotating shards/groups
run_mutation=true
run_miri=true
else
case "$suite" in
all) run_mutation=true; run_miri=true ;;
mutation) run_mutation=true ;;
miri) run_miri=true ;;
esac
fi
echo "run_mutation=$run_mutation" >> "$GITHUB_OUTPUT"
echo "run_miri=$run_miri" >> "$GITHUB_OUTPUT"
echo "mutation_shard=$mutation_shard" >> "$GITHUB_OUTPUT"
echo "miri_group=$miri_group" >> "$GITHUB_OUTPUT"
stress-tests:
name: Stress Tests
if: >-
github.event_name == 'schedule'
|| github.event.inputs.suite == 'auto'
|| github.event.inputs.suite == 'all'
|| github.event.inputs.suite == 'stress'
runs-on: ubuntu-latest
steps:
- name: Free disk space
run: |
sudo rm -rf /usr/share/dotnet
sudo rm -rf /usr/local/lib/android
sudo rm -rf /opt/ghc
sudo rm -rf /opt/hostedtoolcache/CodeQL
sudo docker image prune --all --force
- uses: actions/checkout@v4
- name: Install Rust
uses: dtolnay/rust-toolchain@stable
- name: Cache
uses: Swatinem/rust-cache@v2
- name: Crash soak tests
run: cargo test --test crash_soak_test --features stress-tests
timeout-minutes: 30
- name: Metamorphic property tests
run: cargo test --test metamorphic_test --features stress-tests
timeout-minutes: 15
- name: Concurrency history tests
run: cargo test --test concurrency_history_test --features stress-tests
timeout-minutes: 15
thread-sanitizer:
name: Thread Sanitizer
if: >-
github.event_name == 'schedule'
|| github.event.inputs.suite == 'auto'
|| github.event.inputs.suite == 'all'
|| github.event.inputs.suite == 'sanitizers'
runs-on: ubuntu-latest
steps:
- name: Free disk space
run: |
sudo rm -rf /usr/share/dotnet
sudo rm -rf /usr/local/lib/android
sudo rm -rf /opt/ghc
sudo rm -rf /opt/hostedtoolcache/CodeQL
sudo docker image prune --all --force
- uses: actions/checkout@v4
- name: Install Rust nightly
uses: dtolnay/rust-toolchain@nightly
with:
components: rust-src
- name: Cache
uses: Swatinem/rust-cache@v2
- name: Run with ThreadSanitizer
run: |
echo 'race:std::alloc::System' > /tmp/tsan_suppressions.txt
RUSTFLAGS="-Zsanitizer=thread" \
cargo +nightly test -Zbuild-std --target x86_64-unknown-linux-gnu \
--test dirty_read_test \
--test mvcc_isolation_sql_test \
--test parallel_execution_tests
timeout-minutes: 20
env:
TSAN_OPTIONS: "second_deadlock_stack=1 suppressions=/tmp/tsan_suppressions.txt"
address-sanitizer:
name: Address Sanitizer
if: >-
github.event_name == 'schedule'
|| github.event.inputs.suite == 'auto'
|| github.event.inputs.suite == 'all'
|| github.event.inputs.suite == 'sanitizers'
runs-on: ubuntu-latest
steps:
- name: Free disk space
run: |
sudo rm -rf /usr/share/dotnet
sudo rm -rf /usr/local/lib/android
sudo rm -rf /opt/ghc
sudo rm -rf /opt/hostedtoolcache/CodeQL
sudo docker image prune --all --force
- uses: actions/checkout@v4
- name: Install Rust nightly
uses: dtolnay/rust-toolchain@nightly
with:
components: rust-src
- name: Cache
uses: Swatinem/rust-cache@v2
- name: Run with AddressSanitizer
run: |
RUSTFLAGS="-Zsanitizer=address" \
cargo +nightly test -Zbuild-std --target x86_64-unknown-linux-gnu \
--test durability_test \
--test snapshot_recovery_test \
--test persistence_test
timeout-minutes: 20
env:
ASAN_OPTIONS: "detect_leaks=0"
mutation:
name: Mutation Testing (budgeted shard)
needs: heavy_plan
if: needs.heavy_plan.outputs.run_mutation == 'true'
runs-on: ubuntu-latest
continue-on-error: true
steps:
- name: Free disk space
run: |
sudo rm -rf /usr/share/dotnet
sudo rm -rf /usr/local/lib/android
sudo rm -rf /opt/ghc
sudo rm -rf /opt/hostedtoolcache/CodeQL
sudo docker image prune --all --force
- uses: actions/checkout@v4
- name: Install Rust
uses: dtolnay/rust-toolchain@stable
- name: Cache
uses: Swatinem/rust-cache@v2
- name: Install cargo-mutants and nextest
uses: taiki-e/install-action@v2
with:
tool: cargo-mutants,cargo-nextest
- name: Run mutation testing (rotating shard)
run: |
SHARD_INDEX=${{ needs.heavy_plan.outputs.mutation_shard }}
TOTAL_SHARDS=24
echo "Running shard ${SHARD_INDEX}/${TOTAL_SHARDS}"
cargo mutants --package stoolap \
--in-place \
--baseline=skip \
--timeout 420 \
--shard "${SHARD_INDEX}/${TOTAL_SHARDS}" \
-f src/executor/expression/ \
-f src/executor/aggregation.rs \
-f src/executor/subquery.rs \
-f src/functions/scalar/ \
-f src/storage/mvcc/transaction.rs \
-f src/storage/mvcc/registry.rs \
-- --lib --tests
timeout-minutes: 100
- name: Upload mutation report
if: always()
uses: actions/upload-artifact@v4
with:
name: mutation-report
path: mutants.out/
miri:
name: Miri
needs: heavy_plan
if: needs.heavy_plan.outputs.run_miri == 'true'
runs-on: ubuntu-latest
timeout-minutes: 100
steps:
- name: Free disk space
run: |
sudo rm -rf /usr/share/dotnet
sudo rm -rf /usr/local/lib/android
sudo rm -rf /opt/ghc
sudo rm -rf /opt/hostedtoolcache/CodeQL
sudo docker image prune --all --force
- uses: actions/checkout@v4
- name: Install Rust nightly with Miri
uses: dtolnay/rust-toolchain@nightly
with:
components: miri
- name: Install nextest
uses: taiki-e/install-action@v2
with:
tool: cargo-nextest
- name: Cache
uses: Swatinem/rust-cache@v2
- name: Run Miri (rotating module group)
run: |
GROUP=${{ needs.heavy_plan.outputs.miri_group }}
echo "Miri group: ${GROUP}/5"
# Rotate through 5 module groups (full cycle every 5 days):
# 0: compact_arc + compact_vec + buffer_pool (fast, unsafe data structures)
# 1: cow_hashmap + i64_map (unsafe, recent Stacked Borrows fixes)
# 2: cow_btree (excluding deep-tree tests) (unsafe, moderate speed)
# 3: core:: (value, row, schema, error) (many tests, moderate speed)
# 4: cow_btree deep-tree tests only (slowest tests, ~5-10 min each)
case "$GROUP" in
0) FILTER='test(/^common::(compact_arc|compact_vec|buffer_pool)::/)' ;;
1) FILTER='test(/^common::(cow_hashmap|i64_map)::/)' ;;
2) FILTER='test(/^common::cow_btree::/) - test(/deep_tree/)' ;;
3) FILTER='test(/^core::/)' ;;
4) FILTER='test(/^common::cow_btree::.*deep_tree/)' ;;
esac
echo "Filter: $FILTER"
cargo +nightly miri nextest run --lib -E "$FILTER" --no-fail-fast
env:
MIRIFLAGS: "-Zmiri-disable-isolation"