triviumdb 0.7.0

A high-performance memory-mmap hybrid search engine built for AI, combining dense vector, sparse text, graph relations, and JSON metadata.
Documentation
# TriviumDB CI/CD 管线
#
# 包含:
#   1. 编译检查(stable,x86_64 + aarch64 交叉编译)
#   2. 全量测试(跨平台 stable,含 ARM64 原生)
#   3. 常规 CI 短时 fuzz(PR 级快速模糊测试)
#   4. AddressSanitizer 内存安全检测(nightly, Linux)
#   5. 测试覆盖率报告与 80% 覆盖率门禁
#   6. 代码格式检查 & Clippy 静态分析
#   7. 突变测试(手动触发,发布前质量门禁)
#   8. 基准测试(手动触发)
#
# 注意:索引相关测试(QuIVer/BQ 构图)已从 CI 中移除。
# 原因:并发构图具有平台相关的非确定性(依赖 rayon 线程调度),
# 且 CI runner 缺少 AVX-512 等特定硬件支持。
# 索引测试由本地 bench_cohere1m / bench_quiver_ablation 覆盖。

name: CI

on:
  push:
    branches: [master, dev]
  pull_request:
    branches: [master]
  workflow_dispatch:
    inputs:
      run_mutation:
        description: '是否运行突变测试质量门禁'
        required: false
        default: 'false'
        type: choice
        options:
          - 'false'
          - 'true'
      mutation_files:
        description: '突变测试目标文件,使用空格分隔'
        required: false
        default: 'src/filter.rs src/vector.rs src/storage/wal.rs'
      mutation_timeout:
        description: '单个变异体超时时间(秒)'
        required: false
        default: '60'

env:
  CARGO_TERM_COLOR: always

jobs:
  # ═══════════════════════════════════════════
  #  Job 1: 编译检查(快速门禁)
  # ═══════════════════════════════════════════
  check:
    name: Compile Check
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - uses: dtolnay/rust-toolchain@stable
        with:
          targets: aarch64-unknown-linux-gnu

      - uses: Swatinem/rust-cache@v2

      - name: 编译检查 (x86_64 默认特性)
        run: cargo check --all-targets

      - name: 交叉编译检查 (aarch64)
        run: cargo check --target aarch64-unknown-linux-gnu --lib

  # ═══════════════════════════════════════════
  #  Job 2: 全量测试(跨平台 + ARM64 原生)
  # ═══════════════════════════════════════════
  test:
    name: Test (${{ matrix.name }})
    runs-on: ${{ matrix.runner }}
    needs: check
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: Linux x86_64
            runner: ubuntu-latest
          - name: Windows x86_64
            runner: windows-latest
          - name: macOS x86_64
            runner: macos-latest  # Apple Silicon 交叉编译 x86_64
          - name: macOS ARM64 (M-series)
            runner: macos-latest    # macos-latest = Apple Silicon (M1+)

    steps:
      - uses: actions/checkout@v4

      - uses: dtolnay/rust-toolchain@stable

      - uses: Swatinem/rust-cache@v2

      - name: 运行全量测试(跳过索引相关测试)
        run: |

          cargo test --lib --tests -- --test-threads=1 \
            --skip DET_03 \
            --skip P1_5_BQ \
            --skip bq_ \
            --skip quiver_

      - name: 运行文档测试
        run: cargo test --doc

  # ═══════════════════════════════════════════
  #  Job 2b: ARM64 Linux 原生测试 (QEMU)
  # ═══════════════════════════════════════════
  test-arm64-linux:
    name: Test (Linux ARM64 via QEMU)
    runs-on: ubuntu-latest
    needs: check

    steps:
      - uses: actions/checkout@v4

      - uses: dtolnay/rust-toolchain@stable
        with:
          targets: aarch64-unknown-linux-gnu

      - uses: Swatinem/rust-cache@v2
        with:
          key: arm64-linux

      - name: 安装交叉编译工具链
        run: |

          sudo apt-get update
          sudo apt-get install -y gcc-aarch64-linux-gnu qemu-user

      - name: 设置 Cargo 链接器
        run: |

          mkdir -p .cargo
          cat >> .cargo/config.toml << 'EOF'
          [target.aarch64-unknown-linux-gnu]
          linker = "aarch64-linux-gnu-gcc"
          runner = "qemu-aarch64 -L /usr/aarch64-linux-gnu"
          EOF

      - name: 构建测试 (aarch64)
        run: cargo test --target aarch64-unknown-linux-gnu --lib --tests --no-run

      - name: 运行测试 (aarch64 via QEMU, 跳过索引和硬件测试)
        run: |

          cargo test --target aarch64-unknown-linux-gnu --lib --tests -- \
            --test-threads=1 \
            --skip HW \
            --skip INTRUDE \
            --skip DET_03 \
            --skip P1_5_BQ \
            --skip bq_ \
            --skip quiver_

  # ═══════════════════════════════════════════
  #  Job 3: 常规 CI 短时 fuzz
  #
  #  每个 PR/push 都运行短时 libFuzzer 冒烟测试,快速捕获 parser/WAL
  #  在随机输入下的 panic、OOM、超时和基本内存安全问题。长时 fuzz
  #  仍由 .github/workflows/fuzz.yml 定时运行。
  # ═══════════════════════════════════════════
  short-fuzz:
    name: Short Fuzz (${{ matrix.target }})
    runs-on: ubuntu-latest
    needs: check
    strategy:
      fail-fast: false
      matrix:
        target:
          - fuzz_wal_parse
          - fuzz_query_parse
          - fuzz_filter_parse

    steps:
      - uses: actions/checkout@v4

      - uses: dtolnay/rust-toolchain@nightly

      - uses: Swatinem/rust-cache@v2
        with:
          key: short-fuzz-${{ matrix.target }}

      - name: 安装 cargo-fuzz
        run: cargo install cargo-fuzz --locked

      - name: 运行短时模糊测试 (${{ matrix.target }})
        run: |

          cargo +nightly fuzz run ${{ matrix.target }} \
            -- \
            -max_total_time=30 \
            -max_len=65536 \
            -timeout=10 \
            -rss_limit_mb=2048

      - name: 上传崩溃用例
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: short-fuzz-crash-${{ matrix.target }}
          path: fuzz/artifacts/${{ matrix.target }}/

  # ═══════════════════════════════════════════
  #  Job 4: AddressSanitizer 内存安全检测
  #
  #  使用 nightly + -Z sanitizer=address 检测:
  #  - 堆缓冲区溢出 (heap-buffer-overflow)
  #  - 释放后使用 (use-after-free)
  #  - 栈缓冲区溢出 (stack-buffer-overflow)
  #  - 内存泄漏 (memory leak)
  #
  #  主要覆盖 TriviumDB 中的 unsafe 区域:
  #  mmap 零拷贝、SIMD 指令、bytemuck 转换
  # ═══════════════════════════════════════════
  asan:
    name: AddressSanitizer
    runs-on: ubuntu-latest
    needs: check

    steps:
      - uses: actions/checkout@v4

      - uses: dtolnay/rust-toolchain@nightly
        with:
          components: rust-src  # ASan 需要从源码编译 std

      - uses: Swatinem/rust-cache@v2
        with:
          key: asan

      - name: 运行测试 (ASan 模式)
        env:
          RUSTFLAGS: "-Z sanitizer=address"
          # ASan 运行时选项:检测泄漏 + 首次错误即停止
          ASAN_OPTIONS: "detect_leaks=1:halt_on_error=1"
        run: |

          cargo +nightly test --lib --tests \
            -Z build-std \
            --target x86_64-unknown-linux-gnu \
            -- --test-threads=1 \
            --skip DET_03 \
            --skip P1_5_BQ \
            --skip bq_ \
            --skip quiver_

  # ═══════════════════════════════════════════
  #  Job 5: 测试覆盖率报告与门禁
  #
  #  生成 lcov、HTML 与文本摘要,并用行覆盖率 80% 作为 CI 门禁。
  # ═══════════════════════════════════════════
  coverage:
    name: Coverage Gate
    runs-on: ubuntu-latest
    needs: check

    steps:
      - uses: actions/checkout@v4

      - uses: dtolnay/rust-toolchain@stable
        with:
          components: llvm-tools-preview

      - uses: Swatinem/rust-cache@v2
        with:
          key: coverage

      - name: 安装 cargo-llvm-cov
        uses: taiki-e/install-action@cargo-llvm-cov

      - name: 生成覆盖率报告并执行 80% 门禁
        run: |

          cargo llvm-cov clean --workspace
          # --lib --tests: 只收集库代码和测试的覆盖率,排除 bench target
          # (bench_500k 等重型 benchmark 会导致 CI 超时且不贡献库覆盖率)
          # 跳过索引测试(并发构图依赖平台特定硬件)
          cargo llvm-cov --workspace --lib --tests --lcov --output-path lcov.info --fail-under-lines 80 -- --test-threads=1 \
            --skip DET_03 \
            --skip P1_5_BQ \
            --skip bq_ \
            --skip quiver_
          cargo llvm-cov report --html --output-dir target/llvm-cov/html
          cargo llvm-cov report

      - name: 上传覆盖率报告
        uses: actions/upload-artifact@v4
        with:
          name: coverage-report
          path: |

            lcov.info
            target/llvm-cov/html/

  # ═══════════════════════════════════════════
  #  Job 6: 代码质量检查
  # ═══════════════════════════════════════════
  lint:
    name: Lint & Format
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - uses: dtolnay/rust-toolchain@stable
        with:
          components: rustfmt, clippy

      - uses: Swatinem/rust-cache@v2

      - name: 格式检查 (rustfmt)
        shell: bash
        run: |

          if ! cargo fmt --all -- --check; then
            echo "::warning::rustfmt 检查发现格式差异,请在本地运行 cargo fmt --all 修复。"
            exit 0
          fi

      - name: 静态分析 (Clippy)
        run: cargo clippy --all-targets -- -D warnings

  # ═══════════════════════════════════════════
  #  Job 7: 突变测试(手动触发)
  #
  #  cargo-mutants 成本较高,不放入每个 PR 的必跑路径。发布前可通过
  #  workflow_dispatch 开启 run_mutation=true,对高风险模块进行质量门禁。
  # ═══════════════════════════════════════════
  mutation:
    name: Mutation Testing (Manual)
    runs-on: ubuntu-latest
    needs: check
    if: github.event_name == 'workflow_dispatch' && github.event.inputs.run_mutation == 'true'

    steps:
      - uses: actions/checkout@v4

      - uses: dtolnay/rust-toolchain@stable

      - uses: Swatinem/rust-cache@v2
        with:
          key: mutation

      - name: 安装 cargo-mutants
        run: cargo install cargo-mutants --locked

      - name: 运行突变测试质量门禁
        shell: bash
        run: |

          FILES="${{ github.event.inputs.mutation_files }}"
          TIMEOUT="${{ github.event.inputs.mutation_timeout }}"
          for file in ${FILES}; do
            echo "正在对 ${file} 运行突变测试,单个变异体超时 ${TIMEOUT}s"
            cargo mutants --file "${file}" --timeout "${TIMEOUT}"
          done

  # ═══════════════════════════════════════════
  #  Job 8: 基准测试(仅手动触发)
  # ═══════════════════════════════════════════
  bench:
    name: Benchmark (Manual)
    runs-on: ubuntu-latest
    if: github.event_name == 'workflow_dispatch'

    steps:
      - uses: actions/checkout@v4

      - uses: dtolnay/rust-toolchain@stable

      - uses: Swatinem/rust-cache@v2

      - name: 运行基准测试
        run: cargo bench --workspace

      - name: 上传基准测试结果
        uses: actions/upload-artifact@v4
        with:
          name: benchmark-results
          path: target/criterion/