unitoken 0.1.5

Fast BPE tokenizer/trainer with a Rust core and Python bindings
Documentation
name: Build and Publish Wheels

on:
  push:
    branches: [ master ]
    tags: ['v*']

permissions:
  contents: read

jobs:
  build-wheels:
    name: Build wheels (${{ matrix.os }})
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Set up Rust
        uses: dtolnay/rust-toolchain@v1
        with:
          toolchain: stable

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Build wheels
        uses: PyO3/maturin-action@v1
        with:
          command: build
          args: --release --features py --out dist --strip
          manylinux: auto

      - name: Smoke test (install wheel)
        run: |
          python -m pip install --upgrade pip
          python -c "import glob,sys,subprocess; ws=glob.glob('dist/*.whl'); assert ws, 'no wheel found in dist/'; subprocess.check_call([sys.executable,'-m','pip','install',ws[0]])"

      - name: Smoke test (import)
        run: |
          python -c "import uni_tokenizer; from uni_tokenizer import BpeEncoder, BpeTrainer, PreTokenizer; print('import ok')"
          python ./test.py

      - name: Upload wheels
        uses: actions/upload-artifact@v4
        with:
          name: dist-wheels-${{ matrix.os }}
          path: dist

  build-sdist:
    name: Build sdist
    runs-on: ubuntu-latest

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Set up Rust
        uses: dtolnay/rust-toolchain@v1
        with:
          toolchain: stable

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Build sdist
        uses: PyO3/maturin-action@v1
        with:
          command: sdist
          args: --out dist

      - name: Upload sdist
        uses: actions/upload-artifact@v4
        with:
          name: dist-sdist
          path: dist

  publish:
    name: Publish
    needs: [build-wheels, build-sdist]
    runs-on: ubuntu-latest
    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')

    steps:
      - name: Download artifacts
        uses: actions/download-artifact@v4
        with:
          path: dist
          merge-multiple: true

      - name: Publish to PyPI
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          password: ${{ secrets.PYPI_API_TOKEN }}
          packages-dir: dist