unitoken 0.1.5

Fast BPE tokenizer/trainer with a Rust core and Python bindings
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63

name: CI

on:
  push:
    branches: [ master ]
  pull_request:
    branches: [ master ]

jobs:
  test:
    strategy:
      matrix:
        include:
          - os: ubuntu-latest
            python-version: '3.11'
          - os: ubuntu-latest
            python-version: '3.12'
          - os: ubuntu-latest
            python-version: '3.13'
          - os: macos-latest
            python-version: '3.13'
          # - os: windows-latest
          #   python-version: '3.13'
    runs-on: ${{ matrix.os }}

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Set up Rust
        uses: dtolnay/rust-toolchain@v1
        with:
          toolchain: stable

      - name: Cache cargo registry
        uses: actions/cache@v4
        with:
          path: |
            ~/.cargo/registry
            ~/.cargo/git
          key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.lock') }}

      - name: Build and run Rust tests
        run: cargo test --all --verbose

      - name: Install uv and set the Python version
        uses: astral-sh/setup-uv@v7
        with:
          python-version: ${{ matrix.python-version }}
          enable-cache: true

      - name: Create venv
        run: |
          uv sync --all-extras --dev

      - name: Build
        run: uv build

      - name: Smoke test (wheel)
        run: uv run --isolated --no-project --with dist/uni_tokenizer-*.whl test.py

      - name: Smoke test (source distribution)
        run: uv run --isolated --no-project --with dist/uni_tokenizer-*.tar.gz test.py