ffbpe 0.1.9

Unicode-aware, streaming BPE training and tiktoken-compatible encoding
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144

name: CI

on:
  push:
    branches: [ master ]
  pull_request:
    branches: [ master ]

jobs:
  test:
    strategy:
      matrix:
        include:
          - os: ubuntu-latest
            python-version: '3.11'
          - os: ubuntu-latest
            python-version: '3.12'
          - os: ubuntu-latest
            python-version: '3.13'
          - os: macos-latest
            python-version: '3.13'
          # - os: windows-latest
          #   python-version: '3.13'
    runs-on: ${{ matrix.os }}

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Set up Rust
        uses: dtolnay/rust-toolchain@v1
        with:
          toolchain: stable

      - name: Cache cargo registry
        uses: actions/cache@v4
        with:
          path: |
            ~/.cargo/registry
            ~/.cargo/git
          key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.lock') }}

      - name: Build and run Rust tests
        run: cargo test --all --features cli --verbose

      - name: Run Rust quickstart
        run: cargo run --example quickstart

      - name: Test benchmark comment renderer
        if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.13'
        run: node --test .github/scripts/update-benchmark-comment.test.cjs

      - name: Test benchmark report contract
        if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.13'
        run: bash .github/scripts/validate-benchmark-reports.test.sh

      - name: Install uv and set the Python version
        uses: astral-sh/setup-uv@v7
        with:
          python-version: ${{ matrix.python-version }}
          enable-cache: true

      - name: Create venv
        run: |
          uv sync --all-extras --dev

      - name: Build
        run: uv build

      - name: Smoke test (wheel)
        run: uv run --isolated --no-project --with dist/ffbpe-*.whl test.py

      - name: Smoke test (source distribution)
        run: uv run --isolated --no-project --with dist/ffbpe-*.tar.gz test.py

      - name: Run Python quickstart
        run: uv run python examples/quickstart.py

  wasm-package:
    runs-on: ubuntu-latest

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Verify release metadata
        run: python3 .github/scripts/verify-release-versions.py

      - name: Set up Rust
        uses: dtolnay/rust-toolchain@v1
        with:
          toolchain: stable
          targets: wasm32-unknown-unknown

      - name: Set up Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 22

      - name: Install wasm-pack and pnpm
        run: |
          cargo install wasm-pack --locked --version 0.13.0
          npm install --global pnpm@10

      - name: Install npm package dependencies
        working-directory: packages/ffbpe
        run: pnpm install --frozen-lockfile

      - name: Build and test npm package
        working-directory: packages/ffbpe
        run: |
          pnpm build
          pnpm test
          npm pack --dry-run --ignore-scripts

      - name: Install inspect package dependencies
        working-directory: packages/ffbpe-inspect
        run: pnpm install --frozen-lockfile

      - name: Build and test inspect package
        working-directory: packages/ffbpe-inspect
        run: |
          pnpm build
          pnpm test
          npm pack --dry-run --ignore-scripts

      - name: Install preset package dependencies
        working-directory: packages/ffbpe-presets
        run: pnpm install --frozen-lockfile

      - name: Build and test preset package
        working-directory: packages/ffbpe-presets
        run: |
          pnpm build
          pnpm test
          npm pack --dry-run --ignore-scripts

      - name: Install inspect website dependencies
        working-directory: apps/ffbpe-inspect
        run: npm ci --ignore-scripts --no-audit --no-fund

      - name: Build and test inspect website
        working-directory: apps/ffbpe-inspect
        run: npm test