ffbpe 0.1.9

Unicode-aware, streaming BPE training and tiktoken-compatible encoding
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107

name: Site

on:
  push:
    branches: [ master ]
  pull_request:
    branches: [ master ]
  workflow_dispatch:

permissions:
  contents: read
  pages: write
  id-token: write

concurrency:
  group: pages
  cancel-in-progress: false

jobs:
  build:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Set up Rust
        uses: dtolnay/rust-toolchain@v1
        with:
          toolchain: stable
          targets: wasm32-unknown-unknown

      - name: Set up Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 22

      - name: Install wasm-pack and pnpm
        run: |
          cargo install wasm-pack --locked --version 0.13.0
          npm install --global pnpm@10

      - name: Build inspector packages
        run: |
          pnpm --dir packages/ffbpe install --frozen-lockfile
          pnpm --dir packages/ffbpe build
          pnpm --dir packages/ffbpe-inspect install --frozen-lockfile
          pnpm --dir packages/ffbpe-inspect build
          pnpm --dir packages/ffbpe-presets install --frozen-lockfile
          pnpm --dir packages/ffbpe-presets build

      - name: Build inspector website
        run: |
          npm --prefix apps/ffbpe-inspect ci --ignore-scripts --no-audit --no-fund
          npm --prefix apps/ffbpe-inspect run build:pages

      - name: Stage landing page
        run: |
          mkdir -p site
          cp -R landing/. site/
          mkdir -p site/inspect
          cp -R apps/ffbpe-inspect/dist-pages/. site/inspect/

      - name: Install uv
        uses: astral-sh/setup-uv@v7
        with:
          python-version: "3.13"
          enable-cache: true

      - name: Install documentation dependencies
        run: uv sync --group docs --no-install-project

      - name: Build documentation
        run: uv run --no-sync mkdocs build --strict

      - name: Validate site artifact
        run: |
          test -f site/index.html
          test -f site/assets/site-header.css
          test -f site/docs/index.html
          test -f site/inspect/index.html
          test -f site/inspect/models/cl100k_base.tiktoken
          test -n "$(find site/inspect/assets -name '*.wasm' -print -quit)"
          grep -Fq 'href="./inspect/"' site/index.html
          grep -Fq 'class="site-header"' site/index.html
          grep -Fq 'src="./assets/' site/inspect/index.html

      - name: Configure GitHub Pages
        if: github.event_name != 'pull_request'
        uses: actions/configure-pages@v5

      - name: Upload GitHub Pages artifact
        if: github.event_name != 'pull_request'
        uses: actions/upload-pages-artifact@v3
        with:
          path: site

  deploy:
    if: github.event_name != 'pull_request'
    environment:
      name: github-pages
      url: ${{ steps.deployment.outputs.page_url }}
    runs-on: ubuntu-latest
    needs: build
    steps:
      - name: Deploy to GitHub Pages
        id: deployment
        uses: actions/deploy-pages@v4