ffbpe 0.1.8

Unicode-aware, streaming BPE training and tiktoken-compatible encoding
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115

name: Benchmark

on:
  pull_request:
    branches: [ master ]
    types: [ opened, reopened, synchronize ]

permissions:
  contents: read

concurrency:
  group: benchmark-pr-${{ github.event.pull_request.number }}
  cancel-in-progress: true

jobs:
  benchmark:
    runs-on: ubuntu-latest
    timeout-minutes: 30

    steps:
      - name: Checkout base revision
        uses: actions/checkout@v4
        with:
          path: baseline
          ref: ${{ github.event.pull_request.base.sha }}

      - name: Checkout PR revision
        uses: actions/checkout@v4
        with:
          path: candidate
          ref: ${{ github.event.pull_request.head.sha }}

      - name: Set up Rust
        uses: dtolnay/rust-toolchain@v1
        with:
          toolchain: stable

      - name: Cache cargo registry
        uses: actions/cache@v4
        with:
          path: |
            ~/.cargo/registry
            ~/.cargo/git
          key: ${{ runner.os }}-cargo-registry-${{ hashFiles('baseline/Cargo.lock', 'candidate/Cargo.lock') }}

      - name: Run base benchmarks
        id: baseline
        continue-on-error: true
        run: |
          bash candidate/.github/scripts/run-benchmark-suite.sh \
            "$GITHUB_WORKSPACE/baseline" \
            "$GITHUB_WORKSPACE/benchmark-results/baseline" \
            "$GITHUB_WORKSPACE/baseline/benches/regression/config/smoke.yml"

      - name: Run PR benchmarks
        id: candidate
        if: always()
        continue-on-error: true
        run: |
          bash candidate/.github/scripts/run-benchmark-suite.sh \
            "$GITHUB_WORKSPACE/candidate" \
            "$GITHUB_WORKSPACE/benchmark-results/candidate" \
            "$GITHUB_WORKSPACE/candidate/benches/regression/config/smoke.yml"

      - name: Record benchmark revisions
        if: always()
        env:
          BASE_SHA: ${{ github.event.pull_request.base.sha }}
          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
          PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }}
        run: |
          mkdir -p benchmark-results
          jq -n \
            --arg base_sha "$BASE_SHA" \
            --arg head_sha "$HEAD_SHA" \
            --argjson pull_request_number "$PULL_REQUEST_NUMBER" \
            '{
              schema_version: 1,
              pull_request_number: $pull_request_number,
              base_sha: $base_sha,
              head_sha: $head_sha
            }' > benchmark-results/metadata.json

      - name: Validate benchmark report contract
        id: report_contract
        if: always()
        continue-on-error: true
        run: |
          validator=candidate/.github/scripts/validate-benchmark-reports.sh
          if [[ -f baseline/.github/scripts/validate-benchmark-reports.sh ]]; then
            validator=baseline/.github/scripts/validate-benchmark-reports.sh
          fi
          bash "$validator" \
            "$GITHUB_WORKSPACE/benchmark-results/baseline" \
            "$GITHUB_WORKSPACE/benchmark-results/candidate"

      - name: Upload benchmark reports
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: benchmark-results
          path: benchmark-results/
          if-no-files-found: warn
          retention-days: 14

      - name: Require successful correctness gates
        if: always()
        env:
          BASELINE_OUTCOME: ${{ steps.baseline.outcome }}
          CANDIDATE_OUTCOME: ${{ steps.candidate.outcome }}
          REPORT_CONTRACT_OUTCOME: ${{ steps.report_contract.outcome }}
        run: |
          test "$BASELINE_OUTCOME" = success
          test "$CANDIDATE_OUTCOME" = success
          test "$REPORT_CONTRACT_OUTCOME" = success