ffbpe 0.1.8

Unicode-aware, streaming BPE training and tiktoken-compatible encoding
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55

name: Release

on:
  push:
    tags:
      - v*

jobs:
  publish:
    name: Publish to crates.io
    runs-on: ubuntu-latest
    environment: cargo
    permissions:
      contents: read
      id-token: write

    steps:
      - name: Checkout
        uses: actions/checkout@v7

      - name: Check whether the version is already published
        id: version
        shell: bash
        run: |
          package="$(cargo metadata --no-deps --format-version 1 | jq -r '.packages[0].name')"
          version="$(cargo metadata --no-deps --format-version 1 | jq -r '.packages[0].version')"
          status="$(curl --silent --output /dev/null --write-out '%{http_code}' \
            "https://crates.io/api/v1/crates/${package}/${version}")"
          case "$status" in
            200)
              echo "exists=true" >> "$GITHUB_OUTPUT"
              ;;
            404)
              echo "exists=false" >> "$GITHUB_OUTPUT"
              ;;
            *)
              echo "crates.io returned HTTP $status" >&2
              exit 1
              ;;
          esac

      - name: Authenticate to crates.io
        id: auth
        if: steps.version.outputs.exists != 'true'
        uses: rust-lang/crates-io-auth-action@v1

      - name: Publish
        if: steps.version.outputs.exists != 'true'
        run: cargo publish
        env:
          CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }}

      - name: Version already published
        if: steps.version.outputs.exists == 'true'
        run: echo "This crate version is already available on crates.io."