unitoken 0.1.5

Fast BPE tokenizer/trainer with a Rust core and Python bindings
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

name: Release

on:
  push:
    tags:
      - v*

jobs:
  publish:
    name: Publish to crates.io
    runs-on: ubuntu-latest
    environment: cargo
    permissions:
      contents: read
      id-token: write

    steps:
      - name: Checkout
        uses: actions/checkout@v7

      - name: Authenticate to crates.io
        id: auth
        uses: rust-lang/crates-io-auth-action@v1

      - name: Publish
        run: cargo publish
        env:
          CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }}