simd-tokenizer 1.0.0

Safe SWAR-accelerated whitespace token estimator with optional exact tiktoken backend
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
name: CI
on:
  push:
    branches: [main, master]
  pull_request:

jobs:
  quality:
    uses: WyattAu/engineering-standards/.github/workflows/rust-kit.yml@main
    with:
      tier: c
      all-features: true
      loom: false
      wasm: false
      miri: true
      embedded: true
      bench-regression: true