ffbpe 0.1.9

Unicode-aware, streaming BPE training and tiktoken-compatible encoding
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90

site_name: FFBPE
site_description: Fast and faithful byte-pair encoding for large, multilingual corpora.
site_url: https://tokn-ai.dev/ffbpe/docs/
site_dir: site/docs
repo_name: tokn-ai/ffbpe
repo_url: https://github.com/tokn-ai/ffbpe
edit_uri: edit/master/docs/

theme:
  name: material
  language: en
  features:
    - content.code.copy
    - content.code.annotate
    - navigation.footer
    - navigation.indexes
    - navigation.sections
    - navigation.tabs
    - navigation.top
    - search.highlight
    - search.suggest
  palette:
    - media: "(prefers-color-scheme: light)"
      scheme: default
      primary: black
      accent: deep orange
      toggle:
        icon: material/weather-night
        name: Use dark mode
    - media: "(prefers-color-scheme: dark)"
      scheme: slate
      primary: black
      accent: orange
      toggle:
        icon: material/weather-sunny
        name: Use light mode

nav:
  - Home: index.md
  - Get started:
      - Installation and quickstart: getting-started.md
      - Train a tokenizer: guides/training.md
      - Encode and save models: guides/encoding.md
      - Multilingual corpora: guides/multilingual.md
      - tiktoken compatibility: guides/tiktoken.md
  - Python API:
      - Overview: api/index.md
      - Training shortcut: api/training.md
      - Trainer: api/trainer.md
      - Model: api/model.md
      - Encoder: api/encoder.md
      - Pretokenizer and counters: api/pretokenizer.md
      - tiktoken compatibility: api/tiktoken.md
  - Rust API: https://docs.rs/ffbpe
  - Benchmarks: benchmarks.md

plugins:
  - search
  - mkdocstrings:
      handlers:
        python:
          paths:
            - python
          options:
            docstring_style: numpy
            heading_level: 2
            members_order: source
            merge_init_into_class: true
            separate_signature: true
            show_bases: false
            show_root_heading: true
            show_source: true
            show_symbol_type_heading: true
            show_symbol_type_toc: true
            signature_crossrefs: true

markdown_extensions:
  - admonition
  - attr_list
  - md_in_html
  - pymdownx.details
  - pymdownx.highlight:
      anchor_linenums: true
  - pymdownx.inlinehilite
  - pymdownx.snippets
  - pymdownx.superfences
  - toc:
      permalink: true

strict: true