ffbpe 0.1.10

Unicode-aware, streaming BPE training and tiktoken-compatible encoding
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116

.site-header {
  position: sticky;
  z-index: 20;
  top: 0;
  width: 100%;
  background: var(--paper);
  border-bottom: 1px solid var(--line);
}

.site-header-inner {
  width: min(100% - 3rem, 1240px);
  min-height: 88px;
  display: flex;
  align-items: center;
  justify-content: space-between;
  gap: 2rem;
  margin-inline: auto;
}

.site-brand {
  display: inline-flex;
  align-items: baseline;
  gap: 12px;
  width: fit-content;
  color: inherit;
  text-decoration: none;
}

.site-brand-mark {
  display: inline-flex;
  align-items: baseline;
  font-family: Impact, Haettenschweiler, "Arial Narrow Bold", sans-serif;
  font-size: 29px;
  font-weight: 900;
  letter-spacing: -0.04em;
  line-height: 1;
}

.site-brand-mark i {
  padding-inline: 4px;
  color: var(--orange);
  font-style: normal;
}

.site-brand-section {
  color: var(--muted, #686c6d);
  font-family: var(--mono);
  font-size: 11px;
  font-weight: 700;
  letter-spacing: 0.12em;
}

.site-nav {
  display: flex;
  align-items: center;
  gap: clamp(1.2rem, 3vw, 2.5rem);
  font-family: var(--mono);
  font-size: 0.78rem;
}

.site-nav a {
  position: relative;
  color: inherit;
  text-decoration: none;
}

.site-nav a::after {
  content: "";
  position: absolute;
  right: 0;
  bottom: -0.35rem;
  left: 0;
  height: 1px;
  background: currentColor;
  transform: scaleX(0);
  transform-origin: right;
  transition: transform 180ms ease;
}

.site-nav a:hover::after,
.site-nav a:focus-visible::after,
.site-nav a[aria-current="page"]::after {
  transform: scaleX(1);
  transform-origin: left;
}

.site-nav a[aria-current="page"]::after {
  height: 2px;
  background: var(--orange);
}

@media (max-width: 680px) {
  .site-header-inner {
    width: min(100% - 1.5rem, 1240px);
    min-height: 72px;
    gap: 1rem;
  }

  .site-nav {
    gap: 0.75rem;
  }

  .site-nav a:nth-child(-n + 2) {
    display: none;
  }

  .site-brand-section {
    display: none;
  }
}

@media (prefers-reduced-motion: reduce) {
  .site-nav a::after {
    transition: none;
  }
}