wordchipper 0.9.2

HPC Rust LLM Tokenizer Library
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81

//! # Public `OpenAI` Resources

use crate::support::resources::{
    ConstKeyedResource,
    ConstUrlResource,
};

/// Shared download context key.
const OA_KEY: &str = "openai";

/// Cache-keyed GPT-2 `DataGym` "vocab.bpe" vocabulary resource.
pub const OA_GPT2_VOCAB_BPE_KEYED_RESOURCE: ConstKeyedResource = ConstKeyedResource {
    key: &["openai", "gpt2"],
    resource: OA_GPT2_DATAGYM_VOCAB_BPE_RESOURCE,
};

/// Cache-keyed GPT-2 `DataGym` "encoder.json" encoder resource.
pub const OA_GPT2_ENCODER_JSON_KEYED_RESOURCE: ConstKeyedResource = ConstKeyedResource {
    key: &["openai", "gpt2"],
    resource: OA_GPT2_DATAGYM_ENCODER_JSON_RESOURCE,
};

/// The GPT-2 Data Gym "vocab.bpe" vocabulary resource.
pub const OA_GPT2_DATAGYM_VOCAB_BPE_RESOURCE: ConstUrlResource = ConstUrlResource {
    urls: &["https://openaipublic.blob.core.windows.net/gpt-2/encodings/main/vocab.bpe"],
    hash: Some("1ce1664773c50f3e0cc8842619a93edc4624525b728b188a9e0be33b7726adc5"),
};

/// The GPT-2 Data Gym "encoder.json" vocabulary resource.
pub const OA_GPT2_DATAGYM_ENCODER_JSON_RESOURCE: ConstUrlResource = ConstUrlResource {
    urls: &["https://openaipublic.blob.core.windows.net/gpt-2/encodings/main/encoder.json"],
    hash: Some("196139668be63f3b5d6574427317ae82f612a97c5d1cdaf36ed2256dbf636783"),
};

/// The "`r50k_base.tiktoken`" vocabulary resource.
pub const OA_R50K_BASE_TIKTOKEN_RESOURCE: ConstUrlResource = ConstUrlResource {
    urls: &["https://openaipublic.blob.core.windows.net/encodings/r50k_base.tiktoken"],
    hash: Some("306cd27f03c1a714eca7108e03d66b7dc042abe8c258b44c199a7ed9838dd930"),
};

/// A keyed resource for the "`r50k_base`" tokenizer.
pub const OA_R50K_BASE_TIKTOKEN_KEYED_RESOURCE: ConstKeyedResource = ConstKeyedResource {
    key: &[OA_KEY, "r50k_base"],
    resource: OA_R50K_BASE_TIKTOKEN_RESOURCE,
};

/// The "`p50k_base.tiktoken`" vocabulary resource.
pub const OA_P50K_BASE_TIKTOKEN_RESOURCE: ConstUrlResource = ConstUrlResource {
    urls: &["https://openaipublic.blob.core.windows.net/encodings/p50k_base.tiktoken"],
    hash: Some("94b5ca7dff4d00767bc256fdd1b27e5b17361d7b8a5f968547f9f23eb70d2069"),
};

/// A keyed resource for the "`p50k_base`" tokenizer.
pub const OA_P50K_BASE_TIKTOKEN_KEYED_RESOURCE: ConstKeyedResource = ConstKeyedResource {
    key: &[OA_KEY, "p50k_base"],
    resource: OA_P50K_BASE_TIKTOKEN_RESOURCE,
};

/// The "`cl100k_base.tiktoken`" vocabulary resource.
pub const OA_CL100K_BASE_TIKTOKEN_RESOURCE: ConstUrlResource = ConstUrlResource {
    urls: &["https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken"],
    hash: Some("223921b76ee99bde995b7ff738513eef100fb51d18c93597a113bcffe865b2a7"),
};

/// A keyed resource for the "`cl100k_base`" tokenizer.
pub const OA_CL100K_BASE_TIKTOKEN_KEYED_RESOURCE: ConstKeyedResource = ConstKeyedResource {
    key: &[OA_KEY, "cl100k_base"],
    resource: OA_CL100K_BASE_TIKTOKEN_RESOURCE,
};

/// The "`o200k_base.tiktoken`" vocabulary resource.
pub const OA_O200K_BASE_TIKTOKEN_RESOURCE: ConstUrlResource = ConstUrlResource {
    urls: &["https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken"],
    hash: Some("446a9538cb6c348e3516120d7c08b09f57c36495e2acfffe59a5bf8b0cfb1a2d"),
};

/// A keyed resource for the "`o200k_base`" tokenizer.
pub const OA_O200K_BASE_TIKTOKEN_KEYED_RESOURCE: ConstKeyedResource = ConstKeyedResource {
    key: &[OA_KEY, "o200k_base"],
    resource: OA_O200K_BASE_TIKTOKEN_RESOURCE,
};