sentencepiece-sys 0.13.1

Binding for the sentencepiece tokenizer
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173

/* automatically generated by rust-bindgen 0.66.1 */

pub const __bool_true_false_are_defined: u32 = 1;
pub const true_: u32 = 1;
pub const false_: u32 = 0;
pub const _VCRT_COMPILER_PREPROCESSOR: u32 = 1;
pub const _SAL_VERSION: u32 = 20;
pub const __SAL_H_VERSION: u32 = 180000000;
pub const _USE_DECLSPECS_FOR_SAL: u32 = 0;
pub const _USE_ATTRIBUTES_FOR_SAL: u32 = 0;
pub const _CRT_PACKING: u32 = 8;
pub const _HAS_EXCEPTIONS: u32 = 1;
pub const _STL_LANG: u32 = 0;
pub const _HAS_CXX17: u32 = 0;
pub const _HAS_CXX20: u32 = 0;
pub const _HAS_CXX23: u32 = 0;
pub const _HAS_NODISCARD: u32 = 0;
pub const WCHAR_MIN: u32 = 0;
pub const WCHAR_MAX: u32 = 65535;
pub const WINT_MIN: u32 = 0;
pub const WINT_MAX: u32 = 65535;
pub type wchar_t = ::std::os::raw::c_ushort;
pub type max_align_t = f64;
pub type va_list = *mut ::std::os::raw::c_char;
extern "C" {
    pub fn __va_start(arg1: *mut *mut ::std::os::raw::c_char, ...);
}
pub type __vcrt_bool = bool;
extern "C" {
    pub fn __security_init_cookie();
}
extern "C" {
    pub fn __security_check_cookie(_StackCookie: usize);
}
extern "C" {
    pub fn __report_gsfailure(_StackCookie: usize) -> !;
}
extern "C" {
    pub static mut __security_cookie: usize;
}
pub type int_least8_t = ::std::os::raw::c_schar;
pub type int_least16_t = ::std::os::raw::c_short;
pub type int_least32_t = ::std::os::raw::c_int;
pub type int_least64_t = ::std::os::raw::c_longlong;
pub type uint_least8_t = ::std::os::raw::c_uchar;
pub type uint_least16_t = ::std::os::raw::c_ushort;
pub type uint_least32_t = ::std::os::raw::c_uint;
pub type uint_least64_t = ::std::os::raw::c_ulonglong;
pub type int_fast8_t = ::std::os::raw::c_schar;
pub type int_fast16_t = ::std::os::raw::c_int;
pub type int_fast32_t = ::std::os::raw::c_int;
pub type int_fast64_t = ::std::os::raw::c_longlong;
pub type uint_fast8_t = ::std::os::raw::c_uchar;
pub type uint_fast16_t = ::std::os::raw::c_uint;
pub type uint_fast32_t = ::std::os::raw::c_uint;
pub type uint_fast64_t = ::std::os::raw::c_ulonglong;
pub type intmax_t = ::std::os::raw::c_longlong;
pub type uintmax_t = ::std::os::raw::c_ulonglong;
#[repr(C)]
#[derive(Debug, Copy, Clone)]
pub struct SentencePieceProcessor {
    _unused: [u8; 0],
}
#[repr(C)]
#[derive(Debug, Copy, Clone)]
pub struct SentencePieceText {
    _unused: [u8; 0],
}
extern "C" {
    pub fn spp_decode_piece_ids(
        spp: *mut SentencePieceProcessor,
        pieces: *const u32,
        pieces_len: usize,
        decoded: *mut *mut ::std::os::raw::c_uchar,
        decoded_len: *mut usize,
    ) -> ::std::os::raw::c_int;
}
extern "C" {
    pub fn spp_decode_pieces(
        spp: *mut SentencePieceProcessor,
        pieces: *const *const ::std::os::raw::c_char,
        pieces_len: usize,
        decoded: *mut *mut ::std::os::raw::c_uchar,
        decoded_len: *mut usize,
    ) -> ::std::os::raw::c_int;
}
extern "C" {
    pub fn spp_encode_as_serialized_proto(
        spp: *mut SentencePieceProcessor,
        sentence: *const ::std::os::raw::c_char,
        sentence_len: usize,
        len: *mut usize,
    ) -> *mut ::std::os::raw::c_uchar;
}
extern "C" {
    pub fn spp_sample_encode_as_serialized_proto(
        spp: *mut SentencePieceProcessor,
        sentence: *const ::std::os::raw::c_char,
        sentence_len: usize,
        len: *mut usize,
        nbest: usize,
        alpha: f32,
    ) -> *mut ::std::os::raw::c_uchar;
}
extern "C" {
    pub fn spp_new() -> *mut SentencePieceProcessor;
}
extern "C" {
    pub fn spp_from_serialized_proto(
        spp: *mut SentencePieceProcessor,
        data: *const ::std::os::raw::c_char,
        len: usize,
    ) -> ::std::os::raw::c_int;
}
extern "C" {
    pub fn spp_to_serialized_proto(
        spp: *mut SentencePieceProcessor,
        len: *mut usize,
    ) -> *mut ::std::os::raw::c_uchar;
}
extern "C" {
    pub fn spp_load(
        spp: *mut SentencePieceProcessor,
        filename: *const ::std::os::raw::c_char,
    ) -> ::std::os::raw::c_int;
}
extern "C" {
    pub fn spp_free(spp: *mut SentencePieceProcessor);
}
extern "C" {
    pub fn spp_bos_id(spp: *mut SentencePieceProcessor) -> ::std::os::raw::c_int;
}
extern "C" {
    pub fn spp_eos_id(spp: *mut SentencePieceProcessor) -> ::std::os::raw::c_int;
}
extern "C" {
    pub fn spp_is_unknown(spp: *mut SentencePieceProcessor, id: ::std::os::raw::c_int) -> bool;
}
extern "C" {
    pub fn spp_pad_id(spp: *mut SentencePieceProcessor) -> ::std::os::raw::c_int;
}
extern "C" {
    pub fn spp_piece_to_id(
        spp: *mut SentencePieceProcessor,
        piece: *const ::std::os::raw::c_char,
    ) -> ::std::os::raw::c_int;
}
extern "C" {
    pub fn spp_piece_size(spp: *mut SentencePieceProcessor) -> ::std::os::raw::c_int;
}
extern "C" {
    pub fn spp_unk_id(spp: *mut SentencePieceProcessor) -> ::std::os::raw::c_int;
}
extern "C" {
    pub fn spp_normalize(
        spp: *mut SentencePieceProcessor,
        sentence: *const ::std::os::raw::c_char,
        sentence_len: usize,
        normalized: *mut *mut ::std::os::raw::c_uchar,
        normalized_len: *mut usize,
    ) -> ::std::os::raw::c_int;
}
extern "C" {
    pub fn spp_normalize_with_offsets(
        spp: *mut SentencePieceProcessor,
        sentence: *const ::std::os::raw::c_char,
        sentence_len: usize,
        normalized: *mut *mut ::std::os::raw::c_uchar,
        normalized_len: *mut usize,
        offsets: *mut *mut usize,
        offsets_len: *mut usize,
    ) -> ::std::os::raw::c_int;
}