mtbl_sys/
lib.rs

1// Copyright (c) 2016 Leon Barrett
2//
3// Licensed under the Apache License, Version 2.0
4// <LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0> or the MIT
5// license <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
6// option. All files in the project carrying such notice may not be copied,
7// modified, or distributed except according to those terms.
8//
9//! Rust FFI interface to the [mtbl](https://github.com/farsightsec/mtbl) C
10//! library for dealing with SSTables (write-once sorted map files).
11//!
12//! SSTables are basically constant on-disk maps, like those used by
13//! [CDB](http://www.corpit.ru/mjt/tinycdb.html) (which also has [Rust
14//! bindings](https://github.com/andrew-d/tinycdb-rs), except using sorted maps
15//! instead of hashmaps.
16//!
17//! Version 0.2 of mtbl-sys covers the 0.8 version of the MTBL C library.
18//!
19//! # Function documentation
20//!
21//! For documentation about each function, see MTBL's extensive man pages, e.g.
22//! `man mtbl_reader`.
23//!
24//! # More details about MTBL
25//!
26//! Quoting from the MTBL documentation:
27//!
28//! > mtbl is not a database library. It does not provide an updateable
29//! > key-value data store, but rather exposes primitives for creating,
30//! > searching and merging SSTable files. Unlike databases which use the
31//! > SSTable data structure internally as part of their data store, management
32//! > of SSTable files -- creation, merging, deletion, combining of search
33//! > results from multiple SSTables -- is left to the discretion of the mtbl
34//! > library user.
35//!
36//! > mtbl SSTable files consist of a sequence of data blocks containing sorted
37//! > key-value pairs, where keys and values are arbitrary byte arrays. Data
38//! > blocks are optionally compressed using zlib or the Snappy library. The
39//! > data blocks are followed by an index block, allowing for fast searches
40//! > over the keyspace.
41//!
42//! > The basic mtbl interface is the writer, which receives a sequence of
43//! > key-value pairs in sorted order with no duplicate keys, and writes them
44//! > to data blocks in the SSTable output file. An index containing offsets to
45//! > data blocks and the last key in each data block is buffered in memory
46//! > until the writer object is closed, at which point the index is written to
47//! > the end of the SSTable file. This allows SSTable files to be written in a
48//! > single pass with sequential I/O operations only.
49//!
50//! > Once written, SSTable files can be searched using the mtbl reader
51//! > interface. Searches can retrieve key-value pairs based on an exact key
52//! > match, a key prefix match, or a key range. Results are retrieved using a
53//! > simple iterator interface.
54//!
55//! > The mtbl library also provides two utility interfaces which facilitate a
56//! > sort-and-merge workflow for bulk data loading. The sorter interface
57//! > receives arbitrarily ordered key-value pairs and provides them in sorted
58//! > order, buffering to disk as needed. The merger interface reads from
59//! > multiple SSTables simultaneously and provides the key-value pairs from
60//! > the combined inputs in sorted order. Since mtbl does not allow duplicate
61//! > keys in an SSTable file, both the sorter and merger interfaces require a
62//! > caller-provided merge function which will be called to merge multiple
63//! > values for the same key. These interfaces also make use of sequential I/O
64//! > operations only.
65
66#![crate_name = "mtbl_sys"]
67#![crate_type = "lib"]
68#![allow(dead_code,improper_ctypes)]
69
70extern crate libc;
71use libc::{c_char, c_int, c_uint, c_void, size_t};
72
73/// Compression method used when writing an MTBL file.
74#[derive(Clone,Copy,Debug,PartialEq)]
75#[repr(C)]
76pub enum CompressionType {
77    MTBL_COMPRESSION_NONE = 0,
78    MTBL_COMPRESSION_SNAPPY = 1,
79    MTBL_COMPRESSION_ZLIB = 2,
80    MTBL_COMPRESSION_LZ4 = 3,
81    MTBL_COMPRESSION_LZ4HC = 4,
82}
83
84#[derive(Clone,Copy,Debug,PartialEq)]
85#[repr(C)]
86pub enum MtblRes {
87    mtbl_res_failure = 0,
88    mtbl_res_success = 1,
89}
90
91#[repr(C)]
92pub struct mtbl_iter;
93#[repr(C)]
94pub struct mtbl_source;
95
96#[repr(C)]
97pub struct mtbl_reader;
98#[repr(C)]
99pub struct mtbl_reader_options;
100#[repr(C)]
101pub struct mtbl_metadata;
102#[repr(C)]
103pub struct mtbl_writer;
104#[repr(C)]
105pub struct mtbl_writer_options;
106
107#[repr(C)]
108pub struct mtbl_merger;
109#[repr(C)]
110pub struct mtbl_merger_options;
111#[repr(C)]
112pub struct mtbl_fileset;
113#[repr(C)]
114pub struct mtbl_fileset_options;
115#[repr(C)]
116pub struct mtbl_sorter;
117#[repr(C)]
118pub struct mtbl_sorter_options;
119
120#[link(name="mtbl")]
121extern "C" {
122
123    // iter
124
125    pub fn mtbl_iter_destroy(iter: *mut *mut mtbl_iter);
126
127    pub fn mtbl_iter_next(iter: *mut mtbl_iter,
128                          key: *mut *const u8,
129                          len_key: *mut size_t,
130                          val: *mut *const u8,
131                          len_val: *mut size_t)
132                          -> MtblRes;
133
134    // source
135
136    pub fn mtbl_source_iter(source: *const mtbl_source) -> *mut mtbl_iter;
137
138    pub fn mtbl_source_get(source: *const mtbl_source,
139                           key: *const u8,
140                           len_key: size_t)
141                           -> *mut mtbl_iter;
142
143    pub fn mtbl_source_get_prefix(source: *const mtbl_source,
144                                  key: *const u8,
145                                  len_key: size_t)
146                                  -> *mut mtbl_iter;
147
148    pub fn mtbl_source_get_range(source: *const mtbl_source,
149                                 key0: *const u8,
150                                 len_key0: size_t,
151                                 key1: *const u8,
152                                 len_key1: size_t)
153                                 -> *mut mtbl_iter;
154
155    fn mtbl_source_write(source: *const mtbl_source, writer: *mut mtbl_writer) -> MtblRes;
156
157    // writer
158
159    pub fn mtbl_writer_init(fname: *const c_char,
160                            options: *const mtbl_writer_options)
161                            -> *mut mtbl_writer;
162
163    pub fn mtbl_writer_init_fd(fd: c_int, options: *const mtbl_writer_options) -> *mut mtbl_writer;
164
165    pub fn mtbl_writer_destroy(writer: *mut *mut mtbl_writer);
166
167    pub fn mtbl_writer_add(writer: *mut mtbl_writer,
168                           key: *const u8,
169                           len_key: size_t,
170                           val: *const u8,
171                           len_val: size_t)
172                           -> MtblRes;
173
174    // writer options
175
176    pub fn mtbl_writer_options_init() -> *mut mtbl_writer_options;
177
178    pub fn mtbl_writer_options_destroy(options: *mut *mut mtbl_writer_options);
179
180    pub fn mtbl_writer_options_set_compression(options: *mut mtbl_writer_options,
181                                               compression: CompressionType);
182
183    pub fn mtbl_writer_options_set_block_size(options: *mut mtbl_writer_options, size: size_t);
184
185    pub fn mtbl_writer_options_set_block_restart_interval(options: *mut mtbl_writer_options,
186                                                          size: size_t);
187
188    // reader
189
190    pub fn mtbl_reader_init(fname: *const c_char,
191                            options: *const mtbl_reader_options)
192                            -> *mut mtbl_reader;
193
194    pub fn mtbl_reader_init_fd(fd: c_int, options: *const mtbl_reader_options) -> *mut mtbl_reader;
195
196    pub fn mtbl_reader_destroy(reader: *mut *mut mtbl_reader);
197
198    pub fn mtbl_reader_source(reader: *mut mtbl_reader) -> *const mtbl_source;
199
200    pub fn mtbl_reader_metadata(reader: *mut mtbl_reader) -> *const mtbl_metadata;
201
202    // reader options
203
204    pub fn mtbl_reader_options_init() -> *mut mtbl_reader_options;
205
206    pub fn mtbl_reader_options_destroy(options: *mut *mut mtbl_reader_options);
207
208    // defaults to false
209    pub fn mtbl_reader_options_set_verify_checksums(options: *mut mtbl_reader_options,
210                                                    verify_checksums: bool);
211
212    // defaults to false
213    pub fn mtbl_reader_options_set_madvise_random(options: *mut mtbl_reader_options,
214                                                  madvise_random: bool);
215
216    // reader metadata
217
218    /// Byte offset in the MTBL file where the index begins.
219    pub fn mtbl_metadata_index_block_offset(m: *const mtbl_metadata) -> u64;
220
221    /// Maximum size of an uncompressed data block, see mtbl_writer(3).
222    pub fn mtbl_metadata_data_block_size(m: *const mtbl_metadata) -> u64;
223
224    /// One of the compression values allowed by mtbl_writer(3).
225    pub fn mtbl_metadata_compression_algorithm(m: *const mtbl_metadata) -> CompressionType;
226
227    /// Total number of key-value entries.
228    pub fn mtbl_metadata_count_entries(m: *const mtbl_metadata) -> u64;
229
230    /// Total number of data blocks.
231    pub fn mtbl_metadata_count_data_blocks(m: *const mtbl_metadata) -> u64;
232
233    /// Total number of bytes consumed by data blocks.
234    pub fn mtbl_metadata_bytes_data_blocks(m: *const mtbl_metadata) -> u64;
235
236    /// Total number of bytes consumed by the index.
237    pub fn mtbl_metadata_bytes_index_block(m: *const mtbl_metadata) -> u64;
238
239    /// Total number of bytes that all keys would occupy if stored end-to-end in a byte array with no delimiters.
240    pub fn mtbl_metadata_bytes_keys(m: *const mtbl_metadata) -> u64;
241
242    /// Total number of bytes that all values in the file would occupy if stored end-to-end in a byte array with no delimiters.
243    pub fn mtbl_metadata_bytes_values(m: *const mtbl_metadata) -> u64;
244
245    // merger
246
247    pub fn mtbl_merger_init(options: *const mtbl_merger_options) -> *mut mtbl_merger;
248
249    pub fn mtbl_merger_destroy(merger: *mut *mut mtbl_merger);
250
251    pub fn mtbl_merger_add_source(merger: *mut mtbl_merger, source: *const mtbl_source);
252
253    pub fn mtbl_merger_source(merger: *const mtbl_merger) -> *const mtbl_source;
254
255    // merger options
256
257    pub fn mtbl_merger_options_init() -> *mut mtbl_merger_options;
258
259    pub fn mtbl_merger_options_destroy(options: *mut *mut mtbl_merger_options);
260
261    pub fn mtbl_merger_options_set_merge_func(
262            options: *mut mtbl_merger_options,
263            merge_func: extern "C" fn(clos: *mut c_void,
264                                      key: *const u8, len_key: size_t,
265                                      val0: *const u8, len_val0: size_t,
266                                      val1: *const u8, len_val1: size_t,
267                                      merged_val: *mut *mut u8, len_merged_val: *mut size_t),
268           clos: *mut c_void);
269
270    // fileset
271
272    pub fn mtbl_fileset_init(fname: *const c_char,
273                             options: *const mtbl_fileset_options)
274                             -> *mut mtbl_fileset;
275
276    pub fn mtbl_fileset_destroy(fileset: *mut *mut mtbl_fileset);
277
278    pub fn mtbl_fileset_reload(fileset: *mut mtbl_fileset);
279
280    pub fn mtbl_fileset_reload_now(fileset: *mut mtbl_fileset);
281
282    pub fn mtbl_fileset_source(fileset: *mut mtbl_fileset) -> *const mtbl_source;
283
284    // fileset options
285
286    pub fn mtbl_fileset_options_init() -> *mut mtbl_fileset_options;
287
288    pub fn mtbl_fileset_options_destroy(options: *mut *mut mtbl_fileset_options);
289
290    pub fn mtbl_fileset_options_set_merge_func(
291            options: *mut mtbl_fileset_options,
292            merge_func: extern "C" fn(clos: *mut c_void,
293                                      key: *const u8, len_key: size_t,
294                                      val0: *const u8, len_val0: size_t,
295                                      val1: *const u8, len_val1: size_t,
296                                      merged_val: *mut *mut u8, len_merged_val: *mut size_t),
297            clos: *mut c_void);
298
299    pub fn mtbl_fileset_options_set_reload_interval(options: *mut mtbl_fileset_options,
300                                                    reload_interval: u32);
301
302    // sorter
303
304    pub fn mtbl_sorter_init(options: *const mtbl_sorter_options) -> *mut mtbl_sorter;
305
306    pub fn mtbl_sorter_destroy(sorter: *mut *mut mtbl_sorter);
307
308    pub fn mtbl_sorter_add(sorter: *mut mtbl_sorter,
309                           key: *const u8,
310                           len_key: size_t,
311                           val: *const u8,
312                           len_val: size_t)
313                           -> MtblRes;
314
315    pub fn mtbl_sorter_write(sorter: *mut mtbl_sorter, writer: *mut mtbl_writer) -> MtblRes;
316
317    pub fn mtbl_sorter_iter(sorter: *mut mtbl_sorter) -> *mut mtbl_iter;
318
319    // sorter options
320
321    pub fn mtbl_sorter_options_init() -> *mut mtbl_sorter_options;
322
323    pub fn mtbl_sorter_options_destroy(options: *mut *mut mtbl_sorter_options);
324
325    pub fn mtbl_sorter_options_set_merge_func(options: *mut mtbl_sorter_options,
326                                              merge_fp: extern "C" fn(clos: *mut c_void,
327                                                                      key: *const u8,
328                                                                      len_key: size_t,
329                                                                      val0: *const u8,
330                                                                      len_val0: size_t,
331                                                                      val1: *const u8,
332                                                                      len_val1: size_t,
333                                                                      merged_val: *mut *mut u8,
334                                                                      len_merged_val: *mut size_t)
335                                                                     ,
336                                              clos: *mut c_void);
337
338    pub fn mtbl_sorter_options_set_temp_dir(options: *mut mtbl_sorter_options,
339                                            path: *const c_char);
340
341    pub fn mtbl_sorter_options_set_max_memory(options: *mut mtbl_sorter_options, size: size_t);
342
343    // crc32c
344
345    pub fn mtbl_crc32c(buffer: *const u8, length: size_t) -> u32;
346
347    // fixed
348
349    pub fn mtbl_fixed_encode32(dst: *mut u8, value: u32) -> size_t;
350
351    pub fn mtbl_fixed_encode64(dst: *mut u8, value: u64) -> size_t;
352
353    pub fn mtbl_fixed_decode32(ptr: *const u8) -> u32;
354
355    pub fn mtbl_fixed_decode64(ptr: *const u8) -> u64;
356
357    // varint
358
359    pub fn mtbl_varint_length(v: u64) -> c_uint;
360
361    pub fn mtbl_varint_length_packed(buf: *const u8, len_buf: size_t) -> c_uint;
362
363    pub fn mtbl_varint_encode32(ptr: *mut u8, value: u32) -> size_t;
364
365    pub fn mtbl_varint_encode64(ptr: *mut u8, value: u64) -> size_t;
366
367    pub fn mtbl_varint_decode32(ptr: *const u8, value: *mut u32) -> size_t;
368
369    pub fn mtbl_varint_decode64(ptr: *const u8, value: *mut u64) -> size_t;
370
371}
mtbl_sys/lib.rs

mtbl_sys/
lib.rs
