json_surf/lib.rs
1//! JSON-Surf
2//!
3//! ## Features
4//! * Full Text/Term search (sqlish!!!)
5//! * Easy read, write & delete API
6//! * Serialize __**flat**__ JSON/Struct
7//! * Write multiple documents together
8//! * Support fuzzy word search (see examples)
9//! * Requires no runtime
10//! * No unsafe block
11//! * Run on rust stable
12//! * Coming Soon: Bi-gram suggestion & TF-IDF support
13//!
14//! ## Motivation
15//! * Allow your existing flat rust structs to be searched
16//! * The crate will support arbitary byte stream once it is supported by tantivy (see [here](https://github.com/tantivy-search/tantivy/issues/832))
17//! * This can just act as a container to actual data in databases, keeping indexes light
18//! * Create time-aware containers which could possibly updated/deleted
19//! * Create ephemeral storage for request/response
20//! * This can integrate with any web-server to index and search near real-time
21//! * This crate will focus mostly on user-workflow(s) related problem(s)
22//! * Uses [tantivy](https://github.com/tantivy-search/tantivy) under the hood.
23//!
24//! ## TODO
25//! * Add more examples
26//! * Remove any further copy
27//! * Introduce more housekeeping API (If required)
28//!
29//!
30//! ## Quickstart
31//!
32//! ### Prerequisite:
33//!
34//! ```toml
35//! [dependencies]
36//! json-surf = "*"
37//! ```
38//!
39//! ### Example
40//! ```rust
41//! use std::convert::TryFrom;
42//! use std::hash::{Hash, Hasher};
43//! use std::cmp::{Ord, Ordering, Eq};
44//!
45//!
46//! use std::fs::remove_dir_all;
47//!
48//! use serde::{Serialize, Deserialize};
49//!
50//! use json_surf::prelude::*;
51//!
52//!
53//! // Main struct
54//! #[derive(Serialize, Debug, Deserialize, PartialEq, PartialOrd, Clone)]
55//! struct UserInfo {
56//! first: String,
57//! last: String,
58//! age: u8,
59//! }
60//!
61//! impl UserInfo {
62//! pub fn new(first: String, last: String, age: u8) -> Self {
63//! Self {
64//! first,
65//! last,
66//! age,
67//! }
68//! }
69//! }
70//!
71//! impl Default for UserInfo {
72//! fn default() -> Self {
73//! let first = "".to_string();
74//! let last = "".to_string();
75//! let age = 0u8;
76//! UserInfo::new(first, last, age)
77//! }
78//! }
79//!
80//! fn main() {
81//! // Specify home location for indexes
82//! let home = ".store".to_string();
83//! // Specify index name
84//! let index_name = "usage".to_string();
85//!
86//! // Prepare builder
87//! let mut builder = SurferBuilder::default();
88//! builder.set_home(&home);
89//!
90//! let data = UserInfo::default();
91//! builder.add_struct(index_name.clone(), &data);
92//!
93//! // Prepare Surf
94//! let mut surf = Surf::try_from(builder).unwrap();
95//!
96//! // Prepare data to insert & search
97//!
98//! // User 1: John Doe
99//! let first = "John".to_string();
100//! let last = "Doe".to_string();
101//! let age = 20u8;
102//! let john_doe = UserInfo::new(first, last, age);
103//!
104//! // User 2: Jane Doe
105//! let first = "Jane".to_string();
106//! let last = "Doe".to_string();
107//! let age = 18u8;
108//! let jane_doe = UserInfo::new(first, last, age);
109//!
110//! // See examples for more options
111//! let users = vec![john_doe.clone(), jane_doe.clone()];
112//! let _ = surf.insert(&index_name, &users).unwrap();
113//!
114//! block_thread(1);
115//!
116//! // See examples for more options
117//! // Similar to SELECT * FROM users WHERE (age = 20 AND last = "Doe") OR (first = "Jane")
118//! let conditions = vec![
119//! OrCondition::new(
120//! // (age = 20 AND last = "Doe")
121//! vec![
122//! AndCondition::new("age".to_string(), "20".to_string()),
123//! AndCondition::new("last".to_string(), "doe".to_string())
124//! ]),
125//! OrCondition::new(
126//! // (first = "Jane")
127//! vec![
128//! AndCondition::new("first".to_string(), "jane".to_string())
129//! ])
130//! ];
131//!
132//! // Validate John and Jane Doe
133//! let mut computed = surf.select::<UserInfo>(&index_name, &conditions).unwrap().unwrap();
134//! let mut expected = vec![john_doe.clone(), jane_doe.clone(), ];
135//! expected.sort();
136//! computed.sort();
137//! assert_eq!(expected, computed);
138//!
139//! // Validated John's record - Alternate shortcut for query using one field only
140//! let computed = surf.read_all_structs_by_field(&index_name, "age", "20");
141//! let computed: Vec<UserInfo> = computed.unwrap().unwrap();
142//! assert_eq!(vec![john_doe], computed);
143//!
144//! // Delete John's record
145//! let result = surf.delete(&index_name, "age", "20");
146//! assert!(result.is_ok());
147//!
148//! // John's was removed
149//! let computed = surf.read_all_structs_by_field(&index_name, "age", "20");
150//! let computed: Vec<UserInfo> = computed.unwrap().unwrap();
151//! assert!(computed.is_empty());
152//!
153//!
154//! // Clean-up
155//! let path = surf.which_index(&index_name).unwrap();
156//! let _ = remove_dir_all(&path);
157//! let _ = remove_dir_all(&home);
158//! }
159//!
160//! /// Ignore all of this
161//! /// Convenience method for sorting & likely not required in user code
162//! impl Ord for UserInfo {
163//! fn cmp(&self, other: &Self) -> Ordering {
164//! if self.first == other.first && self.last == other.last {
165//! return Ordering::Equal;
166//! };
167//! if self.first == other.first {
168//! if self.last > other.last {
169//! Ordering::Greater
170//! } else {
171//! Ordering::Less
172//! }
173//! } else {
174//! if self.first > other.first {
175//! Ordering::Greater
176//! } else {
177//! Ordering::Less
178//! }
179//! }
180//! }
181//! }
182//!
183//! /// Ignore all of this
184//! /// Convenience method for sorting & likely not required in user code
185//! impl Eq for UserInfo {}
186//!
187//! /// Ignore all of this
188//! /// Convenience method for sorting & likely not required in user code
189//! impl Hash for UserInfo {
190//! fn hash<H: Hasher>(&self, state: &mut H) {
191//! for i in self.first.as_bytes() {
192//! state.write_u8(*i);
193//! }
194//! for i in self.last.as_bytes() {
195//! state.write_u8(*i);
196//! }
197//! state.write_u8(self.age);
198//! state.finish();
199//! }
200//! }
201//! ```
202
203pub mod prelude;
204pub mod seed;
205pub mod errors;
206pub mod utils;
207pub mod registry;
208pub mod fuzzy;
209
210#[cfg(test)]
211mod tests {
212 use super::*;
213
214 use serde::{Serialize};
215
216 use serde_value;
217 use std::collections::BTreeMap;
218 use std::fmt;
219 use tantivy::schema::{Schema, IntOptions, TEXT, STORED};
220 use std::ops::{Deref, DerefMut};
221 use std::fmt::Debug;
222
223 #[derive(Serialize, Clone, PartialEq)]
224 struct SchemaTest(Schema);
225
226 impl Deref for SchemaTest {
227 type Target = Schema;
228 fn deref(&self) -> &Self::Target {
229 &self.0
230 }
231 }
232
233 impl DerefMut for SchemaTest {
234 fn deref_mut(&mut self) -> &mut Self::Target {
235 &mut self.0
236 }
237 }
238
239 impl Debug for SchemaTest {
240 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
241 let schema = &self.0;
242 let x = schema.fields();
243 let mut fields = Vec::new();
244 for (field, _) in x {
245 fields.push(field);
246 }
247 f.write_str(format!("{:?}", fields).as_str())
248 }
249 }
250
251 impl SchemaTest {
252 fn new(schema: Schema) -> Self {
253 Self(schema)
254 }
255 }
256
257 #[derive(Serialize)]
258 struct Dummy {
259 x: String,
260 y: String,
261 z: u64,
262 }
263
264 #[derive(Serialize)]
265 struct Giant {
266 a: String,
267 b: bool,
268 c: u64,
269 d: u32,
270 e: u16,
271 f: u8,
272 g: i64,
273 h: i32,
274 i: i16,
275 j: i8,
276 k: f64,
277 l: f32,
278 m: Vec<u8>,
279 }
280
281
282 #[test]
283 fn validate_field_names() {
284 let data = Dummy {
285 x: "A".to_owned(),
286 y: "B".to_owned(),
287 z: 100,
288 };
289
290 let value = utils::as_value(&data).unwrap();
291 let computed = utils::field_names(&value).unwrap();
292 let expected = vec!["x", "y", "z"];
293 assert_eq!(expected, computed);
294 }
295
296 #[test]
297 fn validate_as_value() {
298 let data = Dummy {
299 x: "X".to_owned(),
300 y: "Y".to_owned(),
301 z: 100,
302 };
303
304 let mut bmap = BTreeMap::new();
305 bmap.insert(serde_value::Value::String("x".to_owned()), serde_value::Value::String("X".to_owned()));
306 bmap.insert(serde_value::Value::String("y".to_owned()), serde_value::Value::String("Y".to_owned()));
307 bmap.insert(serde_value::Value::String("z".to_owned()), serde_value::Value::U64(100));
308
309 let expected = serde_value::Value::Map(bmap);
310 let computed = utils::as_value(&data).unwrap();
311
312 assert_eq!(expected, computed);
313 }
314
315 #[test]
316 fn validate_basic_schema() {
317 let data = Dummy {
318 x: "X".to_owned(),
319 y: "Y".to_owned(),
320 z: 100u64,
321 };
322
323 let data = utils::as_value(&data).unwrap();
324
325 let (computed, _) = utils::to_schema(&data, None).unwrap();
326 let computed = SchemaTest::new(computed);
327
328 let mut expected = Schema::builder();
329 expected.add_text_field("x", TEXT | STORED);
330 expected.add_text_field("y", TEXT | STORED);
331
332 let options = IntOptions::default();
333 let options = options.set_stored();
334 let options = options.set_indexed();
335 expected.add_u64_field("z", options);
336
337 let expected = expected.build();
338 let expected = SchemaTest::new(expected);
339
340 assert_eq!(expected, computed);
341 }
342
343 #[test]
344 fn validate_full_schema() {
345 let a: String = "Empire Of The Clouds".to_string();
346 let b: bool = true;
347 let c: u64 = 1;
348 let d: u32 = 1;
349 let e: u16 = 1;
350 let f: u8 = 1;
351 let g: i64 = 1;
352 let h: i32 = 1;
353 let i: i16 = 1;
354 let j: i8 = 1;
355 let k: f64 = 1.0;
356 let l: f32 = 1.0;
357 let m: Vec<u8> = "The book of souls".as_bytes().to_vec();
358 let data = Giant {
359 a,
360 b,
361 c,
362 d,
363 e,
364 f,
365 g,
366 h,
367 i,
368 j,
369 k,
370 l,
371 m,
372 };
373 let data = utils::as_value(&data).unwrap();
374 let (computed, _) = utils::to_schema(&data, None).unwrap();
375 let computed = SchemaTest::new(computed);
376
377 let mut expected = Schema::builder();
378 expected.add_text_field("a", TEXT | STORED);
379 expected.add_text_field("b", TEXT | STORED);
380
381 let options = IntOptions::default();
382 let options = options.set_stored();
383 let options = options.set_indexed();
384 expected.add_u64_field("c", options.clone());
385 expected.add_u64_field("d", options.clone());
386 expected.add_u64_field("e", options.clone());
387 expected.add_u64_field("f", options.clone());
388
389 expected.add_i64_field("g", options.clone());
390 expected.add_i64_field("h", options.clone());
391 expected.add_i64_field("i", options.clone());
392 expected.add_i64_field("j", options.clone());
393 expected.add_f64_field("k", options.clone());
394 expected.add_f64_field("l", options.clone());
395 expected.add_bytes_field("m");
396 let expected = expected.build();
397 let expected = SchemaTest::new(expected);
398 assert_eq!(format!("{:?}", expected), format!("{:?}", computed));
399 assert_eq!(expected, computed);
400 }
401
402}