genson_core/genson_rs/
mod.rs

1#![allow(mismatched_lifetime_syntaxes)]
2mod builder;
3mod node;
4mod strategy;
5
6use mimalloc::MiMalloc;
7use rayon::prelude::*;
8use serde_json::json;
9
10// Setting the global allocator to mimalloc for more efficient memory allocation
11#[global_allocator]
12static GLOBAL: MiMalloc = MiMalloc;
13
14pub use builder::SchemaBuilder;
15
16pub fn get_builder(schema_uri: Option<&str>) -> SchemaBuilder {
17    SchemaBuilder::new(schema_uri)
18}
19
20type Schema = serde_json::Value;
21
22/// Configuration for building a JSON schema
23pub struct BuildConfig {
24    /// The delimiter to split the JSON objects in the data
25    pub delimiter: Option<u8>,
26    /// If the outermost array should be ignored if the data is a JSON array of JSON objects
27    pub ignore_outer_array: bool,
28}
29
30/// Parse a single JSON object and add it to the schema builder
31/// * `builder` - the schema builder object
32/// * `object_slice` - the JSON object to parse
33pub fn build_single_json_object_schema(builder: &mut SchemaBuilder, object_slice: &mut [u8]) {
34    let object = simd_json::to_borrowed_value(object_slice).unwrap();
35    builder.add_object(&object);
36}
37
38/// Parse a JSON schema from a JSON object or an array of JSON objects and add it to the schema builder.
39/// Passing delimiter will split the JSON objects more efficiently in parallel, and should result in a 2x speedup
40/// for large JSON object files (either multiple JSON objects concatenated together or a large JSON array).
41/// * `builder` - the schema builder object
42/// * `json_slice` - the JSON object or array of JSON objects to parse
43/// * `config` - the build configuration
44pub fn build_json_schema(
45    builder: &mut SchemaBuilder,
46    json_slice: &mut [u8],
47    config: &BuildConfig,
48) -> Schema {
49    let json_slice = trim_to_object(json_slice);
50    if is_json_object_array(json_slice) {
51        let array_elements = get_json_array_elements(json_slice);
52        build_multi_json_objects_schema(builder, array_elements, None);
53
54        if config.ignore_outer_array {
55            builder.to_schema()
56        } else {
57            let mut inner_schema = builder.to_schema();
58            let schema_uri = inner_schema.as_object_mut().unwrap().remove("$schema");
59            if let Some(schema_uri) = schema_uri {
60                json!({"$schema": schema_uri, "type": "array", "items": inner_schema})
61            } else {
62                json!({"type": "array", "items": inner_schema})
63            }
64        }
65    } else {
66        build_multi_json_objects_schema(builder, json_slice, config.delimiter);
67        builder.to_schema()
68    }
69}
70
71/// Build a JSON schema from multiple JSON objects
72fn build_multi_json_objects_schema(
73    builder: &mut SchemaBuilder,
74    json_slice: &mut [u8],
75    delimiter: Option<u8>,
76) {
77    if let Some(delimiter) = delimiter {
78        let object_iter = json_slice.par_split_mut(|byte| *byte == delimiter);
79        add_schema_from_object_par_iter(object_iter, builder);
80    } else {
81        let mut structure_count = 0;
82        let object_separator = move |byte: &u8| -> bool {
83            if *byte == b'{' {
84                structure_count += 1;
85            } else if *byte == b'}' {
86                structure_count -= 1;
87            }
88            structure_count == 0 && *byte == b'}'
89        };
90        // an vector of pointers to each of the individual JSON object bytes in the data
91        let mut object_slices: Vec<&mut [u8]> = vec![];
92        json_slice
93            .split_inclusive_mut(object_separator)
94            .for_each(|slice| object_slices.push(slice));
95        let object_iter = object_slices.into_par_iter();
96        add_schema_from_object_par_iter(object_iter, builder);
97    }
98}
99
100fn add_schema_from_object_par_iter<'a>(
101    object_iter: impl ParallelIterator<Item = &'a mut [u8]>,
102    builder: &mut SchemaBuilder,
103) {
104    let combined_builder = object_iter
105        .fold(
106            || SchemaBuilder::new(None),
107            |mut chunk_builder, object| {
108                let trimmed_object = trim_to_object(object);
109                if trimmed_object.is_empty() {
110                    return chunk_builder;
111                }
112                build_single_json_object_schema(&mut chunk_builder, trimmed_object);
113                chunk_builder
114            },
115        )
116        .reduce_with(|mut builder1, builder2| {
117            builder1.add_schema(builder2.to_schema());
118            builder1
119        })
120        .unwrap_or(SchemaBuilder::new(None));
121
122    builder.add_schema(combined_builder.to_schema());
123}
124
125/// trim the whitespace and non-JSON object characters from the start and end of the data
126fn trim_to_object(data: &mut [u8]) -> &mut [u8] {
127    let start = data
128        .iter()
129        .position(|&c| !c.is_ascii_whitespace() && (c == b'{' || c == b'['))
130        .unwrap_or(data.len());
131    // if the data is empty or only contains whitespace
132    if start == data.len() {
133        return &mut data[start..];
134    }
135    let end = data
136        .iter()
137        .rev()
138        .position(|&c| !c.is_ascii_whitespace() && (c == b'}' || c == b']'))
139        .map_or(0, |p| data.len() - p);
140    &mut data[start..end]
141}
142
143/// Check if the data is a JSON object array, this function assumes that the data is trimmed
144/// with `trim_to_object` before calling it
145fn is_json_object_array(data: &[u8]) -> bool {
146    data[0] == b'[' && data[data.len() - 1] == b']'
147}
148
149/// Get only the JSON array elements from the JSON array data slice, excluding the square brackets.
150/// This function assumes that the data is a JSON array (i.e. call `is_json_object_array` before calling it)
151fn get_json_array_elements(data: &mut [u8]) -> &mut [u8] {
152    let start = data.iter().position(|&c| c == b'[').unwrap_or(data.len());
153    let end = data.iter().rposition(|&c| c == b']').map_or(0, |p| p);
154    &mut data[start + 1..end]
155}
156
157#[cfg(test)]
158mod tests {
159    use super::*;
160
161    #[test]
162    fn test_trim_whitespace_non_empty() {
163        let mut data = br#"   {"name": "John", "age": 30}   "#.to_vec();
164        let trimmed_data = trim_to_object(&mut data);
165        let expected_data = br#"{"name": "John", "age": 30}"#.to_vec();
166        assert_eq!(trimmed_data, expected_data);
167    }
168
169    #[test]
170    fn test_trim_whitespace_empty() {
171        let mut data = br#"   "#.to_vec();
172        let trimmed_data = trim_to_object(&mut data);
173        let expected_data = br#""#.to_vec();
174        assert_eq!(trimmed_data, expected_data);
175    }
176
177    #[test]
178    fn test_trim_non_json_array_characters() {
179        let mut data = br#"\n  , [1, 2, 3]   \b\t"#.to_vec();
180        let trimmed_data = trim_to_object(&mut data);
181        let expected_data = br#"[1, 2, 3]"#.to_vec();
182        assert_eq!(trimmed_data, expected_data);
183    }
184
185    #[test]
186    fn test_trim_non_json_object_characters() {
187        let mut data = br#"\n  , {"name": "John", "age": 30}   \b\t"#.to_vec();
188        let trimmed_data = trim_to_object(&mut data);
189        let expected_data = br#"{"name": "John", "age": 30}"#.to_vec();
190        assert_eq!(trimmed_data, expected_data);
191    }
192
193    #[test]
194    fn test_is_json_object_array() {
195        let data = br#"[{"name": "John", "age": 30}]"#.to_vec();
196        assert!(is_json_object_array(&data));
197    }
198
199    #[test]
200    fn test_get_json_array_elements_single_elements() {
201        let mut data = br#"[{"name": "John", "age": 30}]"#.to_vec();
202        let elements = get_json_array_elements(&mut data);
203        let expected_elements = br#"{"name": "John", "age": 30}"#.to_vec();
204        assert_eq!(elements, expected_elements);
205    }
206
207    #[test]
208    fn test_get_json_array_elements_multiple_elements() {
209        let mut data = br#"[{"name": "John", "age": 30}, {"name": "Joe", "age": 20}]"#.to_vec();
210        let elements = get_json_array_elements(&mut data);
211        let expected_elements =
212            br#"{"name": "John", "age": 30}, {"name": "Joe", "age": 20}"#.to_vec();
213        assert_eq!(elements, expected_elements);
214    }
215}
genson_core/genson_rs/mod.rs

genson_core/genson_rs/
mod.rs
