genson_core/genson_rs/
mod.rs

1#![allow(mismatched_lifetime_syntaxes)]
2mod builder;
3pub(crate) mod interned;
4mod node;
5mod strategy;
6
7use crate::value::object;
8#[cfg(not(feature = "alloc-stats"))]
9use mimalloc::MiMalloc;
10use rayon::prelude::*;
11
12// Setting the global allocator to mimalloc for more efficient memory allocation
13#[cfg(not(feature = "alloc-stats"))]
14#[global_allocator]
15static GLOBAL: MiMalloc = MiMalloc;
16
17#[cfg(feature = "alloc-stats")]
18#[global_allocator]
19static GLOBAL: crate::phase::Counting = crate::phase::Counting;
20
21pub use builder::SchemaBuilder;
22pub(crate) use node::{DataType, SchemaNode};
23
24/// A parsed JSON value, read from simd-json's tape so every object's keys are visited in
25/// document order. simd-json's `BorrowedValue` stores objects of more than 32 keys in a
26/// randomly seeded hash map, which scrambled the order of wide objects between runs.
27pub(crate) type JsonValue<'a> = simd_json::tape::Value<'a, 'a>;
28
29pub fn get_builder(schema_uri: Option<&str>) -> SchemaBuilder {
30    SchemaBuilder::new(schema_uri)
31}
32
33type Schema = serde_json::Value;
34
35/// Configuration for building a JSON schema
36pub struct BuildConfig {
37    /// The delimiter to split the JSON objects in the data
38    pub delimiter: Option<u8>,
39    /// If the outermost array should be ignored if the data is a JSON array of JSON objects
40    pub ignore_outer_array: bool,
41}
42
43/// Parse a single JSON object and add it to the schema builder
44/// * `builder` - the schema builder object
45/// * `object_slice` - the JSON object to parse
46pub fn build_single_json_object_schema(builder: &mut SchemaBuilder, object_slice: &mut [u8]) {
47    let tape = simd_json::to_tape(object_slice).unwrap();
48    builder.add_object(tape.as_value());
49}
50
51/// Parse a JSON schema from a JSON object or an array of JSON objects and add it to the schema builder.
52/// Passing delimiter will split the JSON objects more efficiently in parallel, and should result in a 2x speedup
53/// for large JSON object files (either multiple JSON objects concatenated together or a large JSON array).
54/// * `builder` - the schema builder object
55/// * `json_slice` - the JSON object or array of JSON objects to parse
56/// * `config` - the build configuration
57pub fn build_json_schema(
58    builder: &mut SchemaBuilder,
59    json_slice: &mut [u8],
60    config: &BuildConfig,
61) -> Schema {
62    let json_slice = trim_to_object(json_slice);
63    let is_array = is_json_object_array(json_slice);
64    build_trimmed(builder, json_slice, is_array, config);
65    if is_array {
66        if config.ignore_outer_array {
67            builder.to_schema()
68        } else {
69            let mut inner_schema = builder.to_schema();
70            let schema_uri = inner_schema
71                .as_object_mut()
72                .unwrap()
73                .shift_remove("$schema");
74            if let Some(schema_uri) = schema_uri {
75                object([
76                    ("$schema", schema_uri),
77                    ("type", "array".into()),
78                    ("items", inner_schema),
79                ])
80            } else {
81                object([("type", "array".into()), ("items", inner_schema)])
82            }
83        }
84    } else {
85        builder.to_schema()
86    }
87}
88
89/// Like [`build_json_schema`] but only accumulates into `builder`, skipping the
90/// (otherwise discarded) conversion of the builder to a schema value.
91pub fn build_json_schema_into(
92    builder: &mut SchemaBuilder,
93    json_slice: &mut [u8],
94    config: &BuildConfig,
95) {
96    let json_slice = trim_to_object(json_slice);
97    let is_array = is_json_object_array(json_slice);
98    build_trimmed(builder, json_slice, is_array, config);
99}
100
101fn build_trimmed(
102    builder: &mut SchemaBuilder,
103    json_slice: &mut [u8],
104    is_array: bool,
105    config: &BuildConfig,
106) {
107    if is_array {
108        let array_elements = get_json_array_elements(json_slice);
109        build_multi_json_objects_schema(builder, array_elements, None);
110    } else {
111        build_multi_json_objects_schema(builder, json_slice, config.delimiter);
112    }
113}
114
115/// Build a JSON schema from multiple JSON objects
116fn build_multi_json_objects_schema(
117    builder: &mut SchemaBuilder,
118    json_slice: &mut [u8],
119    delimiter: Option<u8>,
120) {
121    if let Some(delimiter) = delimiter {
122        let object_iter = json_slice.par_split_mut(|byte| *byte == delimiter);
123        add_schema_from_object_par_iter(object_iter, builder);
124    } else {
125        let mut structure_count = 0;
126        let object_separator = move |byte: &u8| -> bool {
127            if *byte == b'{' {
128                structure_count += 1;
129            } else if *byte == b'}' {
130                structure_count -= 1;
131            }
132            structure_count == 0 && *byte == b'}'
133        };
134        // an vector of pointers to each of the individual JSON object bytes in the data
135        let mut object_slices: Vec<&mut [u8]> = vec![];
136        json_slice
137            .split_inclusive_mut(object_separator)
138            .for_each(|slice| object_slices.push(slice));
139        let object_iter = object_slices.into_par_iter();
140        add_schema_from_object_par_iter(object_iter, builder);
141    }
142}
143
144fn add_schema_from_object_par_iter<'a>(
145    object_iter: impl ParallelIterator<Item = &'a mut [u8]>,
146    builder: &mut SchemaBuilder,
147) {
148    let combined_builder = object_iter
149        .fold(
150            || SchemaBuilder::new(None),
151            |mut chunk_builder, object| {
152                let trimmed_object = trim_to_object(object);
153                if trimmed_object.is_empty() {
154                    return chunk_builder;
155                }
156                build_single_json_object_schema(&mut chunk_builder, trimmed_object);
157                chunk_builder
158            },
159        )
160        .reduce_with(|mut builder1, builder2| {
161            builder1.add_builder(builder2);
162            builder1
163        })
164        .unwrap_or(SchemaBuilder::new(None));
165
166    builder.absorb(combined_builder);
167}
168
169/// trim the whitespace and non-JSON object characters from the start and end of the data
170fn trim_to_object(data: &mut [u8]) -> &mut [u8] {
171    let start = data
172        .iter()
173        .position(|&c| !c.is_ascii_whitespace() && (c == b'{' || c == b'['))
174        .unwrap_or(data.len());
175    // if the data is empty or only contains whitespace
176    if start == data.len() {
177        return &mut data[start..];
178    }
179    let end = data
180        .iter()
181        .rev()
182        .position(|&c| !c.is_ascii_whitespace() && (c == b'}' || c == b']'))
183        .map_or(0, |p| data.len() - p);
184    &mut data[start..end]
185}
186
187/// Check if the data is a JSON object array, this function assumes that the data is trimmed
188/// with `trim_to_object` before calling it
189fn is_json_object_array(data: &[u8]) -> bool {
190    data[0] == b'[' && data[data.len() - 1] == b']'
191}
192
193/// Get only the JSON array elements from the JSON array data slice, excluding the square brackets.
194/// This function assumes that the data is a JSON array (i.e. call `is_json_object_array` before calling it)
195fn get_json_array_elements(data: &mut [u8]) -> &mut [u8] {
196    let start = data.iter().position(|&c| c == b'[').unwrap_or(data.len());
197    let end = data.iter().rposition(|&c| c == b']').unwrap_or(0);
198    &mut data[start + 1..end]
199}
200
201#[cfg(test)]
202mod tests {
203    use super::*;
204
205    #[test]
206    fn test_trim_whitespace_non_empty() {
207        let mut data = br#"   {"name": "John", "age": 30}   "#.to_vec();
208        let trimmed_data = trim_to_object(&mut data);
209        let expected_data = br#"{"name": "John", "age": 30}"#.to_vec();
210        assert_eq!(trimmed_data, expected_data);
211    }
212
213    #[test]
214    fn test_trim_whitespace_empty() {
215        let mut data = br#"   "#.to_vec();
216        let trimmed_data = trim_to_object(&mut data);
217        let expected_data = br#""#.to_vec();
218        assert_eq!(trimmed_data, expected_data);
219    }
220
221    #[test]
222    fn test_trim_non_json_array_characters() {
223        let mut data = br#"\n  , [1, 2, 3]   \b\t"#.to_vec();
224        let trimmed_data = trim_to_object(&mut data);
225        let expected_data = br#"[1, 2, 3]"#.to_vec();
226        assert_eq!(trimmed_data, expected_data);
227    }
228
229    #[test]
230    fn test_trim_non_json_object_characters() {
231        let mut data = br#"\n  , {"name": "John", "age": 30}   \b\t"#.to_vec();
232        let trimmed_data = trim_to_object(&mut data);
233        let expected_data = br#"{"name": "John", "age": 30}"#.to_vec();
234        assert_eq!(trimmed_data, expected_data);
235    }
236
237    #[test]
238    fn test_is_json_object_array() {
239        let data = br#"[{"name": "John", "age": 30}]"#.to_vec();
240        assert!(is_json_object_array(&data));
241    }
242
243    #[test]
244    fn test_get_json_array_elements_single_elements() {
245        let mut data = br#"[{"name": "John", "age": 30}]"#.to_vec();
246        let elements = get_json_array_elements(&mut data);
247        let expected_elements = br#"{"name": "John", "age": 30}"#.to_vec();
248        assert_eq!(elements, expected_elements);
249    }
250
251    #[test]
252    fn test_get_json_array_elements_multiple_elements() {
253        let mut data = br#"[{"name": "John", "age": 30}, {"name": "Joe", "age": 20}]"#.to_vec();
254        let elements = get_json_array_elements(&mut data);
255        let expected_elements =
256            br#"{"name": "John", "age": 30}, {"name": "Joe", "age": 20}"#.to_vec();
257        assert_eq!(elements, expected_elements);
258    }
259}
genson_core/genson_rs/mod.rs

genson_core/genson_rs/
mod.rs
