polywrap_jsonref/
lib.rs

1//! jsonref dereferences JSONSchema `$ref` attributes and creates a new dereferenced schema.
2//!
3//! Dereferencing is normally done by a JSONSchema validator in the process of validation, but
4//! it is sometimes useful to do this independent of the validator for tasks like:
5//!
6//! * Analysing a schema programatically to see what field there are.
7//! * Programatically modifying a schema.
8//! * Passing to tools that create fake JSON data from the schema.
9//! * Passing the schema to form generation tools.
10//!
11//!
12//! Example:
13//! ```
14//! use serde_json::json;
15//! use jsonref::JsonRef;
16//!
17//! let mut simple_example = json!(
18//!           {"properties": {"prop1": {"title": "name"},
19//!                           "prop2": {"$ref": "#/properties/prop1"}}
20//!           }
21//!        );
22//!
23//! let mut jsonref = JsonRef::new();
24//!
25//! jsonref.deref_value(&mut simple_example).unwrap();
26//!
27//! let dereffed_expected = json!(
28//!     {"properties":
29//!         {"prop1": {"title": "name"},
30//!          "prop2": {"title": "name"}}
31//!     }
32//! );
33//! assert_eq!(simple_example, dereffed_expected)
34//! ```
35//!
36//! **Note**:  If the JSONSchema has recursive `$ref` only the first recursion will happen.
37//! This is to stop an infinate loop.
38
39use serde_json::json;
40use serde_json::Value;
41use snafu::{ResultExt, Snafu};
42use std::collections::HashMap;
43use std::env;
44use std::fs;
45use std::io;
46use std::mem;
47use std::path::PathBuf;
48use url::Url;
49
50#[derive(Debug, Snafu)]
51pub enum Error {
52    #[snafu(display("Could not open schema from {}: {}", filename, source))]
53    SchemaFromFile {
54        filename: String,
55        source: std::io::Error,
56    },
57    #[snafu(display("Could not open schema from url {}: {}", url, source))]
58    SchemaFromUrl { url: String, source: ureq::Error },
59    #[snafu(display("Parse error for url {}: {}", url, source))]
60    UrlParseError {
61        url: String,
62        source: url::ParseError,
63    },
64    #[snafu(display("schema from {} not valid JSON: {}", url, source))]
65    SchemaNotJson { url: String, source: std::io::Error },
66    #[snafu(display("schema from {} not valid JSON: {}", url, source))]
67    SchemaNotJsonSerde {
68        url: String,
69        source: serde_json::Error,
70    },
71    #[snafu(display("json pointer {} not found", pointer))]
72    JsonPointerNotFound { pointer: String },
73    #[snafu(display("{}", "Json Ref Error"))]
74    JSONRefError { source: std::io::Error },
75}
76
77/// Trait used to remove Json Value's element
78pub trait Remove {
79    /// Method use to remove element in Json Values
80    fn remove(&mut self, json_pointer: &str) -> io::Result<Option<Value>>;
81}
82
83impl Remove for serde_json::Value {
84    /// # Examples: Remove an element in a table
85    /// ```
86    /// use serde_json::Value;
87    /// use json_value_remove::Remove;
88    ///
89    /// let mut array1: Value = serde_json::from_str(r#"{"my_table":["a","b","c"]}"#).unwrap();
90    /// assert_eq!(Some(Value::String("a".to_string())), array1.remove("/my_table/0").unwrap());
91    /// assert_eq!(r#"{"my_table":["b","c"]}"#, array1.to_string());
92    /// ```
93    /// # Examples: Remove a field from an object
94    /// ```
95    /// use serde_json::Value;
96    /// use json_value_remove::Remove;
97    ///
98    /// let mut object1: Value = serde_json::from_str(r#"{"field1.0":{"field1.1":"value1.1","field1.2":"value1.2"},"field2.0":"value2.0"}"#).unwrap();
99    /// assert_eq!(Some(Value::String("value1.2".to_string())), object1.remove("/field1.0/field1.2").unwrap());
100    /// assert_eq!(r#"{"field1.0":{"field1.1":"value1.1"},"field2.0":"value2.0"}"#,object1.to_string());
101    /// ```
102    fn remove(&mut self, json_pointer: &str) -> io::Result<Option<Value>> {
103        let fields: Vec<&str> = json_pointer.split('/').skip(1).collect();
104
105        remove(self, fields)
106    }
107}
108
109fn remove(json_value: &mut Value, fields: Vec<&str>) -> io::Result<Option<Value>> {
110    if fields.is_empty() {
111        return Ok(None);
112    }
113
114    let mut fields = fields.clone();
115    let field = fields.remove(0);
116
117    if field.is_empty() {
118        return Ok(None);
119    }
120
121    match fields.is_empty() {
122        true => match json_value {
123            Value::Array(vec) => {
124                let index = match field.parse::<usize>() {
125                    Ok(index) => index,
126                    Err(e) => {
127                        return Err(io::Error::new(
128                            io::ErrorKind::InvalidInput,
129                            format!(
130                                "{}. Can't find the field '{}' in {}.",
131                                e,
132                                field,
133                                json_value
134                            ),
135                        ))
136                    }
137                };
138                let len = vec.len();
139                if index >= len {
140                    return Err(io::Error::new(
141                        io::ErrorKind::InvalidInput,
142                        format!(
143                            "removal index (is {}) should be < len (is {}) from {}",
144                            index,
145                            len,
146                            json_value
147                        ),
148                    ));
149                }
150                Ok(Some(vec.remove(index)))
151            }
152            Value::Object(map) => Ok(map.remove(field)),
153            _ => Ok(None),
154        },
155        false => match json_value.pointer_mut(format!("/{}", field).as_str()) {
156            Some(json_targeted) => remove(json_targeted, fields),
157            None => Ok(None),
158        },
159    }
160}
161
162type Result<T, E = Error> = std::result::Result<T, E>;
163
164/// Main struct that holds configuration for a JSONScheama derefferencing.
165///
166/// Instantiate with
167/// ```
168/// use jsonref::JsonRef;
169/// let jsonref = JsonRef::new();
170/// ```
171///
172/// Configuration is done through the `set_` methods on the struct.
173#[derive(Debug)]
174pub struct JsonRef {
175    schema_cache: HashMap<String, Value>,
176    reference_key: Option<String>,
177}
178
179impl JsonRef {
180    /// Create a new instance of JsonRef.
181    pub fn new() -> JsonRef {
182        JsonRef {
183            schema_cache: HashMap::new(),
184            reference_key: None,
185        }
186    }
187
188    /// Set a key to store the data that the `$ref` replaced.
189    ///
190    /// This example uses `__reference__` as the key.
191    ///
192    /// ```
193    /// # use jsonref::JsonRef;
194    /// # let jsonref = JsonRef::new();
195    /// use serde_json::json;
196    ///
197    /// let mut input  = json!(
198    ///     {"properties": {"prop1": {"title": "name"},
199    ///                     "prop2": {"$ref": "#/properties/prop1", "title": "old_title"}}
200    ///     }
201    /// );
202    ///
203    /// let expected = json!(
204    ///     {"properties": {"prop1": {"title": "name"},
205    ///                     "prop2": {"title": "name", "__reference__": {"title": "old_title"}}}
206    ///     }
207    /// );
208    ///                                                                                          
209    /// let mut jsonref = JsonRef::new();
210    ///
211    /// jsonref.set_reference_key("__reference__");
212    ///
213    /// jsonref.deref_value(&mut input).unwrap();
214    ///                                                                                          
215    /// assert_eq!(input, expected)
216    /// ```
217
218    pub fn set_reference_key(&mut self, reference_key: &str) {
219        self.reference_key = Some(reference_key.to_owned());
220    }
221
222    /// deref a serde_json value directly. Uses the current working directory for any relative
223    /// refs.
224    pub fn deref_value(&mut self, value: &mut Value) -> Result<()> {
225        let anon_file_url = format!(
226            "file://{}/anon.json",
227            env::current_dir()
228                .context(JSONRefError {})?
229                .to_string_lossy()
230        );
231        self.schema_cache
232            .insert(anon_file_url.clone(), value.clone());
233
234        let mut definitions = json!({});
235
236        self.deref(value, anon_file_url, &vec![], &mut definitions)?;
237
238        let val = value.as_object_mut().unwrap();
239        val.insert("definitions".to_string(), definitions);
240        Ok(())
241    }
242
243    /// deref from a URL:
244    ///
245    /// ```
246    /// # use jsonref::JsonRef;
247    /// # let jsonref = JsonRef::new();
248    /// # use serde_json::Value;
249    /// # use std::fs;
250    /// let mut jsonref = JsonRef::new();
251    /// # jsonref.set_reference_key("__reference__");
252    /// let input_url = jsonref.deref_url("https://gist.githubusercontent.com/kindly/91e09f88ced65aaca1a15d85a56a28f9/raw/52f8477435cff0b73c54aacc70926c101ce6c685/base.json").unwrap();
253    /// # let file = fs::File::open("fixtures/nested_relative/expected.json").unwrap();
254    /// # let file_expected: Value = serde_json::from_reader(file).unwrap();
255    /// # assert_eq!(input_url, file_expected)
256    /// ```
257    pub fn deref_url(&mut self, url: &str) -> Result<Value> {
258        let mut value: Value = ureq::get(url)
259            .call()
260            .context(SchemaFromUrl {
261                url: url.to_owned(),
262            })?
263            .into_json()
264            .context(SchemaNotJson {
265                url: url.to_owned(),
266            })?;
267
268        self.schema_cache.insert(url.to_string(), value.clone());
269        let mut definitions = json!({});
270        self.deref(&mut value, url.to_string(), &vec![], &mut definitions)?;
271        
272        let val = value.as_object_mut().unwrap();
273        val.insert("definitions".to_string(), definitions);
274
275        Ok(value)
276    }
277
278    /// deref from a File:
279    ///
280    /// ```
281    /// # use jsonref::JsonRef;
282    /// # let jsonref = JsonRef::new();
283    /// # use serde_json::Value;
284    /// # use std::fs;
285    ///
286    /// let mut jsonref = JsonRef::new();
287    /// # jsonref.set_reference_key("__reference__");
288    /// let file_example = jsonref
289    ///     .deref_file("fixtures/nested_relative/base.json")
290    ///     .unwrap();
291    /// # let file = fs::File::open("fixtures/nested_relative/expected.json").unwrap();
292    /// # let file_expected: Value = serde_json::from_reader(file).unwrap();
293    /// # assert_eq!(file_example, file_expected)
294    /// ```
295    pub fn deref_file(&mut self, file_path: &str) -> Result<Value> {
296        let file = fs::File::open(file_path).context(SchemaFromFile {
297            filename: file_path.to_owned(),
298        })?;
299        let mut value: Value = serde_json::from_reader(file).context(SchemaNotJsonSerde {
300            url: file_path.to_owned(),
301        })?;
302        let path = PathBuf::from(file_path);
303        let absolute_path = fs::canonicalize(path).context(JSONRefError {})?;
304        let url = format!("file://{}", absolute_path.to_string_lossy());
305
306        self.schema_cache.insert(url.clone(), value.clone());
307        let mut definitions = json!({});
308        self.deref(&mut value, url, &vec![], &mut definitions)?;
309
310        let val = value.as_object_mut().unwrap();
311        val.insert("definitions".to_string(), definitions);
312
313        Ok(value)
314    }
315
316    fn deref(
317        &mut self,
318        value: &mut Value,
319        id: String,
320        used_refs: &Vec<String>,
321        definitions: &mut Value,
322    ) -> Result<()> {
323        let mut new_id = id;
324        if let Some(id_value) = value.get("$id") {
325            if let Some(id_string) = id_value.as_str() {
326                new_id = id_string.to_string()
327            }
328        }
329
330        if let Some(obj) = value.as_object_mut() {
331            if let Some(defs) = obj.get_mut("definitions") {
332                let mut defs_clone = defs.clone();
333                obj.remove("definitions").unwrap();
334
335                if let Some(def_obj) = defs_clone.as_object_mut() {
336                    let accumulated_defs = definitions.as_object_mut().unwrap();
337                    for (key, val) in def_obj.iter_mut() {
338                        accumulated_defs.insert(key.to_string(), val.clone());
339                    }
340                }
341            }
342
343            if let Some(ref_value) = obj.remove("$ref") {
344                if let Some(ref_string) = ref_value.as_str() {
345                    let id_url = Url::parse(&new_id).context(UrlParseError {
346                        url: new_id.clone(),
347                    })?;
348                    let ref_url = id_url.join(ref_string).context(UrlParseError {
349                        url: ref_string.to_owned(),
350                    })?;
351
352                    let mut ref_url_no_fragment = ref_url.clone();
353                    ref_url_no_fragment.set_fragment(None);
354                    let ref_no_fragment = ref_url_no_fragment.to_string();
355
356                    let mut schema = match self.schema_cache.get(&ref_no_fragment) {
357                        Some(cached_schema) => cached_schema.clone(),
358                        None => {
359                            if ref_no_fragment.starts_with("http") {
360                                ureq::get(&ref_no_fragment)
361                                    .call()
362                                    .context(SchemaFromUrl {
363                                        url: ref_no_fragment.clone(),
364                                    })?
365                                    .into_json()
366                                    .context(SchemaNotJson {
367                                        url: ref_no_fragment.clone(),
368                                    })?
369                            } else if ref_no_fragment.starts_with("file") {
370                                let file = fs::File::open(ref_url_no_fragment.path()).context(
371                                    SchemaFromFile {
372                                        filename: ref_no_fragment.clone(),
373                                    },
374                                )?;
375                                serde_json::from_reader(file).context(SchemaNotJsonSerde {
376                                    url: ref_no_fragment.clone(),
377                                })?
378                            } else {
379                                panic!("need url to be a file or a http based url")
380                            }
381                        }
382                    };
383
384                    if !self.schema_cache.contains_key(&ref_no_fragment) {
385                        self.schema_cache
386                            .insert(ref_no_fragment.clone(), schema.clone());
387                    }
388
389                    let ref_url_string = ref_url.to_string();
390                    if let Some(ref_fragment) = ref_url.fragment() {
391                        schema = schema.pointer(ref_fragment).ok_or(
392                            Error::JsonPointerNotFound {pointer: format!("ref `{}` can not be resolved as pointer `{}` can not be found in the schema", ref_string, ref_fragment)}
393                            )?.clone();
394                    }
395                    if used_refs.contains(&ref_url_string) {
396                        return Ok(());
397                    }
398
399                    let mut new_used_refs = used_refs.clone();
400                    new_used_refs.push(ref_url_string);
401
402                    self.deref(&mut schema, ref_no_fragment, &new_used_refs, definitions)?;
403                    let old_value = mem::replace(value, schema);
404
405                    if let Some(reference_key) = &self.reference_key {
406                        if let Some(new_obj) = value.as_object_mut() {
407                            new_obj.insert(reference_key.clone(), old_value);
408                        }
409                    }
410                }
411            }
412        }
413
414        if let Some(obj) = value.as_object_mut() {
415            for obj_value in obj.values_mut() {
416                self.deref(obj_value, new_id.clone(), used_refs, definitions)?
417            }
418        }
419        Ok(())
420    }
421}
422
423#[cfg(test)]
424mod tests {
425    use super::JsonRef;
426    use serde_json::{json, Value};
427    use std::fs;
428
429    #[test]
430    fn json_no_refs() {
431        let no_ref_example = json!({"properties": {"prop1": {"title": "proptitle"}}});
432
433        let mut jsonref = JsonRef::new();
434
435        let mut input = no_ref_example.clone();
436
437        jsonref.deref_value(&mut input).unwrap();
438
439        assert_eq!(input, no_ref_example)
440    }
441
442    #[test]
443    fn json_with_recursion() {
444        let mut simple_refs_example = json!(
445            {"properties": {"prop1": {"$ref": "#"}}}
446        );
447
448        let simple_refs_expected = json!(
449            {"properties": {"prop1": {"properties": {"prop1": {}}}}
450            }
451        );
452
453        let mut jsonref = JsonRef::new();
454        jsonref.deref_value(&mut simple_refs_example).unwrap();
455        jsonref.set_reference_key("__reference__");
456
457        println!(
458            "{}",
459            serde_json::to_string_pretty(&simple_refs_example).unwrap()
460        );
461
462        assert_eq!(simple_refs_example, simple_refs_expected)
463    }
464
465    #[test]
466    fn simple_from_url() {
467        let mut simple_refs_example = json!(
468            {"properties": {"prop1": {"title": "name"},
469                            "prop2": {"$ref": "https://gist.githubusercontent.com/kindly/35a631d33792413ed8e34548abaa9d61/raw/b43dc7a76cc2a04fde2a2087f0eb389099b952fb/test.json", "title": "old_title"}}
470            }
471        );
472
473        let simple_refs_expected = json!(
474            {"properties": {"prop1": {"title": "name"},
475                            "prop2": {"title": "title from url", "__reference__": {"title": "old_title"}}}
476            }
477        );
478
479        let mut jsonref = JsonRef::new();
480        jsonref.set_reference_key("__reference__");
481        jsonref.deref_value(&mut simple_refs_example).unwrap();
482
483        assert_eq!(simple_refs_example, simple_refs_expected)
484    }
485
486    #[test]
487    fn nested_with_ref_from_url() {
488        let mut simple_refs_example = json!(
489            {"properties": {"prop1": {"title": "name"},
490                            "prop2": {"$ref": "https://gist.githubusercontent.com/kindly/35a631d33792413ed8e34548abaa9d61/raw/0a691c035251f742e8710f71ba92ead307823385/test_nested.json"}}
491            }
492        );
493
494        let simple_refs_expected = json!(
495            {"properties": {"prop1": {"title": "name"},
496                            "prop2": {"__reference__": {},
497                                      "title": "title from url",
498                                      "properties": {"prop1": {"title": "sub property title in url"},
499                                                     "prop2": {"__reference__": {}, "title": "sub property title in url"}}
500                            }}
501            }
502        );
503
504        let mut jsonref = JsonRef::new();
505        jsonref.set_reference_key("__reference__");
506        jsonref.deref_value(&mut simple_refs_example).unwrap();
507
508        assert_eq!(simple_refs_example, simple_refs_expected)
509    }
510
511    #[test]
512    fn nested_ref_from_local_file() {
513        let mut jsonref = JsonRef::new();
514        jsonref.set_reference_key("__reference__");
515        let file_example = jsonref
516            .deref_file("fixtures/nested_relative/base.json")
517            .unwrap();
518
519        let file = fs::File::open("fixtures/nested_relative/expected.json").unwrap();
520        let file_expected: Value = serde_json::from_reader(file).unwrap();
521
522        println!("{}", serde_json::to_string_pretty(&file_example).unwrap());
523
524        assert_eq!(file_example, file_expected)
525    }
526
527    #[test]
528    fn test_defs() {
529        let mut jsonref = JsonRef::new();
530        jsonref.set_reference_key("__reference__");
531        let file_example = jsonref
532            .deref_file("fixtures/definitions/base.json")
533            .unwrap();
534
535        let file = fs::File::open("fixtures/definitions/expected.json").unwrap();
536        let file_expected: Value = serde_json::from_reader(file).unwrap();
537
538        println!("{}", serde_json::to_string_pretty(&file_example).unwrap());
539
540        assert_eq!(file_example, file_expected)
541    }
542}
polywrap_jsonref/lib.rs

polywrap_jsonref/
lib.rs
