jsonc_to_json/
lib.rs

1//! Simple library for converting [JSON with Comments] into [JSON],
2//! in short it removes the following:
3//!
4//! - Line comments, e.g. `// Line Comment`
5//! - Block comments, e.g. `/* Block Comment */`
6//! - Trailing commas, e.g. `[1,2,3,,]` -> `[1,2,3]`
7//!
8//! **Note:** The implementation does not use a full-blown [JSON with Comments]
9//! parser. Instead it uses a [JSON with Comments] tokenizer, which makes
10//! conversion a lot faster.
11//!
12//! Currently `#![no_std]` is not supported. It will however be added, when
13//! some upstream changes have been applied.
14//!
15//! See [`jsonc_to_json()`] for more information.
16//!
17//! # Example
18//!
19//! ```rust
20//! use jsonc_to_json::{jsonc_to_json, jsonc_to_json_into};
21//!
22//! let jsonc = "{\"arr\": [1, 2,/* Comment */ 3, 4,,]}// Line Comment";
23//!
24//! let json = jsonc_to_json(jsonc);
25//! println!("{}", json);
26//! # assert_eq!(json, "{\"arr\": [1, 2, 3, 4]}");
27//!
28//! // Alternatively, use `jsonc_to_json_into()` to reuse an
29//! // already allocated `String`
30//! let mut json = String::new();
31//! jsonc_to_json_into(jsonc, &mut json);
32//! println!("{}", json);
33//! # assert_eq!(json, "{\"arr\": [1, 2, 3, 4]}");
34//! ```
35//!
36//! Both output the following:
37//!
38//! ```text
39//! {"arr": [1, 2, 3, 4]}
40//! ```
41//!
42//! # Serde Example
43//!
44//! ```rust
45//! # fn main() -> Result<(), Box<dyn std::error::Error>> {
46//! use jsonc_to_json::{jsonc_to_json, jsonc_to_json_into};
47//! use serde::Deserialize;
48//!
49//! #[derive(Deserialize, Debug)]
50//! # #[derive(PartialEq)]
51//! struct Data {
52//!     arr: Vec<i32>,
53//! }
54//! let jsonc = "{\"arr\": [1, 2,/* Comment */ 3, 4,,]}// Line Comment";
55//!
56//! let json = jsonc_to_json(jsonc);
57//! let data: Data = serde_json::from_str(&json)?;
58//!
59//! println!("{}", json);
60//! println!("{:?}", data);
61//! # assert_eq!(json, "{\"arr\": [1, 2, 3, 4]}");
62//! # assert_eq!(data, Data { arr: vec![1, 2, 3, 4] });
63//! # Ok(())
64//! # }
65//! ```
66//!
67//! Which outputs the following:
68//!
69//! ```text
70//! {"arr": [1, 2, 3, 4]}
71//! Data { arr: [1, 2, 3, 4] }
72//! ```
73//!
74//! # Non-Allocating & Zero-Copy Iterator Example
75//!
76//! Non-allocating [`Iterator`] that yields string slices of
77//! valid [JSON].
78//!
79//! ```rust
80//! use jsonc_to_json::jsonc_to_json_iter;
81//!
82//! let jsonc = r#"{foo}/**/[1,2,3,,]"bar""#;
83//!
84//! let mut iter = jsonc_to_json_iter(jsonc);
85//! assert_eq!(iter.next(), Some("{foo}")); // Line comment was removed
86//! assert_eq!(iter.next(), Some("[1,2,3")); // Trailing commas was removed
87//! assert_eq!(iter.next(), Some("]\"bar\""));
88//! assert_eq!(iter.next(), None);
89//! ```
90//!
91//! [JSON with Comments]: https://code.visualstudio.com/docs/languages/json#_json-with-comments
92//! [JSON]: https://www.json.org/json-en.html
93
94#![forbid(unsafe_code)]
95#![forbid(elided_lifetimes_in_paths)]
96#![deny(missing_docs)]
97#![deny(missing_debug_implementations)]
98#![warn(clippy::all)]
99
100use std::borrow::Cow;
101use std::iter::FusedIterator;
102use std::ops::Range;
103
104use any_lexer::{JsonCLexer, JsonCToken, Lexer, TokenSpan};
105
106/// Removes all [JSON with Comments] parts from `jsonc`, turning it into
107/// valid [JSON], i.e. removing line comments, block comments, and trailing
108/// commas.
109///
110/// - Line comments, e.g. `// Line Comment`
111/// - Block comments, e.g. `/* Block Comment */`
112/// - Trailing commas, e.g. `[1,2,3,,]` -> `[1,2,3]`
113///
114/// If `jsonc` is already valid [JSON], then <code>[Cow]::[Borrowed]\(jsonc)</code>
115/// is returned, otherwise a new [`String`] is allocated and <code>[Cow]::[Owned]</code>
116/// is returned.
117///
118/// **Warning:** The conversion is infallible and does not validate `jsonc`.
119/// If it contains invalid [JSON] or invalid [JSON with Comments], then the
120/// invalid parts are included in the result, i.e. `{foo,/*comment*/bar,}`
121/// is turned into `{foo,bar}`.
122///
123/// See also [`jsonc_to_json_into()`] for an alternative variant, that reuses
124/// an already allocated [`String`].
125///
126/// # Example
127///
128/// See also [`serde_json` example] in crate root docs.
129///
130/// ```rust
131/// use jsonc_to_json::{jsonc_to_json, jsonc_to_json_into};
132///
133/// let jsonc = "{\"arr\": [1, 2,/* Comment */ 3, 4,,]}// Line Comment";
134///
135/// let json = jsonc_to_json(jsonc);
136/// println!("{}", json);
137/// # assert_eq!(json, "{\"arr\": [1, 2, 3, 4]}");
138/// ```
139///
140/// Which outputs the following:
141///
142/// ```text
143/// {"arr": [1, 2, 3, 4]}
144/// ```
145///
146/// [JSON with Comments]: https://code.visualstudio.com/docs/languages/json#_json-with-comments
147/// [JSON]: https://www.json.org/json-en.html
148/// [Borrowed]: Cow::Borrowed
149/// [Owned]: Cow::Owned
150/// [`serde_json` example]: crate#serde-example
151pub fn jsonc_to_json(jsonc: &str) -> Cow<'_, str> {
152    let mut iter = JsonCToJsonIter::new(jsonc);
153
154    let first = match iter.next() {
155        Some(first) => first,
156        None => return Cow::Borrowed(""),
157    };
158
159    let second = match iter.next() {
160        Some(second) => second,
161        None => return Cow::Borrowed(first),
162    };
163
164    let mut json = String::new();
165    json.push_str(first);
166    json.push_str(second);
167
168    for part in iter {
169        json.push_str(part);
170    }
171
172    Cow::Owned(json)
173}
174
175/// Same as [`jsonc_to_json()`], but instead of allocating a
176/// new [`String`], then the output JSON is appended to `json`.
177///
178/// **Note:** The output [JSON] is appended to `json`, i.e. if `json`
179/// is not empty, then call [`clear()`] beforehand.
180///
181/// See [`jsonc_to_json()`] for more information.
182///
183/// # Example
184///
185/// See also [`serde_json` example] in crate root docs.
186///
187/// ```rust
188/// # use jsonc_to_json::jsonc_to_json_into;
189/// let jsonc = "{\"arr\": [1, 2,/* Comment */ 3, 4,,]}// Line Comment";
190///
191/// let mut json = String::new();
192/// jsonc_to_json_into(jsonc, &mut json);
193/// println!("{}", json);
194/// # assert_eq!(json, "{\"arr\": [1, 2, 3, 4]}");
195/// ```
196///
197/// Which outputs the following:
198///
199/// ```text
200/// {"arr": [1, 2, 3, 4]}
201/// ```
202///
203/// [JSON]: https://www.json.org/json-en.html
204/// [`clear()`]: String::clear
205/// [`serde_json` example]: crate#serde-example
206#[inline]
207pub fn jsonc_to_json_into(jsonc: &str, json: &mut String) {
208    for part in JsonCToJsonIter::new(jsonc) {
209        json.push_str(part);
210    }
211}
212
213/// Non-allocating and zero-copy [`Iterator`] that yields string slices
214/// of valid [JSON].
215///
216/// **Warning:** The conversion is infallible and does not validate `jsonc`.
217/// If it contains invalid [JSON] or invalid [JSON with Comments], then the
218/// invalid parts are included in the result, i.e. `{foo,/*comment*/bar,}`
219/// is turned into `{foo,bar}`.
220///
221/// See [`jsonc_to_json()`] for more information.
222///
223/// # Example
224///
225/// ```rust
226/// # use jsonc_to_json::jsonc_to_json_iter;
227/// let jsonc = r#"{foo}/**/[1,2,3,,]"bar""#;
228///
229/// let mut iter = jsonc_to_json_iter(jsonc);
230/// assert_eq!(iter.next(), Some("{foo}")); // Line comment was removed
231/// assert_eq!(iter.next(), Some("[1,2,3")); // Trailing commas was removed
232/// assert_eq!(iter.next(), Some("]\"bar\""));
233/// assert_eq!(iter.next(), None);
234/// ```
235///
236/// [JSON]: https://www.json.org/json-en.html
237#[inline]
238pub fn jsonc_to_json_iter(jsonc: &str) -> JsonCToJsonIter<'_> {
239    JsonCToJsonIter::new(jsonc)
240}
241
242/// See [`jsonc_to_json_iter()`] for more information.
243#[derive(Clone, Debug)]
244pub struct JsonCToJsonIter<'jsonc> {
245    lexer: JsonCLexer<'jsonc>,
246    next: Option<Range<usize>>,
247}
248
249impl<'jsonc> JsonCToJsonIter<'jsonc> {
250    /// See [`jsonc_to_json_iter()`] for more information.
251    pub fn new(jsonc: &'jsonc str) -> Self {
252        Self {
253            lexer: JsonCLexer::new(jsonc),
254            next: None,
255        }
256    }
257}
258
259impl<'jsonc> Iterator for JsonCToJsonIter<'jsonc> {
260    type Item = &'jsonc str;
261
262    #[inline]
263    fn next(&mut self) -> Option<Self::Item> {
264        let mut span = match self.next.take() {
265            Some(span) => span,
266            None => self.lexer.next_valid_json_token()?,
267        };
268
269        loop {
270            let next = self.lexer.next_valid_json_token();
271            if let Some(next) = next {
272                match span.continue_range(&next) {
273                    Some(new_span) => {
274                        span = new_span;
275                    }
276                    None => {
277                        self.next = Some(next);
278                        break;
279                    }
280                }
281            } else {
282                break;
283            }
284        }
285
286        Some(&self.lexer.scanner().text()[span])
287    }
288}
289
290impl FusedIterator for JsonCToJsonIter<'_> {}
291
292trait JsonCToJsonExt<'jsonc> {
293    fn next_token(&mut self) -> Option<(JsonCToken, &'jsonc str)>;
294    fn next_valid_json_token(&mut self) -> Option<Range<usize>>;
295}
296
297impl<'jsonc, I> JsonCToJsonExt<'jsonc> for I
298where
299    I: Iterator<Item = (JsonCToken, TokenSpan<'jsonc>)>,
300    I: Clone,
301{
302    #[inline]
303    fn next_token(&mut self) -> Option<(JsonCToken, &'jsonc str)> {
304        let (tok, span) = self.next()?;
305        Some((tok, span.as_str()))
306    }
307
308    fn next_valid_json_token(&mut self) -> Option<Range<usize>> {
309        loop {
310            let (tok, span) = self.next()?;
311            let s = span.as_str();
312
313            match tok {
314                JsonCToken::Space => {}
315                JsonCToken::LineComment | JsonCToken::BlockComment => continue,
316                JsonCToken::Punct if s == "," => {
317                    let mut iter = self.clone().filter(|(tok, _span)| {
318                        !matches!(
319                            tok,
320                            JsonCToken::Space | JsonCToken::LineComment | JsonCToken::BlockComment
321                        )
322                    });
323
324                    let (tok, s) = match iter.next_token() {
325                        Some((tok, s)) => (tok, s),
326                        None => continue,
327                    };
328
329                    match tok {
330                        JsonCToken::Punct if s == "," => continue,
331                        JsonCToken::Delim => continue,
332                        JsonCToken::String
333                        | JsonCToken::Number
334                        | JsonCToken::Null
335                        | JsonCToken::True
336                        | JsonCToken::False
337                        | JsonCToken::Punct
338                        | JsonCToken::Unknown => {}
339                        JsonCToken::Space | JsonCToken::LineComment | JsonCToken::BlockComment => {
340                            unreachable!()
341                        }
342                    }
343                }
344                JsonCToken::String
345                | JsonCToken::Number
346                | JsonCToken::Null
347                | JsonCToken::True
348                | JsonCToken::False
349                | JsonCToken::Punct
350                | JsonCToken::Delim
351                | JsonCToken::Unknown => {}
352            }
353
354            return Some(span.range());
355        }
356    }
357}
358
359trait ContinueRange: Sized {
360    fn continue_range(&self, next: &Self) -> Option<Self>;
361}
362
363impl ContinueRange for Range<usize> {
364    #[inline]
365    fn continue_range(&self, next: &Self) -> Option<Self> {
366        if self.end == next.start {
367            Some(self.start..next.end)
368        } else {
369            None
370        }
371    }
372}
373
374#[cfg(test)]
375mod tests {
376    use super::*;
377
378    macro_rules! assert_jsonc_to_json {
379        ($jsonc:expr, $json:expr) => {{
380            let jsonc: &str = $jsonc;
381            let json: Cow<'_, str> = $json;
382            let actual = jsonc_to_json(jsonc);
383            assert_eq!(actual, json);
384            assert_eq!(
385                matches!(actual, Cow::Borrowed(_)),
386                matches!(json, Cow::Borrowed(_))
387            );
388        }};
389    }
390
391    #[test]
392    fn test_empty() {
393        assert_jsonc_to_json!("", Cow::Borrowed(""));
394    }
395
396    #[test]
397    fn test_borrowed() {
398        let jsonc = r#"{"arr": [1, 2, 3, 4]}"#;
399        assert_jsonc_to_json!(jsonc, Cow::Borrowed(jsonc));
400    }
401
402    #[test]
403    fn test_borrowed_ending_removed() {
404        let jsonc = r#"{"arr": [1, 2, 3, 4]} // Line Comment"#;
405        let json = r#"{"arr": [1, 2, 3, 4]} "#;
406        assert_jsonc_to_json!(jsonc, Cow::Borrowed(json));
407    }
408
409    #[test]
410    fn test_line_comment() {
411        let jsonc = r#"// Comment
412{
413    //
414    "arr": [1, 2,
415    // Comment
416    3, 4] // Comment
417    //
418}
419// Comment"#;
420        let json = "\n{\n    \n    \"arr\": [1, 2,\n    \n    3, 4] \n    \n}\n";
421        assert_jsonc_to_json!(jsonc, Cow::Owned(json.to_owned()));
422    }
423
424    #[test]
425    fn test_iter() {
426        let jsonc = r#"{foo}/**/[1,2,3,,]"bar""#;
427        let mut iter = jsonc_to_json_iter(jsonc);
428
429        assert_eq!(iter.next(), Some("{foo}"));
430        assert_eq!(iter.next(), Some("[1,2,3"));
431        assert_eq!(iter.next(), Some("]\"bar\""));
432        assert_eq!(iter.next(), None);
433    }
434}
jsonc_to_json/lib.rs

jsonc_to_json/
lib.rs
