nu_command/formats/from/
json.rs

1use std::io::{BufRead, Cursor};
2
3use nu_engine::command_prelude::*;
4use nu_protocol::{
5    DEFAULT_ERROR_CONTEXT, ListStream, Signals,
6    shell_error::{generic::GenericError, io::IoError},
7    truncated_source_window,
8};
9
10#[derive(Clone)]
11pub struct FromJson;
12
13impl Command for FromJson {
14    fn name(&self) -> &str {
15        "from json"
16    }
17
18    fn description(&self) -> &str {
19        "Convert JSON text into structured data."
20    }
21
22    fn signature(&self) -> nu_protocol::Signature {
23        Signature::build("from json")
24            .input_output_types(vec![(Type::String, Type::Any)])
25            .switch("objects", "Treat each line as a separate value.", Some('o'))
26            .switch(
27                "strict",
28                "Follow the json specification exactly.",
29                Some('s'),
30            )
31            .category(Category::Formats)
32    }
33
34    fn examples(&self) -> Vec<Example<'_>> {
35        vec![
36            Example {
37                example: r#"'{ "a": 1 }' | from json"#,
38                description: "Converts json formatted string to table.",
39                result: Some(Value::test_record(record! {
40                    "a" => Value::test_int(1),
41                })),
42            },
43            Example {
44                example: r#"'{ "a": 1, "b": [1, 2] }' | from json"#,
45                description: "Converts json formatted string to table.",
46                result: Some(Value::test_record(record! {
47                    "a" => Value::test_int(1),
48                    "b" => Value::test_list(vec![Value::test_int(1), Value::test_int(2)]),
49                })),
50            },
51            Example {
52                example: r#"'{ "a": 1, "b": 2 }' | from json -s"#,
53                description: "Parse json strictly which will error on comments and trailing commas.",
54                result: Some(Value::test_record(record! {
55                    "a" => Value::test_int(1),
56                    "b" => Value::test_int(2),
57                })),
58            },
59            Example {
60                example: r#"'{ "a": 1 }
61{ "b": 2 }' | from json --objects"#,
62                description: "Parse a stream of line-delimited JSON values.",
63                result: Some(Value::test_list(vec![
64                    Value::test_record(record! {"a" => Value::test_int(1)}),
65                    Value::test_record(record! {"b" => Value::test_int(2)}),
66                ])),
67            },
68        ]
69    }
70
71    fn run(
72        &self,
73        engine_state: &EngineState,
74        stack: &mut Stack,
75        call: &Call,
76        mut input: PipelineData,
77    ) -> Result<PipelineData, ShellError> {
78        let span = call.head;
79
80        let strict = call.has_flag(engine_state, stack, "strict")?;
81        let metadata = input.take_metadata().map(|md| md.with_content_type(None));
82
83        // TODO: turn this into a structured underline of the nu_json error
84        if call.has_flag(engine_state, stack, "objects")? {
85            // Return a stream of JSON values, one for each non-empty line
86            match input {
87                PipelineData::Value(Value::String { val, .. }, ..) => {
88                    Ok(PipelineData::list_stream(
89                        read_json_lines(
90                            Cursor::new(val),
91                            span,
92                            strict,
93                            engine_state.signals().clone(),
94                        ),
95                        metadata,
96                    ))
97                }
98                PipelineData::ByteStream(stream, ..)
99                    if stream.type_() != ByteStreamType::Binary =>
100                {
101                    if let Some(reader) = stream.reader() {
102                        Ok(PipelineData::list_stream(
103                            read_json_lines(reader, span, strict, engine_state.signals().clone()),
104                            metadata,
105                        ))
106                    } else {
107                        Ok(PipelineData::empty())
108                    }
109                }
110                _ => Err(ShellError::OnlySupportsThisInputType {
111                    exp_input_type: "string".into(),
112                    wrong_type: input.get_type().to_string(),
113                    dst_span: call.head,
114                    src_span: input.span().unwrap_or(call.head),
115                }),
116            }
117        } else {
118            // Return a single JSON value
119            let (string_input, span, ..) = input.collect_string_strict(span)?;
120
121            if string_input.is_empty() {
122                return Ok(Value::nothing(span).into_pipeline_data());
123            }
124
125            Ok(
126                try_str_to_value(&string_input, span, strict, engine_state.signals())?
127                    .into_pipeline_data_with_metadata(metadata),
128            )
129        }
130    }
131}
132
133/// Create a stream of values from a reader that produces line-delimited JSON
134fn read_json_lines(
135    input: impl BufRead + Send + 'static,
136    span: Span,
137    strict: bool,
138    signals: Signals,
139) -> ListStream {
140    let iter_signals = signals.clone();
141    let iter = input
142        .lines()
143        .filter(|line| line.as_ref().is_ok_and(|line| !line.trim().is_empty()) || line.is_err())
144        .map(move |line| {
145            let line = line.map_err(|err| IoError::new(err, span, None))?;
146            try_str_to_value(&line, span, strict, &iter_signals)
147        })
148        .map(move |result| result.unwrap_or_else(|err| Value::error(err, span)));
149
150    ListStream::new(iter, span, signals)
151}
152
153pub fn try_str_to_value(
154    input: &str,
155    span: Span,
156    strict: bool,
157    signals: &Signals,
158) -> Result<Value, ShellError> {
159    match strict {
160        true => try_str_to_value_impl(
161            input,
162            span,
163            signals,
164            |s| serde_json::from_str(s),
165            |err| err.is_syntax().then_some((err.line(), err.column())),
166        ),
167        false => try_str_to_value_impl(input, span, signals, nu_json::from_str, |err| match err {
168            nu_json::Error::Syntax(_, row, col) => Some((*row, *col)),
169            _ => None,
170        }),
171    }
172}
173
174#[inline]
175fn try_str_to_value_impl<E: std::error::Error>(
176    input: &str,
177    span: Span,
178    signals: &Signals,
179    parser: impl Fn(&str) -> Result<nu_json::Value, E>,
180    on_syntax_err: impl Fn(&E) -> Option<(usize, usize)>,
181) -> Result<Value, ShellError> {
182    match parser(input) {
183        Ok(value) => Ok(value.into_value(span)),
184        Err(err) => match on_syntax_err(&err) {
185            Some((row, col)) => {
186                let label = err.to_string();
187                let byte_span = Span::try_from_row_column(row, col, input, &span, signals)?;
188                let (src, label_span) =
189                    truncated_source_window(input, byte_span, DEFAULT_ERROR_CONTEXT);
190                Err(ShellError::Generic(
191                    GenericError::new(
192                        "Error while parsing JSON text",
193                        "error parsing JSON text",
194                        span,
195                    )
196                    .with_inner([ShellError::OutsideSpannedLabeledError {
197                        src,
198                        error: "Error while parsing JSON text".into(),
199                        msg: label,
200                        span: label_span,
201                    }]),
202                ))
203            }
204            None => Err(ShellError::CantConvert {
205                to_type: format!("structured json data ({err})"),
206                from_type: "string".into(),
207                span,
208                help: None,
209            }),
210        },
211    }
212}
213
214#[cfg(test)]
215mod test {
216    use super::*;
217
218    #[test]
219    fn test_examples() -> nu_test_support::Result {
220        nu_test_support::test().examples(FromJson)
221    }
222
223    #[test]
224    fn json_error_source_is_bounded() {
225        // Build a large string (~100KB) with an error near the end
226        let mut valid_part = String::new();
227        valid_part.push('[');
228        for i in 0..2000 {
229            use std::fmt::Write;
230            write!(&mut valid_part, r#""line {i}","#).unwrap();
231        }
232        // Malformed at the end
233        valid_part.push_str("broken]"); // no closing quote
234
235        let signals = Signals::empty();
236        let result = try_str_to_value(&valid_part, Span::test_data(), true, &signals);
237        assert!(result.is_err(), "should fail to parse");
238
239        let err = result.unwrap_err();
240        match &err {
241            ShellError::Generic(GenericError { inner, .. }) => {
242                let inner_err = inner.first().expect("should have inner error");
243                match inner_err {
244                    ShellError::OutsideSpannedLabeledError { src, .. } => {
245                        // src should be bounded well under the 100KB input
246                        assert!(
247                            src.len() < 20_000,
248                            "error source should be bounded, got {} bytes",
249                            src.len()
250                        );
251                    }
252                    other => panic!("expected OutsideSpannedLabeledError, got {other:?}"),
253                }
254            }
255            other => panic!("expected Generic error, got {other:?}"),
256        }
257    }
258
259    #[test]
260    fn json_error_source_not_entire_file() {
261        // With a ~50KB input, the error src should be far smaller
262        let mut input = String::with_capacity(50_000);
263        input.push('[');
264        input.push_str(&"0,".repeat(10_000));
265        input.push_str(":]"); // syntax error: `:]` instead of `]`
266
267        let signals = Signals::empty();
268        let result = try_str_to_value(&input, Span::test_data(), true, &signals);
269        assert!(result.is_err());
270
271        let err = result.unwrap_err();
272        match &err {
273            ShellError::Generic(GenericError { inner, .. }) => {
274                let inner_err = inner.first().expect("should have inner error");
275                match inner_err {
276                    ShellError::OutsideSpannedLabeledError { src, .. } => {
277                        // src should be a window around the error, not the whole 50KB
278                        assert!(
279                            src.len() < 20_000,
280                            "error source should be bounded, got {} bytes",
281                            src.len()
282                        );
283                    }
284                    other => panic!("expected OutsideSpannedLabeledError, got {other:?}"),
285                }
286            }
287            other => panic!("expected Generic error, got {other:?}"),
288        }
289    }
290
291    #[test]
292    fn json_parse_success_not_affected() {
293        let input = r#"{"a": 1, "b": [2, 3]}"#;
294        let signals = Signals::empty();
295        let result = try_str_to_value(input, Span::test_data(), true, &signals);
296        assert!(result.is_ok(), "valid JSON should still parse");
297    }
298}
nu_command/formats/from/json.rs

nu_command/formats/from/
json.rs
