nu_command/formats/from/
json.rs

1use std::io::{BufRead, Cursor};
2
3use nu_engine::command_prelude::*;
4use nu_protocol::{ListStream, Signals, shell_error::io::IoError};
5
6#[derive(Clone)]
7pub struct FromJson;
8
9impl Command for FromJson {
10    fn name(&self) -> &str {
11        "from json"
12    }
13
14    fn description(&self) -> &str {
15        "Convert JSON text into structured data."
16    }
17
18    fn signature(&self) -> nu_protocol::Signature {
19        Signature::build("from json")
20            .input_output_types(vec![(Type::String, Type::Any)])
21            .switch("objects", "Treat each line as a separate value.", Some('o'))
22            .switch(
23                "strict",
24                "Follow the json specification exactly.",
25                Some('s'),
26            )
27            .category(Category::Formats)
28    }
29
30    fn examples(&self) -> Vec<Example<'_>> {
31        vec![
32            Example {
33                example: r#"'{ "a": 1 }' | from json"#,
34                description: "Converts json formatted string to table.",
35                result: Some(Value::test_record(record! {
36                    "a" => Value::test_int(1),
37                })),
38            },
39            Example {
40                example: r#"'{ "a": 1, "b": [1, 2] }' | from json"#,
41                description: "Converts json formatted string to table.",
42                result: Some(Value::test_record(record! {
43                    "a" => Value::test_int(1),
44                    "b" => Value::test_list(vec![Value::test_int(1), Value::test_int(2)]),
45                })),
46            },
47            Example {
48                example: r#"'{ "a": 1, "b": 2 }' | from json -s"#,
49                description: "Parse json strictly which will error on comments and trailing commas.",
50                result: Some(Value::test_record(record! {
51                    "a" => Value::test_int(1),
52                    "b" => Value::test_int(2),
53                })),
54            },
55            Example {
56                example: r#"'{ "a": 1 }
57{ "b": 2 }' | from json --objects"#,
58                description: "Parse a stream of line-delimited JSON values.",
59                result: Some(Value::test_list(vec![
60                    Value::test_record(record! {"a" => Value::test_int(1)}),
61                    Value::test_record(record! {"b" => Value::test_int(2)}),
62                ])),
63            },
64        ]
65    }
66
67    fn run(
68        &self,
69        engine_state: &EngineState,
70        stack: &mut Stack,
71        call: &Call,
72        input: PipelineData,
73    ) -> Result<PipelineData, ShellError> {
74        let span = call.head;
75
76        let strict = call.has_flag(engine_state, stack, "strict")?;
77        let metadata = input.metadata().map(|md| md.with_content_type(None));
78
79        // TODO: turn this into a structured underline of the nu_json error
80        if call.has_flag(engine_state, stack, "objects")? {
81            // Return a stream of JSON values, one for each non-empty line
82            match input {
83                PipelineData::Value(Value::String { val, .. }, ..) => {
84                    Ok(PipelineData::list_stream(
85                        read_json_lines(
86                            Cursor::new(val),
87                            span,
88                            strict,
89                            engine_state.signals().clone(),
90                        ),
91                        metadata,
92                    ))
93                }
94                PipelineData::ByteStream(stream, ..)
95                    if stream.type_() != ByteStreamType::Binary =>
96                {
97                    if let Some(reader) = stream.reader() {
98                        Ok(PipelineData::list_stream(
99                            read_json_lines(reader, span, strict, Signals::empty()),
100                            metadata,
101                        ))
102                    } else {
103                        Ok(PipelineData::empty())
104                    }
105                }
106                _ => Err(ShellError::OnlySupportsThisInputType {
107                    exp_input_type: "string".into(),
108                    wrong_type: input.get_type().to_string(),
109                    dst_span: call.head,
110                    src_span: input.span().unwrap_or(call.head),
111                }),
112            }
113        } else {
114            // Return a single JSON value
115            let (string_input, span, ..) = input.collect_string_strict(span)?;
116
117            if string_input.is_empty() {
118                return Ok(Value::nothing(span).into_pipeline_data());
119            }
120
121            if strict {
122                Ok(convert_string_to_value_strict(&string_input, span)?
123                    .into_pipeline_data_with_metadata(metadata))
124            } else {
125                Ok(convert_string_to_value(&string_input, span)?
126                    .into_pipeline_data_with_metadata(metadata))
127            }
128        }
129    }
130}
131
132/// Create a stream of values from a reader that produces line-delimited JSON
133fn read_json_lines(
134    input: impl BufRead + Send + 'static,
135    span: Span,
136    strict: bool,
137    signals: Signals,
138) -> ListStream {
139    let iter = input
140        .lines()
141        .filter(|line| line.as_ref().is_ok_and(|line| !line.trim().is_empty()) || line.is_err())
142        .map(move |line| {
143            let line = line.map_err(|err| IoError::new(err, span, None))?;
144            if strict {
145                convert_string_to_value_strict(&line, span)
146            } else {
147                convert_string_to_value(&line, span)
148            }
149        })
150        .map(move |result| result.unwrap_or_else(|err| Value::error(err, span)));
151
152    ListStream::new(iter, span, signals)
153}
154
155fn convert_nujson_to_value(value: nu_json::Value, span: Span) -> Value {
156    match value {
157        nu_json::Value::Array(array) => Value::list(
158            array
159                .into_iter()
160                .map(|x| convert_nujson_to_value(x, span))
161                .collect(),
162            span,
163        ),
164        nu_json::Value::Bool(b) => Value::bool(b, span),
165        nu_json::Value::F64(f) => Value::float(f, span),
166        nu_json::Value::I64(i) => Value::int(i, span),
167        nu_json::Value::Null => Value::nothing(span),
168        nu_json::Value::Object(k) => Value::record(
169            k.into_iter()
170                .map(|(k, v)| (k, convert_nujson_to_value(v, span)))
171                .collect(),
172            span,
173        ),
174        nu_json::Value::U64(u) => {
175            if u > i64::MAX as u64 {
176                Value::error(
177                    ShellError::CantConvert {
178                        to_type: "i64 sized integer".into(),
179                        from_type: "value larger than i64".into(),
180                        span,
181                        help: None,
182                    },
183                    span,
184                )
185            } else {
186                Value::int(u as i64, span)
187            }
188        }
189        nu_json::Value::String(s) => Value::string(s, span),
190    }
191}
192
193pub fn convert_string_to_value(string_input: &str, span: Span) -> Result<Value, ShellError> {
194    match nu_json::from_str(string_input) {
195        Ok(value) => Ok(convert_nujson_to_value(value, span)),
196
197        Err(x) => match x {
198            nu_json::Error::Syntax(_, row, col) => {
199                let label = x.to_string();
200                let label_span = Span::from_row_column(row, col, string_input);
201                Err(ShellError::GenericError {
202                    error: "Error while parsing JSON text".into(),
203                    msg: "error parsing JSON text".into(),
204                    span: Some(span),
205                    help: None,
206                    inner: vec![ShellError::OutsideSpannedLabeledError {
207                        src: string_input.into(),
208                        error: "Error while parsing JSON text".into(),
209                        msg: label,
210                        span: label_span,
211                    }],
212                })
213            }
214            x => Err(ShellError::CantConvert {
215                to_type: format!("structured json data ({x})"),
216                from_type: "string".into(),
217                span,
218                help: None,
219            }),
220        },
221    }
222}
223
224fn convert_string_to_value_strict(string_input: &str, span: Span) -> Result<Value, ShellError> {
225    match serde_json::from_str(string_input) {
226        Ok(value) => Ok(convert_nujson_to_value(value, span)),
227        Err(err) => Err(if err.is_syntax() {
228            let label = err.to_string();
229            let label_span = Span::from_row_column(err.line(), err.column(), string_input);
230            ShellError::GenericError {
231                error: "Error while parsing JSON text".into(),
232                msg: "error parsing JSON text".into(),
233                span: Some(span),
234                help: None,
235                inner: vec![ShellError::OutsideSpannedLabeledError {
236                    src: string_input.into(),
237                    error: "Error while parsing JSON text".into(),
238                    msg: label,
239                    span: label_span,
240                }],
241            }
242        } else {
243            ShellError::CantConvert {
244                to_type: format!("structured json data ({err})"),
245                from_type: "string".into(),
246                span,
247                help: None,
248            }
249        }),
250    }
251}
252
253#[cfg(test)]
254mod test {
255    use nu_cmd_lang::eval_pipeline_without_terminal_expression;
256
257    use crate::Reject;
258    use crate::{Metadata, MetadataSet};
259
260    use super::*;
261
262    #[test]
263    fn test_examples() {
264        use crate::test_examples;
265
266        test_examples(FromJson {})
267    }
268
269    #[test]
270    fn test_content_type_metadata() {
271        let mut engine_state = Box::new(EngineState::new());
272        let delta = {
273            let mut working_set = StateWorkingSet::new(&engine_state);
274
275            working_set.add_decl(Box::new(FromJson {}));
276            working_set.add_decl(Box::new(Metadata {}));
277            working_set.add_decl(Box::new(MetadataSet {}));
278            working_set.add_decl(Box::new(Reject {}));
279
280            working_set.render()
281        };
282
283        engine_state
284            .merge_delta(delta)
285            .expect("Error merging delta");
286
287        let cmd = r#"'{"a":1,"b":2}' | metadata set --content-type 'application/json' --path-columns [name] | from json | metadata | reject span | $in"#;
288        let result = eval_pipeline_without_terminal_expression(
289            cmd,
290            std::env::temp_dir().as_ref(),
291            &mut engine_state,
292        );
293        assert_eq!(
294            Value::test_record(
295                record!("path_columns" => Value::test_list(vec![Value::test_string("name")]))
296            ),
297            result.expect("There should be a result")
298        )
299    }
300}
nu_command/formats/from/json.rs

nu_command/formats/from/
json.rs
