nu_command/formats/from/
json.rs

1use std::io::{BufRead, Cursor};
2
3use nu_engine::command_prelude::*;
4use nu_protocol::{ListStream, Signals, shell_error::io::IoError};
5
6#[derive(Clone)]
7pub struct FromJson;
8
9impl Command for FromJson {
10    fn name(&self) -> &str {
11        "from json"
12    }
13
14    fn description(&self) -> &str {
15        "Convert from json to structured data."
16    }
17
18    fn signature(&self) -> nu_protocol::Signature {
19        Signature::build("from json")
20            .input_output_types(vec![(Type::String, Type::Any)])
21            .switch("objects", "treat each line as a separate value", Some('o'))
22            .switch("strict", "follow the json specification exactly", Some('s'))
23            .category(Category::Formats)
24    }
25
26    fn examples(&self) -> Vec<Example> {
27        vec![
28            Example {
29                example: r#"'{ "a": 1 }' | from json"#,
30                description: "Converts json formatted string to table",
31                result: Some(Value::test_record(record! {
32                    "a" => Value::test_int(1),
33                })),
34            },
35            Example {
36                example: r#"'{ "a": 1, "b": [1, 2] }' | from json"#,
37                description: "Converts json formatted string to table",
38                result: Some(Value::test_record(record! {
39                    "a" => Value::test_int(1),
40                    "b" => Value::test_list(vec![Value::test_int(1), Value::test_int(2)]),
41                })),
42            },
43            Example {
44                example: r#"'{ "a": 1, "b": 2 }' | from json -s"#,
45                description: "Parse json strictly which will error on comments and trailing commas",
46                result: Some(Value::test_record(record! {
47                    "a" => Value::test_int(1),
48                    "b" => Value::test_int(2),
49                })),
50            },
51            Example {
52                example: r#"'{ "a": 1 }
53{ "b": 2 }' | from json --objects"#,
54                description: "Parse a stream of line-delimited JSON values",
55                result: Some(Value::test_list(vec![
56                    Value::test_record(record! {"a" => Value::test_int(1)}),
57                    Value::test_record(record! {"b" => Value::test_int(2)}),
58                ])),
59            },
60        ]
61    }
62
63    fn run(
64        &self,
65        engine_state: &EngineState,
66        stack: &mut Stack,
67        call: &Call,
68        input: PipelineData,
69    ) -> Result<PipelineData, ShellError> {
70        let span = call.head;
71
72        let strict = call.has_flag(engine_state, stack, "strict")?;
73        let metadata = input.metadata().map(|md| md.with_content_type(None));
74
75        // TODO: turn this into a structured underline of the nu_json error
76        if call.has_flag(engine_state, stack, "objects")? {
77            // Return a stream of JSON values, one for each non-empty line
78            match input {
79                PipelineData::Value(Value::String { val, .. }, ..) => {
80                    Ok(PipelineData::list_stream(
81                        read_json_lines(
82                            Cursor::new(val),
83                            span,
84                            strict,
85                            engine_state.signals().clone(),
86                        ),
87                        metadata,
88                    ))
89                }
90                PipelineData::ByteStream(stream, ..)
91                    if stream.type_() != ByteStreamType::Binary =>
92                {
93                    if let Some(reader) = stream.reader() {
94                        Ok(PipelineData::list_stream(
95                            read_json_lines(reader, span, strict, Signals::empty()),
96                            metadata,
97                        ))
98                    } else {
99                        Ok(PipelineData::empty())
100                    }
101                }
102                _ => Err(ShellError::OnlySupportsThisInputType {
103                    exp_input_type: "string".into(),
104                    wrong_type: input.get_type().to_string(),
105                    dst_span: call.head,
106                    src_span: input.span().unwrap_or(call.head),
107                }),
108            }
109        } else {
110            // Return a single JSON value
111            let (string_input, span, ..) = input.collect_string_strict(span)?;
112
113            if string_input.is_empty() {
114                return Ok(Value::nothing(span).into_pipeline_data());
115            }
116
117            if strict {
118                Ok(convert_string_to_value_strict(&string_input, span)?
119                    .into_pipeline_data_with_metadata(metadata))
120            } else {
121                Ok(convert_string_to_value(&string_input, span)?
122                    .into_pipeline_data_with_metadata(metadata))
123            }
124        }
125    }
126}
127
128/// Create a stream of values from a reader that produces line-delimited JSON
129fn read_json_lines(
130    input: impl BufRead + Send + 'static,
131    span: Span,
132    strict: bool,
133    signals: Signals,
134) -> ListStream {
135    let iter = input
136        .lines()
137        .filter(|line| line.as_ref().is_ok_and(|line| !line.trim().is_empty()) || line.is_err())
138        .map(move |line| {
139            let line = line.map_err(|err| IoError::new(err, span, None))?;
140            if strict {
141                convert_string_to_value_strict(&line, span)
142            } else {
143                convert_string_to_value(&line, span)
144            }
145        })
146        .map(move |result| result.unwrap_or_else(|err| Value::error(err, span)));
147
148    ListStream::new(iter, span, signals)
149}
150
151fn convert_nujson_to_value(value: nu_json::Value, span: Span) -> Value {
152    match value {
153        nu_json::Value::Array(array) => Value::list(
154            array
155                .into_iter()
156                .map(|x| convert_nujson_to_value(x, span))
157                .collect(),
158            span,
159        ),
160        nu_json::Value::Bool(b) => Value::bool(b, span),
161        nu_json::Value::F64(f) => Value::float(f, span),
162        nu_json::Value::I64(i) => Value::int(i, span),
163        nu_json::Value::Null => Value::nothing(span),
164        nu_json::Value::Object(k) => Value::record(
165            k.into_iter()
166                .map(|(k, v)| (k, convert_nujson_to_value(v, span)))
167                .collect(),
168            span,
169        ),
170        nu_json::Value::U64(u) => {
171            if u > i64::MAX as u64 {
172                Value::error(
173                    ShellError::CantConvert {
174                        to_type: "i64 sized integer".into(),
175                        from_type: "value larger than i64".into(),
176                        span,
177                        help: None,
178                    },
179                    span,
180                )
181            } else {
182                Value::int(u as i64, span)
183            }
184        }
185        nu_json::Value::String(s) => Value::string(s, span),
186    }
187}
188
189pub(crate) fn convert_string_to_value(string_input: &str, span: Span) -> Result<Value, ShellError> {
190    match nu_json::from_str(string_input) {
191        Ok(value) => Ok(convert_nujson_to_value(value, span)),
192
193        Err(x) => match x {
194            nu_json::Error::Syntax(_, row, col) => {
195                let label = x.to_string();
196                let label_span = Span::from_row_column(row, col, string_input);
197                Err(ShellError::GenericError {
198                    error: "Error while parsing JSON text".into(),
199                    msg: "error parsing JSON text".into(),
200                    span: Some(span),
201                    help: None,
202                    inner: vec![ShellError::OutsideSpannedLabeledError {
203                        src: string_input.into(),
204                        error: "Error while parsing JSON text".into(),
205                        msg: label,
206                        span: label_span,
207                    }],
208                })
209            }
210            x => Err(ShellError::CantConvert {
211                to_type: format!("structured json data ({x})"),
212                from_type: "string".into(),
213                span,
214                help: None,
215            }),
216        },
217    }
218}
219
220fn convert_string_to_value_strict(string_input: &str, span: Span) -> Result<Value, ShellError> {
221    match serde_json::from_str(string_input) {
222        Ok(value) => Ok(convert_nujson_to_value(value, span)),
223        Err(err) => Err(if err.is_syntax() {
224            let label = err.to_string();
225            let label_span = Span::from_row_column(err.line(), err.column(), string_input);
226            ShellError::GenericError {
227                error: "Error while parsing JSON text".into(),
228                msg: "error parsing JSON text".into(),
229                span: Some(span),
230                help: None,
231                inner: vec![ShellError::OutsideSpannedLabeledError {
232                    src: string_input.into(),
233                    error: "Error while parsing JSON text".into(),
234                    msg: label,
235                    span: label_span,
236                }],
237            }
238        } else {
239            ShellError::CantConvert {
240                to_type: format!("structured json data ({err})"),
241                from_type: "string".into(),
242                span,
243                help: None,
244            }
245        }),
246    }
247}
248
249#[cfg(test)]
250mod test {
251    use nu_cmd_lang::eval_pipeline_without_terminal_expression;
252
253    use crate::Reject;
254    use crate::{Metadata, MetadataSet};
255
256    use super::*;
257
258    #[test]
259    fn test_examples() {
260        use crate::test_examples;
261
262        test_examples(FromJson {})
263    }
264
265    #[test]
266    fn test_content_type_metadata() {
267        let mut engine_state = Box::new(EngineState::new());
268        let delta = {
269            let mut working_set = StateWorkingSet::new(&engine_state);
270
271            working_set.add_decl(Box::new(FromJson {}));
272            working_set.add_decl(Box::new(Metadata {}));
273            working_set.add_decl(Box::new(MetadataSet {}));
274            working_set.add_decl(Box::new(Reject {}));
275
276            working_set.render()
277        };
278
279        engine_state
280            .merge_delta(delta)
281            .expect("Error merging delta");
282
283        let cmd = r#"'{"a":1,"b":2}' | metadata set --content-type 'application/json' --datasource-ls | from json | metadata | reject span | $in"#;
284        let result = eval_pipeline_without_terminal_expression(
285            cmd,
286            std::env::temp_dir().as_ref(),
287            &mut engine_state,
288        );
289        assert_eq!(
290            Value::test_record(record!("source" => Value::test_string("ls"))),
291            result.expect("There should be a result")
292        )
293    }
294}
nu_command/formats/from/json.rs

nu_command/formats/from/
json.rs
