shannon_nu_command/formats/from/
json.rs

1use std::io::{BufRead, Cursor};
2
3use nu_engine::command_prelude::*;
4use nu_protocol::{
5    ListStream, Signals, shell_error::generic::GenericError, shell_error::io::IoError,
6};
7
8#[derive(Clone)]
9pub struct FromJson;
10
11impl Command for FromJson {
12    fn name(&self) -> &str {
13        "from json"
14    }
15
16    fn description(&self) -> &str {
17        "Convert JSON text into structured data."
18    }
19
20    fn signature(&self) -> nu_protocol::Signature {
21        Signature::build("from json")
22            .input_output_types(vec![(Type::String, Type::Any)])
23            .switch("objects", "Treat each line as a separate value.", Some('o'))
24            .switch(
25                "strict",
26                "Follow the json specification exactly.",
27                Some('s'),
28            )
29            .category(Category::Formats)
30    }
31
32    fn examples(&self) -> Vec<Example<'_>> {
33        vec![
34            Example {
35                example: r#"'{ "a": 1 }' | from json"#,
36                description: "Converts json formatted string to table.",
37                result: Some(Value::test_record(record! {
38                    "a" => Value::test_int(1),
39                })),
40            },
41            Example {
42                example: r#"'{ "a": 1, "b": [1, 2] }' | from json"#,
43                description: "Converts json formatted string to table.",
44                result: Some(Value::test_record(record! {
45                    "a" => Value::test_int(1),
46                    "b" => Value::test_list(vec![Value::test_int(1), Value::test_int(2)]),
47                })),
48            },
49            Example {
50                example: r#"'{ "a": 1, "b": 2 }' | from json -s"#,
51                description: "Parse json strictly which will error on comments and trailing commas.",
52                result: Some(Value::test_record(record! {
53                    "a" => Value::test_int(1),
54                    "b" => Value::test_int(2),
55                })),
56            },
57            Example {
58                example: r#"'{ "a": 1 }
59{ "b": 2 }' | from json --objects"#,
60                description: "Parse a stream of line-delimited JSON values.",
61                result: Some(Value::test_list(vec![
62                    Value::test_record(record! {"a" => Value::test_int(1)}),
63                    Value::test_record(record! {"b" => Value::test_int(2)}),
64                ])),
65            },
66        ]
67    }
68
69    fn run(
70        &self,
71        engine_state: &EngineState,
72        stack: &mut Stack,
73        call: &Call,
74        input: PipelineData,
75    ) -> Result<PipelineData, ShellError> {
76        let span = call.head;
77
78        let strict = call.has_flag(engine_state, stack, "strict")?;
79        let metadata = input.metadata().map(|md| md.with_content_type(None));
80
81        // TODO: turn this into a structured underline of the nu_json error
82        if call.has_flag(engine_state, stack, "objects")? {
83            // Return a stream of JSON values, one for each non-empty line
84            match input {
85                PipelineData::Value(Value::String { val, .. }, ..) => {
86                    Ok(PipelineData::list_stream(
87                        read_json_lines(
88                            Cursor::new(val),
89                            span,
90                            strict,
91                            engine_state.signals().clone(),
92                        ),
93                        metadata,
94                    ))
95                }
96                PipelineData::ByteStream(stream, ..)
97                    if stream.type_() != ByteStreamType::Binary =>
98                {
99                    if let Some(reader) = stream.reader() {
100                        Ok(PipelineData::list_stream(
101                            read_json_lines(reader, span, strict, Signals::empty()),
102                            metadata,
103                        ))
104                    } else {
105                        Ok(PipelineData::empty())
106                    }
107                }
108                _ => Err(ShellError::OnlySupportsThisInputType {
109                    exp_input_type: "string".into(),
110                    wrong_type: input.get_type().to_string(),
111                    dst_span: call.head,
112                    src_span: input.span().unwrap_or(call.head),
113                }),
114            }
115        } else {
116            // Return a single JSON value
117            let (string_input, span, ..) = input.collect_string_strict(span)?;
118
119            if string_input.is_empty() {
120                return Ok(Value::nothing(span).into_pipeline_data());
121            }
122
123            if strict {
124                Ok(convert_string_to_value_strict(&string_input, span)?
125                    .into_pipeline_data_with_metadata(metadata))
126            } else {
127                Ok(convert_string_to_value(&string_input, span)?
128                    .into_pipeline_data_with_metadata(metadata))
129            }
130        }
131    }
132}
133
134/// Create a stream of values from a reader that produces line-delimited JSON
135fn read_json_lines(
136    input: impl BufRead + Send + 'static,
137    span: Span,
138    strict: bool,
139    signals: Signals,
140) -> ListStream {
141    let iter = input
142        .lines()
143        .filter(|line| line.as_ref().is_ok_and(|line| !line.trim().is_empty()) || line.is_err())
144        .map(move |line| {
145            let line = line.map_err(|err| IoError::new(err, span, None))?;
146            if strict {
147                convert_string_to_value_strict(&line, span)
148            } else {
149                convert_string_to_value(&line, span)
150            }
151        })
152        .map(move |result| result.unwrap_or_else(|err| Value::error(err, span)));
153
154    ListStream::new(iter, span, signals)
155}
156
157fn convert_nujson_to_value(value: nu_json::Value, span: Span) -> Value {
158    match value {
159        nu_json::Value::Array(array) => Value::list(
160            array
161                .into_iter()
162                .map(|x| convert_nujson_to_value(x, span))
163                .collect(),
164            span,
165        ),
166        nu_json::Value::Bool(b) => Value::bool(b, span),
167        nu_json::Value::F64(f) => Value::float(f, span),
168        nu_json::Value::I64(i) => Value::int(i, span),
169        nu_json::Value::Null => Value::nothing(span),
170        nu_json::Value::Object(k) => Value::record(
171            k.into_iter()
172                .map(|(k, v)| (k, convert_nujson_to_value(v, span)))
173                .collect(),
174            span,
175        ),
176        nu_json::Value::U64(u) => {
177            if u > i64::MAX as u64 {
178                Value::error(
179                    ShellError::CantConvert {
180                        to_type: "i64 sized integer".into(),
181                        from_type: "value larger than i64".into(),
182                        span,
183                        help: None,
184                    },
185                    span,
186                )
187            } else {
188                Value::int(u as i64, span)
189            }
190        }
191        nu_json::Value::String(s) => Value::string(s, span),
192    }
193}
194
195pub fn convert_string_to_value(string_input: &str, span: Span) -> Result<Value, ShellError> {
196    match nu_json::from_str(string_input) {
197        Ok(value) => Ok(convert_nujson_to_value(value, span)),
198
199        Err(x) => match x {
200            nu_json::Error::Syntax(_, row, col) => {
201                let label = x.to_string();
202                let label_span = Span::from_row_column(row, col, string_input);
203                Err(ShellError::Generic(
204                    GenericError::new(
205                        "Error while parsing JSON text",
206                        "error parsing JSON text",
207                        span,
208                    )
209                    .with_inner([ShellError::OutsideSpannedLabeledError {
210                        src: string_input.into(),
211                        error: "Error while parsing JSON text".into(),
212                        msg: label,
213                        span: label_span,
214                    }]),
215                ))
216            }
217            x => Err(ShellError::CantConvert {
218                to_type: format!("structured json data ({x})"),
219                from_type: "string".into(),
220                span,
221                help: None,
222            }),
223        },
224    }
225}
226
227fn convert_string_to_value_strict(string_input: &str, span: Span) -> Result<Value, ShellError> {
228    match serde_json::from_str(string_input) {
229        Ok(value) => Ok(convert_nujson_to_value(value, span)),
230        Err(err) => Err(if err.is_syntax() {
231            let label = err.to_string();
232            let label_span = Span::from_row_column(err.line(), err.column(), string_input);
233            ShellError::Generic(
234                GenericError::new(
235                    "Error while parsing JSON text",
236                    "error parsing JSON text",
237                    span,
238                )
239                .with_inner([ShellError::OutsideSpannedLabeledError {
240                    src: string_input.into(),
241                    error: "Error while parsing JSON text".into(),
242                    msg: label,
243                    span: label_span,
244                }]),
245            )
246        } else {
247            ShellError::CantConvert {
248                to_type: format!("structured json data ({err})"),
249                from_type: "string".into(),
250                span,
251                help: None,
252            }
253        }),
254    }
255}
256
257#[cfg(test)]
258mod test {
259    use nu_cmd_lang::eval_pipeline_without_terminal_expression;
260
261    use crate::Reject;
262    use crate::{Metadata, MetadataSet};
263
264    use super::*;
265
266    #[test]
267    fn test_examples() -> nu_test_support::Result {
268        nu_test_support::test().examples(FromJson)
269    }
270
271    #[test]
272    fn test_content_type_metadata() {
273        let mut engine_state = Box::new(EngineState::new());
274        let delta = {
275            let mut working_set = StateWorkingSet::new(&engine_state);
276
277            working_set.add_decl(Box::new(FromJson {}));
278            working_set.add_decl(Box::new(Metadata {}));
279            working_set.add_decl(Box::new(MetadataSet {}));
280            working_set.add_decl(Box::new(Reject {}));
281
282            working_set.render()
283        };
284
285        engine_state
286            .merge_delta(delta)
287            .expect("Error merging delta");
288
289        let cmd = r#"'{"a":1,"b":2}' | metadata set --content-type 'application/json' --path-columns [name] | from json | metadata | reject span | $in"#;
290        let result = eval_pipeline_without_terminal_expression(
291            cmd,
292            std::env::temp_dir().as_ref(),
293            &mut engine_state,
294        );
295        assert_eq!(
296            Value::test_record(
297                record!("path_columns" => Value::test_list(vec![Value::test_string("name")]))
298            ),
299            result.expect("There should be a result")
300        )
301    }
302}
shannon_nu_command/formats/from/json.rs

shannon_nu_command/formats/from/
json.rs
