dataprof-json 0.13.1

JSON and JSONL scanning support for dataprof
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
//! The JSON record tokenizer shared by the file reader and the async stream
//! reader, so a payload profiles identically whichever way it arrives.

use std::io::{BufRead, Read};

use dataprof_core::DataProfilerError;
use serde::Deserialize;
use serde_json::Value;

use crate::JsonObject;

/// One record read from a JSON or JSONL source.
pub enum JsonRecord {
    /// A JSON object — the only value with named fields to profile as a row.
    Object(JsonObject),
    /// Valid JSON that is not an object, carrying the value's kind for the
    /// error message.
    NonObject(&'static str),
    /// The bytes are not valid JSON.
    Malformed(serde_json::Error),
    /// An object whose source text holds an integer literal outside the
    /// 64-bit range. serde_json keeps such an integer as the nearest `f64`, so
    /// distinct values would round to one before they are counted (#1103).
    /// The text decides, not the parsed value, so a literal a later duplicate
    /// key overwrites counts too, as it does for the Python bytes reader,
    /// which sees every integer token. The record was read in full, so the
    /// caller can skip it and keep reading.
    WideInteger,
}

/// Read one JSON value and leave the reader positioned immediately after it.
///
/// `first` is the value's first byte, already located by
/// the caller but not yet consumed.
pub fn read_json_record<R: BufRead>(
    reader: &mut R,
    first: u8,
) -> Result<JsonRecord, DataProfilerError> {
    // A number is the only JSON value that is not self-delimiting: serde finds
    // its end by peeking one byte past it and then drops that byte along with
    // the deserializer. Inside an array that byte is the `,` or `]` the scanner
    // needs next, so numbers are read byte-wise instead.
    if first == b'-' || first.is_ascii_digit() {
        let token = read_number_token(reader)?;
        return Ok(match serde_json::from_str::<Value>(&token) {
            Ok(_) => JsonRecord::NonObject("number"),
            Err(err) => JsonRecord::Malformed(err),
        });
    }

    // The source bytes are kept so an integer serde rounded to f64 can be
    // recognised from its literal; the parsed value no longer tells.
    let mut recorded = RecordingReader {
        inner: reader,
        bytes: Vec::new(),
    };
    let parsed = {
        let mut deserializer = serde_json::Deserializer::from_reader(&mut recorded);
        Value::deserialize(&mut deserializer)
    };
    Ok(match parsed {
        Ok(Value::Object(obj)) => object_record(obj, &recorded.bytes),
        Ok(value) => JsonRecord::NonObject(json_value_kind(&value)),
        Err(err) if err.is_io() => {
            // A failed read is not a malformed record that the caller can
            // skip. Recover the original I/O wrapper: serde's Error::source
            // skips it, hiding concrete transport causes from Python.
            let message = format!("JSON record read failed: {err}");
            return Err(DataProfilerError::json_parsing_with_source(
                message,
                std::io::Error::from(err),
            ));
        }
        Err(err) => JsonRecord::Malformed(err),
    })
}

/// A reader that keeps a copy of every byte read through it.
struct RecordingReader<'a, R> {
    inner: &'a mut R,
    bytes: Vec<u8>,
}

impl<R: Read> Read for RecordingReader<'_, R> {
    fn read(&mut self, buffer: &mut [u8]) -> std::io::Result<usize> {
        let read = self.inner.read(buffer)?;
        self.bytes.extend_from_slice(&buffer[..read]);
        Ok(read)
    }
}

/// Whether `value` holds a float large enough to have been read from an
/// integer literal outside the 64-bit range; only the source text can tell.
///
/// An integer within `i64` or `u64` parses exactly; one outside becomes an
/// `f64` of magnitude at least 2^63. A float literal like `1e19` does too, so
/// this alone refuses nothing.
pub fn may_hold_wide_integer(value: &Value) -> bool {
    match value {
        Value::Number(number) => {
            number.as_i64().is_none()
                && number.as_u64().is_none()
                && number
                    .as_f64()
                    .is_some_and(|float| float.abs() >= 2f64.powi(63))
        }
        Value::Array(items) => items.iter().any(may_hold_wide_integer),
        Value::Object(fields) => fields.values().any(may_hold_wide_integer),
        _ => false,
    }
}

/// Whether `raw` JSON text holds an integer literal (no fraction, no exponent)
/// that fits neither `i64` nor `u64`. String contents are skipped, and a token
/// shorter than 19 digits, which always fits, is not parsed.
pub fn has_wide_integer_literal(raw: &[u8]) -> bool {
    integer_literals(raw).any(|(_, token)| is_wide_integer(token))
}

/// The integer literals (no fraction, no exponent) of `raw` JSON text, each
/// with the offset it starts at. String contents are skipped.
fn integer_literals(raw: &[u8]) -> impl Iterator<Item = (usize, &[u8])> {
    let mut index = 0;
    std::iter::from_fn(move || {
        while index < raw.len() {
            match raw[index] {
                b'"' => {
                    index += 1;
                    while index < raw.len() && raw[index] != b'"' {
                        index += if raw[index] == b'\\' { 2 } else { 1 };
                    }
                    index += 1;
                }
                byte if byte == b'-' || byte.is_ascii_digit() => {
                    let start = index;
                    while index < raw.len() && is_json_number_byte(raw[index]) {
                        index += 1;
                    }
                    let token = &raw[start..index];
                    if !token.iter().any(|byte| matches!(byte, b'.' | b'e' | b'E')) {
                        return Some((start, token));
                    }
                }
                _ => index += 1,
            }
        }
        None
    })
}

fn is_wide_integer(token: &[u8]) -> bool {
    let digits = token.len() - usize::from(token.first() == Some(&b'-'));
    if digits < 19 {
        return false;
    }
    // decode-audit: impossible — every number byte is ASCII.
    let text = std::str::from_utf8(token).expect("JSON number bytes are ASCII");
    text.parse::<i64>().is_err() && text.parse::<u64>().is_err()
}

/// The record an object parsed from `raw` profiles as: always
/// [`JsonRecord::Object`] or [`JsonRecord::WideInteger`].
///
/// An object whose text holds an integer outside the 64-bit range is
/// [`JsonRecord::WideInteger`]. serde_json reads the integer literal `-0` as
/// the float `-0.0`, while Python's `json` reads it as the integer `0`, so
/// `{"v":0}` and `{"v":-0}` were duplicates over Python bytes and distinct
/// floats over the file and async readers. Such an object is read again with
/// each `-0` as `0`, which keeps its type and its duplicates the same on every
/// route. A float literal like `-0.0` keeps its sign on every route already.
pub fn object_record(obj: JsonObject, raw: &[u8]) -> JsonRecord {
    let mut negative_zeros = Vec::new();
    for (start, token) in integer_literals(raw) {
        if is_wide_integer(token) {
            return JsonRecord::WideInteger;
        }
        if token == b"-0" {
            negative_zeros.push(start);
        }
    }
    if negative_zeros.is_empty() {
        return JsonRecord::Object(obj);
    }
    // Blanking the sign leaves `0` where `-0` stood; whitespace is legal
    // between any two JSON tokens, so the text stays the same document.
    let mut unsigned = raw.to_vec();
    for start in negative_zeros {
        unsigned[start] = b' ';
    }
    // decode-audit: impossible — `raw` parsed as this object, and only a sign
    // was replaced by whitespace.
    JsonRecord::Object(
        serde_json::from_slice::<JsonObject>(&unsigned)
            .expect("an object with its `-0` signs blanked is still an object"),
    )
}

/// Build the error for a record refused as [`JsonRecord::WideInteger`].
/// `location` names the record ("at position 3", "on line 7"); the value is
/// never included, as with every record error.
pub fn wide_integer_record_error(location: &str) -> DataProfilerError {
    DataProfilerError::json_parsing_error(&format!(
        "JSON record {location} holds an integer outside the 64-bit range, which would be rounded to a float and could merge distinct values; write such integers as strings to profile them"
    ))
}

/// Read the bytes that can spell a JSON number, stopping at the first byte that
/// cannot. The token is returned unvalidated — the caller decides whether it
/// actually parses, so `1.2.3` stays a malformed record rather than a number.
///
/// Leading whitespace is consumed first: callers locate the next value's first
/// byte without consuming what precedes it, because serde needs that whitespace
/// to keep its line and column context.
fn read_number_token<R: BufRead>(reader: &mut R) -> Result<String, DataProfilerError> {
    skip_ascii_whitespace(reader)?;
    let mut token = String::new();
    loop {
        let (consume, token_ended) = {
            let buf = reader.fill_buf().map_err(DataProfilerError::from)?;
            if buf.is_empty() {
                break;
            }
            let consume = buf
                .iter()
                .take_while(|byte| is_json_number_byte(**byte))
                .count();
            // decode-audit: impossible — every byte accepted above is ASCII.
            token.push_str(
                std::str::from_utf8(&buf[..consume]).expect("JSON number bytes are ASCII"),
            );
            (consume, consume < buf.len())
        };
        reader.consume(consume);
        if token_ended {
            break;
        }
    }
    Ok(token)
}

fn is_json_number_byte(byte: u8) -> bool {
    byte.is_ascii_digit() || matches!(byte, b'-' | b'+' | b'.' | b'e' | b'E')
}

/// Advance the reader past any ASCII whitespace at its current position.
pub(crate) fn skip_ascii_whitespace<R: BufRead>(reader: &mut R) -> Result<(), DataProfilerError> {
    loop {
        let (consume, done) = {
            let buf = reader.fill_buf().map_err(DataProfilerError::from)?;
            if buf.is_empty() {
                return Ok(());
            }
            let consume = buf
                .iter()
                .take_while(|byte| byte.is_ascii_whitespace())
                .count();
            (consume, consume < buf.len())
        };
        reader.consume(consume);
        if done {
            return Ok(());
        }
    }
}

/// The text a JSON value is profiled as: scalars as themselves, `null` as the
/// empty string, and an array or object as compact JSON with every object's
/// keys sorted.
///
/// The sort makes the text canonical. With `preserve_order` an object keeps
/// its source key order, so `{"x":1,"y":2}` and `{"y":2,"x":1}` would render
/// differently and two equal rows would not count as duplicates (#767). Keys
/// sort by code point, as Python's `json.dumps(..., sort_keys=True)` does for
/// in-memory inputs.
pub fn json_value_text(value: &Value) -> String {
    match value {
        Value::Null => String::new(),
        Value::Bool(boolean) => boolean.to_string(),
        Value::Number(number) => number.to_string(),
        Value::String(string) => string.to_string(),
        Value::Array(_) | Value::Object(_) => {
            let mut text = String::new();
            write_canonical_json(value, &mut text);
            text
        }
    }
}

fn write_canonical_json(value: &Value, out: &mut String) {
    match value {
        Value::Array(items) => {
            out.push('[');
            for (index, item) in items.iter().enumerate() {
                if index > 0 {
                    out.push(',');
                }
                write_canonical_json(item, out);
            }
            out.push(']');
        }
        Value::Object(fields) => {
            let mut entries: Vec<_> = fields.iter().collect();
            entries.sort_unstable_by_key(|(key, _)| *key);
            out.push('{');
            for (index, (key, field)) in entries.into_iter().enumerate() {
                if index > 0 {
                    out.push(',');
                }
                out.push_str(&scalar_json(key));
                out.push(':');
                write_canonical_json(field, out);
            }
            out.push('}');
        }
        scalar => out.push_str(&scalar_json(scalar)),
    }
}

fn scalar_json<T: serde::Serialize + ?Sized>(value: &T) -> String {
    // decode-audit: impossible — serializing an in-memory string or scalar
    // Value cannot fail; a panic here beats folding "" (= null) into stats.
    serde_json::to_string(value).expect("re-serializing a parsed JSON value cannot fail")
}

pub fn json_value_kind(value: &Value) -> &'static str {
    match value {
        Value::Null => "null",
        Value::Bool(_) => "boolean",
        Value::Number(_) => "number",
        Value::String(_) => "string",
        Value::Array(_) => "array",
        Value::Object(_) => "object",
    }
}

/// Build a strict-mode error for valid JSON that is not a row object. This is a
/// distinct category from a syntax failure: the document parsed, it just does
/// not hold records. `position` is 1-based over the records scanned so far.
pub fn non_object_record_error(kind: &str, position: usize) -> DataProfilerError {
    // No source: the document parsed cleanly, it simply does not hold records,
    // so there is no decoder error underneath to retain.
    DataProfilerError::json_parsing_error(&format!(
        "non-object JSON record at position {position}: \
         expected an object with fields to profile, found {kind}"
    ))
}

pub fn malformed_array_error(message: &str) -> DataProfilerError {
    // Callers here hold only a message, not the decoder error, so there is
    // nothing to retain.
    DataProfilerError::json_parsing_error(&format!("malformed JSON array: {message}"))
}

/// Build a strict-mode error for a standard JSON document that is not exactly
/// one value.
///
/// The most common cause is JSONL read with the JSON grammar — several objects
/// back to back parse as one value plus trailing characters — so the message
/// names the option that would read it correctly.
pub fn json_document_error(err: serde_json::Error) -> DataProfilerError {
    let message = format!(
        // One source line on purpose: rustfmt may join a `\`-continued
        // literal and leave its indentation inside the string, which silently
        // corrupted this very message once.
        "malformed JSON document: {err}. A JSON source must hold exactly one array or object; for one record per line use format=\"jsonl\""
    );
    DataProfilerError::json_parsing_with_source(message, err)
}

/// Consume the rest of the input, returning how many bytes that was.
pub fn drain_to_end<R: BufRead>(reader: &mut R) -> Result<usize, DataProfilerError> {
    let mut consumed = 0;
    loop {
        let available = reader.fill_buf().map_err(DataProfilerError::from)?.len();
        if available == 0 {
            return Ok(consumed);
        }
        reader.consume(available);
        consumed += available;
    }
}

/// Parse one JSONL line as exactly one JSON value.
///
/// `serde_json::from_str` requires the value to fill the whole input, so a line
/// carrying two adjacent or space-separated values is a malformed record rather
/// than two clean ones — the boundary rule that separates JSONL from a stream of
/// JSON values.
pub fn read_jsonl_line(line: &str) -> JsonRecord {
    match serde_json::from_str::<Value>(line.trim()) {
        Ok(Value::Object(obj)) => object_record(obj, line.as_bytes()),
        Ok(value) => JsonRecord::NonObject(json_value_kind(&value)),
        // An integer too long even for f64 fails to parse at all; the line is
        // consumed either way, so name it as what it holds.
        Err(_) if has_wide_integer_literal(line.as_bytes()) => JsonRecord::WideInteger,
        Err(err) => JsonRecord::Malformed(err),
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::error::Error as _;
    use std::io::{self, BufReader, Cursor, Read};

    #[derive(Debug)]
    struct LocalReadFailure;

    impl std::fmt::Display for LocalReadFailure {
        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
            f.write_str("local read failure")
        }
    }

    impl std::error::Error for LocalReadFailure {}

    impl Read for LocalReadFailure {
        fn read(&mut self, _buffer: &mut [u8]) -> io::Result<usize> {
            Err(io::Error::new(io::ErrorKind::PermissionDenied, Self))
        }
    }

    #[test]
    fn integer_negative_zero_reads_as_zero_and_float_negative_zero_keeps_its_sign() {
        let text = r#"{"v":-0,"w":[-0,{"k":-0}],"s":"-0","-0":1,"f":-0.0,"e":-0e0}"#;
        let from_line = match read_jsonl_line(text) {
            JsonRecord::Object(obj) => obj,
            _ => panic!("a JSONL object"),
        };
        let mut reader = Cursor::new(text.as_bytes());
        let from_array = match read_json_record(&mut reader, b'{').unwrap() {
            JsonRecord::Object(obj) => obj,
            _ => panic!("an array element object"),
        };

        for obj in [from_line, from_array] {
            assert_eq!(json_value_text(&obj["v"]), "0");
            assert_eq!(json_value_text(&obj["w"]), r#"[0,{"k":0}]"#);
            assert_eq!(json_value_text(&obj["s"]), "-0");
            assert!(obj.contains_key("-0"));
            assert_eq!(json_value_text(&obj["f"]), "-0.0");
            assert_eq!(json_value_text(&obj["e"]), "-0.0");
        }
    }

    #[test]
    fn json_record_read_failure_keeps_the_original_io_source() {
        let mut reader = BufReader::new(Cursor::new(b"{\"x\":\"").chain(LocalReadFailure));

        let error = read_json_record(&mut reader, b'{')
            .err()
            .expect("a read failure must abort before a caller can skip a malformed record");

        assert!(
            error
                .to_string()
                .contains("JSON record read failed: local read failure")
        );
        let source = error
            .source()
            .and_then(|source| source.downcast_ref::<io::Error>())
            .expect("the original I/O error must survive serde's wrapper");
        assert_eq!(source.kind(), io::ErrorKind::PermissionDenied);
        assert!(source.get_ref().unwrap().is::<LocalReadFailure>());
    }
}