dataprof-json 0.14.1

JSON and JSONL scanning support for dataprof
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
//! The JSON record tokenizer shared by the file reader and the async stream
//! reader, so a payload profiles identically whichever way it arrives.

use std::io::{BufRead, Read};

use dataprof_core::DataProfilerError;
use serde::Deserialize;
use serde_json::Value;

use crate::JsonObject;

/// One record read from a JSON or JSONL source.
pub enum JsonRecord {
    /// A JSON object — the only value with named fields to profile as a row.
    Object(JsonObject),
    /// Valid JSON that is not an object, carrying the value's kind for the
    /// error message.
    NonObject(&'static str),
    /// The bytes are not valid JSON.
    Malformed(serde_json::Error),
    /// An object whose source text holds an integer literal outside the
    /// 64-bit range. serde_json keeps such an integer as the nearest `f64`, so
    /// distinct values would round to one before they are counted (#1103).
    /// The text decides, not the parsed value, so a literal a later duplicate
    /// key overwrites counts too, as it does for the Python bytes reader,
    /// which sees every integer token. The record was read in full, so the
    /// caller can skip it and keep reading.
    WideInteger,
}

/// Read one JSON value and leave the reader positioned immediately after it.
///
/// `first` is the value's first byte, already located by
/// the caller but not yet consumed.
pub fn read_json_record<R: BufRead>(
    reader: &mut R,
    first: u8,
) -> Result<JsonRecord, DataProfilerError> {
    // A number is the only JSON value that is not self-delimiting: serde finds
    // its end by peeking one byte past it and then drops that byte along with
    // the deserializer. Inside an array that byte is the `,` or `]` the scanner
    // needs next, so numbers are read byte-wise instead.
    if first == b'-' || first.is_ascii_digit() {
        let token = read_number_token(reader)?;
        return Ok(match serde_json::from_str::<Value>(&token) {
            Ok(_) => JsonRecord::NonObject("number"),
            Err(err) => JsonRecord::Malformed(err),
        });
    }

    // The source bytes are kept so an integer serde rounded to f64 can be
    // recognised from its literal; the parsed value no longer tells.
    let mut recorded = RecordingReader {
        inner: reader,
        bytes: Vec::new(),
    };
    let parsed = {
        let mut deserializer = serde_json::Deserializer::from_reader(&mut recorded);
        Value::deserialize(&mut deserializer)
    };
    Ok(match parsed {
        Ok(Value::Object(obj)) => object_record(obj, &recorded.bytes),
        Ok(value) => JsonRecord::NonObject(json_value_kind(&value)),
        Err(err) if err.is_io() => {
            // A failed read is not a malformed record that the caller can
            // skip. Recover the original I/O wrapper: serde's Error::source
            // skips it, hiding concrete transport causes from Python.
            let message = format!("JSON record read failed: {err}");
            return Err(DataProfilerError::json_parsing_with_source(
                message,
                std::io::Error::from(err),
            ));
        }
        Err(err) => JsonRecord::Malformed(err),
    })
}

/// A reader that keeps a copy of every byte read through it.
struct RecordingReader<'a, R> {
    inner: &'a mut R,
    bytes: Vec<u8>,
}

impl<R: Read> Read for RecordingReader<'_, R> {
    fn read(&mut self, buffer: &mut [u8]) -> std::io::Result<usize> {
        let read = self.inner.read(buffer)?;
        self.bytes.extend_from_slice(&buffer[..read]);
        Ok(read)
    }
}

/// Whether `value` holds a float large enough to have been read from an
/// integer literal outside the 64-bit range; only the source text can tell.
///
/// An integer within `i64` or `u64` parses exactly; one outside becomes an
/// `f64` of magnitude at least 2^63. A float literal like `1e19` does too, so
/// this alone refuses nothing.
pub fn may_hold_wide_integer(value: &Value) -> bool {
    match value {
        Value::Number(number) => {
            number.as_i64().is_none()
                && number.as_u64().is_none()
                && number
                    .as_f64()
                    .is_some_and(|float| float.abs() >= 2f64.powi(63))
        }
        Value::Array(items) => items.iter().any(may_hold_wide_integer),
        Value::Object(fields) => fields.values().any(may_hold_wide_integer),
        _ => false,
    }
}

/// Whether `raw` JSON text holds an integer literal (no fraction, no exponent)
/// that fits neither `i64` nor `u64`. String contents are skipped, and a token
/// shorter than 19 digits, which always fits, is not parsed.
pub fn has_wide_integer_literal(raw: &[u8]) -> bool {
    integer_literals(raw).any(|(_, token)| is_wide_integer(token))
}

/// Whether `raw` JSON text is a single float literal that names a value its
/// `f64` does not hold, like `1000000000000000.01` (#1156).
///
/// Numbers inside an array or object are not looked at: a container is
/// profiled as its JSON text, with each float as its `f64`, as
/// [`object_record`] keeps it, so equal containers stay equal.
pub fn is_lossy_float_scalar(raw: &[u8]) -> bool {
    number_literals(raw).next().is_some_and(|literal| {
        literal.depth == 0
            && is_float_literal(literal.token)
            && is_lossy_float_literal(literal.token)
    })
}

/// The integer literals (no fraction, no exponent) of `raw` JSON text, each
/// with the offset it starts at. String contents are skipped.
fn integer_literals(raw: &[u8]) -> impl Iterator<Item = (usize, &[u8])> {
    number_literals(raw)
        .filter(|literal| !is_float_literal(literal.token))
        .map(|literal| (literal.start, literal.token))
}

/// A number literal in JSON text.
struct NumberLiteral<'a> {
    /// The offset the literal starts at.
    start: usize,
    /// How many arrays and objects enclose it: a field of a root object is at
    /// depth 1.
    depth: usize,
    token: &'a [u8],
}

/// The number literals of `raw` JSON text. String contents are skipped.
fn number_literals(raw: &[u8]) -> impl Iterator<Item = NumberLiteral<'_>> {
    let mut index = 0;
    let mut depth = 0usize;
    std::iter::from_fn(move || {
        while index < raw.len() {
            match raw[index] {
                b'"' => {
                    index += 1;
                    while index < raw.len() && raw[index] != b'"' {
                        index += if raw[index] == b'\\' { 2 } else { 1 };
                    }
                    index += 1;
                }
                b'{' | b'[' => {
                    depth += 1;
                    index += 1;
                }
                b'}' | b']' => {
                    depth = depth.saturating_sub(1);
                    index += 1;
                }
                byte if byte == b'-' || byte.is_ascii_digit() => {
                    let start = index;
                    while index < raw.len() && is_json_number_byte(raw[index]) {
                        index += 1;
                    }
                    return Some(NumberLiteral {
                        start,
                        depth,
                        token: &raw[start..index],
                    });
                }
                _ => index += 1,
            }
        }
        None
    })
}

fn is_float_literal(token: &[u8]) -> bool {
    token.iter().any(|byte| matches!(byte, b'.' | b'e' | b'E'))
}

/// Whether a float literal names a value its `f64` does not hold, so that two
/// such literals can read as one number (#1156).
///
/// The test is on the decimal value, not the spelling: `1.50` and `1e2` are
/// held exactly and stay numbers, while `1000000000000000.01` reads as
/// `1000000000000000` and `1e-400` as `0`.
fn is_lossy_float_literal(token: &[u8]) -> bool {
    // A zero literal is held exactly.
    let Some(literal) = Decimal::parse(token) else {
        return false;
    };
    // Up to 15 significant digits inside the normal range always survive the
    // round trip, which spares most literals the parse and the format below.
    if literal.digits <= 15 && (-290..=290).contains(&(literal.exponent + literal.digits as i64)) {
        return false;
    }
    // decode-audit: impossible — every number byte is ASCII.
    let text = std::str::from_utf8(token).expect("JSON number bytes are ASCII");
    // A literal serde_json accepted always parses; an infinite one was
    // refused as out of range before the record got here.
    let Ok(value) = text.parse::<f64>() else {
        return false;
    };
    // `{:e}` writes the shortest digits that read back as the same `f64`.
    let mut shortest = ShortBuffer::default();
    // decode-audit: impossible — `{:e}` of an f64 fits in the buffer.
    std::fmt::Write::write_fmt(&mut shortest, format_args!("{value:e}"))
        .expect("an f64 in exponent form fits in 32 bytes");
    Decimal::parse(shortest.as_bytes()).is_none_or(|held| !literal.same_value(&held))
}

/// A nonzero decimal number in JSON number syntax, located in its text
/// without copying, so that equal values compare equal however they are
/// spelled.
struct Decimal<'a> {
    negative: bool,
    /// The mantissa from its first to its last nonzero digit, which may hold
    /// the decimal point.
    significant: &'a [u8],
    /// How many digits `significant` holds.
    digits: usize,
    /// The power of ten of the last significant digit.
    exponent: i64,
}

impl<'a> Decimal<'a> {
    /// `None` for a zero.
    fn parse(text: &'a [u8]) -> Option<Self> {
        let (negative, unsigned) = match text.split_first() {
            Some((b'-', rest)) => (true, rest),
            _ => (false, text),
        };
        let (mantissa, exponent) = match unsigned.iter().position(|b| matches!(b, b'e' | b'E')) {
            Some(at) => {
                let exponent = &unsigned[at + 1..];
                // decode-audit: impossible — every number byte is ASCII.
                let parsed = std::str::from_utf8(exponent)
                    .expect("JSON number bytes are ASCII")
                    .parse::<i64>()
                    // An exponent too long for i64 lies far outside the f64
                    // range either way, so saturating keeps the comparison right.
                    .unwrap_or(if exponent.first() == Some(&b'-') {
                        i64::MIN / 2
                    } else {
                        i64::MAX / 2
                    });
                (&unsigned[..at], parsed)
            }
            None => (unsigned, 0),
        };
        let point = mantissa
            .iter()
            .position(|&b| b == b'.')
            .unwrap_or(mantissa.len());
        let first = mantissa.iter().position(|b| matches!(b, b'1'..=b'9'))?;
        // decode-audit: impossible — `first` found a nonzero digit.
        let last = mantissa
            .iter()
            .rposition(|b| matches!(b, b'1'..=b'9'))
            .expect("a nonzero digit was found");
        let last_power = if last < point {
            (point - last - 1) as i64
        } else {
            -((last - point) as i64)
        };
        let digits = last - first + 1 - usize::from(first < point && point < last);
        Some(Self {
            negative,
            significant: &mantissa[first..=last],
            digits,
            exponent: exponent.saturating_add(last_power),
        })
    }

    fn same_value(&self, other: &Decimal<'_>) -> bool {
        self.negative == other.negative
            && self.exponent == other.exponent
            && self.digits == other.digits
            && self
                .significant
                .iter()
                .filter(|&&b| b != b'.')
                .eq(other.significant.iter().filter(|&&b| b != b'.'))
    }
}

/// A stack buffer for the exponent form of one `f64`.
#[derive(Default)]
struct ShortBuffer {
    bytes: [u8; 32],
    len: usize,
}

impl ShortBuffer {
    fn as_bytes(&self) -> &[u8] {
        &self.bytes[..self.len]
    }
}

impl std::fmt::Write for ShortBuffer {
    fn write_str(&mut self, text: &str) -> std::fmt::Result {
        let end = self.len + text.len();
        self.bytes
            .get_mut(self.len..end)
            .ok_or(std::fmt::Error)?
            .copy_from_slice(text.as_bytes());
        self.len = end;
        Ok(())
    }
}

fn is_wide_integer(token: &[u8]) -> bool {
    let digits = token.len() - usize::from(token.first() == Some(&b'-'));
    if digits < 19 {
        return false;
    }
    // decode-audit: impossible — every number byte is ASCII.
    let text = std::str::from_utf8(token).expect("JSON number bytes are ASCII");
    text.parse::<i64>().is_err() && text.parse::<u64>().is_err()
}

/// The record an object parsed from `raw` profiles as: always
/// [`JsonRecord::Object`] or [`JsonRecord::WideInteger`].
///
/// An object whose text holds an integer outside the 64-bit range is
/// [`JsonRecord::WideInteger`]. serde_json reads the integer literal `-0` as
/// the float `-0.0`, while Python's `json` reads it as the integer `0`, so
/// `{"v":0}` and `{"v":-0}` were duplicates over Python bytes and distinct
/// floats over the file and async readers. Such an object is read again with
/// each `-0` as `0`, which keeps its type and its duplicates the same on every
/// route. A float literal like `-0.0` keeps its sign on every route already.
///
/// A field whose float literal names a value its `f64` does not hold, like
/// `1000000000000000.01`, is given as the literal's text instead of the
/// rounded number, so distinct values stay distinct and the field profiles
/// as the same digits in a CSV do (#1156). Values inside an array or object
/// keep the `f64`: they are profiled as the container's text.
pub fn object_record(obj: JsonObject, raw: &[u8]) -> JsonRecord {
    let mut negative_zeros = Vec::new();
    let mut lossy_fields = Vec::new();
    for literal in number_literals(raw) {
        if is_float_literal(literal.token) {
            if literal.depth == 1 && is_lossy_float_literal(literal.token) {
                lossy_fields.push((literal.start, literal.token.len()));
            }
        } else if is_wide_integer(literal.token) {
            return JsonRecord::WideInteger;
        } else if literal.token == b"-0" {
            negative_zeros.push(literal.start);
        }
    }
    if negative_zeros.is_empty() && lossy_fields.is_empty() {
        return JsonRecord::Object(obj);
    }
    // Blanking the sign leaves `0` where `-0` stood; whitespace is legal
    // between any two JSON tokens, so the text stays the same document.
    let mut unsigned = raw.to_vec();
    for start in negative_zeros {
        unsigned[start] = b' ';
    }
    // Quoting a literal makes it a string of the same digits.
    let mut rewritten = Vec::with_capacity(unsigned.len() + 2 * lossy_fields.len());
    let mut copied = 0;
    for (start, len) in lossy_fields {
        rewritten.extend_from_slice(&unsigned[copied..start]);
        rewritten.push(b'"');
        rewritten.extend_from_slice(&unsigned[start..start + len]);
        rewritten.push(b'"');
        copied = start + len;
    }
    rewritten.extend_from_slice(&unsigned[copied..]);
    // decode-audit: impossible — `raw` parsed as this object, and only signs
    // were replaced by whitespace and number literals quoted.
    JsonRecord::Object(
        serde_json::from_slice::<JsonObject>(&rewritten)
            .expect("an object with its `-0` signs blanked and literals quoted is still an object"),
    )
}

/// Build the error for a record refused as [`JsonRecord::WideInteger`].
/// `location` names the record ("at position 3", "on line 7"); the value is
/// never included, as with every record error.
pub fn wide_integer_record_error(location: &str) -> DataProfilerError {
    DataProfilerError::json_parsing_error(&format!(
        "JSON record {location} holds an integer outside the 64-bit range, which would be rounded to a float and could merge distinct values; write such integers as strings to profile them"
    ))
}

/// Why a JSONL record falls outside the one-value-per-line rule, appended to
/// the decoder's reason for the faults it describes.
const JSONL_BOUNDARY_RULE: &str = "a JSONL record must be one complete JSON value on one line";

/// Build the strict-mode error for a JSONL `line` that [`read_jsonl_line`]
/// refused as malformed, located by its physical `line_number`.
///
/// The reason is serde's own, without the position it appends: each line is
/// decoded alone, so serde's line is always 1. The boundary rule is added only
/// for the faults it describes, data after a complete value and a value the
/// line break cuts short (#1143). The column counts characters of the line as
/// the source has it, the white space [`read_jsonl_line`] trims included, where
/// serde counts bytes of the trimmed text; a record that ends early is located
/// just past its last character. The Python bytes reader takes the same
/// message through [`jsonl_line_error_message`]. The record's text is never
/// included.
pub fn malformed_jsonl_record_error(
    line_number: usize,
    line: &str,
    err: serde_json::Error,
) -> DataProfilerError {
    let message = malformed_jsonl_record_message(line_number, line, &err);
    DataProfilerError::json_parsing_with_source(message, err)
}

/// The message of [`malformed_jsonl_record_error`] for a JSONL `line`, or
/// `None` when [`read_jsonl_line`] does not refuse it as malformed.
///
/// The Python bytes reader decodes with `json` and asks for this message when
/// `json` refuses a line, so all three routes report one reason, one column
/// and one boundary rule for the same fault (#1143).
pub fn jsonl_line_error_message(line_number: usize, line: &str) -> Option<String> {
    match read_jsonl_line(line) {
        JsonRecord::Malformed(err) => Some(malformed_jsonl_record_message(line_number, line, &err)),
        _ => None,
    }
}

fn malformed_jsonl_record_message(
    line_number: usize,
    line: &str,
    err: &serde_json::Error,
) -> String {
    let record = line.trim();
    let leading = line[..line.len() - line.trim_start().len()].chars().count();
    let rendered = err.to_string();
    let position = format!(" at line {} column {}", err.line(), err.column());
    let reason = rendered.strip_suffix(&position).unwrap_or(&rendered);
    let ends_early = err.classify() == serde_json::error::Category::Eof;
    let column = if ends_early {
        leading + record.chars().count() + 1
    } else {
        // serde's column is the 1-based byte of the fault, so the characters
        // that start before it end with the faulting one.
        leading
            + record
                .char_indices()
                .take_while(|(start, _)| *start < err.column())
                .count()
    };
    if ends_early || reason == "trailing characters" {
        format!(
            "malformed JSON record on line {line_number}, column {column}: {reason}; {JSONL_BOUNDARY_RULE}"
        )
    } else {
        format!("malformed JSON record on line {line_number}, column {column}: {reason}")
    }
}

/// Read the bytes that can spell a JSON number, stopping at the first byte that
/// cannot. The token is returned unvalidated — the caller decides whether it
/// actually parses, so `1.2.3` stays a malformed record rather than a number.
///
/// Leading whitespace is consumed first: callers locate the next value's first
/// byte without consuming what precedes it, because serde needs that whitespace
/// to keep its line and column context.
fn read_number_token<R: BufRead>(reader: &mut R) -> Result<String, DataProfilerError> {
    skip_ascii_whitespace(reader)?;
    let mut token = String::new();
    loop {
        let (consume, token_ended) = {
            let buf = reader.fill_buf().map_err(DataProfilerError::from)?;
            if buf.is_empty() {
                break;
            }
            let consume = buf
                .iter()
                .take_while(|byte| is_json_number_byte(**byte))
                .count();
            // decode-audit: impossible — every byte accepted above is ASCII.
            token.push_str(
                std::str::from_utf8(&buf[..consume]).expect("JSON number bytes are ASCII"),
            );
            (consume, consume < buf.len())
        };
        reader.consume(consume);
        if token_ended {
            break;
        }
    }
    Ok(token)
}

fn is_json_number_byte(byte: u8) -> bool {
    byte.is_ascii_digit() || matches!(byte, b'-' | b'+' | b'.' | b'e' | b'E')
}

/// Advance the reader past any ASCII whitespace at its current position.
pub(crate) fn skip_ascii_whitespace<R: BufRead>(reader: &mut R) -> Result<(), DataProfilerError> {
    loop {
        let (consume, done) = {
            let buf = reader.fill_buf().map_err(DataProfilerError::from)?;
            if buf.is_empty() {
                return Ok(());
            }
            let consume = buf
                .iter()
                .take_while(|byte| byte.is_ascii_whitespace())
                .count();
            (consume, consume < buf.len())
        };
        reader.consume(consume);
        if done {
            return Ok(());
        }
    }
}

/// The text a JSON value is profiled as: scalars as themselves, `null` as the
/// empty string, and an array or object as compact JSON with every object's
/// keys sorted.
///
/// The sort makes the text canonical. With `preserve_order` an object keeps
/// its source key order, so `{"x":1,"y":2}` and `{"y":2,"x":1}` would render
/// differently and two equal rows would not count as duplicates (#767). Keys
/// sort by code point, as Python's `json.dumps(..., sort_keys=True)` does for
/// in-memory inputs.
pub fn json_value_text(value: &Value) -> String {
    match value {
        Value::Null => String::new(),
        Value::Bool(boolean) => boolean.to_string(),
        Value::Number(number) => number.to_string(),
        Value::String(string) => string.to_string(),
        Value::Array(_) | Value::Object(_) => {
            let mut text = String::new();
            write_canonical_json(value, &mut text);
            text
        }
    }
}

fn write_canonical_json(value: &Value, out: &mut String) {
    match value {
        Value::Array(items) => {
            out.push('[');
            for (index, item) in items.iter().enumerate() {
                if index > 0 {
                    out.push(',');
                }
                write_canonical_json(item, out);
            }
            out.push(']');
        }
        Value::Object(fields) => {
            let mut entries: Vec<_> = fields.iter().collect();
            entries.sort_unstable_by_key(|(key, _)| *key);
            out.push('{');
            for (index, (key, field)) in entries.into_iter().enumerate() {
                if index > 0 {
                    out.push(',');
                }
                out.push_str(&scalar_json(key));
                out.push(':');
                write_canonical_json(field, out);
            }
            out.push('}');
        }
        scalar => out.push_str(&scalar_json(scalar)),
    }
}

fn scalar_json<T: serde::Serialize + ?Sized>(value: &T) -> String {
    // decode-audit: impossible — serializing an in-memory string or scalar
    // Value cannot fail; a panic here beats folding "" (= null) into stats.
    serde_json::to_string(value).expect("re-serializing a parsed JSON value cannot fail")
}

pub fn json_value_kind(value: &Value) -> &'static str {
    match value {
        Value::Null => "null",
        Value::Bool(_) => "boolean",
        Value::Number(_) => "number",
        Value::String(_) => "string",
        Value::Array(_) => "array",
        Value::Object(_) => "object",
    }
}

/// Build a strict-mode error for valid JSON that is not a row object. This is a
/// distinct category from a syntax failure: the document parsed, it just does
/// not hold records. `position` is 1-based over the records scanned so far.
pub fn non_object_record_error(kind: &str, position: usize) -> DataProfilerError {
    // No source: the document parsed cleanly, it simply does not hold records,
    // so there is no decoder error underneath to retain.
    DataProfilerError::json_parsing_error(&format!(
        "non-object JSON record at position {position}: \
         expected an object with fields to profile, found {kind}"
    ))
}

pub fn malformed_array_error(message: &str) -> DataProfilerError {
    // Callers here hold only a message, not the decoder error, so there is
    // nothing to retain.
    DataProfilerError::json_parsing_error(&format!("malformed JSON array: {message}"))
}

/// Build a strict-mode error for a standard JSON document that is not exactly
/// one value.
///
/// The most common cause is JSONL read with the JSON grammar — several objects
/// back to back parse as one value plus trailing characters — so the message
/// names the option that would read it correctly.
pub fn json_document_error(err: serde_json::Error) -> DataProfilerError {
    let message = format!(
        // One source line on purpose: rustfmt may join a `\`-continued
        // literal and leave its indentation inside the string, which silently
        // corrupted this very message once.
        "malformed JSON document: {err}. A JSON source must hold exactly one array or object; for one record per line use format=\"jsonl\""
    );
    DataProfilerError::json_parsing_with_source(message, err)
}

/// Consume the rest of the input, returning how many bytes that was.
pub fn drain_to_end<R: BufRead>(reader: &mut R) -> Result<usize, DataProfilerError> {
    let mut consumed = 0;
    loop {
        let available = reader.fill_buf().map_err(DataProfilerError::from)?.len();
        if available == 0 {
            return Ok(consumed);
        }
        reader.consume(available);
        consumed += available;
    }
}

/// What [`skip_leading_white_space`] skipped, and what it held back.
///
/// A record is read with `str::trim`, which removes the Unicode White_Space
/// set, for a single object and a JSONL line, so an array accepts the same
/// characters around it: no-break space, NEL, LS, PS, vertical tab and the
/// rest, not only the four JSON whitespace bytes (#1146).
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct LeadingWhiteSpace {
    /// Bytes of white space consumed.
    pub skipped: usize,
    /// The first bytes of a character split across two reads that turned out
    /// not to be white space: at most three, consumed from the reader, which
    /// a caller chains back in front of it. Empty otherwise.
    pub pending: Vec<u8>,
    /// The first byte after the white space, `None` at the end of input.
    pub first: Option<u8>,
}

/// Where a run of white space at the start of `bytes` ends.
enum WhiteSpaceRun {
    /// Every byte is white space.
    All,
    /// White space for this many bytes, then another character.
    Other(usize),
    /// White space for this many bytes, then the start of a character the
    /// slice ends inside.
    Partial(usize),
}

/// Bytes in the UTF-8 encoding a lead byte starts, or 0 for a byte that
/// cannot start one.
fn utf8_width(lead: u8) -> usize {
    match lead {
        0x00..=0x7F => 1,
        0xC2..=0xDF => 2,
        0xE0..=0xEF => 3,
        0xF0..=0xF4 => 4,
        _ => 0,
    }
}

/// Whether `bytes` encode exactly one character that is White_Space.
///
/// Bytes that are not valid UTF-8 are not white space: the skip stops in
/// front of them, and the reader after it reports the decode error.
fn is_white_space_char(bytes: &[u8]) -> bool {
    match std::str::from_utf8(bytes) {
        Ok(text) => text.chars().next().is_some_and(char::is_whitespace),
        Err(_) => false,
    }
}

/// Scan the white space at the start of `bytes` in place, one character at a
/// time, without copying or validating what follows it.
fn white_space_run(bytes: &[u8]) -> WhiteSpaceRun {
    let mut index = 0;
    while index < bytes.len() {
        let width = utf8_width(bytes[index]);
        if width == 0 {
            return WhiteSpaceRun::Other(index);
        }
        if index + width > bytes.len() {
            return WhiteSpaceRun::Partial(index);
        }
        if !is_white_space_char(&bytes[index..index + width]) {
            return WhiteSpaceRun::Other(index);
        }
        index += width;
    }
    WhiteSpaceRun::All
}

/// Finish a character whose first bytes, `pending`, ended the previous read:
/// `Some(true)` when it is white space, `Some(false)` when not, `None` when
/// the next read still does not complete it. Consumes only bytes of that
/// character, and only when it is white space or still incomplete.
fn finish_pending<R: BufRead>(
    reader: &mut R,
    pending: &mut Vec<u8>,
) -> Result<Option<bool>, DataProfilerError> {
    let width = utf8_width(pending[0]);
    let buf = reader.fill_buf().map_err(DataProfilerError::from)?;
    let take = (width - pending.len()).min(buf.len());
    let mut character = pending.clone();
    character.extend_from_slice(&buf[..take]);
    if character.len() < width {
        reader.consume(take);
        *pending = character;
        return Ok(None);
    }
    if is_white_space_char(&character) {
        reader.consume(take);
        pending.clear();
        return Ok(Some(true));
    }
    Ok(Some(false))
}

/// Consume the Unicode White_Space at the reader's position.
///
/// Memory stays bounded: white space is consumed as it is read, and only a
/// character split across two reads is held, at most three bytes, until the
/// next read decides it. Nothing after the white space is copied.
pub fn skip_leading_white_space<R: BufRead>(
    reader: &mut R,
) -> Result<LeadingWhiteSpace, DataProfilerError> {
    let mut pending = Vec::new();
    let mut skipped = 0;
    loop {
        if !pending.is_empty() {
            let width = utf8_width(pending[0]);
            let at_end = reader
                .fill_buf()
                .map_err(DataProfilerError::from)?
                .is_empty();
            if at_end {
                let first = pending.first().copied();
                return Ok(LeadingWhiteSpace {
                    skipped,
                    pending,
                    first,
                });
            }
            match finish_pending(reader, &mut pending)? {
                None => continue,
                Some(true) => {
                    skipped += width;
                    continue;
                }
                Some(false) => {
                    let first = pending.first().copied();
                    return Ok(LeadingWhiteSpace {
                        skipped,
                        pending,
                        first,
                    });
                }
            }
        }
        let buf = reader.fill_buf().map_err(DataProfilerError::from)?;
        if buf.is_empty() {
            return Ok(LeadingWhiteSpace {
                skipped,
                pending,
                first: None,
            });
        }
        match white_space_run(buf) {
            WhiteSpaceRun::Other(len) => {
                let first = Some(buf[len]);
                reader.consume(len);
                skipped += len;
                return Ok(LeadingWhiteSpace {
                    skipped,
                    pending,
                    first,
                });
            }
            WhiteSpaceRun::Partial(len) => {
                pending = buf[len..].to_vec();
                let read = buf.len();
                reader.consume(read);
                skipped += len;
            }
            WhiteSpaceRun::All => {
                let read = buf.len();
                reader.consume(read);
                skipped += read;
            }
        }
    }
}

/// Whether the rest of the input is Unicode White_Space, and how many bytes
/// were consumed to decide: the white space, and the first bytes of a
/// character split across two reads when it turned out to be something else.
/// Reading stops at the first other character, so a caller that goes on
/// draining the input counts the rest itself. Memory stays bounded as in
/// [`skip_leading_white_space`].
pub fn rest_is_white_space<R: BufRead>(reader: &mut R) -> Result<(bool, usize), DataProfilerError> {
    let leading = skip_leading_white_space(reader)?;
    let consumed = leading.skipped + leading.pending.len();
    Ok((leading.first.is_none(), consumed))
}

/// Parse one JSONL line as exactly one JSON value.
///
/// `serde_json::from_str` requires the value to fill the whole input, so a line
/// carrying two adjacent or space-separated values is a malformed record rather
/// than two clean ones — the boundary rule that separates JSONL from a stream of
/// JSON values.
pub fn read_jsonl_line(line: &str) -> JsonRecord {
    match serde_json::from_str::<Value>(line.trim()) {
        Ok(Value::Object(obj)) => object_record(obj, line.as_bytes()),
        Ok(value) => JsonRecord::NonObject(json_value_kind(&value)),
        // An integer too long even for f64 fails to parse at all; the line is
        // consumed either way, so name it as what it holds.
        Err(_) if has_wide_integer_literal(line.as_bytes()) => JsonRecord::WideInteger,
        Err(err) => JsonRecord::Malformed(err),
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::error::Error as _;
    use std::io::{self, BufReader, Cursor, Read};

    /// A one-byte buffer splits every multi-byte character across reads, so a
    /// white-space character is only skipped once all of its bytes are in.
    #[test]
    fn white_space_helpers_decode_characters_split_across_buffers() {
        // (case, input, white-space bytes skipped, first byte after them)
        let leading: [(&str, &str, usize, Option<u8>); 7] = [
            ("none", "[1]", 0, Some(b'[')),
            ("ascii", " \n[1]", 2, Some(b'[')),
            ("nbsp and ideographic", "\u{A0}\u{3000}[1]", 5, Some(b'[')),
            ("all white space", "\u{2028}\u{85}", 5, None),
            ("not white space", "\u{1C}[1]", 0, Some(0x1C)),
            ("a two-byte non-space", "\u{E9}[1]", 0, Some(0xC3)),
            ("a three-byte non-space", "\u{A0}\u{20AC}[1]", 2, Some(0xE2)),
        ];
        let mut wrong = Vec::new();
        for capacity in [1, 2, 64] {
            for (case, input, skipped, first) in leading {
                let mut reader = BufReader::with_capacity(capacity, input.as_bytes());
                let read = skip_leading_white_space(&mut reader).unwrap();
                let mut rest = Vec::new();
                reader.read_to_end(&mut rest).unwrap();
                // Nothing is lost: what was skipped, what was held back and
                // what is left make up the input, and at most one split
                // character is held.
                let mut all = input.as_bytes()[..read.skipped].to_vec();
                all.extend(&read.pending);
                all.extend(rest);
                if (read.skipped, read.first, all.as_slice()) != (skipped, first, input.as_bytes())
                    || read.pending.len() > 3
                {
                    wrong.push(format!("leading, capacity {capacity}, {case}: {read:?}"));
                }
            }
        }
        let rest: [(&str, &str, bool); 5] = [
            ("empty", "", true),
            ("ascii", " \r\n", true),
            ("unicode", "\u{A0}\u{3000}\u{2029}", true),
            ("content after", "\u{A0}x", false),
            ("a truncated character", "\u{A0}\u{E2}", false),
        ];
        for (case, input, white) in rest {
            // A truncated character is not valid UTF-8 text, so it is built
            // from bytes.
            let bytes: Vec<u8> = if case == "a truncated character" {
                vec![0xC2, 0xA0, 0xE2, 0x80]
            } else {
                input.as_bytes().to_vec()
            };
            let mut reader = BufReader::with_capacity(1, bytes.as_slice());
            let (seen, _) = rest_is_white_space(&mut reader).unwrap();
            if seen != white {
                wrong.push(format!("rest, {case}: {seen}"));
            }
        }
        assert!(wrong.is_empty(), "{}", wrong.join("\n"));
    }

    #[derive(Debug)]
    struct LocalReadFailure;

    impl std::fmt::Display for LocalReadFailure {
        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
            f.write_str("local read failure")
        }
    }

    impl std::error::Error for LocalReadFailure {}

    impl Read for LocalReadFailure {
        fn read(&mut self, _buffer: &mut [u8]) -> io::Result<usize> {
            Err(io::Error::new(io::ErrorKind::PermissionDenied, Self))
        }
    }

    #[test]
    fn integer_negative_zero_reads_as_zero_and_float_negative_zero_keeps_its_sign() {
        let text = r#"{"v":-0,"w":[-0,{"k":-0}],"s":"-0","-0":1,"f":-0.0,"e":-0e0}"#;
        let from_line = match read_jsonl_line(text) {
            JsonRecord::Object(obj) => obj,
            _ => panic!("a JSONL object"),
        };
        let mut reader = Cursor::new(text.as_bytes());
        let from_array = match read_json_record(&mut reader, b'{').unwrap() {
            JsonRecord::Object(obj) => obj,
            _ => panic!("an array element object"),
        };

        for obj in [from_line, from_array] {
            assert_eq!(json_value_text(&obj["v"]), "0");
            assert_eq!(json_value_text(&obj["w"]), r#"[0,{"k":0}]"#);
            assert_eq!(json_value_text(&obj["s"]), "-0");
            assert!(obj.contains_key("-0"));
            assert_eq!(json_value_text(&obj["f"]), "-0.0");
            assert_eq!(json_value_text(&obj["e"]), "-0.0");
        }
    }

    #[test]
    fn json_record_read_failure_keeps_the_original_io_source() {
        let mut reader = BufReader::new(Cursor::new(b"{\"x\":\"").chain(LocalReadFailure));

        let error = read_json_record(&mut reader, b'{')
            .err()
            .expect("a read failure must abort before a caller can skip a malformed record");

        assert!(
            error
                .to_string()
                .contains("JSON record read failed: local read failure")
        );
        let source = error
            .source()
            .and_then(|source| source.downcast_ref::<io::Error>())
            .expect("the original I/O error must survive serde's wrapper");
        assert_eq!(source.kind(), io::ErrorKind::PermissionDenied);
        assert!(source.get_ref().unwrap().is::<LocalReadFailure>());
    }

    #[test]
    fn only_a_lossy_float_scalar_is_reported() {
        for raw in [" 1000000000000000.01 ", "1e-400", "-12345678901234567890.0"] {
            assert!(is_lossy_float_scalar(raw.as_bytes()), "{raw}");
        }
        for raw in [
            r#"{"v":1000000000000000.01}"#,
            r#"[1e-400]"#,
            r#""1000000000000000.01""#,
            "1.50",
            "18446744073709551615",
        ] {
            assert!(!is_lossy_float_scalar(raw.as_bytes()), "{raw}");
        }
    }

    #[test]
    fn a_float_literal_is_lossy_only_when_its_value_changes() {
        let held = [
            "0.1", "1.50", "1e2", "1E+2", "-0.0", "0e999", "2.5e-3", "1e308",
        ];
        let lossy = [
            "1000000000000000.01",
            "1e-400",
            "0.30000000000000000001",
            "12345678901234567890.0",
            "1e-99999999999999999999",
            "9007199254740993.0",
            "-1.00000000000000001",
        ];
        for token in held {
            assert!(!is_lossy_float_literal(token.as_bytes()), "{token}");
        }
        for token in lossy {
            assert!(is_lossy_float_literal(token.as_bytes()), "{token}");
        }
    }

    /// The shortcut for 15 significant digits in the normal range must agree
    /// with the full comparison it skips, at both ends of that range.
    #[test]
    fn fifteen_digit_literals_in_the_normal_range_are_held() {
        let mut state = 0x9e37_79b9_7f4a_7c15u64;
        for exponent in (-305..=275).step_by(5) {
            for _ in 0..200 {
                state ^= state << 13;
                state ^= state >> 7;
                state ^= state << 17;
                let mantissa = 100_000_000_000_000 + state % 900_000_000_000_000;
                let token = format!("{mantissa}e{exponent}");
                let literal = Decimal::parse(token.as_bytes()).unwrap();
                let value: f64 = token.parse().unwrap();
                let mut shortest = ShortBuffer::default();
                std::fmt::Write::write_fmt(&mut shortest, format_args!("{value:e}")).unwrap();
                let held = Decimal::parse(shortest.as_bytes()).unwrap();
                assert!(literal.same_value(&held), "{token}");
            }
        }
    }

    #[test]
    fn only_a_lossy_field_of_the_root_object_becomes_text() {
        let raw =
            br#"{"a":1000000000000000.01,"b":[1000000000000000.01],"c":-0,"d":1.5,"e":1e-400}"#;
        let obj = serde_json::from_slice(raw).unwrap();
        let JsonRecord::Object(obj) = object_record(obj, raw) else {
            panic!("not an object record");
        };
        assert_eq!(obj["a"], Value::String("1000000000000000.01".into()));
        assert_eq!(json_value_text(&obj["b"]), "[1000000000000000.0]");
        assert_eq!(json_value_text(&obj["c"]), "0");
        assert_eq!(json_value_text(&obj["d"]), "1.5");
        assert_eq!(obj["e"], Value::String("1e-400".into()));
    }
}