streaming_serde_json 0.1.2

This is a streaming alternative to serde_json for processing JSON data sources that are too large to fit in memory.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132

use std::marker::PhantomData;

use serde::Deserialize;

use crate::{
    de::CharsDeserializer,
    tokenizer::{ErrKind, Error, TokenKind},
};

pub struct ValueStream<T, CharsStream, PassThroughError>
where
    T: for<'de> Deserialize<'de>,
    CharsStream: IntoIterator<Item = Result<char, PassThroughError>>,
    PassThroughError: std::error::Error,
{
    phantom: PhantomData<T>,
    state: ValueStreamState,
    stream: CharsDeserializer<CharsStream, PassThroughError>,
}

enum ValueStreamState {
    Start,
    BeforeComma,
    AfterComma,
    End,
}

impl<T, CharsStream, PassThroughError> ValueStream<T, CharsStream, PassThroughError>
where
    T: for<'de> Deserialize<'de>,
    CharsStream: IntoIterator<Item = Result<char, PassThroughError>>,
    PassThroughError: std::error::Error,
{
    pub(crate) fn new(stream: CharsStream) -> Self {
        Self {
            phantom: PhantomData::default(),
            state: ValueStreamState::Start,
            stream: CharsDeserializer::from_stream(stream),
        }
    }
}

impl<T, CharsStream, PassThroughError> Iterator for ValueStream<T, CharsStream, PassThroughError>
where
    T: for<'de> Deserialize<'de>,
    CharsStream: IntoIterator<Item = Result<char, PassThroughError>>,
    PassThroughError: std::error::Error,
{
    type Item = Result<T, Error<PassThroughError>>;

    fn next(&mut self) -> Option<Self::Item> {
        loop {
            match self.state {
                ValueStreamState::Start => {
                    let token_result = self
                        .stream
                        .next()
                        .unwrap_or_else(|| Err(self.stream.unexpected_eof()));

                    let token = match token_result {
                        Err(err) => {
                            self.state = ValueStreamState::End;
                            return Some(Err(err));
                        }
                        Ok(token) => token,
                    };

                    if let TokenKind::ArrayStart = token.kind {
                        self.state = ValueStreamState::AfterComma;
                    } else {
                        self.state = ValueStreamState::End;
                        return Some(Err(Error {
                            location: token.span.start,
                            kind: ErrKind::UnexpectedToken {
                                expected: vec![TokenKind::ArrayStart],
                                actual: Some(token.kind),
                            },
                        }));
                    }
                }
                ValueStreamState::BeforeComma => {
                    let token_result = self
                        .stream
                        .next()
                        .unwrap_or_else(|| Err(self.stream.unexpected_eof()));

                    let token = match token_result {
                        Err(err) => {
                            self.state = ValueStreamState::End;
                            return Some(Err(err));
                        }
                        Ok(token) => token,
                    };

                    if let TokenKind::ArrayEnd = token.kind {
                        return None;
                    }

                    if let TokenKind::Comma = token.kind {
                        self.state = ValueStreamState::AfterComma;
                    } else {
                        self.state = ValueStreamState::End;
                        return Some(Err(Error {
                            location: token.span.start,
                            kind: ErrKind::UnexpectedToken {
                                expected: vec![TokenKind::Comma],
                                actual: Some(token.kind),
                            },
                        }));
                    }
                }
                ValueStreamState::AfterComma => {
                    let result = T::deserialize(&mut self.stream);
                    match result {
                        Err(err) => {
                            self.state = ValueStreamState::End;
                            return Some(Err(err));
                        }
                        Ok(value) => {
                            self.state = ValueStreamState::BeforeComma;
                            return Some(Ok(value));
                        }
                    }
                }

                ValueStreamState::End => {
                    return None;
                }
            }
        }
    }
}