json_tools/
reader.rs

1use std::{
2    cmp,
3    io::{Read, Result, Write},
4    ptr,
5};
6
7use super::{Buffer, Token, TokenType};
8
9fn copy_memory(src: &[u8], dst: &mut [u8]) {
10    let len_src = src.len();
11    assert!(dst.len() >= len_src);
12    // `dst` is unaliasable, so we know statically it doesn't overlap
13    // with `src`.
14    unsafe {
15        ptr::copy_nonoverlapping(src.as_ptr(), dst.as_mut_ptr(), len_src);
16    }
17}
18
19/// An adapter to convert a stream of `Token`s into bytes by implementing an
20/// `std::io::Read` trait.
21///
22/// Currently it produces output without any whitespace, suitable for efficient
23/// sending and for handling by machines.
24pub struct TokenReader<'a, I: IntoIterator<Item = Token>> {
25    iter: I::IntoIter,
26    src: Option<&'a str>,
27    buf: Vec<u8>,
28    ofs: usize,
29}
30
31impl<'a, I: IntoIterator<Item = Token>> TokenReader<'a, I> {
32    /// Returns a new `TokenReader`
33    /// # Args
34    /// * `iter` - the iterator producing `Token` instances we are to convert
35    /// * `source` - an optional, original string from which the tokens were
36    ///              generated. This offers the best performance when
37    ///              serializing tokens, as they can refer to their original
38    ///              `&str` slice.
39    pub fn new(iter: I, source: Option<&'a str>) -> TokenReader<'a, I> {
40        TokenReader {
41            iter: iter.into_iter(),
42            src: source,
43            buf: Vec::with_capacity(128),
44            ofs: 0,
45        }
46    }
47}
48
49impl<'a, I: IntoIterator<Item = Token>> Read for TokenReader<'a, I> {
50    fn read(&mut self, buf: &mut [u8]) -> Result<usize> {
51        if buf.is_empty() {
52            return Ok(0);
53        }
54
55        // Bytes from Cache
56        let mut bl = buf.len();
57        if !self.buf.is_empty() {
58            let btc = cmp::min(self.buf.len() - self.ofs, buf.len());
59            copy_memory(&self.buf[self.ofs..self.ofs + btc], buf);
60            bl -= btc;
61            self.ofs += btc;
62            if self.ofs == self.buf.len() {
63                self.buf.clear();
64                self.ofs = 0;
65            }
66        }
67        if bl == 0 {
68            return Ok(buf.len());
69        }
70
71        // Generate bytes from tokens
72        while bl > 0 {
73            match self.iter.next() {
74                None => return Ok(buf.len() - bl),
75                Some(t) => {
76                    let bytes: &[u8] = match t.kind {
77                        TokenType::String | TokenType::Number => match t.buf {
78                            Buffer::MultiByte(ref b) => &b,
79                            Buffer::Span(ref s) => match self.src {
80                                Some(b) => b[s.first as usize..s.end as usize].as_bytes(),
81                                None => panic!("Must set source if tokens don't provide byter buffers"),
82                            },
83                        },
84                        TokenType::Invalid => b"",
85                        _ => t.kind.as_ref().as_bytes(),
86                    };
87                    let btc = cmp::min(bytes.len(), bl);
88                    let ofs = buf.len() - bl;
89                    copy_memory(&bytes[..btc], &mut buf[ofs..ofs + btc]);
90                    bl -= btc;
91
92                    if btc < bytes.len() {
93                        debug_assert!(bl == 0);
94                        self.buf.write_all(&bytes[btc..])?;
95                    }
96
97                    if bl == 0 {
98                        return Ok(buf.len());
99                    }
100                }
101            } // match iter.next()
102        } // end while there are bytes to produce
103        unreachable!();
104    }
105}
json_tools/reader.rs

json_tools/
reader.rs
