incremental_writer/json/
mod.rs

1use serde::{Serialize, Deserialize};
2use std::io::{Write, Seek, SeekFrom};
3
4#[cfg(unix)]
5use std::os::unix::fs::FileExt;
6#[cfg(windows)]
7use std::os::windows::fs::FileExt;
8
9type Result<T> = std::result::Result<T, std::io::Error>;
10
11pub struct IncrementalJsonWriter<T: FileExt + Write + Seek> { 
12    buffer: T,
13}
14impl<T: FileExt + Write + Seek> IncrementalJsonWriter<T> {
15    pub fn new(buffer: T) -> Self { 
16        IncrementalJsonWriter::<T> {
17            buffer
18        }
19    }
20
21    pub fn write_json<U: Serialize>(&mut self, element: &U) -> Result<usize> {
22        self.write(serde_json::to_string_pretty(&element)?.as_bytes())
23    }
24    
25    #[cfg(unix)]
26    fn write_at_offset(&mut self, bytes: &[u8], offset: u64) -> Result<usize> {
27        let bytes_written = self.buffer.write_at(bytes, offset)?;
28        self.buffer.seek(SeekFrom::Current((bytes_written - 2) as i64)).map(|_| bytes_written)
29    }
30    #[cfg(windows)]
31    fn write_at_offset(&mut self, bytes: &[u8], offset: u64) -> Result<usize> {
32        self.buffer.seek_write(bytes, offset)
33    }
34}
35
36impl<T: FileExt + Write + Seek> Write for IncrementalJsonWriter<T> { 
37    fn write(&mut self, element: &[u8]) -> Result<usize> { 
38        let mut current = self.buffer.seek(SeekFrom::Current(0))?;
39        let mut bytes = vec![];
40
41        if current == 0 { 
42            self.buffer.write(b"[\n\n]")?;
43            current = self.buffer.seek(SeekFrom::Current(0))?;
44        } else { 
45            bytes.extend(b",\n");
46        }
47
48        bytes.extend(element);
49        bytes.push(b'\n');
50        bytes.push(b']');
51
52        let written = self.write_at_offset(&bytes, current - 2)?;
53
54        let new_position = self.buffer.seek(SeekFrom::Current(0))?;
55        println!("Was: {}, now: {}, total: {}", current, new_position, current + bytes.len() as u64);
56        Ok(written)
57    }
58    fn flush(&mut self) -> Result<()> { 
59        self.buffer.flush()
60    }
61}
62
63#[test]
64fn writer_writes_square_brackets_to_buffer() {
65    use std::io::{Read};
66
67    let expect_one = String::from("[\n{\n  \"name\": \"Test\",\n  \"detail\": 0\n},\
68    \n{\n  \"name\": \"Test\",\n  \"detail\": 1\n}\n]");
69
70    let expect_two = String::from("[\n{\n  \"name\": \"Test\",\n  \"detail\": 0\n},\
71    \n{\n  \"name\": \"Test\",\n  \"detail\": 1\n},\
72    \n{\n  \"name\": \"Test\",\n  \"detail\": 2\n},\
73    \n{\n  \"name\": \"Test\",\n  \"detail\": 3\n}\n]");
74
75    let path = "unittest.json";
76    let rows: Vec<Record> = vec![0, 1, 2, 3]
77        .iter()
78        .map(|num| Record { name: String::from("Test"), detail: *num})
79        .collect();
80    
81    let writer = std::fs::File::create(path).unwrap();
82    let mut json_writer = IncrementalJsonWriter::new(writer);
83
84    let mut reader = std::fs::File::open(path).unwrap();
85    
86    for row in rows.iter().take(2) { 
87        json_writer.write_json(&row).unwrap();
88    }
89    
90    let mut buffer = String::new();
91    reader.read_to_string(&mut buffer).unwrap();
92    assert_eq!(expect_one, buffer);
93
94    for row in rows.iter().skip(2).take(2) { 
95        json_writer.write_json(&row).unwrap();
96    }
97    let mut buffer = String::new();
98    reader.seek(SeekFrom::Start(0)).unwrap();
99    reader.read_to_string(&mut buffer).unwrap();
100    assert_eq!(expect_two, buffer);
101}
102
103#[derive(Serialize, Deserialize, Debug)]
104struct Record { 
105    name: String,
106    detail: u32
107}
incremental_writer/json/mod.rs

incremental_writer/json/
mod.rs
