Skip to main content

ionic/ion/encoder/utilities/
output.rs

1#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
2use std::{
3    fs::{self, File, OpenOptions},
4    io::{BufWriter, Read, Seek, SeekFrom},
5    path::{Path, PathBuf},
6    sync::atomic::{AtomicU64, Ordering},
7};
8
9#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
10use cosmoz::{CompressOptions, Compressor};
11
12#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
13use crate::ion::encoder::encode::get_supported_compression_level;
14use crate::ion::{IonError, IonResult};
15
16pub trait WriteBytes {
17    fn write(&mut self, bytes: &[u8]) -> IonResult<()>;
18    fn patch(&mut self, at: u64, bytes: &[u8]) -> IonResult<()>;
19    fn position(&mut self) -> IonResult<u64>;
20}
21
22#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
23pub struct FileWriter {
24    writer: BufWriter<File>,
25}
26
27#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
28impl FileWriter {
29    pub fn open(path: &str) -> IonResult<Self> {
30        Self::open_path(Path::new(path))
31    }
32
33    pub fn open_path(path: &Path) -> IonResult<Self> {
34        let file = File::create(path).map_err(|err| {
35            IonError::from(format!(
36                "cannot create output file '{}': {err}",
37                path.display()
38            ))
39        })?;
40        Ok(Self {
41            writer: BufWriter::with_capacity(8 * 1024 * 1024, file),
42        })
43    }
44
45    pub fn flush(&mut self) -> IonResult<()> {
46        use std::io::Write as StdWrite;
47        self.writer
48            .flush()
49            .map_err(|err| IonError::from(format!("flush error: {err}")))
50    }
51}
52
53#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
54impl WriteBytes for FileWriter {
55    fn write(&mut self, bytes: &[u8]) -> IonResult<()> {
56        std::io::Write::write_all(&mut self.writer, bytes)
57            .map_err(|err| IonError::from(format!("write error: {err}")))
58    }
59
60    fn patch(&mut self, at: u64, bytes: &[u8]) -> IonResult<()> {
61        use std::io::Write as StdWrite;
62        self.writer
63            .flush()
64            .map_err(|err| IonError::from(format!("flush error: {err}")))?;
65        let resume_position = self
66            .writer
67            .stream_position()
68            .map_err(|err| IonError::from(format!("position error: {err}")))?;
69        self.writer
70            .seek(SeekFrom::Start(at))
71            .map_err(|err| IonError::from(format!("seek error: {err}")))?;
72        std::io::Write::write_all(&mut self.writer, bytes)
73            .map_err(|err| IonError::from(format!("patch write error: {err}")))?;
74        self.writer
75            .seek(SeekFrom::Start(resume_position))
76            .map_err(|err| IonError::from(format!("seek-resume error: {err}")))?;
77        Ok(())
78    }
79
80    fn position(&mut self) -> IonResult<u64> {
81        use std::io::Write as StdWrite;
82        self.writer
83            .flush()
84            .map_err(|err| IonError::from(format!("flush error: {err}")))?;
85        self.writer
86            .stream_position()
87            .map_err(|err| IonError::from(format!("position error: {err}")))
88    }
89}
90
91#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
92static NEXT_SECTION_ID: AtomicU64 = AtomicU64::new(0);
93
94#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
95pub(crate) struct SpilledSection {
96    packer: Option<Compressor<BufWriter<File>>>,
97    path: PathBuf,
98    len: u64,
99}
100
101#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
102impl Drop for SpilledSection {
103    fn drop(&mut self) {
104        let _ = fs::remove_file(&self.path);
105    }
106}
107
108#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
109pub(crate) struct RawSpill {
110    file: File,
111    path: PathBuf,
112    len: u64,
113}
114
115#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
116impl Drop for RawSpill {
117    fn drop(&mut self) {
118        let _ = fs::remove_file(&self.path);
119    }
120}
121
122#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
123impl RawSpill {
124    pub(crate) fn create() -> IonResult<Self> {
125        let path = std::env::temp_dir().join(format!(
126            ".ionic-blocks.{}.{}",
127            std::process::id(),
128            NEXT_SECTION_ID.fetch_add(1, Ordering::Relaxed)
129        ));
130        let file = OpenOptions::new()
131            .read(true)
132            .write(true)
133            .create_new(true)
134            .open(&path)
135            .map_err(|err| IonError::from(format!("cannot create '{}': {err}", path.display())))?;
136        Ok(Self { file, path, len: 0 })
137    }
138
139    pub(crate) fn append(&mut self, bytes: &[u8]) -> IonResult<u64> {
140        let start = self.len;
141        std::io::Write::write_all(&mut self.file, bytes).map_err(|err| {
142            IonError::from(format!(
143                "cannot stage blocks in '{}': {err}",
144                self.path.display()
145            ))
146        })?;
147        self.len += bytes.len() as u64;
148        Ok(start)
149    }
150
151    pub(crate) fn read_at(&mut self, offset: u64, buffer: &mut [u8]) -> IonResult<()> {
152        self.file.seek(SeekFrom::Start(offset)).map_err(|err| {
153            IonError::from(format!(
154                "cannot seek staged blocks in '{}': {err}",
155                self.path.display()
156            ))
157        })?;
158        self.file.read_exact(buffer).map_err(|err| {
159            IonError::from(format!(
160                "cannot read staged blocks from '{}': {err}",
161                self.path.display()
162            ))
163        })
164    }
165
166    #[cfg(test)]
167    pub(crate) fn path(&self) -> &Path {
168        &self.path
169    }
170}
171
172#[derive(Debug, Clone, Copy, PartialEq, Eq)]
173pub enum SectionStorage {
174    Memory,
175    Disk,
176}
177
178pub(crate) enum SectionChunk {
179    Memory(Vec<u8>),
180    #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
181    Spilled(SpilledSection),
182}
183
184impl SectionChunk {
185    pub(crate) fn memory(capacity: usize) -> Self {
186        SectionChunk::Memory(Vec::with_capacity(capacity))
187    }
188
189    #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
190    pub(crate) fn spilled(level: u8) -> IonResult<Self> {
191        let path = std::env::temp_dir().join(format!(
192            ".ionic-section.{}.{}",
193            std::process::id(),
194            NEXT_SECTION_ID.fetch_add(1, Ordering::Relaxed)
195        ));
196        let file = OpenOptions::new()
197            .read(true)
198            .write(true)
199            .create_new(true)
200            .open(&path)
201            .map_err(|err| IonError::from(format!("cannot create '{}': {err}", path.display())))?;
202        let packer = Compressor::to(
203            BufWriter::with_capacity(1 << 20, file),
204            &CompressOptions {
205                level: get_supported_compression_level(level),
206                ..Default::default()
207            },
208        )
209        .map_err(|err| IonError::from(format!("zstd start error: {err}")))?;
210        Ok(SectionChunk::Spilled(SpilledSection {
211            packer: Some(packer),
212            path,
213            len: 0,
214        }))
215    }
216
217    pub(crate) fn write(&mut self, bytes: &[u8]) -> IonResult<()> {
218        match self {
219            SectionChunk::Memory(buffer) => buffer.extend_from_slice(bytes),
220            #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
221            SectionChunk::Spilled(section) => {
222                let packer = section
223                    .packer
224                    .as_mut()
225                    .ok_or_else(|| IonError::from("section is already closed"))?;
226                std::io::Write::write_all(packer, bytes)
227                    .map_err(|err| IonError::from(format!("section write error: {err}")))?;
228                section.len += bytes.len() as u64;
229            }
230        }
231        Ok(())
232    }
233
234    pub(crate) fn len(&self) -> u64 {
235        match self {
236            SectionChunk::Memory(buffer) => buffer.len() as u64,
237            #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
238            SectionChunk::Spilled(section) => section.len,
239        }
240    }
241
242    pub(crate) fn is_spilled(&self) -> bool {
243        #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
244        return matches!(self, SectionChunk::Spilled(_));
245        #[cfg(all(target_arch = "wasm32", not(target_os = "wasi")))]
246        return false;
247    }
248
249    #[cfg(test)]
250    pub(crate) fn as_slice(&self) -> &[u8] {
251        match self {
252            SectionChunk::Memory(buffer) => buffer,
253            #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
254            SectionChunk::Spilled(_) => &[],
255        }
256    }
257
258    pub(crate) fn into_vec(self) -> IonResult<Vec<u8>> {
259        match self {
260            SectionChunk::Memory(buffer) => Ok(buffer),
261            #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
262            SectionChunk::Spilled(_) => Err(IonError::from(
263                "a spilled section cannot be read into memory",
264            )),
265        }
266    }
267
268    #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
269    pub(crate) fn copy_into(self, output: &mut dyn WriteBytes) -> IonResult<(u64, u64, u32)> {
270        let SectionChunk::Spilled(mut section) = self else {
271            return Err(IonError::from("copy_into needs a spilled section"));
272        };
273        if section.len == 0 {
274            let start = pad_to_alignment(output)?;
275            return Ok((start, 0, crc32fast::hash(&[])));
276        }
277        let mut file = section
278            .packer
279            .take()
280            .ok_or_else(|| IonError::from("section is already closed"))?
281            .finish()
282            .map_err(|err| IonError::from(format!("zstd finish error: {err}")))?
283            .into_inner()
284            .map_err(|err| IonError::from(format!("section flush error: {err}")))?;
285        file.seek(SeekFrom::Start(0))
286            .map_err(|err| IonError::from(format!("section seek error: {err}")))?;
287
288        let start = pad_to_alignment(output)?;
289        let mut hasher = crc32fast::Hasher::new();
290        let mut stored = 0u64;
291        let mut buffer = vec![0u8; 1 << 20];
292        loop {
293            let filled = file
294                .read(&mut buffer)
295                .map_err(|err| IonError::from(format!("section read error: {err}")))?;
296            if filled == 0 {
297                break;
298            }
299            hasher.update(&buffer[..filled]);
300            output.write(&buffer[..filled])?;
301            stored += filled as u64;
302        }
303        Ok((start, stored, hasher.finalize()))
304    }
305}
306
307#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
308fn pad_to_alignment(output: &mut dyn WriteBytes) -> IonResult<u64> {
309    static PAD: [u8; 7] = [0u8; 7];
310    let position = output.position()?;
311    let aligned = (position + 7) & !7;
312    if aligned > position {
313        output.write(&PAD[..(aligned - position) as usize])?;
314    }
315    Ok(aligned)
316}
317
318#[cfg(all(test, not(all(target_arch = "wasm32", not(target_os = "wasi")))))]
319mod tests {
320    use cosmoz::{DecompressOptions, Decoder, decompress_into};
321
322    use super::*;
323
324    fn row(seed: u64) -> [u8; 64] {
325        let mut record = [0u8; 64];
326        for (slot, part) in record.chunks_mut(8).enumerate() {
327            part.copy_from_slice(&(seed + slot as u64).to_le_bytes());
328        }
329        record
330    }
331
332    #[test]
333    fn raw_spill_roundtrips_appended_blocks() {
334        let mut spill = RawSpill::create().unwrap();
335        let staged_path = spill.path().to_path_buf();
336        assert!(staged_path.exists());
337
338        let blocks: Vec<Vec<u8>> = (0..8u8)
339            .map(|seed| vec![seed; 100 + seed as usize * 37])
340            .collect();
341        let mut offsets = Vec::new();
342        for block in &blocks {
343            offsets.push(spill.append(block).unwrap());
344        }
345
346        let mut running = 0u64;
347        for (block, offset) in blocks.iter().zip(&offsets) {
348            assert_eq!(*offset, running);
349            running += block.len() as u64;
350        }
351
352        for (block, offset) in blocks.iter().zip(&offsets).rev() {
353            let mut read_back = vec![0u8; block.len()];
354            spill.read_at(*offset, &mut read_back).unwrap();
355            assert_eq!(&read_back, block);
356        }
357
358        drop(spill);
359        assert!(
360            !staged_path.exists(),
361            "the temp file must be deleted on drop"
362        );
363    }
364
365    #[test]
366    fn a_memory_section_never_touches_disk() {
367        let mut chunk = SectionChunk::memory(0);
368        chunk.write(&row(1)).unwrap();
369        assert!(!chunk.is_spilled());
370        assert_eq!(chunk.len(), 64);
371    }
372
373    #[test]
374    fn a_disk_section_keeps_every_byte() {
375        let mut chunk = SectionChunk::spilled(3).unwrap();
376        let rows = 64 * 1024;
377        let mut written = Vec::with_capacity(rows * 64);
378        for seed in 0..rows as u64 {
379            let record = row(seed);
380            chunk.write(&record).unwrap();
381            written.extend_from_slice(&record);
382        }
383
384        assert!(chunk.is_spilled());
385        assert_eq!(chunk.len(), written.len() as u64);
386
387        let mut output: Vec<u8> = Vec::new();
388        let (offset, stored, crc32) = chunk.copy_into(&mut output).unwrap();
389        assert_eq!(offset, 0);
390        assert_eq!(stored, output.len() as u64);
391        assert!(stored < written.len() as u64);
392        assert_eq!(crc32, crc32fast::hash(&output));
393
394        let mut restored = vec![0u8; written.len()];
395        let mut decoder = Decoder::new();
396        let restored_length =
397            decompress_into(&output, &mut restored, &DecompressOptions::default(), &mut decoder)
398                .unwrap();
399        assert_eq!(restored_length, written.len());
400        assert_eq!(restored, written);
401    }
402
403    #[test]
404    fn two_sections_never_share_a_file() {
405        let first = SectionChunk::spilled(3).unwrap();
406        let second = SectionChunk::spilled(3).unwrap();
407        let (SectionChunk::Spilled(first), SectionChunk::Spilled(second)) = (first, second) else {
408            panic!("both must be spilled");
409        };
410        assert_ne!(first.path, second.path);
411    }
412}
413
414impl WriteBytes for Vec<u8> {
415    fn write(&mut self, bytes: &[u8]) -> IonResult<()> {
416        self.extend_from_slice(bytes);
417        Ok(())
418    }
419
420    fn patch(&mut self, at: u64, bytes: &[u8]) -> IonResult<()> {
421        let start = at as usize;
422        let end = start + bytes.len();
423        self.get_mut(start..end)
424            .ok_or_else(|| IonError::from(format!("patch: range {start}..{end} out of bounds")))?
425            .copy_from_slice(bytes);
426        Ok(())
427    }
428
429    fn position(&mut self) -> IonResult<u64> {
430        Ok(self.len() as u64)
431    }
432}