Skip to main content

ionic/ion/encoder/utilities/
output.rs

1#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
2use std::{
3    fs::{self, File, OpenOptions},
4    io::{BufWriter, Read, Seek, SeekFrom},
5    path::{Path, PathBuf},
6    sync::atomic::{AtomicU64, Ordering},
7};
8
9#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
10use cosmoz::{CompressOptions, Compressor};
11
12#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
13use crate::ion::encoder::encode::get_supported_compression_level;
14use crate::ion::{IonError, IonResult};
15
16pub trait WriteBytes {
17    fn write(&mut self, bytes: &[u8]) -> IonResult<()>;
18    fn patch(&mut self, at: u64, bytes: &[u8]) -> IonResult<()>;
19    fn position(&mut self) -> IonResult<u64>;
20}
21
22#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
23pub struct FileWriter {
24    writer: BufWriter<File>,
25}
26
27#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
28impl FileWriter {
29    pub fn open_path(path: &Path) -> IonResult<Self> {
30        let file = File::create(path).map_err(|err| {
31            IonError::from(format!(
32                "cannot create output file '{}': {err}",
33                path.display()
34            ))
35        })?;
36        Ok(Self {
37            writer: BufWriter::with_capacity(8 * 1024 * 1024, file),
38        })
39    }
40
41    pub fn flush(&mut self) -> IonResult<()> {
42        use std::io::Write as StdWrite;
43        self.writer
44            .flush()
45            .map_err(|err| IonError::from(format!("flush error: {err}")))
46    }
47}
48
49#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
50impl WriteBytes for FileWriter {
51    fn write(&mut self, bytes: &[u8]) -> IonResult<()> {
52        std::io::Write::write_all(&mut self.writer, bytes)
53            .map_err(|err| IonError::from(format!("write error: {err}")))
54    }
55
56    fn patch(&mut self, at: u64, bytes: &[u8]) -> IonResult<()> {
57        use std::io::Write as StdWrite;
58        self.writer
59            .flush()
60            .map_err(|err| IonError::from(format!("flush error: {err}")))?;
61        let resume_position = self
62            .writer
63            .stream_position()
64            .map_err(|err| IonError::from(format!("position error: {err}")))?;
65        self.writer
66            .seek(SeekFrom::Start(at))
67            .map_err(|err| IonError::from(format!("seek error: {err}")))?;
68        std::io::Write::write_all(&mut self.writer, bytes)
69            .map_err(|err| IonError::from(format!("patch write error: {err}")))?;
70        self.writer
71            .seek(SeekFrom::Start(resume_position))
72            .map_err(|err| IonError::from(format!("seek-resume error: {err}")))?;
73        Ok(())
74    }
75
76    fn position(&mut self) -> IonResult<u64> {
77        use std::io::Write as StdWrite;
78        self.writer
79            .flush()
80            .map_err(|err| IonError::from(format!("flush error: {err}")))?;
81        self.writer
82            .stream_position()
83            .map_err(|err| IonError::from(format!("position error: {err}")))
84    }
85}
86
87#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
88static NEXT_SECTION_ID: AtomicU64 = AtomicU64::new(0);
89
90#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
91pub(crate) struct SpilledSection {
92    packer: Option<Compressor<BufWriter<File>>>,
93    path: PathBuf,
94    len: u64,
95}
96
97#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
98impl Drop for SpilledSection {
99    fn drop(&mut self) {
100        let _ = fs::remove_file(&self.path);
101    }
102}
103
104#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
105pub(crate) struct RawSpill {
106    file: File,
107    path: PathBuf,
108    len: u64,
109}
110
111#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
112impl Drop for RawSpill {
113    fn drop(&mut self) {
114        let _ = fs::remove_file(&self.path);
115    }
116}
117
118#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
119impl RawSpill {
120    pub(crate) fn create() -> IonResult<Self> {
121        let path = std::env::temp_dir().join(format!(
122            ".ionic-blocks.{}.{}",
123            std::process::id(),
124            NEXT_SECTION_ID.fetch_add(1, Ordering::Relaxed)
125        ));
126        let file = OpenOptions::new()
127            .read(true)
128            .write(true)
129            .create_new(true)
130            .open(&path)
131            .map_err(|err| IonError::from(format!("cannot create '{}': {err}", path.display())))?;
132        Ok(Self { file, path, len: 0 })
133    }
134
135    pub(crate) fn append(&mut self, bytes: &[u8]) -> IonResult<u64> {
136        let start = self.len;
137        std::io::Write::write_all(&mut self.file, bytes).map_err(|err| {
138            IonError::from(format!(
139                "cannot stage blocks in '{}': {err}",
140                self.path.display()
141            ))
142        })?;
143        self.len += bytes.len() as u64;
144        Ok(start)
145    }
146
147    pub(crate) fn read_at(&mut self, offset: u64, buffer: &mut [u8]) -> IonResult<()> {
148        self.file.seek(SeekFrom::Start(offset)).map_err(|err| {
149            IonError::from(format!(
150                "cannot seek staged blocks in '{}': {err}",
151                self.path.display()
152            ))
153        })?;
154        self.file.read_exact(buffer).map_err(|err| {
155            IonError::from(format!(
156                "cannot read staged blocks from '{}': {err}",
157                self.path.display()
158            ))
159        })
160    }
161
162    #[cfg(test)]
163    pub(crate) fn path(&self) -> &Path {
164        &self.path
165    }
166}
167
168#[derive(Debug, Clone, Copy, PartialEq, Eq)]
169pub enum SectionStorage {
170    Memory,
171    Disk,
172}
173
174pub(crate) enum SectionChunk {
175    Memory(Vec<u8>),
176    #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
177    Spilled(SpilledSection),
178}
179
180impl SectionChunk {
181    pub(crate) fn memory(capacity: usize) -> Self {
182        SectionChunk::Memory(Vec::with_capacity(capacity))
183    }
184
185    #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
186    pub(crate) fn spilled(level: u8) -> IonResult<Self> {
187        let path = std::env::temp_dir().join(format!(
188            ".ionic-section.{}.{}",
189            std::process::id(),
190            NEXT_SECTION_ID.fetch_add(1, Ordering::Relaxed)
191        ));
192        let file = OpenOptions::new()
193            .read(true)
194            .write(true)
195            .create_new(true)
196            .open(&path)
197            .map_err(|err| IonError::from(format!("cannot create '{}': {err}", path.display())))?;
198        let packer = Compressor::write_to(
199            BufWriter::with_capacity(1 << 20, file),
200            &CompressOptions {
201                level: get_supported_compression_level(level),
202                ..Default::default()
203            },
204        )
205        .map_err(|err| IonError::from(format!("zstd start error: {err}")))?;
206        Ok(SectionChunk::Spilled(SpilledSection {
207            packer: Some(packer),
208            path,
209            len: 0,
210        }))
211    }
212
213    pub(crate) fn write(&mut self, bytes: &[u8]) -> IonResult<()> {
214        match self {
215            SectionChunk::Memory(buffer) => buffer.extend_from_slice(bytes),
216            #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
217            SectionChunk::Spilled(section) => {
218                let packer = section
219                    .packer
220                    .as_mut()
221                    .ok_or_else(|| IonError::from("section is already closed"))?;
222                std::io::Write::write_all(packer, bytes)
223                    .map_err(|err| IonError::from(format!("section write error: {err}")))?;
224                section.len += bytes.len() as u64;
225            }
226        }
227        Ok(())
228    }
229
230    pub(crate) fn len(&self) -> u64 {
231        match self {
232            SectionChunk::Memory(buffer) => buffer.len() as u64,
233            #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
234            SectionChunk::Spilled(section) => section.len,
235        }
236    }
237
238    #[allow(dead_code)]
239    pub(crate) fn is_spilled(&self) -> bool {
240        #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
241        return matches!(self, SectionChunk::Spilled(_));
242        #[cfg(all(target_arch = "wasm32", not(target_os = "wasi")))]
243        return false;
244    }
245
246    #[cfg(test)]
247    pub(crate) fn as_slice(&self) -> &[u8] {
248        match self {
249            SectionChunk::Memory(buffer) => buffer,
250            #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
251            SectionChunk::Spilled(_) => &[],
252        }
253    }
254
255    pub(crate) fn into_vec(self) -> IonResult<Vec<u8>> {
256        match self {
257            SectionChunk::Memory(buffer) => Ok(buffer),
258            #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
259            SectionChunk::Spilled(_) => Err(IonError::from(
260                "a spilled section cannot be read into memory",
261            )),
262        }
263    }
264
265    #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
266    pub(crate) fn copy_into(self, output: &mut dyn WriteBytes) -> IonResult<(u64, u64, u32)> {
267        let SectionChunk::Spilled(mut section) = self else {
268            return Err(IonError::from("copy_into needs a spilled section"));
269        };
270        if section.len == 0 {
271            let start = pad_to_alignment(output)?;
272            return Ok((start, 0, crc32fast::hash(&[])));
273        }
274        let mut file = section
275            .packer
276            .take()
277            .ok_or_else(|| IonError::from("section is already closed"))?
278            .finish()
279            .map_err(|err| IonError::from(format!("zstd finish error: {err}")))?
280            .into_inner()
281            .map_err(|err| IonError::from(format!("section flush error: {err}")))?;
282        file.seek(SeekFrom::Start(0))
283            .map_err(|err| IonError::from(format!("section seek error: {err}")))?;
284
285        let start = pad_to_alignment(output)?;
286        let mut hasher = crc32fast::Hasher::new();
287        let mut stored = 0u64;
288        let mut buffer = vec![0u8; 1 << 20];
289        loop {
290            let filled = file
291                .read(&mut buffer)
292                .map_err(|err| IonError::from(format!("section read error: {err}")))?;
293            if filled == 0 {
294                break;
295            }
296            hasher.update(&buffer[..filled]);
297            output.write(&buffer[..filled])?;
298            stored += filled as u64;
299        }
300        Ok((start, stored, hasher.finalize()))
301    }
302}
303
304#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
305fn pad_to_alignment(output: &mut dyn WriteBytes) -> IonResult<u64> {
306    static PAD: [u8; 7] = [0u8; 7];
307    let position = output.position()?;
308    let aligned = (position + 7) & !7;
309    if aligned > position {
310        output.write(&PAD[..(aligned - position) as usize])?;
311    }
312    Ok(aligned)
313}
314
315#[cfg(all(test, not(all(target_arch = "wasm32", not(target_os = "wasi")))))]
316mod tests {
317    use cosmoz::{Decoder, DecompressOptions};
318
319    use super::*;
320
321    fn row(seed: u64) -> [u8; 64] {
322        let mut record = [0u8; 64];
323        for (slot, part) in record.chunks_mut(8).enumerate() {
324            part.copy_from_slice(&(seed + slot as u64).to_le_bytes());
325        }
326        record
327    }
328
329    #[test]
330    fn raw_spill_roundtrips_appended_blocks() {
331        let mut spill = RawSpill::create().unwrap();
332        let staged_path = spill.path().to_path_buf();
333        assert!(staged_path.exists());
334
335        let blocks: Vec<Vec<u8>> = (0..8u8)
336            .map(|seed| vec![seed; 100 + seed as usize * 37])
337            .collect();
338        let mut offsets = Vec::new();
339        for block in &blocks {
340            offsets.push(spill.append(block).unwrap());
341        }
342
343        let mut running = 0u64;
344        for (block, offset) in blocks.iter().zip(&offsets) {
345            assert_eq!(*offset, running);
346            running += block.len() as u64;
347        }
348
349        for (block, offset) in blocks.iter().zip(&offsets).rev() {
350            let mut read_back = vec![0u8; block.len()];
351            spill.read_at(*offset, &mut read_back).unwrap();
352            assert_eq!(&read_back, block);
353        }
354
355        drop(spill);
356        assert!(
357            !staged_path.exists(),
358            "the temp file must be deleted on drop"
359        );
360    }
361
362    #[test]
363    fn a_memory_section_never_touches_disk() {
364        let mut chunk = SectionChunk::memory(0);
365        chunk.write(&row(1)).unwrap();
366        assert!(!chunk.is_spilled());
367        assert_eq!(chunk.len(), 64);
368    }
369
370    #[test]
371    fn a_disk_section_keeps_every_byte() {
372        let mut chunk = SectionChunk::spilled(3).unwrap();
373        let rows = 64 * 1024;
374        let mut written = Vec::with_capacity(rows * 64);
375        for seed in 0..rows as u64 {
376            let record = row(seed);
377            chunk.write(&record).unwrap();
378            written.extend_from_slice(&record);
379        }
380
381        assert!(chunk.is_spilled());
382        assert_eq!(chunk.len(), written.len() as u64);
383
384        let mut output: Vec<u8> = Vec::new();
385        let (offset, stored, crc32) = chunk.copy_into(&mut output).unwrap();
386        assert_eq!(offset, 0);
387        assert_eq!(stored, output.len() as u64);
388        assert!(stored < written.len() as u64);
389        assert_eq!(crc32, crc32fast::hash(&output));
390
391        let mut restored = vec![0u8; written.len()];
392        let mut decoder = Decoder::new(&DecompressOptions::default());
393        let restored_length = decoder.decompress_into(&output, &mut restored).unwrap();
394        assert_eq!(restored_length, written.len());
395        assert_eq!(restored, written);
396    }
397
398    #[test]
399    fn two_sections_never_share_a_file() {
400        let first = SectionChunk::spilled(3).unwrap();
401        let second = SectionChunk::spilled(3).unwrap();
402        let (SectionChunk::Spilled(first), SectionChunk::Spilled(second)) = (first, second) else {
403            panic!("both must be spilled");
404        };
405        assert_ne!(first.path, second.path);
406    }
407}
408
409impl WriteBytes for Vec<u8> {
410    fn write(&mut self, bytes: &[u8]) -> IonResult<()> {
411        self.extend_from_slice(bytes);
412        Ok(())
413    }
414
415    fn patch(&mut self, at: u64, bytes: &[u8]) -> IonResult<()> {
416        let start = at as usize;
417        let end = start + bytes.len();
418        self.get_mut(start..end)
419            .ok_or_else(|| IonError::from(format!("patch: range {start}..{end} out of bounds")))?
420            .copy_from_slice(bytes);
421        Ok(())
422    }
423
424    fn position(&mut self) -> IonResult<u64> {
425        Ok(self.len() as u64)
426    }
427}