ionic/ion/encoder/utilities/
output.rs1#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
2use std::{
3 fs::{self, File, OpenOptions},
4 io::{BufWriter, Read, Seek, SeekFrom},
5 path::{Path, PathBuf},
6 sync::atomic::{AtomicU64, Ordering},
7};
8
9#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
10use cosmoz::{CompressOptions, Compressor};
11
12#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
13use crate::ion::encoder::encode::get_supported_compression_level;
14use crate::ion::{IonError, IonResult};
15
16pub trait WriteBytes {
17 fn write(&mut self, bytes: &[u8]) -> IonResult<()>;
18 fn patch(&mut self, at: u64, bytes: &[u8]) -> IonResult<()>;
19 fn position(&mut self) -> IonResult<u64>;
20}
21
22#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
23pub struct FileWriter {
24 writer: BufWriter<File>,
25}
26
27#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
28impl FileWriter {
29 pub fn open_path(path: &Path) -> IonResult<Self> {
30 let file = File::create(path).map_err(|err| {
31 IonError::from(format!(
32 "cannot create output file '{}': {err}",
33 path.display()
34 ))
35 })?;
36 Ok(Self {
37 writer: BufWriter::with_capacity(8 * 1024 * 1024, file),
38 })
39 }
40
41 pub fn flush(&mut self) -> IonResult<()> {
42 use std::io::Write as StdWrite;
43 self.writer
44 .flush()
45 .map_err(|err| IonError::from(format!("flush error: {err}")))
46 }
47}
48
49#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
50impl WriteBytes for FileWriter {
51 fn write(&mut self, bytes: &[u8]) -> IonResult<()> {
52 std::io::Write::write_all(&mut self.writer, bytes)
53 .map_err(|err| IonError::from(format!("write error: {err}")))
54 }
55
56 fn patch(&mut self, at: u64, bytes: &[u8]) -> IonResult<()> {
57 use std::io::Write as StdWrite;
58 self.writer
59 .flush()
60 .map_err(|err| IonError::from(format!("flush error: {err}")))?;
61 let resume_position = self
62 .writer
63 .stream_position()
64 .map_err(|err| IonError::from(format!("position error: {err}")))?;
65 self.writer
66 .seek(SeekFrom::Start(at))
67 .map_err(|err| IonError::from(format!("seek error: {err}")))?;
68 std::io::Write::write_all(&mut self.writer, bytes)
69 .map_err(|err| IonError::from(format!("patch write error: {err}")))?;
70 self.writer
71 .seek(SeekFrom::Start(resume_position))
72 .map_err(|err| IonError::from(format!("seek-resume error: {err}")))?;
73 Ok(())
74 }
75
76 fn position(&mut self) -> IonResult<u64> {
77 use std::io::Write as StdWrite;
78 self.writer
79 .flush()
80 .map_err(|err| IonError::from(format!("flush error: {err}")))?;
81 self.writer
82 .stream_position()
83 .map_err(|err| IonError::from(format!("position error: {err}")))
84 }
85}
86
87#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
88static NEXT_SECTION_ID: AtomicU64 = AtomicU64::new(0);
89
90#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
91pub(crate) struct SpilledSection {
92 packer: Option<Compressor<BufWriter<File>>>,
93 path: PathBuf,
94 len: u64,
95}
96
97#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
98impl Drop for SpilledSection {
99 fn drop(&mut self) {
100 let _ = fs::remove_file(&self.path);
101 }
102}
103
104#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
105pub(crate) struct RawSpill {
106 file: File,
107 path: PathBuf,
108 len: u64,
109}
110
111#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
112impl Drop for RawSpill {
113 fn drop(&mut self) {
114 let _ = fs::remove_file(&self.path);
115 }
116}
117
118#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
119impl RawSpill {
120 pub(crate) fn create() -> IonResult<Self> {
121 let path = std::env::temp_dir().join(format!(
122 ".ionic-blocks.{}.{}",
123 std::process::id(),
124 NEXT_SECTION_ID.fetch_add(1, Ordering::Relaxed)
125 ));
126 let file = OpenOptions::new()
127 .read(true)
128 .write(true)
129 .create_new(true)
130 .open(&path)
131 .map_err(|err| IonError::from(format!("cannot create '{}': {err}", path.display())))?;
132 Ok(Self { file, path, len: 0 })
133 }
134
135 pub(crate) fn append(&mut self, bytes: &[u8]) -> IonResult<u64> {
136 let start = self.len;
137 std::io::Write::write_all(&mut self.file, bytes).map_err(|err| {
138 IonError::from(format!(
139 "cannot stage blocks in '{}': {err}",
140 self.path.display()
141 ))
142 })?;
143 self.len += bytes.len() as u64;
144 Ok(start)
145 }
146
147 pub(crate) fn read_at(&mut self, offset: u64, buffer: &mut [u8]) -> IonResult<()> {
148 self.file.seek(SeekFrom::Start(offset)).map_err(|err| {
149 IonError::from(format!(
150 "cannot seek staged blocks in '{}': {err}",
151 self.path.display()
152 ))
153 })?;
154 self.file.read_exact(buffer).map_err(|err| {
155 IonError::from(format!(
156 "cannot read staged blocks from '{}': {err}",
157 self.path.display()
158 ))
159 })
160 }
161
162 #[cfg(test)]
163 pub(crate) fn path(&self) -> &Path {
164 &self.path
165 }
166}
167
168#[derive(Debug, Clone, Copy, PartialEq, Eq)]
169pub enum SectionStorage {
170 Memory,
171 Disk,
172}
173
174pub(crate) enum SectionChunk {
175 Memory(Vec<u8>),
176 #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
177 Spilled(SpilledSection),
178}
179
180impl SectionChunk {
181 pub(crate) fn memory(capacity: usize) -> Self {
182 SectionChunk::Memory(Vec::with_capacity(capacity))
183 }
184
185 #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
186 pub(crate) fn spilled(level: u8) -> IonResult<Self> {
187 let path = std::env::temp_dir().join(format!(
188 ".ionic-section.{}.{}",
189 std::process::id(),
190 NEXT_SECTION_ID.fetch_add(1, Ordering::Relaxed)
191 ));
192 let file = OpenOptions::new()
193 .read(true)
194 .write(true)
195 .create_new(true)
196 .open(&path)
197 .map_err(|err| IonError::from(format!("cannot create '{}': {err}", path.display())))?;
198 let packer = Compressor::write_to(
199 BufWriter::with_capacity(1 << 20, file),
200 &CompressOptions {
201 level: get_supported_compression_level(level),
202 ..Default::default()
203 },
204 )
205 .map_err(|err| IonError::from(format!("zstd start error: {err}")))?;
206 Ok(SectionChunk::Spilled(SpilledSection {
207 packer: Some(packer),
208 path,
209 len: 0,
210 }))
211 }
212
213 pub(crate) fn write(&mut self, bytes: &[u8]) -> IonResult<()> {
214 match self {
215 SectionChunk::Memory(buffer) => buffer.extend_from_slice(bytes),
216 #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
217 SectionChunk::Spilled(section) => {
218 let packer = section
219 .packer
220 .as_mut()
221 .ok_or_else(|| IonError::from("section is already closed"))?;
222 std::io::Write::write_all(packer, bytes)
223 .map_err(|err| IonError::from(format!("section write error: {err}")))?;
224 section.len += bytes.len() as u64;
225 }
226 }
227 Ok(())
228 }
229
230 pub(crate) fn len(&self) -> u64 {
231 match self {
232 SectionChunk::Memory(buffer) => buffer.len() as u64,
233 #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
234 SectionChunk::Spilled(section) => section.len,
235 }
236 }
237
238 #[allow(dead_code)]
239 pub(crate) fn is_spilled(&self) -> bool {
240 #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
241 return matches!(self, SectionChunk::Spilled(_));
242 #[cfg(all(target_arch = "wasm32", not(target_os = "wasi")))]
243 return false;
244 }
245
246 #[cfg(test)]
247 pub(crate) fn as_slice(&self) -> &[u8] {
248 match self {
249 SectionChunk::Memory(buffer) => buffer,
250 #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
251 SectionChunk::Spilled(_) => &[],
252 }
253 }
254
255 pub(crate) fn into_vec(self) -> IonResult<Vec<u8>> {
256 match self {
257 SectionChunk::Memory(buffer) => Ok(buffer),
258 #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
259 SectionChunk::Spilled(_) => Err(IonError::from(
260 "a spilled section cannot be read into memory",
261 )),
262 }
263 }
264
265 #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
266 pub(crate) fn copy_into(self, output: &mut dyn WriteBytes) -> IonResult<(u64, u64, u32)> {
267 let SectionChunk::Spilled(mut section) = self else {
268 return Err(IonError::from("copy_into needs a spilled section"));
269 };
270 if section.len == 0 {
271 let start = pad_to_alignment(output)?;
272 return Ok((start, 0, crc32fast::hash(&[])));
273 }
274 let mut file = section
275 .packer
276 .take()
277 .ok_or_else(|| IonError::from("section is already closed"))?
278 .finish()
279 .map_err(|err| IonError::from(format!("zstd finish error: {err}")))?
280 .into_inner()
281 .map_err(|err| IonError::from(format!("section flush error: {err}")))?;
282 file.seek(SeekFrom::Start(0))
283 .map_err(|err| IonError::from(format!("section seek error: {err}")))?;
284
285 let start = pad_to_alignment(output)?;
286 let mut hasher = crc32fast::Hasher::new();
287 let mut stored = 0u64;
288 let mut buffer = vec![0u8; 1 << 20];
289 loop {
290 let filled = file
291 .read(&mut buffer)
292 .map_err(|err| IonError::from(format!("section read error: {err}")))?;
293 if filled == 0 {
294 break;
295 }
296 hasher.update(&buffer[..filled]);
297 output.write(&buffer[..filled])?;
298 stored += filled as u64;
299 }
300 Ok((start, stored, hasher.finalize()))
301 }
302}
303
304#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
305fn pad_to_alignment(output: &mut dyn WriteBytes) -> IonResult<u64> {
306 static PAD: [u8; 7] = [0u8; 7];
307 let position = output.position()?;
308 let aligned = (position + 7) & !7;
309 if aligned > position {
310 output.write(&PAD[..(aligned - position) as usize])?;
311 }
312 Ok(aligned)
313}
314
315#[cfg(all(test, not(all(target_arch = "wasm32", not(target_os = "wasi")))))]
316mod tests {
317 use cosmoz::{DecompressOptions, Decoder};
318
319 use super::*;
320
321 fn row(seed: u64) -> [u8; 64] {
322 let mut record = [0u8; 64];
323 for (slot, part) in record.chunks_mut(8).enumerate() {
324 part.copy_from_slice(&(seed + slot as u64).to_le_bytes());
325 }
326 record
327 }
328
329 #[test]
330 fn raw_spill_roundtrips_appended_blocks() {
331 let mut spill = RawSpill::create().unwrap();
332 let staged_path = spill.path().to_path_buf();
333 assert!(staged_path.exists());
334
335 let blocks: Vec<Vec<u8>> = (0..8u8)
336 .map(|seed| vec![seed; 100 + seed as usize * 37])
337 .collect();
338 let mut offsets = Vec::new();
339 for block in &blocks {
340 offsets.push(spill.append(block).unwrap());
341 }
342
343 let mut running = 0u64;
344 for (block, offset) in blocks.iter().zip(&offsets) {
345 assert_eq!(*offset, running);
346 running += block.len() as u64;
347 }
348
349 for (block, offset) in blocks.iter().zip(&offsets).rev() {
350 let mut read_back = vec![0u8; block.len()];
351 spill.read_at(*offset, &mut read_back).unwrap();
352 assert_eq!(&read_back, block);
353 }
354
355 drop(spill);
356 assert!(
357 !staged_path.exists(),
358 "the temp file must be deleted on drop"
359 );
360 }
361
362 #[test]
363 fn a_memory_section_never_touches_disk() {
364 let mut chunk = SectionChunk::memory(0);
365 chunk.write(&row(1)).unwrap();
366 assert!(!chunk.is_spilled());
367 assert_eq!(chunk.len(), 64);
368 }
369
370 #[test]
371 fn a_disk_section_keeps_every_byte() {
372 let mut chunk = SectionChunk::spilled(3).unwrap();
373 let rows = 64 * 1024;
374 let mut written = Vec::with_capacity(rows * 64);
375 for seed in 0..rows as u64 {
376 let record = row(seed);
377 chunk.write(&record).unwrap();
378 written.extend_from_slice(&record);
379 }
380
381 assert!(chunk.is_spilled());
382 assert_eq!(chunk.len(), written.len() as u64);
383
384 let mut output: Vec<u8> = Vec::new();
385 let (offset, stored, crc32) = chunk.copy_into(&mut output).unwrap();
386 assert_eq!(offset, 0);
387 assert_eq!(stored, output.len() as u64);
388 assert!(stored < written.len() as u64);
389 assert_eq!(crc32, crc32fast::hash(&output));
390
391 let mut restored = vec![0u8; written.len()];
392 let mut decoder = Decoder::new(&DecompressOptions::default());
393 let restored_length = decoder.decompress_into(&output, &mut restored).unwrap();
394 assert_eq!(restored_length, written.len());
395 assert_eq!(restored, written);
396 }
397
398 #[test]
399 fn two_sections_never_share_a_file() {
400 let first = SectionChunk::spilled(3).unwrap();
401 let second = SectionChunk::spilled(3).unwrap();
402 let (SectionChunk::Spilled(first), SectionChunk::Spilled(second)) = (first, second) else {
403 panic!("both must be spilled");
404 };
405 assert_ne!(first.path, second.path);
406 }
407}
408
409impl WriteBytes for Vec<u8> {
410 fn write(&mut self, bytes: &[u8]) -> IonResult<()> {
411 self.extend_from_slice(bytes);
412 Ok(())
413 }
414
415 fn patch(&mut self, at: u64, bytes: &[u8]) -> IonResult<()> {
416 let start = at as usize;
417 let end = start + bytes.len();
418 self.get_mut(start..end)
419 .ok_or_else(|| IonError::from(format!("patch: range {start}..{end} out of bounds")))?
420 .copy_from_slice(bytes);
421 Ok(())
422 }
423
424 fn position(&mut self) -> IonResult<u64> {
425 Ok(self.len() as u64)
426 }
427}