ionic/ion/encoder/utilities/
output.rs1#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
2use std::{
3 fs::{self, File, OpenOptions},
4 io::{BufWriter, Read, Seek, SeekFrom},
5 path::{Path, PathBuf},
6 sync::atomic::{AtomicU64, Ordering},
7};
8
9#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
10use cosmoz::{CompressOptions, Compressor};
11
12#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
13use crate::ion::encoder::encode::get_supported_compression_level;
14use crate::ion::{IonError, IonResult};
15
16pub trait WriteBytes {
17 fn write(&mut self, bytes: &[u8]) -> IonResult<()>;
18 fn patch(&mut self, at: u64, bytes: &[u8]) -> IonResult<()>;
19 fn position(&mut self) -> IonResult<u64>;
20}
21
22#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
23pub struct FileWriter {
24 writer: BufWriter<File>,
25}
26
27#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
28impl FileWriter {
29 pub fn open(path: &str) -> IonResult<Self> {
30 Self::open_path(Path::new(path))
31 }
32
33 pub fn open_path(path: &Path) -> IonResult<Self> {
34 let file = File::create(path).map_err(|err| {
35 IonError::from(format!(
36 "cannot create output file '{}': {err}",
37 path.display()
38 ))
39 })?;
40 Ok(Self {
41 writer: BufWriter::with_capacity(8 * 1024 * 1024, file),
42 })
43 }
44
45 pub fn flush(&mut self) -> IonResult<()> {
46 use std::io::Write as StdWrite;
47 self.writer
48 .flush()
49 .map_err(|err| IonError::from(format!("flush error: {err}")))
50 }
51}
52
53#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
54impl WriteBytes for FileWriter {
55 fn write(&mut self, bytes: &[u8]) -> IonResult<()> {
56 std::io::Write::write_all(&mut self.writer, bytes)
57 .map_err(|err| IonError::from(format!("write error: {err}")))
58 }
59
60 fn patch(&mut self, at: u64, bytes: &[u8]) -> IonResult<()> {
61 use std::io::Write as StdWrite;
62 self.writer
63 .flush()
64 .map_err(|err| IonError::from(format!("flush error: {err}")))?;
65 let resume_position = self
66 .writer
67 .stream_position()
68 .map_err(|err| IonError::from(format!("position error: {err}")))?;
69 self.writer
70 .seek(SeekFrom::Start(at))
71 .map_err(|err| IonError::from(format!("seek error: {err}")))?;
72 std::io::Write::write_all(&mut self.writer, bytes)
73 .map_err(|err| IonError::from(format!("patch write error: {err}")))?;
74 self.writer
75 .seek(SeekFrom::Start(resume_position))
76 .map_err(|err| IonError::from(format!("seek-resume error: {err}")))?;
77 Ok(())
78 }
79
80 fn position(&mut self) -> IonResult<u64> {
81 use std::io::Write as StdWrite;
82 self.writer
83 .flush()
84 .map_err(|err| IonError::from(format!("flush error: {err}")))?;
85 self.writer
86 .stream_position()
87 .map_err(|err| IonError::from(format!("position error: {err}")))
88 }
89}
90
91#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
92static NEXT_SECTION_ID: AtomicU64 = AtomicU64::new(0);
93
94#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
95pub(crate) struct SpilledSection {
96 packer: Option<Compressor<BufWriter<File>>>,
97 path: PathBuf,
98 len: u64,
99}
100
101#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
102impl Drop for SpilledSection {
103 fn drop(&mut self) {
104 let _ = fs::remove_file(&self.path);
105 }
106}
107
108#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
109pub(crate) struct RawSpill {
110 file: File,
111 path: PathBuf,
112 len: u64,
113}
114
115#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
116impl Drop for RawSpill {
117 fn drop(&mut self) {
118 let _ = fs::remove_file(&self.path);
119 }
120}
121
122#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
123impl RawSpill {
124 pub(crate) fn create() -> IonResult<Self> {
125 let path = std::env::temp_dir().join(format!(
126 ".ionic-blocks.{}.{}",
127 std::process::id(),
128 NEXT_SECTION_ID.fetch_add(1, Ordering::Relaxed)
129 ));
130 let file = OpenOptions::new()
131 .read(true)
132 .write(true)
133 .create_new(true)
134 .open(&path)
135 .map_err(|err| IonError::from(format!("cannot create '{}': {err}", path.display())))?;
136 Ok(Self { file, path, len: 0 })
137 }
138
139 pub(crate) fn append(&mut self, bytes: &[u8]) -> IonResult<u64> {
140 let start = self.len;
141 std::io::Write::write_all(&mut self.file, bytes).map_err(|err| {
142 IonError::from(format!(
143 "cannot stage blocks in '{}': {err}",
144 self.path.display()
145 ))
146 })?;
147 self.len += bytes.len() as u64;
148 Ok(start)
149 }
150
151 pub(crate) fn read_at(&mut self, offset: u64, buffer: &mut [u8]) -> IonResult<()> {
152 self.file.seek(SeekFrom::Start(offset)).map_err(|err| {
153 IonError::from(format!(
154 "cannot seek staged blocks in '{}': {err}",
155 self.path.display()
156 ))
157 })?;
158 self.file.read_exact(buffer).map_err(|err| {
159 IonError::from(format!(
160 "cannot read staged blocks from '{}': {err}",
161 self.path.display()
162 ))
163 })
164 }
165
166 #[cfg(test)]
167 pub(crate) fn path(&self) -> &Path {
168 &self.path
169 }
170}
171
172#[derive(Debug, Clone, Copy, PartialEq, Eq)]
173pub enum SectionStorage {
174 Memory,
175 Disk,
176}
177
178pub(crate) enum SectionChunk {
179 Memory(Vec<u8>),
180 #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
181 Spilled(SpilledSection),
182}
183
184impl SectionChunk {
185 pub(crate) fn memory(capacity: usize) -> Self {
186 SectionChunk::Memory(Vec::with_capacity(capacity))
187 }
188
189 #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
190 pub(crate) fn spilled(level: u8) -> IonResult<Self> {
191 let path = std::env::temp_dir().join(format!(
192 ".ionic-section.{}.{}",
193 std::process::id(),
194 NEXT_SECTION_ID.fetch_add(1, Ordering::Relaxed)
195 ));
196 let file = OpenOptions::new()
197 .read(true)
198 .write(true)
199 .create_new(true)
200 .open(&path)
201 .map_err(|err| IonError::from(format!("cannot create '{}': {err}", path.display())))?;
202 let packer = Compressor::to(
203 BufWriter::with_capacity(1 << 20, file),
204 &CompressOptions {
205 level: get_supported_compression_level(level),
206 ..Default::default()
207 },
208 )
209 .map_err(|err| IonError::from(format!("zstd start error: {err}")))?;
210 Ok(SectionChunk::Spilled(SpilledSection {
211 packer: Some(packer),
212 path,
213 len: 0,
214 }))
215 }
216
217 pub(crate) fn write(&mut self, bytes: &[u8]) -> IonResult<()> {
218 match self {
219 SectionChunk::Memory(buffer) => buffer.extend_from_slice(bytes),
220 #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
221 SectionChunk::Spilled(section) => {
222 let packer = section
223 .packer
224 .as_mut()
225 .ok_or_else(|| IonError::from("section is already closed"))?;
226 std::io::Write::write_all(packer, bytes)
227 .map_err(|err| IonError::from(format!("section write error: {err}")))?;
228 section.len += bytes.len() as u64;
229 }
230 }
231 Ok(())
232 }
233
234 pub(crate) fn len(&self) -> u64 {
235 match self {
236 SectionChunk::Memory(buffer) => buffer.len() as u64,
237 #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
238 SectionChunk::Spilled(section) => section.len,
239 }
240 }
241
242 pub(crate) fn is_spilled(&self) -> bool {
243 #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
244 return matches!(self, SectionChunk::Spilled(_));
245 #[cfg(all(target_arch = "wasm32", not(target_os = "wasi")))]
246 return false;
247 }
248
249 #[cfg(test)]
250 pub(crate) fn as_slice(&self) -> &[u8] {
251 match self {
252 SectionChunk::Memory(buffer) => buffer,
253 #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
254 SectionChunk::Spilled(_) => &[],
255 }
256 }
257
258 pub(crate) fn into_vec(self) -> IonResult<Vec<u8>> {
259 match self {
260 SectionChunk::Memory(buffer) => Ok(buffer),
261 #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
262 SectionChunk::Spilled(_) => Err(IonError::from(
263 "a spilled section cannot be read into memory",
264 )),
265 }
266 }
267
268 #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
269 pub(crate) fn copy_into(self, output: &mut dyn WriteBytes) -> IonResult<(u64, u64, u32)> {
270 let SectionChunk::Spilled(mut section) = self else {
271 return Err(IonError::from("copy_into needs a spilled section"));
272 };
273 if section.len == 0 {
274 let start = pad_to_alignment(output)?;
275 return Ok((start, 0, crc32fast::hash(&[])));
276 }
277 let mut file = section
278 .packer
279 .take()
280 .ok_or_else(|| IonError::from("section is already closed"))?
281 .finish()
282 .map_err(|err| IonError::from(format!("zstd finish error: {err}")))?
283 .into_inner()
284 .map_err(|err| IonError::from(format!("section flush error: {err}")))?;
285 file.seek(SeekFrom::Start(0))
286 .map_err(|err| IonError::from(format!("section seek error: {err}")))?;
287
288 let start = pad_to_alignment(output)?;
289 let mut hasher = crc32fast::Hasher::new();
290 let mut stored = 0u64;
291 let mut buffer = vec![0u8; 1 << 20];
292 loop {
293 let filled = file
294 .read(&mut buffer)
295 .map_err(|err| IonError::from(format!("section read error: {err}")))?;
296 if filled == 0 {
297 break;
298 }
299 hasher.update(&buffer[..filled]);
300 output.write(&buffer[..filled])?;
301 stored += filled as u64;
302 }
303 Ok((start, stored, hasher.finalize()))
304 }
305}
306
307#[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))]
308fn pad_to_alignment(output: &mut dyn WriteBytes) -> IonResult<u64> {
309 static PAD: [u8; 7] = [0u8; 7];
310 let position = output.position()?;
311 let aligned = (position + 7) & !7;
312 if aligned > position {
313 output.write(&PAD[..(aligned - position) as usize])?;
314 }
315 Ok(aligned)
316}
317
318#[cfg(all(test, not(all(target_arch = "wasm32", not(target_os = "wasi")))))]
319mod tests {
320 use cosmoz::{DecompressOptions, Decoder, decompress_into};
321
322 use super::*;
323
324 fn row(seed: u64) -> [u8; 64] {
325 let mut record = [0u8; 64];
326 for (slot, part) in record.chunks_mut(8).enumerate() {
327 part.copy_from_slice(&(seed + slot as u64).to_le_bytes());
328 }
329 record
330 }
331
332 #[test]
333 fn raw_spill_roundtrips_appended_blocks() {
334 let mut spill = RawSpill::create().unwrap();
335 let staged_path = spill.path().to_path_buf();
336 assert!(staged_path.exists());
337
338 let blocks: Vec<Vec<u8>> = (0..8u8)
339 .map(|seed| vec![seed; 100 + seed as usize * 37])
340 .collect();
341 let mut offsets = Vec::new();
342 for block in &blocks {
343 offsets.push(spill.append(block).unwrap());
344 }
345
346 let mut running = 0u64;
347 for (block, offset) in blocks.iter().zip(&offsets) {
348 assert_eq!(*offset, running);
349 running += block.len() as u64;
350 }
351
352 for (block, offset) in blocks.iter().zip(&offsets).rev() {
353 let mut read_back = vec![0u8; block.len()];
354 spill.read_at(*offset, &mut read_back).unwrap();
355 assert_eq!(&read_back, block);
356 }
357
358 drop(spill);
359 assert!(
360 !staged_path.exists(),
361 "the temp file must be deleted on drop"
362 );
363 }
364
365 #[test]
366 fn a_memory_section_never_touches_disk() {
367 let mut chunk = SectionChunk::memory(0);
368 chunk.write(&row(1)).unwrap();
369 assert!(!chunk.is_spilled());
370 assert_eq!(chunk.len(), 64);
371 }
372
373 #[test]
374 fn a_disk_section_keeps_every_byte() {
375 let mut chunk = SectionChunk::spilled(3).unwrap();
376 let rows = 64 * 1024;
377 let mut written = Vec::with_capacity(rows * 64);
378 for seed in 0..rows as u64 {
379 let record = row(seed);
380 chunk.write(&record).unwrap();
381 written.extend_from_slice(&record);
382 }
383
384 assert!(chunk.is_spilled());
385 assert_eq!(chunk.len(), written.len() as u64);
386
387 let mut output: Vec<u8> = Vec::new();
388 let (offset, stored, crc32) = chunk.copy_into(&mut output).unwrap();
389 assert_eq!(offset, 0);
390 assert_eq!(stored, output.len() as u64);
391 assert!(stored < written.len() as u64);
392 assert_eq!(crc32, crc32fast::hash(&output));
393
394 let mut restored = vec![0u8; written.len()];
395 let mut decoder = Decoder::new();
396 let restored_length =
397 decompress_into(&output, &mut restored, &DecompressOptions::default(), &mut decoder)
398 .unwrap();
399 assert_eq!(restored_length, written.len());
400 assert_eq!(restored, written);
401 }
402
403 #[test]
404 fn two_sections_never_share_a_file() {
405 let first = SectionChunk::spilled(3).unwrap();
406 let second = SectionChunk::spilled(3).unwrap();
407 let (SectionChunk::Spilled(first), SectionChunk::Spilled(second)) = (first, second) else {
408 panic!("both must be spilled");
409 };
410 assert_ne!(first.path, second.path);
411 }
412}
413
414impl WriteBytes for Vec<u8> {
415 fn write(&mut self, bytes: &[u8]) -> IonResult<()> {
416 self.extend_from_slice(bytes);
417 Ok(())
418 }
419
420 fn patch(&mut self, at: u64, bytes: &[u8]) -> IonResult<()> {
421 let start = at as usize;
422 let end = start + bytes.len();
423 self.get_mut(start..end)
424 .ok_or_else(|| IonError::from(format!("patch: range {start}..{end} out of bounds")))?
425 .copy_from_slice(bytes);
426 Ok(())
427 }
428
429 fn position(&mut self) -> IonResult<u64> {
430 Ok(self.len() as u64)
431 }
432}