Skip to main content

aff4_image/
writer.rs

1use std::collections::BTreeMap;
2use std::fs::{self, File};
3#[path = "collect.rs"]
4mod collect;
5#[path = "collection_budget.rs"]
6mod collection_budget;
7pub use collection_budget::CollectionLimits;
8#[path = "logical_metadata.rs"]
9mod logical_metadata;
10pub use collect::{CollectionIssue, CollectionOptions, CollectionReport};
11pub use logical_metadata::{CaseMetadata, LogicalMetadata, SubstreamKind};
12use std::io::{Read, Write};
13use std::ops::ControlFlow;
14use std::path::{Path, PathBuf};
15
16use md5::{Digest, Md5};
17use rayon::prelude::*;
18use sha1::Sha1;
19use sha2::{Sha256, Sha512};
20use tempfile::NamedTempFile;
21use zip::{ZipWriter, write::SimpleFileOptions};
22
23use crate::{Error, Result, malformed, reader::hex};
24
25/// Container profile chosen before any evidence bytes are written.
26#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)]
27pub enum Profile {
28    /// AFF4 Standard 1.0 physical images with ImageStream storage.
29    Physical,
30    /// Legacy AFF4-L 1.1 logical files with ZIP segment storage.
31    Logical,
32}
33
34/// Physical chunk codec. Logical ZIP segments use Stored or Deflate.
35#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)]
36pub enum Compression {
37    /// Uncompressed bytes.
38    Stored,
39    /// Zlib chunk compression / ZIP Deflate.
40    Zlib,
41    /// Raw Snappy chunks (physical profile only).
42    Snappy,
43    /// Raw LZ4 blocks (physical profile only).
44    Lz4,
45}
46
47/// Bounded writer geometry. Output is always a new single ZIP volume.
48#[derive(Debug, Clone, serde::Serialize)]
49pub struct WriteOptions {
50    /// Physical chunk bytes (1 through 16 MiB).
51    pub chunk_bytes: u32,
52    /// Physical chunks per bevy. A bevy cannot exceed 128 MiB uncompressed.
53    pub chunks_per_bevy: u32,
54    /// Physical codec or logical ZIP codec.
55    pub compression: Compression,
56}
57impl Default for WriteOptions {
58    fn default() -> Self {
59        Self {
60            chunk_bytes: 32768,
61            chunks_per_bevy: 1024,
62            compression: Compression::Zlib,
63        }
64    }
65}
66
67/// Digests of source bytes accepted for one image or file.
68#[derive(Debug, Clone, serde::Serialize)]
69pub struct AcquiredStream {
70    /// Image or FileImage resource identifier.
71    pub id: String,
72    /// Exact byte count, excluding final chunk padding.
73    pub size: u64,
74    /// Hexadecimal MD5.
75    pub md5: String,
76    /// Hexadecimal SHA1.
77    pub sha1: String,
78    /// Hexadecimal SHA256.
79    pub sha256: String,
80}
81
82/// Published acquisition result. Caller may reopen to verify independently.
83#[derive(Debug, serde::Serialize)]
84pub struct WriteResult {
85    /// Final, exclusively created path.
86    pub path: PathBuf,
87    /// Per-resource source hashes.
88    pub streams: Vec<AcquiredStream>,
89    /// SHA256 of the exact information.turtle bytes; record externally.
90    pub metadata_sha256: String,
91}
92
93/// Streaming writer with exclusive publication and no full-source spool.
94///
95/// A failed/cancelled input poisons the writer; finish then refuses publication.
96/// Dropping the writer removes its temporary file. Crash leftovers may remain.
97/// Inputs must be stable snapshots; device opening, retries, splitting, resume,
98/// encryption, and metadata/ACL collection are outside this API.
99pub struct Writer {
100    zip: Option<ZipWriter<File>>,
101    temporary: NamedTempFile,
102    path: PathBuf,
103    profile: Profile,
104    volume: String,
105    options: WriteOptions,
106    metadata: String,
107    archive_entries: u64,
108    streams: Vec<AcquiredStream>,
109    poisoned: bool,
110    logical_zip_threshold: u64,
111    folders: BTreeMap<String, (String, Vec<String>)>,
112    roots: Vec<String>,
113    path_separator: char,
114}
115
116impl Writer {
117    /// Creates private staging beside a new destination. Existing files,
118    /// including symlinks, are never overwritten.
119    pub fn create(path: impl AsRef<Path>, profile: Profile, options: WriteOptions) -> Result<Self> {
120        if options.chunk_bytes == 0
121            || options.chunk_bytes > 16 * 1024 * 1024
122            || options.chunks_per_bevy == 0
123            || u64::from(options.chunk_bytes) * u64::from(options.chunks_per_bevy)
124                > 128 * 1024 * 1024
125        {
126            return Err(malformed("invalid writer chunk geometry"));
127        }
128        if profile == Profile::Logical
129            && !matches!(options.compression, Compression::Stored | Compression::Zlib)
130        {
131            return Err(Error::Unsupported(
132                "logical ZIP compression must be Stored or Zlib".into(),
133            ));
134        }
135        let input_path = path.as_ref();
136        let name = input_path
137            .file_name()
138            .ok_or_else(|| malformed("missing output filename"))?;
139        #[cfg(windows)]
140        if name.to_string_lossy().contains(':') {
141            return Err(malformed("alternate-stream output is unsupported"));
142        }
143        let parent = fs::canonicalize(
144            input_path
145                .parent()
146                .filter(|p| !p.as_os_str().is_empty())
147                .unwrap_or_else(|| Path::new(".")),
148        )?;
149        let path = parent.join(name);
150        match fs::symlink_metadata(&path) {
151            Ok(_) => return Err(malformed("output already exists")),
152            Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
153            Err(e) => return Err(e.into()),
154        }
155        let temporary = tempfile::Builder::new()
156            .prefix(".aff4-acquire-")
157            .tempfile_in(&parent)?;
158        let mut zip = ZipWriter::new(temporary.as_file().try_clone()?);
159        let volume = identifier();
160        zip.set_comment(volume.as_str())?;
161        write_member(&mut zip, "container.description", volume.as_bytes())?;
162        let version = format!(
163            "major=1\nminor={}\ntool=aff4-image {}\n",
164            match profile {
165                Profile::Physical => 0,
166                Profile::Logical => 1,
167            },
168            env!("CARGO_PKG_VERSION")
169        );
170        write_member(&mut zip, "version.txt", version.as_bytes())?;
171        let metadata = format!(
172            "@prefix a: <http://aff4.org/Schema#> .\n@prefix l: <https://aff4.org/Schema/2022/#> .\n<{volume}> a a:ZipVolume .\n"
173        );
174        Ok(Self {
175            zip: Some(zip),
176            temporary,
177            path,
178            profile,
179            volume,
180            options,
181            metadata,
182            archive_entries: 2,
183            streams: Vec::new(),
184            poisoned: false,
185            logical_zip_threshold: 1024 * 1024,
186            folders: BTreeMap::new(),
187            roots: Vec::new(),
188            path_separator: '/',
189        })
190    }
191
192    /// Acquires exactly `size` bytes as a physical ImageStream, preserving
193    /// trailing input bytes for the caller. Progress is before/after chunks.
194    pub fn add_image(
195        &mut self,
196        size: u64,
197        input: &mut impl Read,
198        progress: impl FnMut(u64, u64) -> ControlFlow<()>,
199    ) -> Result<String> {
200        self.healthy()?;
201        if self.profile != Profile::Physical {
202            return Err(Error::Unsupported(
203                "physical image in a logical writer".into(),
204            ));
205        }
206        self.add_chunked(size, input, progress, None, "")
207    }
208
209    /// Adds a physical image with its known logical sector size (`blockSize`).
210    /// The size must be nonzero, sector aligned, and the sector size a power of two.
211    pub fn add_image_with_sector_size(
212        &mut self,
213        size: u64,
214        sector_size: u32,
215        input: &mut impl Read,
216        progress: impl FnMut(u64, u64) -> ControlFlow<()>,
217    ) -> Result<String> {
218        self.healthy()?;
219        if !sector_size.is_power_of_two()
220            || size == 0
221            || !size.is_multiple_of(u64::from(sector_size))
222        {
223            return Err(malformed("invalid physical sector geometry"));
224        }
225        let id = self.add_image(size, input, progress)?;
226        self.metadata
227            .push_str(&format!("<{id}> a:blockSize {sector_size} .\n"));
228        Ok(id)
229    }
230
231    /// Sets the largest logical file stored as one ZIP segment (at most 1 GiB).
232    /// Larger files use chunked ImageStreams with Maps and block integrity hashes.
233    pub fn set_logical_zip_threshold(&mut self, bytes: u64) -> Result<()> {
234        self.healthy()?;
235        if bytes > 1024 * 1024 * 1024 {
236            return Err(malformed("ZIP threshold exceeds 1 GiB"));
237        }
238        self.logical_zip_threshold = bytes;
239        Ok(())
240    }
241
242    fn add_chunked(
243        &mut self,
244        size: u64,
245        input: &mut impl Read,
246        mut progress: impl FnMut(u64, u64) -> ControlFlow<()>,
247        logical_path: Option<&str>,
248        extra_properties: &str,
249    ) -> Result<String> {
250        self.poisoned = true;
251        let entries = size
252            .div_ceil(u64::from(self.options.chunk_bytes))
253            .div_ceil(u64::from(self.options.chunks_per_bevy))
254            .checked_mul(4)
255            .and_then(|n| n.checked_add(if logical_path.is_some() { 0 } else { 2 }))
256            .ok_or_else(|| malformed("archive entry count overflow"))?;
257        self.archive_entries = self
258            .archive_entries
259            .checked_add(entries)
260            .ok_or_else(|| malformed("archive entry count overflow"))?;
261        let id = if logical_path.is_some() {
262            format!("{}/files/{}", self.volume, uuid::Uuid::new_v4())
263        } else {
264            identifier()
265        };
266        let stream = if logical_path.is_some() {
267            id.clone()
268        } else {
269            identifier()
270        };
271        let map = identifier();
272        let storage = if logical_path.is_some() {
273            id.strip_prefix(&format!("{}/", self.volume))
274                .unwrap()
275                .to_owned()
276        } else {
277            format!("aff4%3A%2F%2F{}", stream.strip_prefix("aff4://").unwrap())
278        };
279        let mut hashes = Hashes::new();
280        let mut done = 0;
281        let mut chunk = 0u64;
282        let mut index = Vec::new();
283        let mut md5_blocks = Vec::new();
284        let mut sha256_blocks = Vec::new();
285        let mut md5_tree = Sha512::new();
286        let mut sha256_tree = Sha512::new();
287        let mut index_tree = Sha512::new();
288        let zip = self.zip.as_mut().unwrap();
289        while done < size {
290            let bevy = chunk / u64::from(self.options.chunks_per_bevy);
291            zip.start_file(format!("{storage}/{bevy:08}"), stored())?;
292            let mut bevy_bytes = 0u64;
293            index.clear();
294            md5_blocks.clear();
295            sha256_blocks.clear();
296
297            let bevy_end = chunk + u64::from(self.options.chunks_per_bevy);
298            while done < size && chunk < bevy_end {
299                let batch_chunks = automatic_encode_batch_chunks(self.options.chunk_bytes as usize)
300                    .min(usize::try_from(bevy_end - chunk).unwrap_or(usize::MAX));
301                let mut buffers = Vec::with_capacity(batch_chunks);
302                let mut lengths = Vec::with_capacity(batch_chunks);
303                while done < size && chunk + (buffers.len() as u64) < bevy_end {
304                    if buffers.len() == batch_chunks {
305                        break;
306                    }
307                    if progress(done, size).is_break() {
308                        return Err(Error::Aborted);
309                    }
310                    let length = (size - done).min(u64::from(self.options.chunk_bytes)) as usize;
311                    let mut buffer = vec![0; self.options.chunk_bytes as usize];
312                    input.read_exact(&mut buffer[..length])?;
313                    hashes.update(&buffer[..length]);
314                    done += length as u64;
315                    buffers.push(buffer);
316                    lengths.push(length);
317                }
318
319                let prepared = prepare_chunks(&buffers, &lengths, self.options.compression)?;
320                for (buffer, prepared) in buffers.iter().zip(prepared) {
321                    md5_blocks.extend_from_slice(&prepared.md5);
322                    sha256_blocks.extend_from_slice(&prepared.sha256);
323                    let encoded = if self.options.compression != Compression::Stored
324                        && prepared.encoded.len() < buffer.len().saturating_sub(16)
325                    {
326                        &prepared.encoded
327                    } else {
328                        buffer
329                    };
330                    zip.write_all(encoded)?;
331                    index.extend_from_slice(&bevy_bytes.to_le_bytes());
332                    index.extend_from_slice(&(encoded.len() as u32).to_le_bytes());
333                    bevy_bytes += encoded.len() as u64;
334                    chunk += 1;
335                }
336            }
337            write_member(zip, &format!("{storage}/{bevy:08}.index"), &index)?;
338            write_member(
339                zip,
340                &format!("{storage}/{bevy:08}.blockHash.md5"),
341                &md5_blocks,
342            )?;
343            write_member(
344                zip,
345                &format!("{storage}/{bevy:08}.blockHash.sha256"),
346                &sha256_blocks,
347            )?;
348            md5_tree.update(&md5_blocks);
349            sha256_tree.update(&sha256_blocks);
350            index_tree.update(&index);
351        }
352        if progress(done, size).is_break() {
353            return Err(Error::Aborted);
354        }
355        let result = hashes.finish(id.clone(), size);
356        let map_storage = format!("aff4%3A%2F%2F{}", map.strip_prefix("aff4://").unwrap());
357        let mut range = Vec::new();
358        if size != 0 {
359            range.extend_from_slice(&0u64.to_le_bytes());
360            range.extend_from_slice(&size.to_le_bytes());
361            range.extend_from_slice(&0u64.to_le_bytes());
362            range.extend_from_slice(&0u32.to_le_bytes());
363        }
364        if logical_path.is_none() {
365            write_member(zip, &format!("{map_storage}/map"), &range)?;
366            write_member(
367                zip,
368                &format!("{map_storage}/idx"),
369                format!("{stream}\n").as_bytes(),
370            )?;
371        }
372        let index_hash = hex(&index_tree.finalize());
373        let md5_tree = md5_tree.finalize();
374        let sha256_tree = sha256_tree.finalize();
375        let points_hash = Sha512::digest(&range);
376        let target_index = format!("{stream}\n");
377        let target_hash = Sha512::digest(target_index.as_bytes());
378        let empty_hash = Sha512::digest([]);
379        let mut composite = Sha512::new();
380        if size != 0 {
381            composite.update(md5_tree);
382            composite.update(sha256_tree);
383        }
384        composite.update(points_hash);
385        composite.update(target_hash);
386        composite.update(empty_hash);
387        let composite = hex(&composite.finalize());
388        let mut map_hash = Sha512::new();
389        map_hash.update(&range);
390        map_hash.update(target_index.as_bytes());
391        let map_hash = hex(&map_hash.finalize());
392        let compression: String = match self.options.compression {
393            Compression::Stored => {
394                "a:compressionMethod <http://aff4.org/Schema#compression/stored>;".into()
395            }
396            Compression::Zlib => {
397                "a:compressionMethod <https://www.ietf.org/rfc/rfc1950.txt>;".into()
398            }
399            Compression::Snappy => "a:compressionMethod <http://code.google.com/p/snappy/>;".into(),
400            Compression::Lz4 => "a:compressionMethod <https://code.google.com/p/lz4/>;".into(),
401        };
402        let kind = if logical_path.is_some() {
403            "FileImage"
404        } else {
405            "DiskImage"
406        };
407        let logical_properties = logical_path
408            .map(|path| {
409                let path_literal = oxrdf::Literal::new_simple_literal(path).to_string();
410                let name = oxrdf::Literal::new_simple_literal(
411                    path.rsplit(['/', '\\']).next().unwrap_or(path),
412                )
413                .to_string();
414                format!("a:originalFileName {path_literal}; a:fileName {name};")
415            })
416            .unwrap_or_default();
417        let volume = &self.volume;
418        if logical_path.is_some() {
419            self.metadata.push_str(&format!("<{id}> a a:FileImage, a:Image, a:ImageStream; {extra_properties} {logical_properties} a:size {size}; a:chunkSize {}; a:chunksInSegment {}; {compression} a:stored <{volume}>; a:imageStreamIndexHash \"{index_hash}\"^^a:SHA512; {} .\n", self.options.chunk_bytes, self.options.chunks_per_bevy, hash_triples(&result)));
420        } else {
421            self.metadata.push_str(&format!("<{stream}> a a:ImageStream; a:size {size}; a:chunkSize {}; a:chunksInSegment {}; {compression} a:target <{map}>; a:stored <{volume}>; a:imageStreamIndexHash \"{index_hash}\"^^a:SHA512 .\n<{id}> a a:Image, a:ContiguousImage, a:{kind}; {logical_properties} a:size {size}; a:dataStream <{map}>; a:stored <{volume}>; {}; a:hash \"{composite}\"^^a:blockMapHashSHA512 .\n<{map}> a a:Map; a:size {size}; a:dependentStream <{stream}>; a:target <{id}>; a:stored <{volume}>; a:blockMapHash \"{composite}\"^^a:SHA512; a:mapPointHash \"{}\"^^a:SHA512; a:mapIdxHash \"{}\"^^a:SHA512; a:mapPathHash \"{}\"^^a:SHA512; a:mapHash \"{map_hash}\"^^a:SHA512 .\n", self.options.chunk_bytes, self.options.chunks_per_bevy, hash_triples(&result), hex(&points_hash), hex(&target_hash), hex(&empty_hash)));
422        }
423        if size != 0 {
424            self.metadata.push_str(&format!("<{stream}/blockhash.md5> a a:BlockHashes; a:hash \"{}\"^^a:SHA512 .\n<{stream}/blockhash.sha256> a a:BlockHashes; a:hash \"{}\"^^a:SHA512 .\n", hex(&md5_tree), hex(&sha256_tree)));
425        }
426        self.streams.push(result);
427        self.poisoned = false;
428        Ok(id)
429    }
430
431    /// Acquires one logical file as a ZIP segment with recorded original path.
432    /// Names stay in metadata; ZIP storage uses a generated identifier.
433    /// Files above the configured threshold use ImageStreams with block hashes.
434    pub fn add_file(
435        &mut self,
436        original_path: &str,
437        size: u64,
438        input: &mut impl Read,
439        progress: impl FnMut(u64, u64) -> ControlFlow<()>,
440    ) -> Result<String> {
441        self.add_file_impl(original_path, size, input, progress, "")
442    }
443
444    fn add_file_impl(
445        &mut self,
446        original_path: &str,
447        size: u64,
448        input: &mut impl Read,
449        mut progress: impl FnMut(u64, u64) -> ControlFlow<()>,
450        extra_properties: &str,
451    ) -> Result<String> {
452        self.healthy()?;
453        if self.profile != Profile::Logical {
454            return Err(Error::Unsupported(
455                "logical file in a physical writer".into(),
456            ));
457        }
458        if original_path.is_empty() || original_path.chars().any(char::is_control) {
459            return Err(malformed(
460                "logical path must be nonempty UTF-8 without control characters",
461            ));
462        }
463        if size > self.logical_zip_threshold {
464            return self.add_chunked(size, input, progress, Some(original_path), extra_properties);
465        }
466        self.poisoned = true;
467        let storage = format!("files/{}", uuid::Uuid::new_v4());
468        let id = format!("{}/{storage}", self.volume);
469        let zip = self.zip.as_mut().unwrap();
470        let method = if self.options.compression == Compression::Stored {
471            zip::CompressionMethod::Stored
472        } else {
473            zip::CompressionMethod::Deflated
474        };
475        self.archive_entries = self
476            .archive_entries
477            .checked_add(1)
478            .ok_or_else(|| malformed("archive entry count overflow"))?;
479        zip.start_file(&storage, stored().compression_method(method))?;
480        let mut hashes = Hashes::new();
481        let mut buffer = vec![0; 1024 * 1024];
482        let mut done = 0;
483        loop {
484            if progress(done, size).is_break() {
485                return Err(Error::Aborted);
486            }
487            if done == size {
488                break;
489            }
490            let length = (size - done).min(buffer.len() as u64) as usize;
491            input.read_exact(&mut buffer[..length])?;
492            zip.write_all(&buffer[..length])?;
493            hashes.update(&buffer[..length]);
494            done += length as u64;
495        }
496        let result = hashes.finish(id.clone(), size);
497        let name = original_path
498            .rsplit(['/', '\\'])
499            .next()
500            .unwrap_or(original_path);
501        let name = oxrdf::Literal::new_simple_literal(name).to_string();
502        let path = oxrdf::Literal::new_simple_literal(original_path).to_string();
503        self.metadata.push_str(&format!("<{id}> a a:FileImage, a:Image, a:zip_segment; {extra_properties} a:size {size}; a:fileName {name}; a:originalFileName {path}; a:stored <{}>; {} .\n", self.volume, hash_triples(&result)));
504        self.streams.push(result);
505        self.poisoned = false;
506        Ok(id)
507    }
508
509    /// Finalizes ZIP metadata, synchronizes bytes, and publishes exclusively.
510    /// Unix additionally synchronizes the parent directory. This API does not
511    /// certify power-loss durability on every filesystem or device.
512    /// `Error::PublishedButUnsynced` retains the result if that last sync fails.
513    pub fn finish(self) -> Result<WriteResult> {
514        self.finish_checked(|_, _| Ok(()))
515            .map(|(result, ())| result)
516    }
517
518    /// Verifies the finalized temporary container under explicit reader budgets
519    /// before exclusive publication. Limit, integrity, or cancellation failures
520    /// remove the temporary file and leave the destination absent.
521    /// `VerificationFailed` retains the complete available verification report.
522    /// `PublishedButUnsynced` means verification passed but directory sync failed.
523    pub fn finish_verified(
524        self,
525        limits: crate::Limits,
526        mut progress: impl FnMut(&str, u64, u64) -> ControlFlow<()>,
527    ) -> Result<(WriteResult, crate::ContainerVerification)> {
528        self.finish_checked(|path, digest| {
529            if progress("staged verification", 0, 0).is_break() {
530                return Err(Error::Aborted);
531            }
532            let verification = crate::Container::open_with_limits(path, limits)?
533                .verify_all(Some(digest), &mut progress)?;
534            if !verification.all_match() {
535                return Err(Error::VerificationFailed {
536                    report: Box::new(verification),
537                });
538            }
539            if progress("before publication", 0, 0).is_break() {
540                return Err(Error::Aborted);
541            }
542            Ok(verification)
543        })
544    }
545
546    fn finish_checked<T>(
547        mut self,
548        check: impl FnOnce(&Path, &str) -> Result<T>,
549    ) -> Result<(WriteResult, T)> {
550        self.healthy()?;
551        self.finish_logical_metadata();
552        let mut zip = self.zip.take().unwrap();
553        write_member(&mut zip, "information.turtle", self.metadata.as_bytes())?;
554        let metadata_sha256 = hex(&Sha256::digest(self.metadata.as_bytes()));
555        let hashes = format!(
556            "@prefix a: <http://aff4.org/Schema#> .\n<{}/information.turtle> a:hash \"{}\"^^a:SHA256 .\n",
557            self.volume, metadata_sha256
558        );
559        write_member(&mut zip, "information.turtle.hashes", hashes.as_bytes())?;
560        #[cfg(test)]
561        failure_tests::boundary("metadata");
562        let file = zip.finish()?;
563        #[cfg(test)]
564        failure_tests::boundary("zip_closed");
565        file.sync_all()?;
566        drop(file);
567        // The finalized ZIP now owns these bytes. Release the write-side graph
568        // before staged verification allocates its bounded read-side graph.
569        self.metadata = String::new();
570        self.folders.clear();
571        self.roots = Vec::new();
572        #[cfg(test)]
573        failure_tests::boundary("file_synced");
574        let checked = check(self.temporary.path(), &metadata_sha256)?;
575        self.temporary
576            .persist_noclobber(&self.path)
577            .map_err(|error| Error::Io(error.error))?;
578        let result = WriteResult {
579            path: self.path,
580            streams: self.streams,
581            metadata_sha256,
582        };
583        #[cfg(test)]
584        failure_tests::boundary("published");
585        sync_published_directory(&result.path).map_err(|source| Error::PublishedButUnsynced {
586            result: Box::new(WriteResult {
587                path: result.path.clone(),
588                streams: result.streams.clone(),
589                metadata_sha256: result.metadata_sha256.clone(),
590            }),
591            source,
592        })?;
593        #[cfg(test)]
594        failure_tests::boundary("directory_synced");
595        Ok((result, checked))
596    }
597
598    fn healthy(&self) -> Result<()> {
599        if self.poisoned {
600            return Err(Error::Unsupported(
601                "acquisition failed; discard this writer".into(),
602            ));
603        }
604        Ok(())
605    }
606}
607
608fn sync_published_directory(path: &Path) -> std::io::Result<()> {
609    #[cfg(test)]
610    if failure_tests::FAIL_DIRECTORY_SYNC.get() {
611        return Err(std::io::Error::other("injected directory sync failure"));
612    }
613    #[cfg(unix)]
614    File::open(path.parent().unwrap())?.sync_all()?;
615    #[cfg(not(unix))]
616    let _ = path;
617    Ok(())
618}
619
620#[cfg(test)]
621#[path = "writer_failure_tests.rs"]
622mod failure_tests;
623
624fn identifier() -> String {
625    format!("aff4://{}", uuid::Uuid::new_v4())
626}
627fn stored() -> SimpleFileOptions {
628    SimpleFileOptions::default()
629        .compression_method(zip::CompressionMethod::Stored)
630        .large_file(true)
631}
632fn write_member(zip: &mut ZipWriter<File>, name: &str, data: &[u8]) -> Result<()> {
633    zip.start_file(name, stored())?;
634    zip.write_all(data)?;
635    Ok(())
636}
637fn hash_triples(result: &AcquiredStream) -> String {
638    format!(
639        "a:hash \"{}\"^^a:MD5, \"{}\"^^a:SHA1, \"{}\"^^a:SHA256",
640        result.md5, result.sha1, result.sha256
641    )
642}
643
644struct PreparedChunk {
645    encoded: Vec<u8>,
646    md5: [u8; 16],
647    sha256: [u8; 32],
648}
649
650fn prepare_chunks(
651    buffers: &[Vec<u8>],
652    lengths: &[usize],
653    codec: Compression,
654) -> Result<Vec<PreparedChunk>> {
655    let prepare = |(buffer, length): (&Vec<u8>, &usize)| {
656        Ok(PreparedChunk {
657            encoded: if codec == Compression::Stored {
658                Vec::new()
659            } else {
660                encode(buffer, codec)?
661            },
662            md5: Md5::digest(&buffer[..*length]).into(),
663            sha256: Sha256::digest(&buffer[..*length]).into(),
664        })
665    };
666    if buffers.len() > 1 {
667        buffers.par_iter().zip(lengths).map(prepare).collect()
668    } else {
669        buffers.iter().zip(lengths).map(prepare).collect()
670    }
671}
672
673fn automatic_encode_batch_chunks(chunk_size: usize) -> usize {
674    const MAXIMUM_BATCH_BYTES: usize = 8 * 1024 * 1024;
675    let memory_limit = MAXIMUM_BATCH_BYTES
676        .checked_div(chunk_size)
677        .unwrap_or(0)
678        .max(1);
679    rayon::current_num_threads()
680        .saturating_mul(2)
681        .min(memory_limit)
682        .max(1)
683}
684
685fn encode(data: &[u8], codec: Compression) -> Result<Vec<u8>> {
686    match codec {
687        Compression::Stored => Ok(data.to_vec()),
688        Compression::Zlib => {
689            let mut encoder =
690                flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default());
691            encoder.write_all(data)?;
692            Ok(encoder.finish()?)
693        }
694        Compression::Snappy => snap::raw::Encoder::new()
695            .compress_vec(data)
696            .map_err(|e| malformed(e.to_string())),
697        Compression::Lz4 => Ok(lz4_flex::block::compress(data)),
698    }
699}
700struct Hashes {
701    md5: Md5,
702    sha1: Sha1,
703    sha256: Sha256,
704}
705impl Hashes {
706    fn new() -> Self {
707        Self {
708            md5: Md5::new(),
709            sha1: Sha1::new(),
710            sha256: Sha256::new(),
711        }
712    }
713    fn update(&mut self, data: &[u8]) {
714        self.md5.update(data);
715        self.sha1.update(data);
716        self.sha256.update(data);
717    }
718    fn finish(self, id: String, size: u64) -> AcquiredStream {
719        AcquiredStream {
720            id,
721            size,
722            md5: hex(&self.md5.finalize()),
723            sha1: hex(&self.sha1.finalize()),
724            sha256: hex(&self.sha256.finalize()),
725        }
726    }
727}