nix_nar/
enc.rs

1use std::{
2    cmp::min,
3    fs::File,
4    io::{self, Read},
5    path::Path,
6};
7
8use camino::{Utf8Path, Utf8PathBuf};
9
10use crate::{coder, NarError};
11
12mod filesystem;
13pub use filesystem::{FileSystem, FileSystemMetadata, NativeFileSystem};
14
15/// Encoder which can archive a given path as a NAR file.
16///
17/// The encoder will read the data to archive using its [`FileSystem`]
18/// implementation. By default, it reads the host filesystem using
19/// [`std::fs`].
20pub struct Encoder<FS: FileSystem = NativeFileSystem> {
21    stack: Vec<CurrentActivity<FS>>,
22    internal_buffer_size: usize,
23    fs: FS,
24}
25
26/// Builder for [`Encoder`].
27pub struct EncoderBuilder<P: AsRef<Path>, FS: FileSystem = NativeFileSystem> {
28    path: P,
29    internal_buffer_size: usize,
30    fs: FS,
31}
32
33#[derive(Debug)]
34enum CurrentActivity<FS: FileSystem> {
35    StartArchive,
36    StartEntry,
37    Toplevel {
38        path: Utf8PathBuf,
39    },
40    WalkingDir {
41        dir_path: Utf8PathBuf,
42        files_rev: Vec<String>,
43    },
44    EncodingFile {
45        file: FS::File,
46        length_remaining: u64,
47    },
48    WritePadding {
49        padding: u64,
50    },
51    WriteMoreBytes {
52        bytes: Vec<u8>,
53    },
54    CloseDirEntry,
55    CloseEntry,
56}
57
58impl Encoder<NativeFileSystem> {
59    /// Create a new encoder for file hierarchy at the given path
60    /// using default filesystem implementation.
61    ///
62    /// # Errors
63    ///
64    /// Returns an error if the path is not valid UTF-8.
65    pub fn new<P: AsRef<Path>>(path: P) -> Result<Self, NarError> {
66        Self::new_with_filesystem(path, NativeFileSystem {})
67    }
68
69    /// Create a builder for this encoder that can take additional
70    /// options.
71    ///
72    /// ```
73    /// # use std::{io, fs::File};
74    /// use nix_nar::{Encoder, NativeFileSystem};
75    /// let mut enc = Encoder::builder("./test-data/")
76    ///     .internal_buffer_size(2048)
77    ///     .filesystem(NativeFileSystem {})
78    ///     .build().unwrap();
79    /// let mut nar = File::create("output2.nar")?;
80    /// io::copy(&mut enc, &mut nar)?;
81    /// # std::fs::remove_file("output2.nar")?;
82    /// # Ok::<(), nix_nar::NarError>(())
83    /// ```
84    pub fn builder<P: AsRef<Path>>(path: P) -> EncoderBuilder<P> {
85        EncoderBuilder {
86            path,
87            internal_buffer_size: 1024,
88            fs: NativeFileSystem {},
89        }
90    }
91}
92
93impl<FS: FileSystem> Encoder<FS> {
94    /// Create a new encoder for file hierarchy at the given path and
95    /// provided filesystem implementation.
96    ///
97    /// # Errors
98    ///
99    /// Returns an error if the path is not valid UTF-8.
100    pub fn new_with_filesystem<P: AsRef<Path>>(
101        path: P,
102        fs: FS,
103    ) -> Result<Self, NarError> {
104        let path = to_utf8_path(path)?;
105        Ok(Self {
106            stack: vec![
107                CurrentActivity::CloseEntry,
108                CurrentActivity::Toplevel { path },
109                CurrentActivity::StartEntry,
110                CurrentActivity::StartArchive,
111            ],
112            internal_buffer_size: 1024,
113            fs,
114        })
115    }
116
117    /// Archive to the given path.
118    ///
119    /// This is equivalent to creating the file, and [`io::copy`]ing
120    /// to it.
121    ///
122    /// # Errors
123    ///
124    /// Returns an error if the path already exists or an I/O error occurs.
125    pub fn pack<P: AsRef<Path>>(&mut self, dst: P) -> Result<(), NarError> {
126        let dst = to_utf8_path(dst)?;
127        if dst.symlink_metadata().is_ok() {
128            return Err(NarError::PackError(format!(
129                "Destination {dst} already exists. Delete it first."
130            )));
131        }
132        let mut nar = File::create(&dst)?;
133        io::copy(self, &mut nar)?;
134        Ok(())
135    }
136}
137
138impl<P: AsRef<Path>, FS: FileSystem> EncoderBuilder<P, FS> {
139    /// Build the encoder.
140    ///
141    /// # Errors
142    ///
143    /// Returns an error if the path is not valid UTF-8.
144    pub fn build(self) -> Result<Encoder<FS>, NarError> {
145        let Self {
146            path,
147            internal_buffer_size,
148            fs,
149        } = self;
150        let mut enc = Encoder::new_with_filesystem(path, fs)?;
151        enc.internal_buffer_size = internal_buffer_size;
152        Ok(enc)
153    }
154
155    /// Configure the internal buffer size.  This should be at least
156    /// 200 bytes larger than the longest filename.
157    ///
158    /// # Panics
159    ///
160    /// Panics if the given number is smaller than 200.
161    #[must_use]
162    pub fn internal_buffer_size(mut self, x: usize) -> Self {
163        assert!(
164            x >= 200,
165            "internal_buffer_size should be at least 200 bytes larger than the longest filename you have"
166        );
167        self.internal_buffer_size = x;
168        self
169    }
170
171    /// Configure filesystem implementation.
172    #[must_use]
173    pub fn filesystem<FS2: FileSystem>(self, fs: FS2) -> EncoderBuilder<P, FS2> {
174        let Self {
175            path,
176            internal_buffer_size,
177            fs: _,
178        } = self;
179        EncoderBuilder {
180            path,
181            internal_buffer_size,
182            fs,
183        }
184    }
185}
186
187impl<FS: FileSystem> Encoder<FS> {
188    fn start_encoding<P: AsRef<Utf8Path>>(
189        &mut self,
190        buf: &mut [u8],
191        path: P,
192        metadata: FileSystemMetadata,
193        dir_entry: Option<String>,
194    ) -> Result<usize, io::Error> {
195        match metadata {
196            FileSystemMetadata::Directory => {
197                self.start_encoding_dir(buf, path, dir_entry)
198            }
199            FileSystemMetadata::File { executable, length } => {
200                self.start_encoding_file(buf, path, executable, length, dir_entry)
201            }
202            FileSystemMetadata::Symlink { target_path } => {
203                self.start_encoding_symlink(buf, &target_path, dir_entry)
204            }
205        }
206    }
207
208    fn start_encoding_file<P: AsRef<Utf8Path>>(
209        &mut self,
210        buf: &mut [u8],
211        path: P,
212        executable: bool,
213        length: u64,
214        dir_entry: Option<String>,
215    ) -> Result<usize, io::Error> {
216        let path = path.as_ref();
217        let file_handle = self.fs.open(path).map_err(annotate_err_with_path(&path))?;
218        let file_len_rounded_up = (length + 7) & !7;
219        if file_len_rounded_up > length {
220            self.stack.push(CurrentActivity::WritePadding {
221                padding: file_len_rounded_up - length,
222            });
223        }
224        self.stack.push(CurrentActivity::EncodingFile {
225            file: file_handle,
226            length_remaining: length,
227        });
228        self.write_with_buffer(buf, move |buf| {
229            let mut len = 0;
230            if let Some(ref file) = dir_entry {
231                len += coder::start_dir_entry(&mut buf[len..], file)?;
232            }
233            len += coder::write_file_regular(&mut buf[len..], executable)?;
234            len += coder::write_u64_le(&mut buf[len..], length)?;
235            Ok(len)
236        })
237    }
238
239    fn start_encoding_dir<P: AsRef<Utf8Path>>(
240        &mut self,
241        buf: &mut [u8],
242        path: P,
243        dir_entry: Option<String>,
244    ) -> Result<usize, io::Error> {
245        let path = path.as_ref();
246        let mut files_rev = self
247            .fs
248            .read_dir(path)
249            .map_err(annotate_err_with_path(path))?;
250        files_rev.sort_by(|a, b| b.cmp(a));
251
252        self.stack.push(CurrentActivity::WalkingDir {
253            files_rev,
254            dir_path: path.into(),
255        });
256        self.write_with_buffer(buf, move |buf| {
257            let mut len = 0;
258            if let Some(ref file) = dir_entry {
259                len += coder::start_dir_entry(&mut buf[len..], file)?;
260            }
261            len += coder::start_dir(&mut buf[len..])?;
262            Ok(len)
263        })
264    }
265
266    fn start_encoding_symlink<P: AsRef<Utf8Path>>(
267        &mut self,
268        buf: &mut [u8],
269        target_path: P,
270        dir_entry: Option<String>,
271    ) -> Result<usize, io::Error> {
272        self.write_with_buffer(buf, move |buf| {
273            let mut len = 0;
274            if let Some(ref file) = dir_entry {
275                len += coder::start_dir_entry(buf, file)?;
276            }
277            len += coder::write_symlink(&mut buf[len..], target_path)?;
278            Ok(len)
279        })
280    }
281
282    /// Execute a write-into-buffer operation.  If the given buffer is
283    /// big enough, then we just write into that.  Otherwise, we
284    /// create a temporary buffer, we write into that, we copy as much
285    /// data as we can to the given buffer, and store the remainer
286    /// into a `CurrentActivity::WriteMoreBytes` on the `self.stack`.
287    fn write_with_buffer<F>(&mut self, dst_buf: &mut [u8], f: F) -> io::Result<usize>
288    where
289        F: FnOnce(&mut [u8]) -> io::Result<usize>,
290    {
291        if dst_buf.len() >= 1024 {
292            f(dst_buf)
293        } else {
294            let mut buf = vec![0; self.internal_buffer_size];
295            let len = f(&mut buf)?;
296            let to_write_len = min(len, dst_buf.len());
297            dst_buf[..to_write_len].copy_from_slice(&buf[..to_write_len]);
298            if len > to_write_len {
299                buf.truncate(len);
300                self.stack.push(CurrentActivity::WriteMoreBytes {
301                    bytes: buf.split_off(to_write_len),
302                });
303            }
304            Ok(to_write_len)
305        }
306    }
307}
308
309/// Read the encoded NAR file.
310///
311/// # Errors
312///
313/// [`Self::read`] will return underlying I/O errors returned by [`FileSystem`].
314///
315/// It will also return an error if the data read from an archived
316/// file is not the same length as declared by
317/// [`FileSystem::metadata`]. The length from the metadata was already
318/// written to the NAR output at this point, so it's not possible to
319/// recover and produce a valid archive.
320impl<FS: FileSystem> Read for Encoder<FS> {
321    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
322        match self.stack.pop() {
323            None => Ok(0),
324            Some(CurrentActivity::StartArchive) => {
325                self.write_with_buffer(buf, coder::start_archive)
326            }
327            Some(CurrentActivity::StartEntry) => {
328                self.write_with_buffer(buf, coder::start_entry)
329            }
330            Some(CurrentActivity::CloseDirEntry) => {
331                self.write_with_buffer(buf, coder::close_dir_entry)
332            }
333            Some(CurrentActivity::CloseEntry) => {
334                self.write_with_buffer(buf, coder::close_entry)
335            }
336            Some(CurrentActivity::Toplevel { path }) => {
337                let metadata = self
338                    .fs
339                    .metadata(&path)
340                    .map_err(annotate_err_with_path(&path))?;
341                self.start_encoding(buf, path, metadata, None)
342            }
343            Some(CurrentActivity::WalkingDir {
344                dir_path,
345                mut files_rev,
346            }) => match files_rev.pop() {
347                None => self.read(buf),
348                Some(file) => {
349                    let path = dir_path.join(&file);
350
351                    self.stack.push(CurrentActivity::WalkingDir {
352                        dir_path,
353                        files_rev,
354                    });
355
356                    self.stack.push(CurrentActivity::CloseDirEntry);
357
358                    let metadata = self
359                        .fs
360                        .metadata(&path)
361                        .map_err(annotate_err_with_path(&path))?;
362                    self.start_encoding(buf, path, metadata, Some(file))
363                }
364            },
365            Some(CurrentActivity::EncodingFile {
366                mut file,
367                length_remaining,
368            }) => {
369                let len = file.read(buf)?;
370                if len != 0 {
371                    self.stack.push(CurrentActivity::EncodingFile {
372                        file,
373                        length_remaining: length_remaining
374                            .checked_sub(len.try_into().map_err(io::Error::other)?)
375                            .ok_or_else(|| {
376                                other_io_error("File was longer than declared length")
377                            })?,
378                    });
379                    Ok(len)
380                } else if length_remaining > 0 {
381                    Err(other_io_error("File was shorter than declared length"))
382                } else {
383                    self.read(buf)
384                }
385            }
386            Some(CurrentActivity::WritePadding { padding }) => {
387                #[allow(clippy::cast_possible_truncation)]
388                let len = min(padding, buf.len() as u64) as usize;
389                buf.fill(0);
390                if (len as u64) < padding {
391                    self.stack.push(CurrentActivity::WritePadding {
392                        padding: padding - len as u64,
393                    });
394                }
395                Ok(len)
396            }
397            Some(CurrentActivity::WriteMoreBytes { bytes }) => {
398                let len = min(bytes.len(), buf.len());
399                buf[..len].copy_from_slice(&bytes[..len]);
400                if len < bytes.len() {
401                    self.stack.push(CurrentActivity::WriteMoreBytes {
402                        bytes: bytes[len..].to_vec(),
403                    });
404                }
405                Ok(len)
406            }
407        }
408    }
409}
410
411fn annotate_err_with_path<P: AsRef<Utf8Path>>(
412    path: P,
413) -> impl FnOnce(io::Error) -> io::Error {
414    let path = path.as_ref().to_path_buf();
415    move |err: io::Error| other_io_error(format!("IO error on {path}: {err}"))
416}
417
418fn other_io_error<S: AsRef<str>>(message: S) -> io::Error {
419    io::Error::other(message.as_ref())
420}
421
422fn to_utf8_path<P: AsRef<Path>>(path: P) -> Result<Utf8PathBuf, NarError> {
423    let path = path.as_ref();
424    path.try_into()
425        .map(|x: &Utf8Path| x.to_path_buf())
426        .map_err(|err| {
427            NarError::Utf8PathError(format!(
428                "Failed to convert '{}' to UTF-8: {err}",
429                path.display()
430            ))
431        })
432}
nix_nar/enc.rs

nix_nar/
enc.rs
