molframe_core/io/input/
buffer.rs

1//! Shared storage and public input construction.
2
3#[cfg(feature = "mmap")]
4use super::collect::map_checked;
5use super::collect::{OutputLimit, collect_checked, path_origin, read_failure};
6#[cfg(all(feature = "gzip", feature = "mmap"))]
7use super::decode::map_gzip_reader;
8#[cfg(all(feature = "zstd", feature = "mmap"))]
9use super::decode::map_zstd_reader;
10use super::decode::{compressed_file_size, expand_gzip_reader, expand_zstd_reader, sniff_reader};
11#[cfg(not(feature = "mmap"))]
12use super::decode::{decompress, read_bounded};
13use super::{Compression, Limits};
14#[cfg(not(feature = "mmap"))]
15use crate::diagnostic::Code;
16use crate::diagnostic::Diagnostic;
17use std::fmt;
18use std::fs::File;
19use std::io::Read;
20use std::path::Path;
21use std::sync::Arc;
22
23#[cfg(feature = "mmap")]
24pub(super) const MMAP_MIN_BYTES: u64 = 1 << 20;
25
26/// The storage that backs an input byte slice.
27#[derive(Clone, Copy, PartialEq, Eq, Debug)]
28pub enum InputKind {
29    /// Bytes owned by the process.
30    Owned,
31    /// Stable bytes retained through an external owner without copying.
32    OwnerBacked,
33    /// Read-only pages mapped from a local file.
34    #[cfg(feature = "mmap")]
35    Mapped,
36}
37
38trait ByteOwner: AsRef<[u8]> + fmt::Debug + Send + Sync {}
39
40impl<T> ByteOwner for T where T: AsRef<[u8]> + fmt::Debug + Send + Sync {}
41
42#[derive(Clone, Debug)]
43enum InputBytes {
44    Owned(Arc<Vec<u8>>),
45    OwnerBacked(Arc<dyn ByteOwner>),
46    #[cfg(feature = "mmap")]
47    Mapped(Arc<molframe_mmap::MappedFile>),
48}
49
50impl InputBytes {
51    fn as_slice(&self) -> &[u8] {
52        match self {
53            Self::Owned(bytes) => bytes.as_slice(),
54            Self::OwnerBacked(owner) => owner.as_ref().as_ref(),
55            #[cfg(feature = "mmap")]
56            Self::Mapped(mapped) => mapped.as_bytes(),
57        }
58    }
59
60    const fn kind(&self) -> InputKind {
61        match self {
62            Self::Owned(_) => InputKind::Owned,
63            Self::OwnerBacked(_) => InputKind::OwnerBacked,
64            #[cfg(feature = "mmap")]
65            Self::Mapped(_) => InputKind::Mapped,
66        }
67    }
68}
69
70/// Bytes to read, however they were obtained.
71#[derive(Clone, Debug)]
72pub struct InputBuffer {
73    bytes: InputBytes,
74    origin: Option<Arc<str>>,
75}
76
77impl InputBuffer {
78    /// Takes ownership of bytes already in hand.
79    #[must_use]
80    pub fn from_bytes(bytes: Vec<u8>) -> Self {
81        Self {
82            bytes: InputBytes::Owned(Arc::new(bytes)),
83            origin: None,
84        }
85    }
86
87    /// Retains stable externally owned bytes without copying their payload.
88    ///
89    /// The owner is held for the complete lifetime of this buffer and every
90    /// clone. This is the adoption path for language-runtime byte objects and
91    /// existing immutable buffers whose address remains stable.
92    #[must_use]
93    pub fn from_owner<T>(owner: T) -> Self
94    where
95        T: AsRef<[u8]> + fmt::Debug + Send + Sync + 'static,
96    {
97        Self {
98            bytes: InputBytes::OwnerBacked(Arc::new(owner)),
99            origin: None,
100        }
101    }
102
103    /// Opens and transparently decompresses a bounded local file.
104    ///
105    /// # Errors
106    ///
107    /// Returns a registered resource or limit diagnostic.
108    pub fn open(path: impl AsRef<Path>, limits: Limits) -> Result<Self, Diagnostic> {
109        let path = path.as_ref();
110        let file = File::open(path)
111            .map_err(|error| read_failure("input could not be opened", Some(path), &error))?;
112        let compressed_size = file.metadata().map(|metadata| metadata.len());
113        #[cfg(feature = "mmap")]
114        let file_length = compressed_size.as_ref().copied().ok();
115
116        let (compression, reader) = sniff_reader(file, path)?;
117        let compressed = match compression {
118            Compression::None => 0,
119            Compression::Gzip | Compression::Zstd => compressed_file_size(compressed_size, path)?,
120        };
121        #[cfg(feature = "mmap")]
122        if compression != Compression::None
123            || file_length.is_some_and(|length| length >= MMAP_MIN_BYTES)
124        {
125            let mapped = match compression {
126                Compression::None => map_checked(
127                    reader,
128                    OutputLimit::uncompressed(limits),
129                    "input could not be read",
130                    Some(path),
131                )?,
132                #[cfg(feature = "gzip")]
133                Compression::Gzip => map_gzip_reader(reader, compressed, limits, Some(path))?,
134                #[cfg(not(feature = "gzip"))]
135                Compression::Gzip => return Err(super::decode::unsupported("gzip")),
136                #[cfg(feature = "zstd")]
137                Compression::Zstd => map_zstd_reader(reader, compressed, limits, Some(path))?,
138                #[cfg(not(feature = "zstd"))]
139                Compression::Zstd => return Err(super::decode::unsupported("zstd")),
140            };
141            return Ok(Self {
142                bytes: InputBytes::Mapped(Arc::new(mapped)),
143                origin: Some(path_origin(path)),
144            });
145        }
146
147        let bytes = match compression {
148            Compression::None => collect_checked(
149                reader,
150                OutputLimit::uncompressed(limits),
151                "input could not be read",
152                Some(path),
153            )?,
154            Compression::Gzip => expand_gzip_reader(reader, compressed, limits, Some(path))?,
155            Compression::Zstd => expand_zstd_reader(reader, compressed, limits, Some(path))?,
156        };
157        Ok(Self {
158            bytes: InputBytes::Owned(Arc::new(bytes)),
159            origin: Some(path_origin(path)),
160        })
161    }
162
163    /// Wraps a mapping the caller has already established.
164    ///
165    /// [`Self::open`] streams a source through a bounded buffer into private
166    /// storage before mapping it, so opening a hundred-gigabyte entry reads a
167    /// hundred gigabytes, writes a hundred gigabytes of scratch, and only then
168    /// begins. A caller who can guarantee the file will not change underneath
169    /// the mapping can avoid all of that by mapping it in place and handing the
170    /// result here.
171    ///
172    /// Establishing such a mapping requires `unsafe`, because no operating
173    /// system can enforce the guarantee. This crate forbids `unsafe` and so
174    /// cannot offer it: the call belongs to
175    /// [`molframe_mmap::MappedFile::map_file_unchecked`], in the crate that
176    /// audits it, written by the caller who is in a position to make the
177    /// promise. By the time a [`MappedFile`](molframe_mmap::MappedFile) exists
178    /// the obligation is discharged, which is why this constructor is safe.
179    ///
180    /// The mapping is advised as a single forward pass, which is how every
181    /// parser above this crate reads. A platform that declines the hint is not
182    /// an error.
183    #[cfg(feature = "mmap")]
184    #[must_use]
185    pub fn from_mapped(mapped: molframe_mmap::MappedFile) -> Self {
186        let _advised = mapped.advise_sequential();
187        Self {
188            bytes: InputBytes::Mapped(Arc::new(mapped)),
189            origin: None,
190        }
191    }
192
193    /// Reads a bounded stream and transparently decompresses its bytes.
194    ///
195    /// # Errors
196    ///
197    /// Returns a registered resource or limit diagnostic.
198    pub fn from_reader(reader: impl Read, limits: Limits) -> Result<Self, Diagnostic> {
199        #[cfg(feature = "mmap")]
200        {
201            Self::map_reader(reader, limits)
202        }
203        #[cfg(not(feature = "mmap"))]
204        {
205            let raw = read_bounded(reader, limits).map_err(|error| {
206                Diagnostic::new(Code::E1901)
207                    .with_message("input stream could not be read")
208                    .with_context("reason", error.to_string())
209            })?;
210            let bytes = decompress(raw, limits)?;
211            Ok(Self {
212                bytes: InputBytes::Owned(Arc::new(bytes)),
213                origin: None,
214            })
215        }
216    }
217
218    #[cfg(feature = "mmap")]
219    fn map_reader(reader: impl Read, limits: Limits) -> Result<Self, Diagnostic> {
220        let raw = map_checked(
221            reader,
222            OutputLimit::uncompressed(limits),
223            "input stream could not be read",
224            None,
225        )?;
226        #[cfg(any(feature = "gzip", feature = "zstd"))]
227        let compressed = u64::try_from(raw.as_bytes().len())
228            .map_err(|_| Limits::exceeded("compressed bytes", "more than u64::MAX"))?;
229        let bytes = match Compression::sniff(raw.as_bytes()) {
230            Compression::None => raw,
231            #[cfg(feature = "gzip")]
232            Compression::Gzip => map_gzip_reader(raw.as_bytes(), compressed, limits, None)?,
233            #[cfg(not(feature = "gzip"))]
234            Compression::Gzip => return Err(super::decode::unsupported("gzip")),
235            #[cfg(feature = "zstd")]
236            Compression::Zstd => map_zstd_reader(raw.as_bytes(), compressed, limits, None)?,
237            #[cfg(not(feature = "zstd"))]
238            Compression::Zstd => return Err(super::decode::unsupported("zstd")),
239        };
240        Ok(Self {
241            bytes: InputBytes::Mapped(Arc::new(bytes)),
242            origin: None,
243        })
244    }
245
246    /// The bytes.
247    #[must_use]
248    pub fn as_bytes(&self) -> &[u8] {
249        self.bytes.as_slice()
250    }
251
252    /// The number of bytes.
253    #[must_use]
254    pub fn len(&self) -> usize {
255        self.bytes.as_slice().len()
256    }
257
258    /// Returns true when there is nothing to read.
259    #[must_use]
260    pub fn is_empty(&self) -> bool {
261        self.bytes.as_slice().is_empty()
262    }
263
264    /// How this buffer owns its bytes.
265    #[must_use]
266    pub const fn kind(&self) -> InputKind {
267        self.bytes.kind()
268    }
269
270    /// Where the bytes came from.
271    #[must_use]
272    pub fn origin(&self) -> Option<&str> {
273        self.origin.as_deref()
274    }
275
276    /// Names where the bytes came from.
277    #[must_use]
278    pub fn with_origin(mut self, origin: &str) -> Self {
279        self.origin = Some(Arc::from(origin));
280        self
281    }
282}
molframe_core/io/input/buffer.rs

molframe_core/io/input/
buffer.rs
