ic_host_tools/artifact/
mod.rs

1//! Bounded reads and raw SHA-256 identities for files and downloaded bytes.
2//!
3//! Consumers own trusted paths, admitted digests, and byte limits. An identity
4//! describes the bytes read; it does not freeze a path for later execution or
5//! certify an executable's version. Copying writes only to a caller-owned sink;
6//! paths, confinement, synchronization, publication and cleanup remain local.
7
8use sha2::{Digest, Sha256};
9use std::{
10    fmt,
11    fs::File,
12    io::{self, Read},
13    path::Path,
14    str::FromStr,
15};
16
17mod copy;
18mod gzip;
19#[cfg(unix)]
20mod no_follow;
21#[cfg(test)]
22mod tests;
23mod writer;
24
25pub use copy::{CopyError, copy_reader};
26pub use gzip::{GzipError, decode_gzip};
27#[cfg(unix)]
28pub use no_follow::read_file_no_follow;
29pub use writer::{BoundedWriter, WriterError};
30
31/// Raw SHA-256 identity, without a product-specific prefix or wire format.
32#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
33pub struct Sha256Digest([u8; 32]);
34
35impl Sha256Digest {
36    /// Construct an identity from exact digest bytes.
37    #[must_use]
38    pub const fn from_bytes(bytes: [u8; 32]) -> Self {
39        Self(bytes)
40    }
41
42    /// Borrow the digest bytes.
43    #[must_use]
44    pub const fn as_bytes(&self) -> &[u8; 32] {
45        &self.0
46    }
47
48    /// Hash bytes already held by the caller.
49    #[must_use]
50    pub fn compute(bytes: &[u8]) -> Self {
51        Self(Sha256::digest(bytes).into())
52    }
53}
54
55impl fmt::Display for Sha256Digest {
56    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
57        for byte in self.0 {
58            write!(f, "{byte:02x}")?;
59        }
60        Ok(())
61    }
62}
63
64/// Invalid lowercase hexadecimal digest authority.
65#[derive(Clone, Copy, Debug, Eq, PartialEq)]
66pub enum DigestParseError {
67    /// The digest does not contain exactly 64 ASCII bytes.
68    Length {
69        /// Observed byte length.
70        actual: usize,
71    },
72    /// A byte is not a lowercase hexadecimal digit.
73    Digit {
74        /// Offset of the invalid byte.
75        offset: usize,
76    },
77}
78
79impl fmt::Display for DigestParseError {
80    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
81        match self {
82            Self::Length { actual } => {
83                write!(f, "SHA-256 requires 64 hex bytes, received {actual}")
84            }
85            Self::Digit { offset } => write!(f, "invalid lowercase SHA-256 digit at byte {offset}"),
86        }
87    }
88}
89impl std::error::Error for DigestParseError {}
90
91impl FromStr for Sha256Digest {
92    type Err = DigestParseError;
93
94    fn from_str(text: &str) -> Result<Self, Self::Err> {
95        if text.len() != 64 {
96            return Err(DigestParseError::Length { actual: text.len() });
97        }
98        let mut bytes = [0; 32];
99        for (offset, digit) in text.bytes().enumerate() {
100            let nibble = match digit {
101                b'0'..=b'9' => digit - b'0',
102                b'a'..=b'f' => digit - b'a' + 10,
103                _ => return Err(DigestParseError::Digit { offset }),
104            };
105            bytes[offset / 2] |= nibble << if offset % 2 == 0 { 4 } else { 0 };
106        }
107        Ok(Self(bytes))
108    }
109}
110
111/// Exact size and digest of one bounded byte stream.
112#[derive(Clone, Copy, Debug, Eq, PartialEq)]
113pub struct ArtifactIdentity {
114    /// Number of bytes successfully read.
115    pub bytes: u64,
116    /// SHA-256 of those bytes.
117    pub sha256: Sha256Digest,
118}
119
120/// A bounded read or digest verification failed.
121#[derive(Debug)]
122pub enum ArtifactError {
123    /// The reader or filesystem failed. No partial identity is returned.
124    Io(io::Error),
125    /// The selected path is not a regular file.
126    NotRegularFile,
127    /// A stream exceeded the caller's byte allowance.
128    LimitExceeded {
129        /// Maximum permitted bytes.
130        limit: u64,
131    },
132    /// The complete bounded stream does not match the admitted digest.
133    DigestMismatch {
134        /// Caller-selected authority.
135        expected: Sha256Digest,
136        /// Observed size and digest.
137        actual: ArtifactIdentity,
138    },
139    /// Storage could not be allocated for a bounded file read.
140    Allocation(std::collections::TryReserveError),
141}
142
143impl fmt::Display for ArtifactError {
144    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
145        match self {
146            Self::Io(source) => write!(f, "artifact read failed: {source}"),
147            Self::NotRegularFile => f.write_str("artifact path is not a regular file"),
148            Self::LimitExceeded { limit } => write!(f, "artifact exceeds {limit} bytes"),
149            Self::DigestMismatch { expected, actual } => write!(
150                f,
151                "artifact SHA-256 is {}, expected {expected}",
152                actual.sha256
153            ),
154            Self::Allocation(source) => write!(f, "artifact allocation failed: {source}"),
155        }
156    }
157}
158impl std::error::Error for ArtifactError {
159    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
160        match self {
161            Self::Io(source) => Some(source),
162            Self::Allocation(source) => Some(source),
163            _ => None,
164        }
165    }
166}
167impl From<io::Error> for ArtifactError {
168    fn from(source: io::Error) -> Self {
169        Self::Io(source)
170    }
171}
172
173/// Hash one stream with constant working memory and no reliance on metadata.
174///
175/// Reads at most `max_bytes + 1` bytes, using the extra byte to detect overflow.
176/// Retries interrupted reads only; no subprocess or network retry is performed.
177/// A blocking reader's timeouts remain the caller's responsibility.
178///
179/// # Errors
180/// Returns [`ArtifactError::LimitExceeded`] or the underlying read error.
181/// An impossible reader byte count returns IO [`io::ErrorKind::InvalidData`].
182pub fn hash_reader(
183    mut reader: impl Read,
184    max_bytes: u64,
185) -> Result<ArtifactIdentity, ArtifactError> {
186    let mut hasher = Sha256::new();
187    let bytes = visit_reader::<ArtifactError>(&mut reader, max_bytes, |chunk| {
188        hasher.update(chunk);
189        Ok(())
190    })?;
191    Ok(ArtifactIdentity {
192        bytes,
193        sha256: Sha256Digest(hasher.finalize().into()),
194    })
195}
196
197/// Verify a complete bounded stream before the caller extracts or uses its bytes.
198///
199/// # Errors
200/// Returns read/limit failures or a typed mismatch with the observed identity.
201pub fn verify_reader(
202    reader: impl Read,
203    max_bytes: u64,
204    expected: Sha256Digest,
205) -> Result<ArtifactIdentity, ArtifactError> {
206    let actual = hash_reader(reader, max_bytes)?;
207    if actual.sha256 != expected {
208        return Err(ArtifactError::DigestMismatch { expected, actual });
209    }
210    Ok(actual)
211}
212
213/// Hash a regular file without loading it into memory.
214///
215/// Trusted path selection and protection against concurrent replacement remain
216/// caller-owned. This does not admit a path for later execution.
217///
218/// # Errors
219/// Returns filesystem, non-regular-file, or byte-limit failures.
220pub fn hash_file(path: &Path, max_bytes: u64) -> Result<ArtifactIdentity, ArtifactError> {
221    hash_reader(open_file(path, max_bytes)?, max_bytes)
222}
223
224/// Read a regular artifact into bounded, fallibly allocated storage.
225///
226/// Metadata is an early rejection only; the stream is bounded independently to
227/// detect growth. Paths must be selected from a caller-controlled filesystem.
228///
229/// # Errors
230/// Returns filesystem, non-regular-file, allocation, or byte-limit failures.
231pub fn read_file(path: &Path, max_bytes: usize) -> Result<Vec<u8>, ArtifactError> {
232    read_reader(open_file(path, max_bytes as u64)?, max_bytes)
233}
234
235/// Validate an already-open regular file and read it into bounded storage.
236///
237/// Consumes the descriptor without reopening a pathname. Its metadata must
238/// describe a regular file whose complete length fits `max_bytes`; the stream
239/// is bounded independently to detect growth. Reading begins at the descriptor's
240/// current position. Supply a newly opened or explicitly rewound descriptor to
241/// read the complete file. Storage is allocated fallibly by [`read_reader`].
242///
243/// Opening policy, confinement, symlink provenance and concurrent modification
244/// remain caller-owned. An already-open special file is rejected before reading;
245/// this cannot undo effects or blocking that occurred when the caller opened it.
246/// An open descriptor retains the selected inode, not immutable contents.
247///
248/// # Errors
249/// Returns metadata, non-regular-file, byte-limit, read or allocation failures.
250pub fn read_opened_file(file: File, max_bytes: usize) -> Result<Vec<u8>, ArtifactError> {
251    check_metadata(&file.metadata()?, max_bytes as u64)?;
252    read_reader(file, max_bytes)
253}
254
255/// Read one stream into bounded, fallibly allocated storage.
256///
257/// Observes at most `max_bytes + 1` bytes to detect overflow. A blocking reader's
258/// deadline remains caller-owned. Only interrupted reads are retried.
259///
260/// # Errors
261/// Returns read, allocation, or byte-limit failures. An impossible reader byte
262/// count returns IO [`io::ErrorKind::InvalidData`].
263pub fn read_reader(mut reader: impl Read, max_bytes: usize) -> Result<Vec<u8>, ArtifactError> {
264    let mut bytes = Vec::new();
265    visit_reader::<ArtifactError>(&mut reader, max_bytes as u64, |chunk| {
266        bytes
267            .try_reserve_exact(chunk.len())
268            .map_err(ArtifactError::Allocation)?;
269        bytes.extend_from_slice(chunk);
270        Ok(())
271    })?;
272    Ok(bytes)
273}
274
275fn open_file(path: &Path, limit: u64) -> Result<File, ArtifactError> {
276    // Reject ordinary special-file inputs before open; callers still own races
277    // in their filesystem and must not supply attacker-controlled path trees.
278    check_metadata(&std::fs::metadata(path)?, limit)?;
279    let file = File::open(path)?;
280    check_metadata(&file.metadata()?, limit)?;
281    Ok(file)
282}
283
284fn check_metadata(metadata: &std::fs::Metadata, limit: u64) -> Result<(), ArtifactError> {
285    if !metadata.is_file() {
286        return Err(ArtifactError::NotRegularFile);
287    }
288    if metadata.len() > limit {
289        return Err(ArtifactError::LimitExceeded { limit });
290    }
291    Ok(())
292}
293
294fn visit_reader<E: From<ArtifactError>>(
295    reader: &mut impl Read,
296    limit: u64,
297    mut visit: impl FnMut(&[u8]) -> Result<(), E>,
298) -> Result<u64, E> {
299    let mut bytes = 0_u64;
300    let mut buffer = [0_u8; 16 * 1024];
301    loop {
302        let remaining = usize::try_from(limit - bytes).unwrap_or(usize::MAX);
303        let allowance = remaining.saturating_add(1).min(buffer.len());
304        let count = match reader.read(&mut buffer[..allowance]) {
305            Ok(count) => count,
306            Err(source) if source.kind() == io::ErrorKind::Interrupted => continue,
307            Err(source) => return Err(ArtifactError::Io(source).into()),
308        };
309        if count > allowance {
310            return Err(ArtifactError::Io(io::Error::new(
311                io::ErrorKind::InvalidData,
312                "reader returned more bytes than its buffer can hold",
313            ))
314            .into());
315        }
316        if count == 0 {
317            return Ok(bytes);
318        }
319        if count as u64 > limit - bytes {
320            return Err(ArtifactError::LimitExceeded { limit }.into());
321        }
322        visit(&buffer[..count])?;
323        bytes += count as u64;
324    }
325}
ic_host_tools/artifact/mod.rs

ic_host_tools/artifact/
mod.rs
