ic_host_artifacts/artifact/
mod.rs

1//! Bounded streams and raw SHA-256 identities for artifact bytes.
2//!
3//! Consumers own trusted paths, admitted digests, and byte limits. An identity
4//! describes the bytes read; it does not freeze a path for later execution or
5//! certify an executable's version. Copying writes only to a caller-owned sink;
6//! paths, confinement, synchronization, publication and cleanup remain local.
7
8use sha2::{Digest, Sha256};
9use std::{
10    fmt,
11    io::{self, Read},
12    str::FromStr,
13};
14
15pub mod buffer;
16mod chunks;
17mod copy;
18#[cfg(feature = "gzip")]
19mod gzip;
20mod hashing;
21mod matching;
22#[cfg(test)]
23mod test_support;
24#[cfg(test)]
25mod tests;
26pub mod tree;
27mod writer;
28
29pub use chunks::{ChunkDigestError, chunk_digests};
30pub use copy::{CopyError, copy_reader};
31#[cfg(feature = "gzip")]
32pub use gzip::{
33    GzipError, GzipReadError, copy_gzip, decode_gzip, encode_gzip, gzip_matches, hash_gzip,
34    with_gzip_reader,
35};
36pub use hashing::HashingWriter;
37pub use matching::MatchingWriter;
38pub use writer::{BoundedWriter, WriterError};
39
40use buffer::reserve_bounded;
41
42/// Raw SHA-256 identity, without a product-specific prefix or wire format.
43#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
44pub struct Sha256Digest([u8; 32]);
45
46impl Sha256Digest {
47    /// Construct an identity from exact digest bytes.
48    #[must_use]
49    pub const fn from_bytes(bytes: [u8; 32]) -> Self {
50        Self(bytes)
51    }
52
53    /// Borrow the digest bytes.
54    #[must_use]
55    pub const fn as_bytes(&self) -> &[u8; 32] {
56        &self.0
57    }
58
59    /// Encode the digest as exactly 64 lowercase ASCII hexadecimal bytes.
60    ///
61    /// Returns a fixed-size buffer without allocation or a product-specific
62    /// prefix. Leading zeroes are retained, matching this type's Display output.
63    #[must_use]
64    pub const fn hex_bytes(self) -> [u8; 64] {
65        const HEX: &[u8; 16] = b"0123456789abcdef";
66        let mut encoded = [0; 64];
67        let mut index = 0;
68        while index < self.0.len() {
69            let byte = self.0[index];
70            encoded[index * 2] = HEX[(byte >> 4) as usize];
71            encoded[index * 2 + 1] = HEX[(byte & 0x0f) as usize];
72            index += 1;
73        }
74        encoded
75    }
76
77    /// Hash bytes already held by the caller.
78    #[must_use]
79    pub fn compute(bytes: &[u8]) -> Self {
80        Self(Sha256::digest(bytes).into())
81    }
82}
83
84impl fmt::Display for Sha256Digest {
85    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
86        f.write_str(std::str::from_utf8(&self.hex_bytes()).expect("hex encoding is ASCII"))
87    }
88}
89
90/// Invalid lowercase hexadecimal digest authority.
91#[derive(Clone, Copy, Debug, Eq, PartialEq)]
92pub enum DigestParseError {
93    /// The digest does not contain exactly 64 ASCII bytes.
94    Length {
95        /// Observed byte length.
96        actual: usize,
97    },
98    /// A byte is not a lowercase hexadecimal digit.
99    Digit {
100        /// Offset of the invalid byte.
101        offset: usize,
102    },
103}
104
105impl fmt::Display for DigestParseError {
106    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
107        match self {
108            Self::Length { actual } => {
109                write!(f, "SHA-256 requires 64 hex bytes, received {actual}")
110            }
111            Self::Digit { offset } => write!(f, "invalid lowercase SHA-256 digit at byte {offset}"),
112        }
113    }
114}
115impl std::error::Error for DigestParseError {}
116
117impl FromStr for Sha256Digest {
118    type Err = DigestParseError;
119
120    fn from_str(text: &str) -> Result<Self, Self::Err> {
121        if text.len() != 64 {
122            return Err(DigestParseError::Length { actual: text.len() });
123        }
124        let mut bytes = [0; 32];
125        for (offset, digit) in text.bytes().enumerate() {
126            let nibble = match digit {
127                b'0'..=b'9' => digit - b'0',
128                b'a'..=b'f' => digit - b'a' + 10,
129                _ => return Err(DigestParseError::Digit { offset }),
130            };
131            bytes[offset / 2] |= nibble << if offset % 2 == 0 { 4 } else { 0 };
132        }
133        Ok(Self(bytes))
134    }
135}
136
137/// Exact size and digest of one bounded byte stream.
138#[derive(Clone, Copy, Debug, Eq, PartialEq)]
139pub struct ArtifactIdentity {
140    /// Number of bytes successfully read.
141    pub bytes: u64,
142    /// SHA-256 of those bytes.
143    pub sha256: Sha256Digest,
144}
145
146impl ArtifactIdentity {
147    /// Compare this observed digest with caller-selected authority.
148    ///
149    /// Does not read, hash, freeze or validate the origin of the identity.
150    /// Size admission and selection of the expected digest remain caller-owned.
151    ///
152    /// # Errors
153    /// Returns a typed mismatch retaining the expected digest and this identity.
154    pub fn verify(self, expected: Sha256Digest) -> Result<Self, ArtifactError> {
155        if self.sha256 != expected {
156            return Err(ArtifactError::DigestMismatch {
157                expected,
158                actual: self,
159            });
160        }
161        Ok(self)
162    }
163}
164
165/// A bounded read or digest verification failed.
166#[derive(Debug)]
167pub enum ArtifactError {
168    /// The reader or filesystem failed. No partial identity is returned.
169    Io(io::Error),
170    /// The selected path is not a regular file.
171    NotRegularFile,
172    /// A stream exceeded the caller's byte allowance.
173    LimitExceeded {
174        /// Maximum permitted bytes.
175        limit: u64,
176    },
177    /// A bounded stream ended before the caller's exact byte count.
178    LengthMismatch {
179        /// Required bytes.
180        expected: u64,
181        /// Successfully read bytes.
182        actual: u64,
183    },
184    /// The complete bounded stream does not match the admitted digest.
185    DigestMismatch {
186        /// Caller-selected authority.
187        expected: Sha256Digest,
188        /// Observed size and digest.
189        actual: ArtifactIdentity,
190    },
191    /// Storage could not be allocated for a bounded file read.
192    Allocation(std::collections::TryReserveError),
193}
194
195impl fmt::Display for ArtifactError {
196    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
197        match self {
198            Self::Io(source) => write!(f, "artifact read failed: {source}"),
199            Self::NotRegularFile => f.write_str("artifact path is not a regular file"),
200            Self::LimitExceeded { limit } => write!(f, "artifact exceeds {limit} bytes"),
201            Self::LengthMismatch { expected, actual } => {
202                write!(f, "artifact has {actual} bytes, expected {expected}")
203            }
204            Self::DigestMismatch { expected, actual } => write!(
205                f,
206                "artifact SHA-256 is {}, expected {expected}",
207                actual.sha256
208            ),
209            Self::Allocation(source) => write!(f, "artifact allocation failed: {source}"),
210        }
211    }
212}
213impl std::error::Error for ArtifactError {
214    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
215        match self {
216            Self::Io(source) => Some(source),
217            Self::Allocation(source) => Some(source),
218            _ => None,
219        }
220    }
221}
222impl From<io::Error> for ArtifactError {
223    fn from(source: io::Error) -> Self {
224        Self::Io(source)
225    }
226}
227
228/// Preserve native I/O identity or retain the typed artifact cause at an I/O boundary.
229impl From<ArtifactError> for io::Error {
230    fn from(error: ArtifactError) -> Self {
231        match error {
232            ArtifactError::Io(source) => source,
233            other => Self::other(other),
234        }
235    }
236}
237
238/// Hash one stream with constant working memory and no reliance on metadata.
239///
240/// Reads at most `max_bytes + 1` bytes, using the extra byte to detect overflow.
241/// Retries interrupted reads only; no subprocess or network retry is performed.
242/// A blocking reader's timeouts remain the caller's responsibility.
243/// Admit the returned identity against a selected digest with [`ArtifactIdentity::verify`].
244///
245/// # Errors
246/// Returns [`ArtifactError::LimitExceeded`] or the underlying read error.
247/// An impossible reader byte count returns IO [`io::ErrorKind::InvalidData`].
248pub fn hash_reader(
249    mut reader: impl Read,
250    max_bytes: u64,
251) -> Result<ArtifactIdentity, ArtifactError> {
252    let mut hasher = Sha256::new();
253    let bytes =
254        visit_reader::<ArtifactError>(&mut reader, &mut [0; 16 * 1024], max_bytes, |chunk| {
255            hasher.update(chunk);
256            Ok(())
257        })?;
258    Ok(ArtifactIdentity {
259        bytes,
260        sha256: Sha256Digest(hasher.finalize().into()),
261    })
262}
263
264/// Hash a stream whose complete length must equal `expected_bytes`.
265///
266/// Reads at most one byte beyond the selected length to distinguish overflow.
267/// The caller owns the reader's origin, deadline and byte-budget selection.
268///
269/// # Errors
270/// Returns a typed short-length or limit error, or the underlying read error.
271pub fn hash_exact_reader(
272    reader: impl Read,
273    expected_bytes: u64,
274) -> Result<ArtifactIdentity, ArtifactError> {
275    let identity = hash_reader(reader, expected_bytes)?;
276    if identity.bytes != expected_bytes {
277        return Err(ArtifactError::LengthMismatch {
278            expected: expected_bytes,
279            actual: identity.bytes,
280        });
281    }
282    Ok(identity)
283}
284
285/// Read one stream into bounded, fallibly allocated storage.
286///
287/// Observes at most `max_bytes + 1` bytes to detect overflow. A blocking reader's
288/// deadline remains caller-owned. Only interrupted reads are retried.
289///
290/// # Errors
291/// Returns read, allocation, or byte-limit failures. An impossible reader byte
292/// count returns IO [`io::ErrorKind::InvalidData`].
293pub fn read_reader(mut reader: impl Read, max_bytes: usize) -> Result<Vec<u8>, ArtifactError> {
294    let mut bytes = Vec::new();
295    visit_reader::<ArtifactError>(
296        &mut reader,
297        &mut [0; 16 * 1024],
298        max_bytes as u64,
299        |chunk| {
300            reserve_bounded(&mut bytes, chunk.len(), max_bytes)
301                .map_err(ArtifactError::Allocation)?;
302            bytes.extend_from_slice(chunk);
303            Ok(())
304        },
305    )?;
306    Ok(bytes)
307}
308
309/// Read a complete stream into bounded storage at an exact selected length.
310///
311/// The caller owns the reader's origin, deadline and byte-budget selection.
312/// Reads at most one byte beyond the selected length to distinguish overflow.
313///
314/// # Errors
315/// Returns a typed short-length or limit error, allocation failure, or read error.
316pub fn read_exact_reader(
317    reader: impl Read,
318    expected_bytes: usize,
319) -> Result<Vec<u8>, ArtifactError> {
320    let bytes = read_reader(reader, expected_bytes)?;
321    if bytes.len() != expected_bytes {
322        return Err(ArtifactError::LengthMismatch {
323            expected: expected_bytes as u64,
324            actual: bytes.len() as u64,
325        });
326    }
327    Ok(bytes)
328}
329
330/// Visit bounded stream chunks using nonempty caller-owned scratch space.
331///
332/// Returns the complete byte count, observing at most `limit + 1` bytes. Only
333/// interrupted reads are retried. An overflowing chunk is not passed to `visit`;
334/// earlier callback effects remain when a later read, limit or callback fails.
335/// Chunks borrow `buffer` only during each call. Its contents after return are
336/// unspecified. Callers own framing, retained observations and blocking deadlines.
337/// No allocation, digest framing, filesystem or lifecycle policy is supplied.
338///
339/// # Errors
340/// Returns original callback errors or converted [`ArtifactError`] read/limit
341/// errors. Empty scratch is invalid input; impossible read counts are invalid data.
342pub fn visit_reader<E: From<ArtifactError>>(
343    reader: &mut impl Read,
344    buffer: &mut [u8],
345    limit: u64,
346    mut visit: impl FnMut(&[u8]) -> Result<(), E>,
347) -> Result<u64, E> {
348    let mut bytes = 0_u64;
349    if buffer.is_empty() {
350        return Err(ArtifactError::Io(io::Error::new(
351            io::ErrorKind::InvalidInput,
352            "stream scratch buffer must not be empty",
353        ))
354        .into());
355    }
356    loop {
357        let remaining = usize::try_from(limit - bytes).unwrap_or(usize::MAX);
358        let allowance = remaining.saturating_add(1).min(buffer.len());
359        let count = match reader.read(&mut buffer[..allowance]) {
360            Ok(count) => count,
361            Err(source) if source.kind() == io::ErrorKind::Interrupted => continue,
362            Err(source) => return Err(ArtifactError::Io(source).into()),
363        };
364        if count > allowance {
365            return Err(ArtifactError::Io(io::Error::new(
366                io::ErrorKind::InvalidData,
367                "reader returned more bytes than its buffer can hold",
368            ))
369            .into());
370        }
371        if count == 0 {
372            return Ok(bytes);
373        }
374        if count as u64 > limit - bytes {
375            return Err(ArtifactError::LimitExceeded { limit }.into());
376        }
377        visit(&buffer[..count])?;
378        bytes += count as u64;
379    }
380}
ic_host_artifacts/artifact/mod.rs

ic_host_artifacts/artifact/
mod.rs
