Skip to main content

shaul_core/
checksum.rs

1use std::fmt;
2use std::io::Read;
3use std::path::{Path, PathBuf};
4use std::str::FromStr;
5
6use digest::Digest;
7use serde::{Deserialize, Serialize};
8use sha1::Sha1;
9use sha2::{Sha256, Sha512};
10
11use crate::error::{Error, Result};
12
13/// Digest algorithms supported for checksum verification.
14#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
15#[serde(rename_all = "lowercase")]
16#[non_exhaustive]
17pub enum ChecksumAlgorithm {
18    /// SHA-256, 64 hex characters.
19    Sha256,
20    /// SHA-512, 128 hex characters.
21    Sha512,
22    /// SHA-1, 40 hex characters.
23    Sha1,
24    /// MD5, 32 hex characters.
25    Md5,
26}
27
28impl ChecksumAlgorithm {
29    /// Length of the digest in hex characters.
30    pub fn hex_len(self) -> usize {
31        match self {
32            Self::Sha256 => 64,
33            Self::Sha512 => 128,
34            Self::Sha1 => 40,
35            Self::Md5 => 32,
36        }
37    }
38
39    /// Lowercase algorithm name as used in checksum strings.
40    pub fn as_str(self) -> &'static str {
41        match self {
42            Self::Sha256 => "sha256",
43            Self::Sha512 => "sha512",
44            Self::Sha1 => "sha1",
45            Self::Md5 => "md5",
46        }
47    }
48
49    fn from_token(token: &str) -> Option<Self> {
50        match token.to_ascii_lowercase().replace('-', "").as_str() {
51            "sha256" => Some(Self::Sha256),
52            "sha512" => Some(Self::Sha512),
53            "sha1" => Some(Self::Sha1),
54            "md5" => Some(Self::Md5),
55            _ => None,
56        }
57    }
58
59    fn from_hex_len(len: usize) -> Option<Self> {
60        match len {
61            64 => Some(Self::Sha256),
62            128 => Some(Self::Sha512),
63            40 => Some(Self::Sha1),
64            32 => Some(Self::Md5),
65            _ => None,
66        }
67    }
68}
69
70impl FromStr for ChecksumAlgorithm {
71    type Err = Error;
72
73    fn from_str(value: &str) -> Result<Self> {
74        Self::from_token(value)
75            .ok_or_else(|| Error::Config(format!("unknown checksum algorithm: {value}")))
76    }
77}
78
79impl fmt::Display for ChecksumAlgorithm {
80    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
81        f.write_str(self.as_str())
82    }
83}
84
85/// An expected content digest, for example `sha256:abc...`.
86#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
87pub struct Checksum {
88    /// Algorithm that produced the digest.
89    pub algorithm: ChecksumAlgorithm,
90    /// Lowercase hexadecimal digest.
91    pub hex: String,
92}
93
94impl Checksum {
95    /// Validates `hex` against `algorithm` and builds a checksum.
96    pub fn new(algorithm: ChecksumAlgorithm, hex: impl AsRef<str>) -> Result<Self> {
97        let hex = hex.as_ref().trim().trim_matches('"');
98        let hex = hex.strip_prefix("0x").unwrap_or(hex).to_ascii_lowercase();
99        if hex.len() != algorithm.hex_len() || hex::decode(&hex).is_err() {
100            return Err(Error::Config(format!(
101                "invalid {} checksum: {hex}",
102                algorithm.as_str()
103            )));
104        }
105        Ok(Self { algorithm, hex })
106    }
107
108    /// Accepts `sha256:<hex>`, `sha-256:<hex>` or a bare hex digest whose
109    /// length identifies the algorithm.
110    pub fn parse(value: &str) -> Result<Self> {
111        let value = value.trim();
112        if let Some((algorithm, hex)) = value.split_once(':') {
113            let algorithm = ChecksumAlgorithm::from_str(algorithm)?;
114            return Self::new(algorithm, hex);
115        }
116
117        let algorithm = ChecksumAlgorithm::from_hex_len(value.len()).ok_or_else(|| {
118            Error::Config(format!(
119                "cannot infer checksum algorithm from {value}; use <algorithm>:<hex>"
120            ))
121        })?;
122        Self::new(algorithm, value)
123    }
124
125    /// Hugging Face LFS exposes the SHA-256 digest through `X-Linked-Etag`
126    /// (usually quoted). Weak ETags are not checksums.
127    pub fn from_etag(value: &str) -> Option<Self> {
128        let value = value.trim();
129        if value.starts_with("W/") {
130            return None;
131        }
132        let value = value.trim_matches('"');
133        let algorithm = ChecksumAlgorithm::from_hex_len(value.len())?;
134        Self::new(algorithm, value).ok()
135    }
136
137    /// Returns whether both checksums use the same algorithm and digest,
138    /// ignoring hex case.
139    pub fn matches(&self, actual: &Checksum) -> bool {
140        self.algorithm == actual.algorithm && self.hex.eq_ignore_ascii_case(&actual.hex)
141    }
142}
143
144impl fmt::Display for Checksum {
145    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
146        write!(f, "{}:{}", self.algorithm, self.hex)
147    }
148}
149
150impl FromStr for Checksum {
151    type Err = Error;
152
153    fn from_str(value: &str) -> Result<Self> {
154        Self::parse(value)
155    }
156}
157
158/// Compute a file digest without blocking the async runtime.
159pub async fn compute_file_checksum(
160    path: PathBuf,
161    algorithm: ChecksumAlgorithm,
162) -> Result<Checksum> {
163    tokio::task::spawn_blocking(move || compute_file_checksum_blocking(&path, algorithm))
164        .await
165        .map_err(|error| Error::Other(format!("checksum task failed: {error}")))?
166}
167
168/// Compute a file digest on the calling thread.
169///
170/// Use [`compute_file_checksum`] inside async code so the read does not block
171/// the runtime.
172pub fn compute_file_checksum_blocking(
173    path: &Path,
174    algorithm: ChecksumAlgorithm,
175) -> Result<Checksum> {
176    let file = std::fs::File::open(path)?;
177    let mut reader = std::io::BufReader::with_capacity(1 << 20, file);
178    let mut hasher = AnyHasher::new(algorithm);
179    let mut buffer = vec![0u8; 1 << 20];
180    loop {
181        let read = reader.read(&mut buffer)?;
182        if read == 0 {
183            break;
184        }
185        hasher.update(&buffer[..read]);
186    }
187    Ok(Checksum {
188        algorithm,
189        hex: hasher.finalize_hex(),
190    })
191}
192
193enum AnyHasher {
194    Sha256(Sha256),
195    Sha512(Sha512),
196    Sha1(Sha1),
197    Md5(md5::Md5),
198}
199
200impl AnyHasher {
201    fn new(algorithm: ChecksumAlgorithm) -> Self {
202        match algorithm {
203            ChecksumAlgorithm::Sha256 => Self::Sha256(Sha256::new()),
204            ChecksumAlgorithm::Sha512 => Self::Sha512(Sha512::new()),
205            ChecksumAlgorithm::Sha1 => Self::Sha1(Sha1::new()),
206            ChecksumAlgorithm::Md5 => Self::Md5(md5::Md5::new()),
207        }
208    }
209
210    fn update(&mut self, data: &[u8]) {
211        match self {
212            Self::Sha256(hasher) => hasher.update(data),
213            Self::Sha512(hasher) => hasher.update(data),
214            Self::Sha1(hasher) => hasher.update(data),
215            Self::Md5(hasher) => hasher.update(data),
216        }
217    }
218
219    fn finalize_hex(self) -> String {
220        match self {
221            Self::Sha256(hasher) => hex::encode(hasher.finalize()),
222            Self::Sha512(hasher) => hex::encode(hasher.finalize()),
223            Self::Sha1(hasher) => hex::encode(hasher.finalize()),
224            Self::Md5(hasher) => hex::encode(hasher.finalize()),
225        }
226    }
227}
228
229#[cfg(test)]
230mod tests {
231    use super::*;
232
233    #[test]
234    fn parses_prefixed_and_bare_checksums() {
235        let prefixed = Checksum::parse(
236            "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
237        )
238        .unwrap();
239        assert_eq!(prefixed.algorithm, ChecksumAlgorithm::Sha256);
240        let bare = Checksum::parse("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").unwrap();
241        assert_eq!(bare.algorithm, ChecksumAlgorithm::Md5);
242    }
243
244    #[test]
245    fn rejects_invalid_checksum() {
246        assert!(Checksum::parse("sha256:xyz").is_err());
247    }
248
249    #[test]
250    fn etag_is_sha256() {
251        let etag = "\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"";
252        let checksum = Checksum::from_etag(etag).unwrap();
253        assert_eq!(checksum.algorithm, ChecksumAlgorithm::Sha256);
254    }
255}