use serde::{Deserialize, Deserializer, Serialize, Serializer, de};
use sha2::{Digest, Sha256};
use unicode_normalization::UnicodeNormalization;
pub const CONTENT_HASH_VERSION: u32 = 1;
const DOMAIN_PREFIX: &str = "trusty-memory/content-hash/v";
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct ContentHash([u8; 32]);
impl ContentHash {
pub const UNSET: Self = Self([0u8; 32]);
pub fn is_unset(&self) -> bool {
self.0 == [0u8; 32]
}
pub fn as_bytes(&self) -> &[u8; 32] {
&self.0
}
pub fn to_hex(self) -> String {
hex::encode(self.0)
}
pub fn from_hex(s: &str) -> Result<Self, ContentHashParseError> {
let bytes = hex::decode(s).map_err(|_| ContentHashParseError {
got: s.chars().take(72).collect(),
})?;
let arr: [u8; 32] = bytes.try_into().map_err(|_| ContentHashParseError {
got: s.chars().take(72).collect(),
})?;
Ok(Self(arr))
}
}
#[derive(Debug, Clone, thiserror::Error)]
#[error("not a 64-character hex SHA-256 digest: {got:?}")]
pub struct ContentHashParseError {
pub got: String,
}
impl std::fmt::Display for ContentHash {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.to_hex())
}
}
impl Default for ContentHash {
fn default() -> Self {
Self::UNSET
}
}
impl Serialize for ContentHash {
fn serialize<S: Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
s.serialize_str(&self.to_hex())
}
}
impl<'de> Deserialize<'de> for ContentHash {
fn deserialize<D: Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
let s = String::deserialize(d)?;
Self::from_hex(&s).map_err(de::Error::custom)
}
}
pub fn normalize_for_hash(content: &str) -> String {
let unix: String = content.replace("\r\n", "\n").replace('\r', "\n");
let visible: String = unix
.chars()
.filter_map(|c| match c {
'\u{200B}' | '\u{200C}' | '\u{200D}' | '\u{FEFF}' => None,
'\u{00A0}' => Some(' '),
other => Some(other),
})
.collect();
let composed: String = visible.nfc().collect();
let mut out = String::with_capacity(composed.len());
for (i, line) in composed.split('\n').enumerate() {
if i > 0 {
out.push('\n');
}
out.push_str(line.trim_end());
}
let trimmed_len = out.trim_end().len();
out.truncate(trimmed_len);
out
}
pub fn memory_content_hash(content: &str) -> ContentHash {
let mut hasher = Sha256::new();
hasher.update(DOMAIN_PREFIX.as_bytes());
hasher.update(CONTENT_HASH_VERSION.to_string().as_bytes());
hasher.update([0u8]);
hasher.update(normalize_for_hash(content).as_bytes());
let digest = hasher.finalize();
let mut out = [0u8; 32];
out.copy_from_slice(&digest);
ContentHash(out)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn normalize_collapses_line_endings() {
assert_eq!(normalize_for_hash("a\r\nb"), "a\nb");
assert_eq!(normalize_for_hash("a\rb"), "a\nb");
assert_eq!(normalize_for_hash("a\nb"), "a\nb");
}
#[test]
fn normalize_trims_trailing_space_per_line() {
assert_eq!(normalize_for_hash("a \nb\t\nc"), "a\nb\nc");
}
#[test]
fn normalize_collapses_trailing_newlines() {
assert_eq!(normalize_for_hash("fact"), "fact");
assert_eq!(normalize_for_hash("fact\n"), "fact");
assert_eq!(normalize_for_hash("fact\n\n\n"), "fact");
assert_eq!(normalize_for_hash("fact\n \n\t\n"), "fact");
}
#[test]
fn normalize_applies_nfc() {
let decomposed = "caf\u{0065}\u{0301}";
let composed = "caf\u{00e9}";
assert_ne!(decomposed, composed, "the inputs must differ as bytes");
assert_eq!(normalize_for_hash(decomposed), normalize_for_hash(composed));
}
#[test]
fn normalize_preserves_leading_whitespace_and_interior_blanks() {
assert_eq!(normalize_for_hash(" indented"), " indented");
assert_eq!(normalize_for_hash("a\n\nb"), "a\n\nb");
assert_ne!(normalize_for_hash(" a"), normalize_for_hash("a"));
}
#[test]
fn normalize_of_only_whitespace_is_empty() {
assert_eq!(normalize_for_hash(" \n\t\r\n "), "");
assert_eq!(normalize_for_hash(""), "");
}
#[test]
fn normalize_strips_zero_width_characters() {
let clean = "the daemon binds loopback only";
assert_eq!(
normalize_for_hash("the daemon\u{200B} binds"),
"the daemon binds"
);
assert_eq!(normalize_for_hash("the\u{200C} daemon"), "the daemon");
assert_eq!(normalize_for_hash("the\u{200D} daemon"), "the daemon");
assert_eq!(normalize_for_hash("\u{FEFF}the daemon"), "the daemon");
assert_eq!(normalize_for_hash("the daemon\u{FEFF}"), "the daemon");
let pasted = "\u{FEFF}the\u{200B} daemon\u{200C} binds\u{200D} loopback only";
assert_eq!(normalize_for_hash(pasted), clean);
assert_eq!(memory_content_hash(pasted), memory_content_hash(clean));
assert_eq!(normalize_for_hash("\u{200B}\u{200C}\u{200D}\u{FEFF}"), "");
}
#[test]
fn normalize_folds_nbsp_to_a_space() {
assert_eq!(normalize_for_hash("a\u{00A0}b"), "a b");
assert_eq!(
memory_content_hash("MSRV\u{00A0}1.94"),
memory_content_hash("MSRV 1.94")
);
assert_eq!(normalize_for_hash("a\u{00A0}\nb\u{00A0}"), "a\nb");
assert_eq!(normalize_for_hash("\u{00A0}\u{00A0}"), "");
assert_eq!(normalize_for_hash("\u{00A0}a"), " a");
}
#[test]
fn normalize_preserves_bidi_marks() {
assert_eq!(normalize_for_hash("a\u{200E}b"), "a\u{200E}b");
assert_eq!(normalize_for_hash("a\u{200F}b"), "a\u{200F}b");
assert_ne!(memory_content_hash("a\u{200E}b"), memory_content_hash("ab"));
}
#[test]
fn normalize_preserves_non_bmp_codepoints() {
assert_eq!(normalize_for_hash("ship it 🚀"), "ship it 🚀");
assert_eq!(normalize_for_hash("𝔘𝔫𝔦𝔠𝔬𝔡𝔢"), "𝔘𝔫𝔦𝔠𝔬𝔡𝔢");
assert_eq!(normalize_for_hash("𝔘\u{200B}𝔫"), "𝔘𝔫");
assert_eq!(
memory_content_hash("cafe\u{0301}\u{200B} time"),
memory_content_hash("caf\u{00E9} time")
);
}
#[test]
fn a_zero_width_between_base_and_mark_still_composes() {
let wedged = "caf\u{0065}\u{200B}\u{0301} time";
let composed = "caf\u{00E9} time";
assert_ne!(wedged, composed, "the inputs must differ as bytes");
assert_eq!(normalize_for_hash(wedged), composed);
assert_eq!(memory_content_hash(wedged), memory_content_hash(composed));
for zw in ['\u{200C}', '\u{200D}', '\u{FEFF}'] {
let body = format!("caf\u{0065}{zw}\u{0301} time");
assert_eq!(
memory_content_hash(&body),
memory_content_hash(composed),
"{zw:?} between base and mark blocked the composition"
);
}
}
#[test]
fn normalize_of_invisible_only_bodies_is_one_identity() {
let empty = memory_content_hash("");
for body in [
"",
" ",
"\n\n",
"\u{200B}",
"\u{FEFF}\u{00A0}\n\u{200D} \r\n",
] {
assert_eq!(
memory_content_hash(body),
empty,
"body {body:?} must hash as the empty body"
);
}
}
#[test]
fn hash_hex_round_trips() {
let h = memory_content_hash("a fact");
let hex = h.to_hex();
assert_eq!(hex.len(), 64);
assert_eq!(ContentHash::from_hex(&hex).unwrap(), h);
assert_eq!(h.to_string(), hex);
}
#[test]
fn parse_rejects_a_short_or_non_hex_digest() {
assert!(ContentHash::from_hex("deadbeef").is_err());
assert!(ContentHash::from_hex("").is_err());
assert!(ContentHash::from_hex(&"z".repeat(64)).is_err());
}
#[test]
fn unset_is_distinguishable_from_a_real_digest() {
assert!(ContentHash::UNSET.is_unset());
assert!(ContentHash::default().is_unset());
assert!(!memory_content_hash("").is_unset());
assert!(!memory_content_hash("x").is_unset());
}
#[test]
fn hash_is_stable_for_a_known_body() {
assert_eq!(
memory_content_hash("the daemon binds loopback only").to_hex(),
"5b00cdfb7e5932bd483cdb66a70fbc680693a056d6bc708ef3182cfdff0f31da"
);
}
#[test]
fn hash_ignores_line_ending_and_trailing_newline() {
let base = memory_content_hash("line one\nline two");
assert_eq!(memory_content_hash("line one\r\nline two"), base);
assert_eq!(memory_content_hash("line one\nline two\n"), base);
assert_eq!(memory_content_hash("line one\r\nline two\r\n\r\n"), base);
assert_eq!(memory_content_hash("line one \nline two\t"), base);
}
#[test]
fn hash_ignores_unicode_composition_form() {
assert_eq!(
memory_content_hash("caf\u{0065}\u{0301} rules"),
memory_content_hash("caf\u{00e9} rules")
);
}
#[test]
fn hash_distinguishes_different_bodies() {
assert_ne!(memory_content_hash("a"), memory_content_hash("b"));
assert_ne!(memory_content_hash("Fact"), memory_content_hash("fact"));
assert_ne!(memory_content_hash(" fact"), memory_content_hash("fact"));
}
#[test]
fn hash_is_not_a_bare_sha256_of_the_body() {
let bare = {
let mut h = Sha256::new();
h.update(b"a fact");
hex::encode(h.finalize())
};
assert_ne!(memory_content_hash("a fact").to_hex(), bare);
}
#[test]
fn domain_separator_pins_the_version() {
assert_eq!(CONTENT_HASH_VERSION, 1);
let expected = {
let mut h = Sha256::new();
h.update(b"trusty-memory/content-hash/v1");
h.update([0u8]);
h.update(b"a fact");
hex::encode(h.finalize())
};
assert_eq!(memory_content_hash("a fact").to_hex(), expected);
}
}