webdataset_core/
fields.rs

1//! The map type used for sample fields.
2//!
3//! Fields are keyed by short file extensions such as `png` or `cls`, and their
4//! order matters — it is the order the files appeared in the archive. That
5//! calls for an insertion-ordered map, and for a hasher that is cheap on short
6//! keys rather than one hardened against adversarial input.
7//!
8//! [`FieldHasher`] is FNV-1a: deterministic, fast on a handful of bytes, and
9//! available without the standard library, which is what lets the data model
10//! build for `no_std` targets.
11
12use core::hash::{BuildHasherDefault, Hasher};
13
14use indexmap::IndexMap;
15
16use crate::prelude::*;
17use crate::value::Value;
18
19/// The hasher used for field maps.
20pub type FieldHasher = BuildHasherDefault<Fnv1a>;
21
22/// An insertion-ordered map from field name to value.
23pub type Fields = IndexMap<String, Value, FieldHasher>;
24
25/// An insertion-ordered map with the same hasher, over any value type.
26pub type Map<V> = IndexMap<String, V, FieldHasher>;
27
28/// An empty field map.
29pub fn fields() -> Fields {
30    Fields::default()
31}
32
33/// The FNV-1a hash, 64-bit variant.
34#[derive(Debug, Clone, Copy)]
35pub struct Fnv1a(u64);
36
37impl Default for Fnv1a {
38    fn default() -> Fnv1a {
39        Fnv1a(0xcbf2_9ce4_8422_2325)
40    }
41}
42
43impl Hasher for Fnv1a {
44    fn finish(&self) -> u64 {
45        self.0
46    }
47
48    fn write(&mut self, bytes: &[u8]) {
49        for byte in bytes {
50            self.0 ^= *byte as u64;
51            self.0 = self.0.wrapping_mul(0x0000_0100_0000_01b3);
52        }
53    }
54}
55
56#[cfg(test)]
57mod tests {
58    use super::*;
59
60    fn hash(text: &str) -> u64 {
61        let mut hasher = Fnv1a::default();
62        hasher.write(text.as_bytes());
63        hasher.finish()
64    }
65
66    #[test]
67    fn hashes_distinctly_and_stably() {
68        assert_eq!(hash("png"), hash("png"));
69        assert_ne!(hash("png"), hash("cls"));
70        assert_ne!(hash("png"), hash("jpg"));
71    }
72
73    #[test]
74    fn matches_the_reference_fnv1a_vector() {
75        // The published FNV-1a 64 test vector for "a".
76        assert_eq!(hash("a"), 0xaf63_dc4c_8601_ec8c);
77    }
78
79    #[test]
80    fn keeps_insertion_order() {
81        let mut map = fields();
82        for name in ["zzz", "aaa", "mmm"] {
83            map.insert(name.to_string(), Value::Null);
84        }
85        assert_eq!(map.keys().map(String::as_str).collect::<Vec<_>>(), ["zzz", "aaa", "mmm"]);
86    }
87}
webdataset_core/fields.rs

webdataset_core/
fields.rs
