malwaredb_server/db/
types.rs

1// SPDX-License-Identifier: Apache-2.0
2
3use malwaredb_api::{SupportedFileType, SupportedFileTypes};
4use malwaredb_types::utils::EntropyCalc;
5
6use fuzzyhash::FuzzyHash;
7use human_hash::humanize;
8use magic::cookie::DatabasePaths;
9use malwaredb_lzjd::{LZDict, Murmur3HashState};
10use md5::Md5;
11use sha1::Sha1;
12use sha2::{Digest, Sha256, Sha384, Sha512};
13use tlsh_fixed::TlshBuilder;
14use tracing::error;
15use uuid::Uuid;
16
17/// Metadata about a file for storing as a record in Malware DB
18#[derive(Debug, Clone)]
19pub struct FileMetadata {
20    /// File name
21    pub name: Option<String>,
22
23    /// File size in bytes
24    pub size: u64,
25
26    /// Entropy of the file
27    pub entropy: f32,
28
29    /// SHA-1 hash
30    pub sha1: Vec<u8>,
31
32    /// SHA-256 hash
33    pub sha256: Vec<u8>,
34
35    /// SHA-384 hash
36    pub sha384: Vec<u8>,
37
38    /// SHA-512 hash
39    pub sha512: Vec<u8>,
40
41    /// MD5 hash
42    pub md5: Uuid,
43
44    /// `LZJD` similarity hash
45    pub lzjd: Option<String>,
46
47    /// `SSDeep` similarity hash, if the file is large enough
48    pub ssdeep: Option<String>,
49
50    /// Trend Micro's similarity hash (distance metric)
51    pub tlsh: Option<String>,
52
53    /// Human Hash, based on <https://github.com/zacharyvoase/humanhash>
54    pub humanhash: String,
55
56    /// File command (or libmagic) description of the file
57    pub file_command: String,
58}
59
60impl FileMetadata {
61    /// Get the collection of file measurements given a byte sequence
62    ///
63    /// # Panics
64    ///
65    /// This won't actually panic despite a call to `.unwrap()` because the input to that function
66    /// is known to always be the correct size.
67    pub fn new(contents: &[u8], name: Option<&str>) -> Self {
68        let mut sha1 = Sha1::new();
69        sha1.update(contents);
70        let sha1 = sha1.finalize();
71
72        let mut sha256 = Sha256::new();
73        sha256.update(contents);
74        let sha256 = sha256.finalize();
75
76        let mut sha384 = Sha384::new();
77        sha384.update(contents);
78        let sha384 = sha384.finalize();
79
80        let mut sha512 = Sha512::new();
81        sha512.update(contents);
82        let sha512 = sha512.finalize();
83
84        let mut md5 = Md5::new();
85        md5.update(contents);
86        let md5 = md5.finalize();
87
88        let build_hasher = Murmur3HashState::default();
89        let lzjd_str =
90            LZDict::from_bytes_stream(contents.iter().copied(), &build_hasher).to_string();
91
92        let mut builder = TlshBuilder::new(
93            tlsh_fixed::BucketKind::Bucket256,
94            tlsh_fixed::ChecksumKind::ThreeByte,
95            tlsh_fixed::Version::Version4,
96        );
97
98        builder.update(contents);
99
100        let tlsh = if let Ok(hasher) = builder.build() {
101            Some(hasher.hash())
102        } else {
103            None
104        };
105
106        // This won't panic since the MD5 hash is 16 bytes long
107        let md5 = Uuid::from_bytes(uuid::Bytes::from(md5));
108
109        let file_command = {
110            if let Ok(cookie) = magic::Cookie::open(magic::cookie::Flags::ERROR) {
111                let db_paths = DatabasePaths::default();
112                if let Ok(cookie) = cookie.load(&db_paths) {
113                    if let Ok(output) = cookie.buffer(contents) {
114                        output
115                    } else {
116                        error!("LibMagic: failed to get output for buffer");
117                        String::new()
118                    }
119                } else {
120                    error!("LibMagic: failed to load signature database");
121                    String::new()
122                }
123            } else {
124                error!("LibMagic: failed to get handle");
125                String::new()
126            }
127        };
128
129        Self {
130            name: name.map(str::to_ascii_lowercase),
131            size: contents.len() as u64,
132            entropy: contents.entropy(),
133            sha1: sha1.to_vec(),
134            sha256: sha256.to_vec(),
135            sha384: sha384.to_vec(),
136            sha512: sha512.to_vec(),
137            md5,
138            lzjd: Some(lzjd_str),
139            ssdeep: Some(FuzzyHash::new(contents).to_string()),
140            tlsh,
141            humanhash: humanize(&md5, 4),
142            file_command,
143        }
144    }
145}
146
147/// File Types known to Malware DB; a magic number has to be matched to a database ID.
148#[derive(Debug, Clone)]
149pub struct FileType {
150    /// Database ID number
151    pub id: u32,
152
153    /// Friendly name
154    pub name: String,
155
156    /// Description of the type
157    pub description: Option<String>,
158
159    /// Magic numbers as bytes
160    /// These are the first few bytes of the file which identify it's type
161    /// Some types have more than one possible magic number, though it's rare
162    pub magic: Vec<Vec<u8>>,
163
164    /// Whether or not this file is executable on some system
165    /// Assumption: if not executable, it's a document
166    pub executable: bool,
167}
168
169/// File types, just simple Vec wrapper around [`FileType`].
170pub struct FileTypes(pub Vec<FileType>);
171
172impl From<FileType> for SupportedFileType {
173    fn from(value: FileType) -> Self {
174        Self {
175            name: value.name,
176            magic: value.magic.iter().map(hex::encode).collect(),
177            is_executable: value.executable,
178            description: value.description,
179        }
180    }
181}
182
183impl From<FileTypes> for SupportedFileTypes {
184    fn from(value: FileTypes) -> Self {
185        Self {
186            types: value.0.into_iter().map(std::convert::Into::into).collect(),
187            message: None,
188        }
189    }
190}
191
192#[cfg(test)]
193mod test {
194    use super::*;
195    use std::str::FromStr;
196
197    #[test]
198    fn meta_and_sim_hashes() {
199        let contents = include_bytes!("../../../types/testdata/elf/elf_haiku_x86").to_vec();
200        let meta = FileMetadata::new(&contents, Some("elf_haiku_x86"));
201        assert!(meta.lzjd.is_some());
202        assert!(meta.tlsh.is_some());
203        assert!(meta.ssdeep.is_some());
204
205        let ssdeep = meta.ssdeep.unwrap();
206        let tlsh = meta.tlsh.unwrap();
207        let lzjd = meta.lzjd.unwrap();
208
209        println!("LZJD: {lzjd}");
210        println!("Tlsh: {tlsh}");
211        println!("SSDeep: {ssdeep}");
212        println!("Human hash: {}", meta.humanhash);
213        println!("File command: {}", meta.file_command);
214
215        assert_eq!(FuzzyHash::compare(ssdeep.clone(), ssdeep).unwrap(), 100);
216
217        let tlsh =
218            tlsh_fixed::Tlsh::from_str(&tlsh).expect("failed to convert tlsh string to object");
219        assert_eq!(tlsh.diff(&tlsh, true), 0);
220
221        let lzjd =
222            LZDict::from_base64_string(&lzjd).expect("failed to convert lzjd string to object");
223        assert!(lzjd.jaccard_similarity(&lzjd) - 1.0f64 <= f64::EPSILON);
224    }
225}
malwaredb_server/db/types.rs

malwaredb_server/db/
types.rs
