mithril_cardano_node_internal_database/entities/
immutable_file.rs

1use digest::{Digest, Output};
2use std::{
3    cmp::Ordering,
4    fs::File,
5    io,
6    num::ParseIntError,
7    path::{Path, PathBuf},
8};
9use thiserror::Error;
10use walkdir::{DirEntry, WalkDir};
11
12use mithril_common::entities::{ImmutableFileName, ImmutableFileNumber};
13
14use crate::entities::ImmutableFileListingError::{MissingImmutableFiles, MissingImmutableFolder};
15use crate::IMMUTABLE_DIR;
16
17const IMMUTABLE_FILE_EXTENSIONS: [&str; 3] = ["chunk", "primary", "secondary"];
18
19fn is_immutable(entry: &walkdir::DirEntry) -> bool {
20    let is_file = entry.file_type().is_file();
21    let extension = entry.path().extension().map(|e| e.to_string_lossy());
22
23    is_file && extension.is_some_and(|e| IMMUTABLE_FILE_EXTENSIONS.contains(&e.as_ref()))
24}
25
26/// Walk the given path and return the first directory named "immutable" it finds
27fn find_immutables_dir(path_to_walk: &Path) -> Option<PathBuf> {
28    WalkDir::new(path_to_walk)
29        .into_iter()
30        .filter_entry(|e| e.file_type().is_dir())
31        .filter_map(|e| e.ok())
32        .find(|f| f.file_name() == IMMUTABLE_DIR)
33        .map(|e| e.into_path())
34}
35
36/// Walk the given immutable directory and return an iterator over its files (no subdirectories)
37fn walk_immutables_in_dir<P: AsRef<Path>>(immutable_dir: P) -> impl Iterator<Item = DirEntry> {
38    WalkDir::new(immutable_dir)
39        .min_depth(1)
40        .max_depth(1)
41        .into_iter()
42        .filter_entry(is_immutable)
43        .filter_map(|file| file.ok())
44}
45
46/// Represent an immutable file in a Cardano node database directory
47#[derive(Debug, PartialEq, Eq, Clone)]
48pub struct ImmutableFile {
49    /// The path to the immutable file
50    pub path: PathBuf,
51
52    /// The immutable file number
53    pub number: ImmutableFileNumber,
54
55    /// The filename
56    pub filename: ImmutableFileName,
57}
58
59/// [ImmutableFile::new] related errors.
60#[derive(Error, Debug)]
61pub enum ImmutableFileCreationError {
62    /// Raised when the immutable file stem extraction fails.
63    #[error("Couldn't extract the file stem for '{path:?}'")]
64    FileStemExtraction {
65        /// Path for which file stem extraction failed.
66        path: PathBuf,
67    },
68
69    /// Raised when the immutable file filename extraction fails.
70    #[error("Couldn't extract the filename as string for '{path:?}'")]
71    FileNameExtraction {
72        /// Path for which filename extraction failed.
73        path: PathBuf,
74    },
75
76    /// Raised when the immutable file number parsing, from the filename, fails.
77    #[error("Error while parsing immutable file number")]
78    FileNumberParsing(#[from] ParseIntError),
79}
80
81/// [ImmutableFile::list_completed_in_dir] related errors.
82#[derive(Error, Debug)]
83pub enum ImmutableFileListingError {
84    /// Raised when the metadata of a file could not be read.
85    #[error("metadata parsing failed")]
86    MetadataParsing(#[from] io::Error),
87
88    /// Raised when [ImmutableFile::new] fails.
89    #[error("immutable file creation error")]
90    ImmutableFileCreation(#[from] ImmutableFileCreationError),
91
92    /// Raised when the "immutable" folder could not be found in a file structure.
93    #[error("Couldn't find the 'immutable' folder in '{0:?}'")]
94    MissingImmutableFolder(PathBuf),
95
96    /// Raised when no immutable files could be found in the 'immutable' folder.
97    #[error("There are no immutable files in '{0:?}'")]
98    MissingImmutableFiles(PathBuf),
99}
100
101impl ImmutableFile {
102    /// ImmutableFile factory
103    pub fn new(path: PathBuf) -> Result<ImmutableFile, ImmutableFileCreationError> {
104        let filename = path
105            .file_name()
106            .ok_or(ImmutableFileCreationError::FileNameExtraction { path: path.clone() })?
107            .to_str()
108            .ok_or(ImmutableFileCreationError::FileNameExtraction { path: path.clone() })?
109            .to_string();
110
111        let filestem = path
112            .file_stem()
113            .ok_or(ImmutableFileCreationError::FileStemExtraction { path: path.clone() })?
114            .to_str()
115            .ok_or(ImmutableFileCreationError::FileNameExtraction { path: path.clone() })?;
116        let immutable_file_number = filestem.parse::<ImmutableFileNumber>()?;
117
118        Ok(Self {
119            path,
120            number: immutable_file_number,
121            filename,
122        })
123    }
124
125    /// Compute the hash of this immutable file.
126    pub fn compute_raw_hash<D>(&self) -> Result<Output<D>, io::Error>
127    where
128        D: Digest + io::Write,
129    {
130        let mut hasher = D::new();
131        let mut file = File::open(&self.path)?;
132        io::copy(&mut file, &mut hasher)?;
133        Ok(hasher.finalize())
134    }
135
136    /// List all [`ImmutableFile`] in a given directory.
137    pub fn list_all_in_dir(dir: &Path) -> Result<Vec<ImmutableFile>, ImmutableFileListingError> {
138        let immutable_dir = find_immutables_dir(dir).ok_or(
139            ImmutableFileListingError::MissingImmutableFolder(dir.to_path_buf()),
140        )?;
141        let mut files: Vec<ImmutableFile> = vec![];
142
143        for path in walk_immutables_in_dir(&immutable_dir) {
144            let immutable_file = ImmutableFile::new(path.into_path())?;
145            files.push(immutable_file);
146        }
147        files.sort();
148
149        Ok(files)
150    }
151
152    /// List all complete [`ImmutableFile`] in a given directory.
153    ///
154    /// Important Note: It will skip the last chunk / primary / secondary trio since they're not yet
155    /// complete.
156    pub fn list_completed_in_dir(
157        dir: &Path,
158    ) -> Result<Vec<ImmutableFile>, ImmutableFileListingError> {
159        let files = Self::list_all_in_dir(dir)?;
160
161        match files.last() {
162            // empty list
163            None => Ok(files),
164            // filter out the last immutable file(s)
165            Some(last_file) => {
166                let last_number = last_file.number;
167                Ok(files
168                    .into_iter()
169                    .filter(|f| f.number < last_number)
170                    .collect())
171            }
172        }
173    }
174
175    /// Check if at least one immutable file exists in the given directory
176    pub fn at_least_one_immutable_files_exist_in_dir(
177        dir: &Path,
178    ) -> Result<(), ImmutableFileListingError> {
179        let immutable_dir =
180            find_immutables_dir(dir).ok_or(MissingImmutableFolder(dir.to_path_buf()))?;
181        if walk_immutables_in_dir(immutable_dir).next().is_some() {
182            Ok(())
183        } else {
184            Err(MissingImmutableFiles(dir.to_path_buf().join(IMMUTABLE_DIR)))
185        }
186    }
187}
188
189impl PartialOrd for ImmutableFile {
190    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
191        Some(self.cmp(other))
192    }
193}
194
195impl Ord for ImmutableFile {
196    fn cmp(&self, other: &Self) -> Ordering {
197        self.number
198            .cmp(&other.number)
199            .then(self.path.cmp(&other.path))
200    }
201}
202
203#[cfg(test)]
204mod tests {
205    use std::fs;
206    use std::io::prelude::*;
207
208    use mithril_common::temp_dir_create;
209    use mithril_common::test_utils::TempDir;
210
211    use super::*;
212
213    fn get_test_dir(subdir_name: &str) -> PathBuf {
214        TempDir::create("immutable_file", subdir_name)
215    }
216
217    fn create_fake_files(parent_dir: &Path, child_filenames: &[&str]) {
218        for filename in child_filenames {
219            let file = parent_dir.join(Path::new(filename));
220            let mut source_file = File::create(file).unwrap();
221            write!(source_file, "This is a test file named '{filename}'").unwrap();
222        }
223    }
224
225    fn extract_filenames(immutables: &[ImmutableFile]) -> Vec<String> {
226        immutables
227            .iter()
228            .map(|i| i.path.file_name().unwrap().to_str().unwrap().to_owned())
229            .collect()
230    }
231
232    #[test]
233    fn list_completed_immutable_file_fail_if_not_in_immutable_dir() {
234        let target_dir = get_test_dir("list_immutable_file_fail_if_not_in_immutable_dir/invalid");
235        let entries = vec![];
236        create_fake_files(&target_dir, &entries);
237
238        ImmutableFile::list_completed_in_dir(target_dir.parent().unwrap())
239            .expect_err("ImmutableFile::list_in_dir should have Failed");
240    }
241
242    #[test]
243    fn list_all_immutable_file_should_not_skip_last_number() {
244        let target_dir =
245            get_test_dir("list_all_immutable_file_should_not_skip_last_number/immutable");
246        let entries = vec![
247            "123.chunk",
248            "123.primary",
249            "123.secondary",
250            "125.chunk",
251            "125.primary",
252            "125.secondary",
253            "0124.chunk",
254            "0124.primary",
255            "0124.secondary",
256            "223.chunk",
257            "223.primary",
258            "223.secondary",
259            "0423.chunk",
260            "0423.primary",
261            "0423.secondary",
262            "0424.chunk",
263            "0424.primary",
264            "0424.secondary",
265            "21.chunk",
266            "21.primary",
267            "21.secondary",
268        ];
269        create_fake_files(&target_dir, &entries);
270        let result = ImmutableFile::list_all_in_dir(target_dir.parent().unwrap())
271            .expect("ImmutableFile::list_in_dir Failed");
272
273        assert_eq!(result.last().unwrap().number, 424);
274        let expected_entries_length = 21;
275        assert_eq!(
276            expected_entries_length,
277            result.len(),
278            "Expected to find {} files but found {}",
279            entries.len(),
280            result.len(),
281        );
282    }
283
284    #[test]
285    fn list_completed_immutable_file_should_skip_last_number() {
286        let target_dir = get_test_dir("list_immutable_file_should_skip_last_number/immutable");
287        let entries = vec![
288            "123.chunk",
289            "123.primary",
290            "123.secondary",
291            "125.chunk",
292            "125.primary",
293            "125.secondary",
294            "0124.chunk",
295            "0124.primary",
296            "0124.secondary",
297            "223.chunk",
298            "223.primary",
299            "223.secondary",
300            "0423.chunk",
301            "0423.primary",
302            "0423.secondary",
303            "0424.chunk",
304            "0424.primary",
305            "0424.secondary",
306            "21.chunk",
307            "21.primary",
308            "21.secondary",
309        ];
310        create_fake_files(&target_dir, &entries);
311        let result = ImmutableFile::list_completed_in_dir(target_dir.parent().unwrap())
312            .expect("ImmutableFile::list_in_dir Failed");
313
314        assert_eq!(result.last().unwrap().number, 423);
315        assert_eq!(
316            result.len(),
317            entries.len() - 3,
318            "Expected to find {} files since the last (chunk, primary, secondary) trio is skipped, but found {}",
319            entries.len() - 3,
320            result.len(),
321        );
322    }
323
324    #[test]
325    fn list_completed_immutable_file_should_works_in_a_empty_folder() {
326        let target_dir =
327            get_test_dir("list_immutable_file_should_works_even_in_a_empty_folder/immutable");
328        let entries = vec![];
329        create_fake_files(&target_dir, &entries);
330        let result = ImmutableFile::list_completed_in_dir(target_dir.parent().unwrap())
331            .expect("ImmutableFile::list_in_dir Failed");
332
333        assert!(result.is_empty());
334    }
335
336    #[test]
337    fn list_completed_immutable_file_order_should_be_deterministic() {
338        let target_dir =
339            get_test_dir("list_completed_immutable_file_order_should_be_deterministic/immutable");
340        let entries = vec![
341            "21.chunk",
342            "21.primary",
343            "21.secondary",
344            "123.chunk",
345            "123.primary",
346            "123.secondary",
347            "124.chunk",
348            "124.primary",
349            "124.secondary",
350            "125.chunk",
351            "125.primary",
352            "125.secondary",
353            "223.chunk",
354            "223.primary",
355            "223.secondary",
356            "423.chunk",
357            "423.primary",
358            "423.secondary",
359            "424.chunk",
360            "424.primary",
361            "424.secondary",
362        ];
363        create_fake_files(&target_dir, &entries);
364        let immutables = ImmutableFile::list_completed_in_dir(target_dir.parent().unwrap())
365            .expect("ImmutableFile::list_in_dir Failed");
366        let immutables_names: Vec<String> = extract_filenames(&immutables);
367
368        let expected: Vec<&str> = entries.into_iter().rev().skip(3).rev().collect();
369        assert_eq!(expected, immutables_names);
370    }
371
372    #[test]
373    fn list_completed_immutable_file_should_work_with_non_immutable_files() {
374        let target_dir =
375            get_test_dir("list_immutable_file_should_work_with_non_immutable_files/immutable");
376        let entries = vec![
377            "123.chunk",
378            "123.primary",
379            "123.secondary",
380            "124.chunk",
381            "124.primary",
382            "124.secondary",
383            "README.md",
384            "124.secondary.back",
385        ];
386        create_fake_files(&target_dir, &entries);
387        let immutables = ImmutableFile::list_completed_in_dir(target_dir.parent().unwrap())
388            .expect("ImmutableFile::list_in_dir Failed");
389        let immutables_names: Vec<String> = extract_filenames(&immutables);
390
391        let expected: Vec<&str> = entries.into_iter().rev().skip(5).rev().collect();
392        assert_eq!(expected, immutables_names);
393    }
394
395    #[test]
396    fn list_completed_immutable_file_can_list_incomplete_trio() {
397        let target_dir = get_test_dir("list_immutable_file_can_list_incomplete_trio/immutable");
398        let entries = vec![
399            "21.chunk",
400            "21.primary",
401            "21.secondary",
402            "123.chunk",
403            "123.secondary",
404            "124.chunk",
405            "124.primary",
406            "125.primary",
407            "125.secondary",
408            "223.chunk",
409            "224.primary",
410            "225.secondary",
411            "226.chunk",
412        ];
413        create_fake_files(&target_dir, &entries);
414        let immutables = ImmutableFile::list_completed_in_dir(target_dir.parent().unwrap())
415            .expect("ImmutableFile::list_in_dir Failed");
416        let immutables_names: Vec<String> = extract_filenames(&immutables);
417
418        let expected: Vec<&str> = entries.into_iter().rev().skip(1).rev().collect();
419        assert_eq!(expected, immutables_names);
420    }
421
422    #[test]
423    fn at_least_one_immutable_files_exist_in_dir_throw_error_if_immutable_dir_does_not_exist() {
424        let database_path = temp_dir_create!();
425
426        let error = ImmutableFile::at_least_one_immutable_files_exist_in_dir(&database_path)
427            .expect_err("check_presence_of_immutables should fail");
428        assert_eq!(
429            error.to_string(),
430            format!("Couldn't find the 'immutable' folder in '{database_path:?}'")
431        );
432    }
433
434    #[test]
435    fn at_least_one_immutable_files_exist_in_dir_throw_error_if_immutable_dir_is_empty() {
436        let database_path = temp_dir_create!();
437        fs::create_dir(database_path.join(IMMUTABLE_DIR)).unwrap();
438
439        let error = ImmutableFile::at_least_one_immutable_files_exist_in_dir(&database_path)
440            .expect_err("check_presence_of_immutables should fail");
441        assert_eq!(
442            error.to_string(),
443            format!(
444                "There are no immutable files in '{:?}'",
445                database_path.join(IMMUTABLE_DIR)
446            )
447        );
448    }
449
450    #[test]
451    fn at_least_one_immutable_files_exist_in_dir_is_ok_if_immutable_dir_contains_at_least_one_file()
452    {
453        let database_dir = temp_dir_create!();
454        let database_path = database_dir.as_path();
455        let immutable_file_path = database_dir.join(IMMUTABLE_DIR).join("00001.chunk");
456        fs::create_dir(database_dir.join(IMMUTABLE_DIR)).unwrap();
457        File::create(immutable_file_path).unwrap();
458
459        ImmutableFile::at_least_one_immutable_files_exist_in_dir(database_path)
460            .expect("check_presence_of_immutables should succeed");
461    }
462}
mithril_cardano_node_internal_database/entities/immutable_file.rs

mithril_cardano_node_internal_database/entities/
immutable_file.rs
