mithril_cardano_node_internal_database/test/
dummy_cardano_db.rs

1use std::fs::File;
2use std::io::prelude::Write;
3use std::path::{Path, PathBuf};
4
5use mithril_common::entities::{ImmutableFileNumber, SlotNumber};
6
7use crate::entities::{ImmutableFile, LedgerStateSnapshot};
8use crate::{IMMUTABLE_DIR, LEDGER_DIR, VOLATILE_DIR};
9
10/// Dummy cardano db 'immutables' subdirectory content
11struct DummyImmutableDb {
12    /// The dummy cardano db 'immutables' directory path.
13    dir: PathBuf,
14    /// The [immutables files][ImmutableFile] in the dummy cardano db 'immutables' subdirectory.
15    immutables_files: Vec<ImmutableFile>,
16    /// Files that doesn't follow the immutable file name scheme in the dummy cardano db 'immutables' subdirectory.
17    non_immutables_files: Vec<PathBuf>,
18}
19
20/// Dummy cardano db 'ledger' subdirectory content
21struct DummyLedgerStateFolder {
22    /// The dummy cardano db 'ledger' directory path.
23    dir: PathBuf,
24    /// The [ledger state snapshots][LedgerStateSnapshot] in the dummy cardano db 'ledger' subdirectory.
25    ledger_state_snapshots: Vec<LedgerStateSnapshot>,
26    /// Files that are not ledger snapshots in the dummy cardano db 'ledger' subdirectory.
27    non_ledger_state_snapshots: Vec<PathBuf>,
28}
29
30impl DummyImmutableDb {
31    /// Add an immutable chunk file and its primary & secondary to the dummy DB.
32    pub fn add_immutable_file(&mut self) -> ImmutableFileNumber {
33        let new_file_number = self.last_immutable_number().unwrap_or(0) + 1;
34        let mut new_files = write_immutable_trio(None, &self.dir, new_file_number);
35
36        self.immutables_files.append(&mut new_files);
37
38        new_file_number
39    }
40
41    /// Return the file number of the last immutable
42    pub fn last_immutable_number(&self) -> Option<ImmutableFileNumber> {
43        self.immutables_files.last().map(|f| f.number)
44    }
45}
46
47/// A dummy cardano db.
48pub struct DummyCardanoDb {
49    /// The dummy cardano db directory path.
50    dir: PathBuf,
51
52    /// Dummy immutable db
53    immutable_db: DummyImmutableDb,
54
55    /// Dummy ledger state folder
56    ledger_state_folder: DummyLedgerStateFolder,
57}
58
59impl DummyCardanoDb {
60    /// Return the cardano db directory path.
61    pub fn get_dir(&self) -> &PathBuf {
62        &self.dir
63    }
64
65    /// Return the immutable db directory path.
66    pub fn get_immutable_dir(&self) -> &Path {
67        &self.immutable_db.dir
68    }
69
70    /// Return the ledger directory path.
71    pub fn get_ledger_dir(&self) -> &Path {
72        &self.ledger_state_folder.dir
73    }
74
75    /// Return the volatile directory path.
76    pub fn get_volatile_dir(&self) -> PathBuf {
77        self.dir.join(VOLATILE_DIR)
78    }
79
80    /// Return the file number of the last immutable
81    pub fn get_immutable_files(&self) -> &Vec<ImmutableFile> {
82        &self.immutable_db.immutables_files
83    }
84
85    /// Return the path of the files in the ledger directory.
86    pub fn get_ledger_state_snapshots(&self) -> &Vec<LedgerStateSnapshot> {
87        &self.ledger_state_folder.ledger_state_snapshots
88    }
89
90    /// Return the non-ledger state snapshot files in the ledger directory
91    pub fn get_non_ledger_state_snapshots(&self) -> &Vec<PathBuf> {
92        &self.ledger_state_folder.non_ledger_state_snapshots
93    }
94
95    /// Add an immutable chunk file and its primary & secondary to the dummy DB.
96    pub fn add_immutable_file(&mut self) -> ImmutableFileNumber {
97        self.immutable_db.add_immutable_file()
98    }
99
100    /// Return the file number of the last immutable
101    pub fn last_immutable_number(&self) -> Option<ImmutableFileNumber> {
102        self.immutable_db.last_immutable_number()
103    }
104
105    /// Return the non-immutables files in the immutables directory
106    pub fn get_non_immutables_files(&self) -> &Vec<PathBuf> {
107        &self.immutable_db.non_immutables_files
108    }
109}
110
111/// A [DummyCardanoDbBuilder] builder.
112pub struct DummyCardanoDbBuilder {
113    sub_dir: String,
114    immutables_to_write: Vec<ImmutableFileNumber>,
115    non_immutables_to_write: Vec<String>,
116    append_uncompleted_trio: bool,
117    immutable_file_size: Option<u64>,
118    ledger_snapshot_in_memory_to_write: Vec<SlotNumber>,
119    ledger_snapshot_legacy_to_write: Vec<SlotNumber>,
120    non_ledger_snapshot_to_write: Vec<String>,
121    ledger_file_size: Option<u64>,
122    volatile_files_to_write: Vec<String>,
123    volatile_file_size: Option<u64>,
124}
125
126impl DummyCardanoDbBuilder {
127    /// [DummyCardanoDbBuilder] factory, will create a folder with the given `dirname` in the
128    /// system temp directory, if it exists already it will be cleaned.
129    pub fn new(dir_name: &str) -> Self {
130        Self {
131            sub_dir: dir_name.to_string(),
132            immutables_to_write: vec![],
133            non_immutables_to_write: vec![],
134            append_uncompleted_trio: false,
135            immutable_file_size: None,
136            non_ledger_snapshot_to_write: vec![],
137            ledger_snapshot_in_memory_to_write: vec![],
138            ledger_snapshot_legacy_to_write: vec![],
139            ledger_file_size: None,
140            volatile_files_to_write: vec![],
141            volatile_file_size: None,
142        }
143    }
144
145    /// Set the immutables file number that will be used to generate the immutable files, for each
146    /// number three files will be generated (a 'chunk', a 'primary' and a 'secondary' file).
147    pub fn with_immutables(&mut self, immutables: &[ImmutableFileNumber]) -> &mut Self {
148        self.immutables_to_write = immutables.to_vec();
149        self
150    }
151
152    /// Set filenames to write to the db that doesn't follow the immutable file name scheme.
153    pub fn with_non_immutables(&mut self, non_immutables: &[&str]) -> &mut Self {
154        self.non_immutables_to_write = non_immutables.iter().map(|f| f.to_string()).collect();
155        self
156    }
157
158    /// Set legacy ledger state snapshot slot numbers to write to the db in the 'ledger' subdirectory.
159    pub fn with_legacy_ledger_snapshots(&mut self, snapshot_slot_numbers: &[u64]) -> &mut Self {
160        self.ledger_snapshot_legacy_to_write =
161            snapshot_slot_numbers.iter().map(|s| SlotNumber(*s)).collect();
162        self
163    }
164
165    /// Set the slot numbers of utxo-hd in-memory snapshot folders to write to the db in the 'ledger' subdirectory.
166    pub fn with_in_memory_ledger_snapshots(&mut self, snapshot_slot_numbers: &[u64]) -> &mut Self {
167        self.ledger_snapshot_in_memory_to_write =
168            snapshot_slot_numbers.iter().map(|s| SlotNumber(*s)).collect();
169        self
170    }
171
172    /// Set filenames to write to the 'ledger' subdirectory that are not ledger snapshots.
173    pub fn with_non_ledger_files(&mut self, files: &[&str]) -> &mut Self {
174        self.non_ledger_snapshot_to_write = files.iter().map(|name| name.to_string()).collect();
175        self
176    }
177
178    /// Set the size of all ledger files written by [build][Self::build] to the given `file_size` in bytes.
179    ///
180    /// Note: for types of ledger state snapshots that have more than one file, the size is evenly
181    /// split across all files:
182    /// * Legacy snapshots: not divided as it's a single file
183    /// * In-memory snapshots: divided by three
184    pub fn set_ledger_file_size(&mut self, file_size: u64) -> &mut Self {
185        self.ledger_file_size = Some(file_size);
186        self
187    }
188
189    /// Set volatile files to write to the db in the 'volatile' subdirectory.
190    pub fn with_volatile_files(&mut self, files: &[&str]) -> &mut Self {
191        self.volatile_files_to_write = files.iter().map(|f| f.to_string()).collect();
192        self
193    }
194
195    /// Set the size of all volatile files written by [build][Self::build] to the given `file_size` in bytes.
196    pub fn set_volatile_file_size(&mut self, file_size: u64) -> &mut Self {
197        self.volatile_file_size = Some(file_size);
198        self
199    }
200
201    /// Makes [build][Self::build] add another trio of immutables file, that won't be included
202    /// in its returned vec, to simulate the last 3 'uncompleted / wip' files that can be found in
203    /// a cardano immutable db.
204    pub fn append_immutable_trio(&mut self) -> &mut Self {
205        self.append_uncompleted_trio = true;
206        self
207    }
208
209    /// Set the size of all immutable files written by [build][Self::build] to the given `file_size` in bytes.
210    ///
211    /// Note: by default, the size of the produced files is less than 1 kb.
212    pub fn set_immutable_trio_file_size(&mut self, trio_file_size: u64) -> &mut Self {
213        assert!(
214            trio_file_size.is_multiple_of(3),
215            "'trio_file_size' must be a multiple of 3"
216        );
217
218        self.immutable_file_size = Some(trio_file_size / 3);
219        self
220    }
221
222    /// Build a [DummyCardanoDb].
223    pub fn build(&self) -> DummyCardanoDb {
224        let dir = get_test_dir(&self.sub_dir);
225
226        let mut non_immutables_files = vec![];
227        let mut ledger_state_snapshots = vec![];
228        let mut non_ledger_state_snapshots = vec![];
229        let mut immutable_numbers = self.immutables_to_write.clone();
230        immutable_numbers.sort();
231
232        if self.append_uncompleted_trio {
233            write_immutable_trio(
234                self.immutable_file_size,
235                &dir.join(IMMUTABLE_DIR),
236                match immutable_numbers.last() {
237                    None => 0,
238                    Some(last) => last + 1,
239                },
240            );
241        }
242
243        for non_immutable in &self.non_immutables_to_write {
244            non_immutables_files.push(write_dummy_file(
245                self.immutable_file_size,
246                &dir.join(IMMUTABLE_DIR),
247                non_immutable,
248            ));
249        }
250
251        for filename in &self.non_ledger_snapshot_to_write {
252            let ledger_file_path =
253                write_dummy_file(self.ledger_file_size, &dir.join(LEDGER_DIR), filename);
254            non_ledger_state_snapshots.push(ledger_file_path);
255        }
256
257        for slot_number in &self.ledger_snapshot_legacy_to_write {
258            let ledger_file = write_dummy_file(
259                self.ledger_file_size,
260                &dir.join(LEDGER_DIR),
261                &slot_number.to_string(),
262            );
263
264            ledger_state_snapshots.push(LedgerStateSnapshot::legacy(
265                ledger_file,
266                *slot_number,
267                slot_number.to_string().into(),
268            ));
269        }
270
271        for slot_number in &self.ledger_snapshot_in_memory_to_write {
272            let ledger_state_snapshot =
273                write_in_memory_ledger_snapshot(*slot_number, &dir, self.ledger_file_size);
274            ledger_state_snapshots.push(ledger_state_snapshot);
275        }
276
277        for filename in &self.volatile_files_to_write {
278            write_dummy_file(self.volatile_file_size, &dir.join(VOLATILE_DIR), filename);
279        }
280
281        let immutable_db = DummyImmutableDb {
282            dir: dir.join(IMMUTABLE_DIR),
283            immutables_files: immutable_numbers
284                .into_iter()
285                .flat_map(|ifn| {
286                    write_immutable_trio(self.immutable_file_size, &dir.join(IMMUTABLE_DIR), ifn)
287                })
288                .collect::<Vec<_>>(),
289            non_immutables_files,
290        };
291
292        let ledger_state_folder = DummyLedgerStateFolder {
293            dir: dir.join(LEDGER_DIR),
294            ledger_state_snapshots,
295            non_ledger_state_snapshots,
296        };
297
298        DummyCardanoDb {
299            dir,
300            immutable_db,
301            ledger_state_folder,
302        }
303    }
304}
305
306fn write_immutable_trio(
307    optional_size: Option<u64>,
308    dir: &Path,
309    immutable: ImmutableFileNumber,
310) -> Vec<ImmutableFile> {
311    let mut result = vec![];
312    for filename in [
313        format!("{immutable:05}.chunk"),
314        format!("{immutable:05}.primary"),
315        format!("{immutable:05}.secondary"),
316    ] {
317        let file = write_dummy_file(optional_size, dir, &filename);
318        result.push(ImmutableFile {
319            number: immutable.to_owned(),
320            path: file,
321            filename: filename.to_string(),
322        });
323    }
324    result
325}
326
327fn write_in_memory_ledger_snapshot(
328    slot_number: SlotNumber,
329    dir: &Path,
330    optional_size: Option<u64>,
331) -> LedgerStateSnapshot {
332    let ledger_folder_name = slot_number.to_string();
333    let ledger_folder_path = dir.join(LEDGER_DIR).join(&ledger_folder_name);
334    let optional_file_size = optional_size.map(|s| s / 3);
335
336    std::fs::create_dir_all(ledger_folder_path.join(LedgerStateSnapshot::IN_MEMORY_TABLES))
337        .unwrap();
338    write_dummy_file(
339        optional_file_size,
340        &ledger_folder_path,
341        LedgerStateSnapshot::IN_MEMORY_STATE,
342    );
343    write_in_memory_meta_file(optional_file_size, &ledger_folder_path);
344    write_dummy_file(
345        optional_file_size,
346        &ledger_folder_path.join(LedgerStateSnapshot::IN_MEMORY_TABLES),
347        LedgerStateSnapshot::IN_MEMORY_TVAR,
348    );
349
350    LedgerStateSnapshot::InMemoryUpTo10_6 {
351        path: ledger_folder_path,
352        slot_number,
353        folder_name: ledger_folder_name.into(),
354    }
355}
356
357/// Write a valid UTxO-HD in-memory ledger snapshot `meta` file with `{"backend": "utxohd-mem"}` to
358/// the given directory, optionally padding it to the given size with whitespace so the content
359/// remains valid JSON.
360fn write_in_memory_meta_file(optional_size: Option<u64>, dir: &Path) -> PathBuf {
361    const META_JSON: &str = r#"{"backend": "utxohd-mem"}"#;
362
363    let file_path = dir.join(LedgerStateSnapshot::IN_MEMORY_META);
364    let mut file = File::create(&file_path).unwrap();
365    write!(file, "{META_JSON}").unwrap();
366    if let Some(file_size) = optional_size {
367        let json_size = META_JSON.len() as u64;
368        if file_size > json_size {
369            let padding = vec![b' '; (file_size - json_size) as usize];
370            file.write_all(&padding).unwrap();
371        }
372    }
373    file_path
374}
375
376/// Create a file with the given name in the given dir, write some text to it, and then
377/// return its path.
378fn write_dummy_file(optional_size: Option<u64>, dir: &Path, filename: &str) -> PathBuf {
379    let file = dir.join(Path::new(filename));
380    let mut source_file = File::create(&file).unwrap();
381
382    write!(source_file, "This is a test file named '{filename}'").unwrap();
383
384    if let Some(file_size) = optional_size {
385        writeln!(source_file).unwrap();
386        source_file.set_len(file_size).unwrap();
387    }
388
389    file
390}
391
392fn get_test_dir(subdir_name: &str) -> PathBuf {
393    // Note: used the common `TestDir` api before, but as the DummyCardanoDb is public, `test_tools` can't be used.
394    let db_dir = std::env::temp_dir()
395        .join("mithril_test")
396        .join("test_cardano_db")
397        .join(subdir_name);
398
399    if db_dir.exists() {
400        std::fs::remove_dir_all(&db_dir)
401            .unwrap_or_else(|e| panic!("Could not remove dir {db_dir:?}: {e}"));
402    }
403    std::fs::create_dir_all(&db_dir)
404        .unwrap_or_else(|e| panic!("Could not create dir {db_dir:?}: {e}"));
405
406    for subdir_name in [LEDGER_DIR, IMMUTABLE_DIR, VOLATILE_DIR] {
407        std::fs::create_dir(db_dir.join(subdir_name)).unwrap();
408    }
409
410    db_dir
411}
412
413#[cfg(test)]
414mod tests {
415    use mithril_common::{assert_dir_eq, current_function};
416
417    use super::*;
418
419    #[test]
420    fn writing_empty_dummy_cardano_db_structure() {
421        let db = DummyCardanoDbBuilder::new(current_function!()).build();
422
423        assert_eq!(db.get_immutable_dir(), db.get_dir().join(IMMUTABLE_DIR));
424        assert_eq!(db.get_ledger_dir(), db.get_dir().join(LEDGER_DIR));
425        assert_eq!(db.get_volatile_dir(), db.get_dir().join(VOLATILE_DIR));
426        assert_eq!(db.last_immutable_number(), None);
427        assert_dir_eq!(
428            &db.dir,
429            format!(
430                "* {IMMUTABLE_DIR}/
431                 * {LEDGER_DIR}/
432                 * {VOLATILE_DIR}/"
433            )
434        );
435    }
436
437    #[test]
438    fn writing_immutable_files() {
439        let db = DummyCardanoDbBuilder::new(current_function!())
440            .with_immutables(&[1, 2])
441            .build();
442
443        assert_eq!(db.last_immutable_number(), Some(2));
444        assert_eq!(db.get_immutable_files().len(), 6); // Each immutable is a trio of files
445        assert_dir_eq!(
446            &db.dir,
447            format!(
448                "* {IMMUTABLE_DIR}/
449                 ** 00001.chunk
450                 ** 00001.primary
451                 ** 00001.secondary
452                 ** 00002.chunk
453                 ** 00002.primary
454                 ** 00002.secondary
455                 * {LEDGER_DIR}/
456                 * {VOLATILE_DIR}/"
457            )
458        );
459    }
460
461    #[test]
462    fn adding_non_completed_immutable_files_trio_is_not_take_in_account_in_resulting_db_metadata() {
463        let db = DummyCardanoDbBuilder::new(current_function!())
464            .with_immutables(&[1])
465            .append_immutable_trio()
466            .build();
467
468        assert_eq!(db.last_immutable_number(), Some(1));
469        assert_eq!(db.get_immutable_files().len(), 3); // Each immutable is a trio of files
470        assert_dir_eq!(
471            &db.dir,
472            format!(
473                "* {IMMUTABLE_DIR}/
474                 ** 00001.chunk
475                 ** 00001.primary
476                 ** 00001.secondary
477                 ** 00002.chunk
478                 ** 00002.primary
479                 ** 00002.secondary
480                 * {LEDGER_DIR}/
481                 * {VOLATILE_DIR}/"
482            )
483        );
484    }
485
486    #[test]
487    fn writing_non_immutable_files() {
488        let db = DummyCardanoDbBuilder::new(current_function!())
489            .with_non_immutables(&["test1.txt", "test2.txt"])
490            .build();
491
492        let non_immutable_files = db.get_non_immutables_files();
493        assert_eq!(
494            non_immutable_files,
495            &vec![
496                db.get_dir().join(IMMUTABLE_DIR).join("test1.txt"),
497                db.get_dir().join(IMMUTABLE_DIR).join("test2.txt"),
498            ]
499        );
500        assert_dir_eq!(
501            &db.dir,
502            format!(
503                "* {IMMUTABLE_DIR}/
504                 ** test1.txt
505                 ** test2.txt
506                 * {LEDGER_DIR}/
507                 * {VOLATILE_DIR}/"
508            )
509        );
510    }
511
512    #[test]
513    fn writing_ledger_snapshots() {
514        let db = DummyCardanoDbBuilder::new(current_function!())
515            .with_legacy_ledger_snapshots(&[100])
516            .with_in_memory_ledger_snapshots(&[200])
517            .build();
518
519        let snapshots = db.get_ledger_state_snapshots();
520        assert_eq!(snapshots.len(), 2);
521        assert!(snapshots.iter().any(|s| s.slot_number() == SlotNumber(100)));
522        assert!(snapshots.iter().any(|s| s.slot_number() == SlotNumber(200)));
523        assert_dir_eq!(
524            &db.dir,
525            format!(
526                "* {IMMUTABLE_DIR}/
527                 * {LEDGER_DIR}/
528                 ** 200/
529                 *** {}/
530                 **** {}
531                 *** {}
532                 *** {}
533                 ** 100
534                 * {VOLATILE_DIR}/",
535                LedgerStateSnapshot::IN_MEMORY_TABLES,
536                LedgerStateSnapshot::IN_MEMORY_TVAR,
537                LedgerStateSnapshot::IN_MEMORY_META,
538                LedgerStateSnapshot::IN_MEMORY_STATE,
539            )
540        );
541    }
542
543    #[test]
544    fn setting_file_sizes() {
545        fn get_file_size(path: &Path) -> u64 {
546            std::fs::metadata(path).unwrap().len()
547        }
548
549        let (immutable_file_size, ledger_file_size, volatile_file_size) = (3000, 6000, 9000);
550        let db = DummyCardanoDbBuilder::new(current_function!())
551            .with_immutables(&[1])
552            .with_legacy_ledger_snapshots(&[100])
553            .with_in_memory_ledger_snapshots(&[200])
554            .with_volatile_files(&["test.txt"])
555            .set_immutable_trio_file_size(immutable_file_size)
556            .set_ledger_file_size(ledger_file_size)
557            .set_volatile_file_size(volatile_file_size)
558            .build();
559
560        for file in db.get_immutable_files() {
561            assert_eq!(get_file_size(&file.path), immutable_file_size / 3);
562        }
563
564        for ledger_state_snapshot in db.get_ledger_state_snapshots() {
565            match ledger_state_snapshot {
566                LedgerStateSnapshot::Legacy { path, .. } => {
567                    assert_eq!(get_file_size(path), ledger_file_size);
568                }
569                LedgerStateSnapshot::InMemoryUpTo10_6 { path, .. } => {
570                    assert_eq!(
571                        get_file_size(&path.join(LedgerStateSnapshot::IN_MEMORY_STATE)),
572                        ledger_file_size / 3
573                    );
574                    assert_eq!(
575                        get_file_size(&path.join(LedgerStateSnapshot::IN_MEMORY_META)),
576                        ledger_file_size / 3
577                    );
578                    assert_eq!(
579                        get_file_size(
580                            &path
581                                .join(LedgerStateSnapshot::IN_MEMORY_TABLES)
582                                .join(LedgerStateSnapshot::IN_MEMORY_TVAR)
583                        ),
584                        ledger_file_size / 3
585                    );
586                }
587                LedgerStateSnapshot::InMemoryFrom10_7 { .. } => {
588                    unreachable!(
589                        "DummyCardanoDbBuilder does not create InMemoryFrom10_7 snapshots in this test"
590                    )
591                }
592            }
593        }
594
595        assert_eq!(
596            get_file_size(&db.get_volatile_dir().join("test.txt")),
597            volatile_file_size
598        );
599    }
600}
mithril_cardano_node_internal_database/test/dummy_cardano_db.rs

mithril_cardano_node_internal_database/test/
dummy_cardano_db.rs
