solana-runtime 4.5.0-alpha.2

#[cfg(test)]
mod serde_snapshot_tests {
    use {
        crate::{
            bank::{Bank, BankHashStats, BankId, test_utils as bank_test_utils},
            epoch_stakes::{
                EpochAuthorizedVoters, EpochStakes, NodeIdToVoteAccounts, VersionedEpochStakes,
            },
            genesis_utils::{
                GenesisConfigInfo, activate_all_features, create_genesis_config_with_leader,
            },
            runtime_config::RuntimeConfig,
            serde_snapshot::{
                self, AccountsDbFields, ExtraFieldsToSerialize, SerializableAccountsDb,
                SnapshotAccountsDbFields, SnapshotStreams, deserialize_wincode_from,
                reconstruct_accountsdb_from_fields, remap_append_vec_file, serialize_into,
            },
            snapshot_bank_utils,
            snapshot_utils::{StorageAndNextAccountsFileId, create_tmp_accounts_dir_for_tests},
        },
        agave_fs::{FileInfo, buffered_reader::FileBufRead as _, io_setup::IoSetupState},
        agave_snapshots::snapshot_config::SnapshotConfig,
        rand::{Rng, rng},
        solana_account::{AccountSharedData, ReadableAccount},
        solana_accounts_db::{
            ObsoleteAccounts,
            account_storage::AccountStorageMap,
            account_storage_entry::AccountStorageEntry,
            account_storage_reader::{
                AccountStorageReader, TombstonesFilter, open_storage_files, storage_file_buf_reader,
            },
            accounts::Accounts,
            accounts_db::{
                ACCOUNTS_DB_CONFIG_FOR_TESTING, AccountsDb, AccountsDbConfig, AtomicAccountsFileId,
                get_temp_accounts_paths,
            },
            accounts_file::{AccountsFile, AccountsFileError},
            ancestors::Ancestors,
        },
        solana_clock::Slot,
        solana_epoch_schedule::EpochSchedule,
        solana_hash::Hash,
        solana_native_token::LAMPORTS_PER_SOL,
        solana_pubkey::Pubkey,
        std::{
            fs::File,
            io::{self, BufReader, BufWriter, Cursor, Read, Write},
            mem,
            ops::RangeFull,
            path::{Path, PathBuf},
            sync::{Arc, OnceLock, atomic::Ordering},
        },
        tempfile::TempDir,
        test_case::test_case,
    };

    fn linear_ancestors(end_slot: u64) -> Ancestors {
        let mut ancestors = Ancestors::from(vec![0]);
        for i in 1..end_slot {
            ancestors.insert(i);
        }
        ancestors
    }

    fn context_accountsdb_from_stream<R>(
        stream: &mut BufReader<R>,
        account_paths: &[PathBuf],
        storage_and_next_append_vec_id: StorageAndNextAccountsFileId,
        accounts_db_config: AccountsDbConfig,
    ) -> agave_snapshots::Result<AccountsDb>
    where
        R: Read,
    {
        // read and deserialise the accounts database directly from the stream
        let accounts_db_fields: AccountsDbFields = deserialize_wincode_from(&mut *stream)?;
        let snapshot_accounts_db_fields = SnapshotAccountsDbFields {
            full_snapshot_accounts_db_fields: accounts_db_fields,
            incremental_snapshot_accounts_db_fields: None,
        };
        let (accounts_db, _) = reconstruct_accountsdb_from_fields(
            snapshot_accounts_db_fields,
            account_paths,
            storage_and_next_append_vec_id,
            None,
            false,
            accounts_db_config,
            None,
            Arc::default(),
        )?;
        Ok(accounts_db)
    }

    fn accountsdb_from_stream<R>(
        stream: &mut BufReader<R>,
        account_paths: &[PathBuf],
        storage_and_next_append_vec_id: StorageAndNextAccountsFileId,
        accounts_db_config: AccountsDbConfig,
    ) -> agave_snapshots::Result<AccountsDb>
    where
        R: Read,
    {
        context_accountsdb_from_stream::<R>(
            stream,
            account_paths,
            storage_and_next_append_vec_id,
            accounts_db_config,
        )
    }

    fn accounts_db_fields_to_stream<W>(stream: &mut W, slot: Slot) -> wincode::WriteResult<()>
    where
        W: Write,
    {
        let bank_hash_stats = BankHashStats::default();
        serialize_into(stream, &SerializableAccountsDb::new(slot, bank_hash_stats))
    }

    /// Simulates the unpacking & storage reconstruction done during snapshot unpacking
    fn copy_append_vecs(
        accounts_db: &AccountsDb,
        output_dir: impl AsRef<Path>,
    ) -> Result<StorageAndNextAccountsFileId, AccountsFileError> {
        let storage_entries = accounts_db.get_storages(RangeFull).0;
        let storage: AccountStorageMap = AccountStorageMap::with_capacity(storage_entries.len());
        let mut next_append_vec_id = 0;
        const MAX_BUFFER_SIZE: usize = 2 * 1024 * 1024;
        let storage_files = open_storage_files(storage_entries.iter().map(|s| s.as_ref()), false)
            .collect::<io::Result<Vec<_>>>()?;
        let mut buf_reader =
            storage_file_buf_reader(MAX_BUFFER_SIZE, false, &IoSetupState::default())?;
        for (storage_entry, file) in storage_entries.iter().zip(storage_files.iter()) {
            // Copy file to new directory
            let file_name = AccountsFile::file_name(storage_entry.slot(), storage_entry.id());
            let output_path = output_dir.as_ref().join(file_name);
            buf_reader.set_file(file.as_ref(), storage_entry.accounts.len() as u64)?;
            let reader = AccountStorageReader::new(
                storage_entry,
                None,
                TombstonesFilter::Include,
                &mut buf_reader,
            )
            .unwrap();
            let mut writer = File::create(&output_path)?;
            reader.write_to(&mut writer)?;

            // Read new file into append-vec and build new entry
            let file_info = FileInfo::new_from_path(output_path)?;
            let accounts_file = AccountsFile::new_for_startup(file_info)?;
            let new_storage_entry = AccountStorageEntry::new_existing(
                storage_entry.slot(),
                storage_entry.id(),
                accounts_file,
                ObsoleteAccounts::default(),
            );
            next_append_vec_id = next_append_vec_id.max(new_storage_entry.id());
            storage.insert(new_storage_entry.slot(), Arc::new(new_storage_entry));
        }

        Ok(StorageAndNextAccountsFileId {
            storage,
            next_append_vec_id: AtomicAccountsFileId::new(next_append_vec_id + 1),
        })
    }

    fn reconstruct_accounts_db_via_serialization(
        accounts: &AccountsDb,
        slot: Slot,
        accounts_db_config: AccountsDbConfig,
    ) -> AccountsDb {
        let mut writer = Cursor::new(vec![]);
        accounts_db_fields_to_stream(&mut writer, slot).unwrap();

        let buf = writer.into_inner();
        let mut reader = BufReader::new(&buf[..]);
        let copied_accounts = TempDir::new().unwrap();

        // Simulate obtaining a copy of the AppendVecs from a tarball
        let storage_and_next_append_vec_id =
            copy_append_vecs(accounts, copied_accounts.path()).unwrap();
        let mut accounts_db = accountsdb_from_stream(
            &mut reader,
            &[],
            storage_and_next_append_vec_id,
            accounts_db_config,
        )
        .unwrap();

        // The append vecs will be used from `copied_accounts` directly by the new AccountsDb so keep
        // its TempDir alive
        accounts_db
            .temp_paths
            .as_mut()
            .unwrap()
            .push(copied_accounts);

        accounts_db
    }

    fn check_accounts_local(accounts: &Accounts, pubkeys: &[Pubkey], num: usize) {
        for _ in 1..num {
            let idx = rng().random_range(0..num - 1);
            let ancestors = Ancestors::from(vec![0]);
            let account = accounts.load_without_fixed_root(&ancestors, &pubkeys[idx]);
            let account1 = Some((
                AccountSharedData::new((idx + 1) as u64, 0, AccountSharedData::default().owner()),
                0,
            ));
            assert_eq!(account, account1);
        }
    }

    #[test]
    fn test_accounts_serialize() {
        agave_logger::setup();
        let (_accounts_dir, paths) = get_temp_accounts_paths(4).unwrap();
        let accounts_db =
            AccountsDb::new_for_tests_with_config(paths, ACCOUNTS_DB_CONFIG_FOR_TESTING);
        let accounts = Accounts::new(Arc::new(accounts_db));

        let slot = 0;
        let pubkeys: Vec<_> = std::iter::repeat_with(solana_pubkey::new_rand)
            .take(100)
            .collect();
        let ancestors = Ancestors::from(vec![slot]);

        for (i, pubkey) in pubkeys.iter().enumerate() {
            let account = AccountSharedData::new(i as u64 + 1, 0, &Pubkey::default());
            accounts.store_accounts(
                (slot, [(pubkey, &account)].as_slice()),
                BankId::new(0),
                None,
                &ancestors,
            );
        }
        check_accounts_local(&accounts, &pubkeys, 100);
        accounts.accounts_db.add_root_and_flush_write_cache(slot);
        let accounts_hash = accounts
            .accounts_db
            .calculate_accounts_lt_hash_at_startup_from_index(&Ancestors::default());

        let mut writer = Cursor::new(vec![]);
        accounts_db_fields_to_stream(&mut writer, slot).unwrap();

        let copied_accounts = TempDir::new().unwrap();

        // Simulate obtaining a copy of the AppendVecs from a tarball
        let storage_and_next_append_vec_id =
            copy_append_vecs(&accounts.accounts_db, copied_accounts.path()).unwrap();

        let buf = writer.into_inner();
        let mut reader = BufReader::new(&buf[..]);
        let (_accounts_dir, daccounts_paths) = get_temp_accounts_paths(2).unwrap();
        let daccounts = Accounts::new(Arc::new(
            accountsdb_from_stream(
                &mut reader,
                &daccounts_paths,
                storage_and_next_append_vec_id,
                ACCOUNTS_DB_CONFIG_FOR_TESTING,
            )
            .unwrap(),
        ));
        check_accounts_local(&daccounts, &pubkeys, 100);
        let daccounts_hash = accounts
            .accounts_db
            .calculate_accounts_lt_hash_at_startup_from_index(&Ancestors::default());
        assert_eq!(accounts_hash, daccounts_hash);
    }

    #[test]
    fn test_remove_unrooted_slot_snapshot() {
        agave_logger::setup();
        let unrooted_slot = 9;
        let unrooted_bank_id = BankId::new(9);
        let db = AccountsDb::default_for_tests();
        let key = solana_pubkey::new_rand();
        let account0 = AccountSharedData::new(1, 0, &key);
        db.store_for_tests((unrooted_slot, [(&key, &account0)].as_slice()));

        // Purge the slot
        db.remove_unrooted_slots(&[(unrooted_slot, unrooted_bank_id)]);

        // Add a new root
        let key2 = solana_pubkey::new_rand();
        let new_root = unrooted_slot + 1;
        db.store_for_tests((new_root, [(&key2, &account0)].as_slice()));
        db.add_root_and_flush_write_cache(new_root);

        // Simulate reconstruction from snapshot
        let db = reconstruct_accounts_db_via_serialization(
            &db,
            new_root,
            ACCOUNTS_DB_CONFIG_FOR_TESTING,
        );

        // Check root account exists
        db.assert_load_account(new_root, key2, 1);

        // Check purged account stays gone
        db.assert_not_load_account(unrooted_slot, key);
    }

    #[test]
    fn test_accounts_db_serialize1() {
        for pass in 0..2 {
            agave_logger::setup();
            let accounts = AccountsDb::default_for_tests();
            let mut pubkeys: Vec<Pubkey> = vec![];

            // Create 100 accounts in slot 0
            accounts.create_account(&mut pubkeys, 0, 100, 0, 0);
            if pass == 0 {
                accounts.add_root_and_flush_write_cache(0);
                accounts.check_storage(0, 100, 100);
                accounts.clean_accounts_for_tests();
                accounts.check_accounts(&pubkeys, 0, 100, 1);
                // clean should have done nothing
                continue;
            }

            // do some updates to those accounts and re-check
            accounts.modify_accounts(&pubkeys, 0, 100, 2);
            accounts.add_root_and_flush_write_cache(0);
            accounts.check_storage(0, 100, 100);
            accounts.check_accounts(&pubkeys, 0, 100, 2);

            let mut pubkeys1: Vec<Pubkey> = vec![];

            // CREATE SLOT 1
            let latest_slot = 1;

            // Modify the first 10 of the accounts from slot 0 in slot 1
            accounts.modify_accounts(&pubkeys, latest_slot, 10, 3);
            // Overwrite account 30 from slot 0 with lamports=0 into slot 1.
            // Slot 1 should now have 10 + 1 = 11 accounts
            let account = AccountSharedData::new(0, 0, AccountSharedData::default().owner());
            accounts.store_for_tests((latest_slot, [(&pubkeys[30], &account)].as_slice()));

            // Create 10 new accounts in slot 1, should now have 11 + 10 = 21
            // accounts
            accounts.create_account(&mut pubkeys1, latest_slot, 10, 0, 0);

            accounts.add_root_and_flush_write_cache(latest_slot);
            accounts.check_storage(1, 21, 21);

            // CREATE SLOT 2
            let latest_slot = 2;
            let mut pubkeys2: Vec<Pubkey> = vec![];

            // Modify first 20 of the accounts from slot 0 in slot 2
            accounts.modify_accounts(&pubkeys, latest_slot, 20, 4);
            accounts.clean_accounts_for_tests();
            // Overwrite account 31 from slot 0 with lamports=0 into slot 2.
            // Slot 2 should now have 20 + 1 = 21 accounts
            let account = AccountSharedData::new(0, 0, AccountSharedData::default().owner());
            accounts.store_for_tests((latest_slot, [(&pubkeys[31], &account)].as_slice()));

            // Create 10 new accounts in slot 2. Slot 2 should now have
            // 21 + 10 = 31 accounts
            accounts.create_account(&mut pubkeys2, latest_slot, 10, 0, 0);

            accounts.add_root_and_flush_write_cache(latest_slot);
            accounts.check_storage(2, 31, 31);

            let ancestors = linear_ancestors(latest_slot);

            accounts.clean_accounts_for_tests();
            // The first 20 accounts of slot 0 have been updated in slot 2, as well as
            // accounts 30 and  31 (overwritten with zero-lamport accounts in slot 1 and
            // slot 2 respectively), so only 78 accounts are left in slot 0's storage entries.
            accounts.check_storage(0, 78, 100);
            // 10 of the 21 accounts have been modified in slot 2, so only 11
            // accounts left in slot 1.
            accounts.check_storage(1, 11, 21);
            accounts.check_storage(2, 31, 31);

            let daccounts = reconstruct_accounts_db_via_serialization(
                &accounts,
                latest_slot,
                ACCOUNTS_DB_CONFIG_FOR_TESTING,
            );

            assert_eq!(
                daccounts.write_version.load(Ordering::Acquire),
                accounts.write_version.load(Ordering::Acquire)
            );

            daccounts.print_count_and_status("daccounts");

            // Don't check the first 35 accounts which have not been modified on slot 0
            daccounts.check_accounts(&pubkeys[35..], 0, 65, 37);
            daccounts.check_accounts(&pubkeys1, 1, 10, 1);

            daccounts.check_storage(0, 78, 78);
            daccounts.check_storage(1, 11, 11);

            daccounts.check_storage(2, 31, 31);

            assert_eq!(
                daccounts.calculate_accounts_lt_hash_at_startup_from_index(&ancestors),
                accounts.calculate_accounts_lt_hash_at_startup_from_index(&ancestors),
            );
        }
    }

    #[test]
    fn test_accounts_db_serialize_zero_and_free() {
        agave_logger::setup();

        let some_lamport = 223;
        let zero_lamport = 0;
        let no_data = 0;
        let owner = *AccountSharedData::default().owner();

        let account = AccountSharedData::new(some_lamport, no_data, &owner);
        let pubkey = solana_pubkey::new_rand();
        let zero_lamport_account = AccountSharedData::new(zero_lamport, no_data, &owner);

        let account2 = AccountSharedData::new(some_lamport + 1, no_data, &owner);
        let pubkey2 = solana_pubkey::new_rand();

        let accounts = AccountsDb::default_for_tests();

        let mut current_slot = 1;
        accounts.store_for_tests((current_slot, [(&pubkey, &account)].as_slice()));
        accounts.add_root_and_flush_write_cache(current_slot);

        current_slot += 1;
        accounts.store_for_tests((current_slot, [(&pubkey, &zero_lamport_account)].as_slice()));
        accounts.store_for_tests((current_slot, [(&pubkey2, &account2)].as_slice()));

        accounts.add_root_and_flush_write_cache(current_slot);

        accounts.assert_not_load_account(current_slot, pubkey);

        accounts.print_accounts_stats("accounts");

        accounts.clean_accounts_for_tests();

        accounts.print_accounts_stats("accounts_post_purge");

        let accounts = reconstruct_accounts_db_via_serialization(
            &accounts,
            current_slot,
            ACCOUNTS_DB_CONFIG_FOR_TESTING,
        );

        accounts.print_accounts_stats("reconstructed");

        accounts.assert_load_account(current_slot, pubkey, zero_lamport);
    }

    fn with_chained_zero_lamport_accounts<F>(f: F)
    where
        F: Fn(AccountsDb, Slot) -> AccountsDb,
    {
        let some_lamport = 223;
        let zero_lamport = 0;
        let dummy_lamport = 999;
        let no_data = 0;
        let owner = *AccountSharedData::default().owner();

        let account = AccountSharedData::new(some_lamport, no_data, &owner);
        let account2 = AccountSharedData::new(some_lamport + 100_001, no_data, &owner);
        let account3 = AccountSharedData::new(some_lamport + 100_002, no_data, &owner);
        let zero_lamport_account = AccountSharedData::new(zero_lamport, no_data, &owner);

        let pubkey = solana_pubkey::new_rand();
        let purged_pubkey1 = solana_pubkey::new_rand();
        let purged_pubkey2 = solana_pubkey::new_rand();

        let dummy_account = AccountSharedData::new(dummy_lamport, no_data, &owner);
        let dummy_pubkey = Pubkey::default();

        let accounts = AccountsDb::default_for_tests();

        let mut current_slot = 1;
        accounts.store_for_tests((current_slot, [(&pubkey, &account)].as_slice()));
        accounts.store_for_tests((current_slot, [(&purged_pubkey1, &account2)].as_slice()));
        accounts.add_root_and_flush_write_cache(current_slot);

        current_slot += 1;
        accounts.store_for_tests((
            current_slot,
            [(&purged_pubkey1, &zero_lamport_account)].as_slice(),
        ));
        accounts.store_for_tests((current_slot, [(&purged_pubkey2, &account3)].as_slice()));
        accounts.add_root_and_flush_write_cache(current_slot);

        current_slot += 1;
        accounts.store_for_tests((
            current_slot,
            [(&purged_pubkey2, &zero_lamport_account)].as_slice(),
        ));
        accounts.add_root_and_flush_write_cache(current_slot);

        current_slot += 1;
        accounts.store_for_tests((current_slot, [(&dummy_pubkey, &dummy_account)].as_slice()));
        accounts.add_root_and_flush_write_cache(current_slot);

        accounts.print_accounts_stats("pre_f");

        let accounts = f(accounts, current_slot);

        accounts.print_accounts_stats("post_f");

        // The live accounts survive the chained zero-lamport purges and snapshot round-trip.
        accounts.assert_load_account(current_slot, pubkey, some_lamport);
        accounts.assert_load_account(current_slot, dummy_pubkey, dummy_lamport);
        // Both purged accounts were converted to tombstones at flush; with no full snapshot
        // taken by then, their tombstone-only storages were purged and neither is in the index.
        accounts.assert_not_load_account(current_slot, purged_pubkey1);
        accounts.assert_not_load_account(current_slot, purged_pubkey2);

        let calculated_capitalization =
            accounts.calculate_capitalization_at_startup_from_index(&Ancestors::default());
        let expected_capitalization = 1_222;
        assert_eq!(calculated_capitalization, expected_capitalization);
    }

    #[test]
    fn test_accounts_purge_chained_purge_before_snapshot_restore() {
        agave_logger::setup();
        with_chained_zero_lamport_accounts(|accounts, current_slot| {
            // If there is no latest full snapshot, zero lamport accounts can be cleaned and
            // removed immediately. Set latest full snapshot slot to zero to avoid cleaning
            // zero lamport accounts
            accounts.set_latest_full_snapshot_slot(0);
            accounts.clean_accounts_for_tests();
            reconstruct_accounts_db_via_serialization(
                &accounts,
                current_slot,
                ACCOUNTS_DB_CONFIG_FOR_TESTING,
            )
        });
    }

    #[test]
    fn test_accounts_purge_chained_purge_after_snapshot_restore() {
        agave_logger::setup();
        with_chained_zero_lamport_accounts(|accounts, current_slot| {
            let accounts = reconstruct_accounts_db_via_serialization(
                &accounts,
                current_slot,
                ACCOUNTS_DB_CONFIG_FOR_TESTING,
            );
            accounts.print_accounts_stats("after_reconstruct");
            accounts.set_latest_full_snapshot_slot(0);
            accounts.clean_accounts_for_tests();
            reconstruct_accounts_db_via_serialization(
                &accounts,
                current_slot,
                ACCOUNTS_DB_CONFIG_FOR_TESTING,
            )
        });
    }

    #[test]
    fn test_accounts_purge_long_chained_after_snapshot_restore() {
        agave_logger::setup();
        let old_lamport = 223;
        let zero_lamport = 0;
        let no_data = 0;
        let owner = *AccountSharedData::default().owner();

        let account = AccountSharedData::new(old_lamport, no_data, &owner);
        let account2 = AccountSharedData::new(old_lamport + 100_001, no_data, &owner);
        let account3 = AccountSharedData::new(old_lamport + 100_002, no_data, &owner);
        let dummy_account = AccountSharedData::new(99_999_999, no_data, &owner);
        let zero_lamport_account = AccountSharedData::new(zero_lamport, no_data, &owner);

        let pubkey = solana_pubkey::new_rand();
        let dummy_pubkey = solana_pubkey::new_rand();
        let purged_pubkey1 = solana_pubkey::new_rand();
        let purged_pubkey2 = solana_pubkey::new_rand();

        let mut current_slot = 0;
        let accounts = AccountsDb::default_for_tests();

        // create intermediate updates to purged_pubkey1 so that
        // generate_index must add slots as root last at once
        current_slot += 1;
        accounts.store_for_tests((current_slot, [(&pubkey, &account)].as_slice()));
        accounts.store_for_tests((current_slot, [(&purged_pubkey1, &account2)].as_slice()));
        accounts.add_root_and_flush_write_cache(current_slot);

        current_slot += 1;
        accounts.store_for_tests((current_slot, [(&purged_pubkey1, &account2)].as_slice()));
        accounts.add_root_and_flush_write_cache(current_slot);

        current_slot += 1;
        accounts.store_for_tests((current_slot, [(&purged_pubkey1, &account2)].as_slice()));
        accounts.add_root_and_flush_write_cache(current_slot);

        current_slot += 1;
        accounts.store_for_tests((
            current_slot,
            [(&purged_pubkey1, &zero_lamport_account)].as_slice(),
        ));
        accounts.store_for_tests((current_slot, [(&purged_pubkey2, &account3)].as_slice()));
        accounts.add_root_and_flush_write_cache(current_slot);

        current_slot += 1;
        accounts.store_for_tests((
            current_slot,
            [(&purged_pubkey2, &zero_lamport_account)].as_slice(),
        ));
        accounts.add_root_and_flush_write_cache(current_slot);

        current_slot += 1;
        accounts.store_for_tests((current_slot, [(&dummy_pubkey, &dummy_account)].as_slice()));
        accounts.add_root_and_flush_write_cache(current_slot);

        accounts.print_count_and_status("before reconstruct");
        let accounts = reconstruct_accounts_db_via_serialization(
            &accounts,
            current_slot,
            ACCOUNTS_DB_CONFIG_FOR_TESTING,
        );
        accounts.set_latest_full_snapshot_slot(0);
        accounts.print_count_and_status("before purge zero");
        accounts.clean_accounts_for_tests();
        accounts.print_count_and_status("after purge zero");

        accounts.assert_load_account(current_slot, pubkey, old_lamport);
        // Both purged accounts were converted to tombstones at flush; with no full snapshot
        // taken by then, their tombstone-only storages were purged and neither is in the index.
        accounts.assert_not_load_account(current_slot, purged_pubkey1);
        accounts.assert_not_load_account(current_slot, purged_pubkey2);
    }

    #[test]
    fn test_accounts_clean_after_snapshot_restore_then_old_revives() {
        agave_logger::setup();
        let old_lamport = 223;
        let zero_lamport = 0;
        let no_data = 0;
        let dummy_lamport = 999_999;
        let owner = *AccountSharedData::default().owner();

        let account = AccountSharedData::new(old_lamport, no_data, &owner);
        let account2 = AccountSharedData::new(old_lamport + 100_001, no_data, &owner);
        let account3 = AccountSharedData::new(old_lamport + 100_002, no_data, &owner);
        let dummy_account = AccountSharedData::new(dummy_lamport, no_data, &owner);
        let zero_lamport_account = AccountSharedData::new(zero_lamport, no_data, &owner);

        let pubkey1 = solana_pubkey::new_rand();
        let pubkey2 = solana_pubkey::new_rand();
        let dummy_pubkey = solana_pubkey::new_rand();

        let mut current_slot = 0;
        let accounts = AccountsDb::default_for_tests();

        accounts.set_latest_full_snapshot_slot(0);

        // A: Initialize AccountsDb with pubkey1 and pubkey2
        current_slot += 1;
        accounts.store_for_tests((current_slot, [(&pubkey1, &account)].as_slice()));
        accounts.store_for_tests((current_slot, [(&pubkey2, &account)].as_slice()));
        accounts.add_root(current_slot);

        // B: Test multiple updates to pubkey1 in a single slot/storage
        current_slot += 1;
        assert_eq!(0, accounts.alive_account_count_in_slot(current_slot));
        accounts.add_root_and_flush_write_cache(current_slot - 1);
        assert!(accounts.contains(&pubkey1));
        accounts.store_for_tests((current_slot, [(&pubkey1, &account2)].as_slice()));
        accounts.store_for_tests((current_slot, [(&pubkey1, &account2)].as_slice()));
        accounts.add_root_and_flush_write_cache(current_slot);
        assert_eq!(1, accounts.alive_account_count_in_slot(current_slot));
        // pubkey1 is still alive; the older version in the previous slot was marked obsolete
        assert!(accounts.contains(&pubkey1));

        // C: Yet more update to trigger lazy clean of step A
        current_slot += 1;
        assert!(accounts.contains(&pubkey1));
        accounts.store_for_tests((current_slot, [(&pubkey1, &account3)].as_slice()));
        accounts.add_root_and_flush_write_cache(current_slot);
        assert!(accounts.contains(&pubkey1));
        accounts.add_root_and_flush_write_cache(current_slot);

        // D: Make pubkey1 0-lamport; also triggers clean of step B
        current_slot += 1;
        assert!(accounts.contains(&pubkey1));
        accounts.store_for_tests((current_slot, [(&pubkey1, &zero_lamport_account)].as_slice()));
        accounts.add_root_and_flush_write_cache(current_slot);

        // The zero lamport account was converted to a tombstone, so pubkey1 is out of the index
        assert!(!accounts.contains(&pubkey1));
        accounts.add_root(current_slot);

        // E: Avoid missing bank hash error
        current_slot += 1;
        accounts.store_for_tests((current_slot, [(&dummy_pubkey, &dummy_account)].as_slice()));
        accounts.add_root(current_slot);

        accounts.assert_not_load_account(current_slot, pubkey1);
        accounts.assert_load_account(current_slot, pubkey2, old_lamport);
        accounts.assert_load_account(current_slot, dummy_pubkey, dummy_lamport);

        // At this point, there is no index entries for A and B
        // If step C and step D should be purged, snapshot restore would cause
        // pubkey1 to be revived as the state of step A.
        // So, prevent that from happening by introducing refcount
        ((current_slot - 1)..=current_slot).for_each(|slot| accounts.flush_root_write_cache(slot));
        accounts.clean_accounts_for_tests();

        // Reconstruct the database from the snapshot to simulate a restore
        let accounts = reconstruct_accounts_db_via_serialization(
            &accounts,
            current_slot,
            ACCOUNTS_DB_CONFIG_FOR_TESTING,
        );

        // At this point pubkey 1 has been restored as a zero lamport account
        accounts.assert_load_account(current_slot, pubkey1, zero_lamport);
        accounts.assert_load_account(current_slot, pubkey2, old_lamport);
        accounts.assert_load_account(current_slot, dummy_pubkey, dummy_lamport);

        // Regardless of the snapshot slot the zero-lamport pubkey1 is turned into a tombstone
        // by clean
        accounts.set_latest_full_snapshot_slot(0);
        accounts.clean_accounts_for_tests();

        accounts.assert_not_load_account(current_slot, pubkey1);
        accounts.assert_load_account(current_slot, pubkey2, old_lamport);
        accounts.assert_load_account(current_slot, dummy_pubkey, dummy_lamport);
    }

    #[test]
    fn test_shrink_stale_slots_processed() {
        agave_logger::setup();

        let accounts = AccountsDb::default_for_tests();

        let pubkey_count = 100;
        let pubkeys: Vec<_> = (0..pubkey_count)
            .map(|_| solana_pubkey::new_rand())
            .collect();

        let some_lamport = 223;
        let no_data = 0;
        let owner = *AccountSharedData::default().owner();

        let account = AccountSharedData::new(some_lamport, no_data, &owner);

        let mut current_slot = 0;

        current_slot += 1;
        for pubkey in &pubkeys {
            accounts.store_for_tests((current_slot, [(pubkey, &account)].as_slice()));
        }
        let shrink_slot = current_slot;
        accounts.add_root_and_flush_write_cache(current_slot);

        current_slot += 1;
        let pubkey_count_after_shrink = 10;
        let updated_pubkeys = &pubkeys[0..pubkey_count - pubkey_count_after_shrink];

        for pubkey in updated_pubkeys {
            accounts.store_for_tests((current_slot, [(pubkey, &account)].as_slice()));
        }
        accounts.add_root_and_flush_write_cache(current_slot);

        accounts.clean_accounts_for_tests();

        assert_eq!(
            pubkey_count,
            accounts.all_account_count_in_accounts_file(shrink_slot)
        );
        accounts.shrink_candidate_slots(&EpochSchedule::default());
        assert_eq!(
            pubkey_count_after_shrink,
            accounts.all_account_count_in_accounts_file(shrink_slot)
        );

        let no_ancestors = Ancestors::default();

        let calculated_capitalization =
            accounts.calculate_capitalization_at_startup_from_index(&no_ancestors);
        let expected_capitalization = 22_300;
        assert_eq!(calculated_capitalization, expected_capitalization);

        let accounts_lt_hash_pre =
            accounts.calculate_accounts_lt_hash_at_startup_from_index(&no_ancestors);
        let accounts = reconstruct_accounts_db_via_serialization(
            &accounts,
            current_slot,
            ACCOUNTS_DB_CONFIG_FOR_TESTING,
        );
        let accounts_lt_hash_post =
            accounts.calculate_accounts_lt_hash_at_startup_from_index(&no_ancestors);
        assert_eq!(accounts_lt_hash_pre, accounts_lt_hash_post);
    }

    // no remap needed
    #[test_case(456, 456, 456, 0, |_| {})]
    // remap from 456 to 457, no collisions
    #[test_case(456, 457, 457, 0, |_| {})]
    // attempt to remap from 456 to 457, but there's a collision, so we get 458
    #[test_case(456, 457, 458, 1, |tmp| {
        File::create(tmp.join("123.457")).unwrap();
    })]
    fn test_remap_append_vec_file(
        old_id: usize,
        next_id: usize,
        expected_remapped_id: usize,
        expected_collisions: usize,
        become_ungovernable: impl FnOnce(&Path),
    ) {
        let tmp = tempfile::tempdir().unwrap();
        let old_path = tmp.path().join(format!("123.{old_id}"));
        let old_file_info = FileInfo {
            file: File::create(&old_path).unwrap(),
            path: old_path,
            size: 0,
        };
        let expected_remapped_path = tmp.path().join(format!("123.{expected_remapped_id}"));

        become_ungovernable(tmp.path());

        let next_append_vec_id = AtomicAccountsFileId::new(next_id as u32);
        let mut num_collisions = 0;
        let (remapped_id, remapped_file_info) = remap_append_vec_file(
            123,
            old_id,
            old_file_info,
            &next_append_vec_id,
            &mut num_collisions,
        )
        .unwrap();
        assert_eq!(remapped_id as usize, expected_remapped_id);
        assert_eq!(&remapped_file_info.path, &expected_remapped_path);
        assert_eq!(num_collisions, expected_collisions);
    }

    #[test]
    #[should_panic(expected = "No such file or directory")]
    fn test_remap_append_vec_file_error() {
        let tmp = tempfile::tempdir().unwrap();
        let original_path = tmp.path().join("123.456");
        // there won't be any file at `original_path`, so rename should generate error
        let original_file_info = FileInfo {
            file: File::create(tmp.path().join("wrong_name")).unwrap(),
            path: original_path,
            size: 0,
        };

        // In remap_append_vec_file() we want to handle EEXIST (collisions), but we want to return all
        // other errors
        let next_append_vec_id = AtomicAccountsFileId::new(457);
        let mut num_collisions = 0;
        remap_append_vec_file(
            123,
            456,
            original_file_info,
            &next_append_vec_id,
            &mut num_collisions,
        )
        .unwrap();
    }

    /// Test roundtrip serialize/deserialize of a bank
    #[test]
    fn test_serialize_bank_snapshot() {
        let leader_id = Pubkey::new_unique();
        let GenesisConfigInfo {
            mut genesis_config, ..
        } = create_genesis_config_with_leader(500, &leader_id, LAMPORTS_PER_SOL);
        genesis_config.epoch_schedule = EpochSchedule::custom(400, 400, false);
        let (bank0, bank_forks) =
            Bank::new_for_tests(&genesis_config).wrap_with_bank_forks_for_tests();
        let deposit_amount = bank0.get_minimum_balance_for_rent_exemption(0);
        let bank1 = Bank::new_from_parent_with_bank_forks(
            bank_forks.as_ref(),
            bank0.clone(),
            *bank0.leader(),
            1,
        );

        // Create an account on a non-root fork
        let key1 = Pubkey::new_unique();
        bank_test_utils::deposit(&bank1, &key1, deposit_amount).unwrap();

        let bank2_slot = 2;
        let bank0_leader = *bank0.leader();
        let bank2 = Bank::new_from_parent_with_bank_forks(
            bank_forks.as_ref(),
            bank0,
            bank0_leader,
            bank2_slot,
        );

        // Test new account
        let key2 = Pubkey::new_unique();
        bank_test_utils::deposit(&bank2, &key2, deposit_amount).unwrap();
        assert_eq!(bank2.get_balance(&key2), deposit_amount);

        let key3 = Pubkey::new_unique();
        bank_test_utils::deposit(&bank2, &key3, 0).unwrap();

        let accounts_db = &bank2.rc.accounts.accounts_db;

        bank2.set_block_id(Some(Hash::default()));
        bank2.squash();
        bank2.force_flush_accounts_cache();

        let mut buf = Vec::new();
        let cursor = Cursor::new(&mut buf);
        let mut writer = BufWriter::new(cursor);
        {
            let mut bank_fields = bank2.get_fields_to_serialize();
            let versioned_epoch_stakes = mem::take(&mut bank_fields.versioned_epoch_stakes);
            let accounts_lt_hash = Some(bank_fields.accounts_lt_hash.clone().into());
            let block_id = Some(bank_fields.block_id);
            serde_snapshot::serialize_bank_snapshot_into(
                &mut writer,
                bank_fields,
                bank2.get_bank_hash_stats(),
                ExtraFieldsToSerialize {
                    lamports_per_signature: bank2.fee_rate_governor.lamports_per_signature,
                    unused_incremental_snapshot_persistence: None,
                    unused_epoch_accounts_hash: None,
                    versioned_epoch_stakes,
                    accounts_lt_hash,
                    block_id,
                },
            )
            .unwrap();
        }
        drop(writer);

        // Now deserialize the serialized bank and ensure it matches the original bank

        // Create a new set of directories for this bank's accounts
        let (_accounts_dir, dbank_paths) = get_temp_accounts_paths(4).unwrap();
        // Create a directory to simulate AppendVecs unpackaged from a snapshot tar
        let copied_accounts = TempDir::new().unwrap();
        let storage_and_next_append_vec_id =
            copy_append_vecs(accounts_db, copied_accounts.path()).unwrap();

        let cursor = Cursor::new(buf.as_slice());
        let mut reader = BufReader::new(cursor);
        let mut snapshot_streams = SnapshotStreams {
            full_snapshot_stream: &mut reader,
            incremental_snapshot_stream: None,
        };
        let (dbank, _) = serde_snapshot::bank_from_streams(
            &mut snapshot_streams,
            &dbank_paths,
            storage_and_next_append_vec_id,
            &genesis_config,
            &RuntimeConfig::default(),
            None,
            None,
            false,
            ACCOUNTS_DB_CONFIG_FOR_TESTING,
            None,
            Arc::default(),
        )
        .unwrap();
        assert_eq!(dbank.get_balance(&key1), 0);
        assert_eq!(dbank.get_balance(&key2), deposit_amount);
        assert_eq!(dbank.get_balance(&key3), 0);
        assert_eq!(dbank.get_bank_hash_stats(), bank2.get_bank_hash_stats());
        assert_eq!(&dbank, bank2.as_ref());
    }

    #[test]
    fn test_extra_fields_eof() {
        agave_logger::setup();
        let leader_id = Pubkey::new_unique();
        let GenesisConfigInfo { genesis_config, .. } =
            create_genesis_config_with_leader(500, &leader_id, LAMPORTS_PER_SOL);

        let (bank0, _bank_forks) =
            Bank::new_for_tests(&genesis_config).wrap_with_bank_forks_for_tests();
        bank0.squash();
        let mut bank = Bank::new_from_parent(bank0.clone(), *bank0.leader(), 1);
        bank.set_block_id(Some(Hash::default()));
        bank.freeze();
        bank.rc.accounts.add_root(bank.slot());
        bank.force_flush_accounts_cache();

        // Set extra fields
        bank.fee_rate_governor.lamports_per_signature = 7000;
        // Note that epoch_stakes already has two epoch stakes entries for epochs 0 and 1
        // which will also be serialized to the versioned epoch stakes extra field, so add a
        // third entry to exercise round-tripping the extra field.
        bank.set_epoch_stakes_for_test(
            42,
            VersionedEpochStakes::Current {
                stakes: EpochStakes::default(),
                total_stake: 42,
                node_id_to_vote_accounts: Arc::<NodeIdToVoteAccounts>::default(),
                epoch_authorized_voters: Arc::<EpochAuthorizedVoters>::default(),
                bls_pubkey_to_rank_map: OnceLock::new(),
            },
        );
        assert_eq!(bank.epoch_stakes_map().len(), 3);

        // Serialize
        let mut buf = vec![];
        let mut writer = Cursor::new(&mut buf);

        crate::serde_snapshot::bank_to_stream(&mut std::io::BufWriter::new(&mut writer), &bank)
            .unwrap();

        // Deserialize
        let rdr = Cursor::new(&buf[..]);
        let mut reader = std::io::BufReader::new(&buf[rdr.position() as usize..]);
        let mut snapshot_streams = SnapshotStreams {
            full_snapshot_stream: &mut reader,
            incremental_snapshot_stream: None,
        };
        let (_accounts_dir, dbank_paths) = get_temp_accounts_paths(4).unwrap();
        let copied_accounts = TempDir::new().unwrap();
        let storage_and_next_append_vec_id =
            copy_append_vecs(&bank.rc.accounts.accounts_db, copied_accounts.path()).unwrap();
        let (dbank, _) = crate::serde_snapshot::bank_from_streams(
            &mut snapshot_streams,
            &dbank_paths,
            storage_and_next_append_vec_id,
            &genesis_config,
            &RuntimeConfig::default(),
            None,
            None,
            false,
            ACCOUNTS_DB_CONFIG_FOR_TESTING,
            None,
            Arc::default(),
        )
        .unwrap();

        assert_eq!(bank.epoch_stakes_map(), dbank.epoch_stakes_map());
        assert_eq!(
            bank.fee_rate_governor.lamports_per_signature,
            dbank.fee_rate_governor.lamports_per_signature
        );
    }

    #[test]
    fn test_extra_fields_full_snapshot_archive() {
        agave_logger::setup();

        let leader_id = Pubkey::new_unique();
        let GenesisConfigInfo {
            mut genesis_config, ..
        } = create_genesis_config_with_leader(500, &leader_id, LAMPORTS_PER_SOL);
        activate_all_features(&mut genesis_config);

        let (bank0, _bank_forks) =
            Bank::new_for_tests(&genesis_config).wrap_with_bank_forks_for_tests();
        let bank0_leader = *bank0.leader();
        let mut bank = Bank::new_from_parent(bank0, bank0_leader, 1);
        while !bank.is_complete() {
            bank.fill_bank_with_ticks_for_tests();
        }
        bank.set_block_id(Some(Hash::default()));

        // Set extra field
        bank.fee_rate_governor.lamports_per_signature = 7000;

        let (_tmp_dir, accounts_dir) = create_tmp_accounts_dir_for_tests();
        let bank_snapshots_dir = TempDir::new().unwrap();
        let full_snapshot_archives_dir = TempDir::new().unwrap();
        let incremental_snapshot_archives_dir = TempDir::new().unwrap();

        // Serialize
        let snapshot_config = SnapshotConfig {
            full_snapshot_archives_dir: full_snapshot_archives_dir.path().to_path_buf(),
            incremental_snapshot_archives_dir: incremental_snapshot_archives_dir
                .path()
                .to_path_buf(),
            bank_snapshots_dir: bank_snapshots_dir.path().to_path_buf(),
            ..SnapshotConfig::default()
        };
        let snapshot_archive_info =
            snapshot_bank_utils::bank_to_full_snapshot_archive(&snapshot_config, &bank).unwrap();

        // Deserialize
        let dbank = snapshot_bank_utils::bank_from_snapshot_archives(
            &[accounts_dir],
            &snapshot_archive_info,
            None,
            &snapshot_config,
            &genesis_config,
            &RuntimeConfig::default(),
            None,
            None, // leader_for_tests
            None,
            false,
            ACCOUNTS_DB_CONFIG_FOR_TESTING,
            None,
            Arc::default(),
        )
        .unwrap();

        assert_eq!(
            bank.fee_rate_governor.lamports_per_signature,
            dbank.fee_rate_governor.lamports_per_signature
        );
    }
}