casc 0.1.6

/*****************************************************************************/
/* CascIndexFiles.cpp                     Copyright (c) Ladislav Zezula 2019 */
/*---------------------------------------------------------------------------*/
/* Index file support                                                        */
/*---------------------------------------------------------------------------*/
/*   Date    Ver   Who  Comment                                              */
/* --------  ----  ---  -------                                              */
/* 23.05.19  1.00  Lad  Created                                              */
/*****************************************************************************/

#define __CASCLIB_SELF__
#include "CascLib.h"
#include "CascCommon.h"

//-----------------------------------------------------------------------------
// Local variables

static LPCTSTR szAllowedHexChars = _T("0123456789aAbBcCdDeEfF");
static LPCTSTR szIndexFormat_V1 = _T("data.i%x%x");
static LPCTSTR szIndexFormat_V2 = _T("%02x%08x.idx");

// Limit for "orphaned" items - those that are in index files, but are not in ENCODING manifest
#define CASC_MAX_ORPHANED_ITEMS 0x100

typedef bool (*EKEY_ENTRY_CALLBACK)(TCascStorage * hs, CASC_INDEX_HEADER & InHeader, LPBYTE pbEKeyEntry);

//-----------------------------------------------------------------------------
// Local functions

// Check a file name with mask
static bool IsFileNameMask(LPCTSTR szFileName, LPCTSTR szMask)
{
    size_t i;

    for(i = 0; szFileName[i] != 0 && szMask[i] != 0; i++)
    {
        // '#' in the mask means any hexa number
        if(szMask[i] == _T('#'))
        {
            if(!IsHexadecimalDigit(szFileName[i]))
                return false;
            continue;
        }

        // Compare both characters, converted to lowercase
        if(AsciiToUpperTable_BkSlash[(BYTE)(szMask[i])] != AsciiToUpperTable_BkSlash[(BYTE)(szFileName[i])])
            return false;
    }

    // If we found end of both strings, it's a match
    return (szFileName[i] == 0 && szMask[i] == 0);
}

// Heroes of the Storm, build 29049: "data.i##"
static bool IsIndexFileName_V1(LPCTSTR szFileName)
{
    return IsFileNameMask(szFileName, _T("data.i##"));
}

// Current CASC storages: ##########.idx
static bool IsIndexFileName_V2(LPCTSTR szFileName)
{
    return IsFileNameMask(szFileName, _T("##########.idx"));
}

static bool IndexDirectory_OnFileFound(
    LPCTSTR szFileName,
    void * pvContext)
{
    TCascStorage * hs = (TCascStorage *)pvContext;
    PCASC_INDEX pIndexFile;
    DWORD IndexVersion = 0;
    DWORD IndexValue = 0;

    // Auto-detect the format of the index file name
    if(hs->szIndexFormat == NULL)
    {
        if(IsIndexFileName_V2(szFileName))
            hs->szIndexFormat = szIndexFormat_V2;
        else if(IsIndexFileName_V1(szFileName))
            hs->szIndexFormat = szIndexFormat_V1;
        else
            return true;
    }

    if(hs->szIndexFormat == szIndexFormat_V2)
    {
        // Check the index file name format
        if(!IsIndexFileName_V2(szFileName))
            return true;

        // Get the main index from the first two digits
        if(ConvertStringToInt(szFileName + 0, 2, IndexValue) != ERROR_SUCCESS)
            return true;
        if(ConvertStringToInt(szFileName + 2, 8, IndexVersion) != ERROR_SUCCESS)
            return true;
    }
    else if(hs->szIndexFormat == szIndexFormat_V1)
    {
        // Check the index file name format
        if(!IsIndexFileName_V1(szFileName))
            return true;

        // Get the main index from the first two digits
        if(ConvertStringToInt(szFileName + 6, 1, IndexValue) != ERROR_SUCCESS)
            return true;
        if(ConvertStringToInt(szFileName + 7, 1, IndexVersion) != ERROR_SUCCESS)
            return true;
    }
    else
    {
        // Should never happen
        assert(false);
        return false;
    }

    // The index value must not be greater than 0x0F
    if(IndexValue >= CASC_INDEX_COUNT)
        return false;
    pIndexFile = &hs->IndexFiles[IndexValue];

    // If the new subindex is greater than the previous one, use this one instead
    if(IndexVersion > pIndexFile->NewSubIndex)
    {
        pIndexFile->OldSubIndex = pIndexFile->NewSubIndex;
        pIndexFile->NewSubIndex = IndexVersion;
    }
    else if(IndexVersion > pIndexFile->OldSubIndex)
    {
        pIndexFile->OldSubIndex = IndexVersion;
    }

    // Note: WoW6 only keeps last two index files
    // Any additional index files are deleted at this point
    return true;
}

static LPTSTR CreateIndexFileName(TCascStorage * hs, DWORD IndexValue, DWORD IndexVersion)
{
    TCHAR szPlainName[0x40];

    // Sanity checks
    assert(hs->szIndexFormat != NULL);
    assert(hs->szIndexPath != NULL);
    assert(IndexValue <= 0x0F);

    // Create the plain name of the index file
    CascStrPrintf(szPlainName, _countof(szPlainName), hs->szIndexFormat, IndexValue, IndexVersion);

    // Allocate path
    return CASC_PATH<TCHAR>(hs->szIndexPath, szPlainName, NULL).New();
}

static void SaveFileOffsetBitsAndEKeyLength(TCascStorage * hs, BYTE FileOffsetBits, BYTE EKeyLength)
{
    if(hs->FileOffsetBits == 0)
        hs->FileOffsetBits = FileOffsetBits;
    assert(hs->FileOffsetBits == FileOffsetBits);

    if(hs->EKeyLength == 0)
        hs->EKeyLength = EKeyLength;
    assert(hs->EKeyLength == EKeyLength);
}

// Verifies a guarded block - data availability and checksum match
static LPBYTE CaptureGuardedBlock1(LPBYTE pbFileData, LPBYTE pbFileEnd)
{
    PFILE_INDEX_GUARDED_BLOCK pBlock = (PFILE_INDEX_GUARDED_BLOCK)pbFileData;

    // Check the guarded block
    if((pbFileData + sizeof(FILE_INDEX_GUARDED_BLOCK)) >= pbFileEnd)
        return NULL;
    if((pbFileData + sizeof(FILE_INDEX_GUARDED_BLOCK) + pBlock->BlockSize) > pbFileEnd)
        return NULL;
    
    // Verify the hash
    if(hashlittle(pbFileData + sizeof(FILE_INDEX_GUARDED_BLOCK), pBlock->BlockSize, 0) != pBlock->BlockHash)
        return NULL;

    // Give the output
    return (LPBYTE)(pBlock + 1);
}

// Second method of checking a guarded block; hash is calculated entry-by-entry.
// Unfortunately, due to implementation in hashlittle2(), we cannot calculate the hash of a continuous block
static LPBYTE CaptureGuardedBlock2(LPBYTE pbFileData, LPBYTE pbFileEnd, size_t EntryLength, PDWORD PtrBlockSize = NULL)
{
    PFILE_INDEX_GUARDED_BLOCK pBlock = (PFILE_INDEX_GUARDED_BLOCK)pbFileData;
    LPBYTE pbEntryPtr;
    size_t EntryCount;
    unsigned int HashBlizzGet = 0;
    unsigned int HashHigh = 0;
    unsigned int HashLow = 0;

    // Check the guarded block. There must be enough bytes to contain FILE_INDEX_GUARDED_BLOCK
    // and also the block length must not be NULL
    if((pbFileData + sizeof(FILE_INDEX_GUARDED_BLOCK)) >= pbFileEnd)
        return NULL;
    if(pBlock->BlockSize == 0 || (pbFileData + sizeof(FILE_INDEX_GUARDED_BLOCK) + pBlock->BlockSize) > pbFileEnd)
        return NULL;

    //
    // Verify the hash from the Blizzard Downloader
    //

    pbEntryPtr = pbFileData + sizeof(FILE_INDEX_GUARDED_BLOCK);
    EntryCount = pBlock->BlockSize / EntryLength;
    for(size_t i = 0; i < EntryCount; i++)
    {
        hashlittle2(pbEntryPtr, EntryLength, &HashHigh, &HashLow);
        pbEntryPtr += EntryLength;
    }

    // Verify hash
    if(HashHigh == pBlock->BlockHash)
    {
        // Give the output
        if(PtrBlockSize != NULL)
            PtrBlockSize[0] = pBlock->BlockSize;
        return (LPBYTE)(pBlock + 1);
    }

    //
    // Verify the hash from the Blizzget tool, which calculates the hash differently
    // https://github.com/d07RiV/blizzget/blob/master/src/ngdp.cpp
    // Function void DataStorage::writeIndex()
    //

    pbEntryPtr = pbFileData + sizeof(FILE_INDEX_GUARDED_BLOCK);
    EntryCount = pBlock->BlockSize / EntryLength;
    for(size_t i = 0; i < EntryCount; i++)
    {
        HashBlizzGet = hashlittle(pbEntryPtr, EntryLength, HashBlizzGet);
        pbEntryPtr += EntryLength;
    }

    // Verify hash
    if(HashBlizzGet == pBlock->BlockHash)
    {
        // Give the output
        if(PtrBlockSize != NULL)
            PtrBlockSize[0] = pBlock->BlockSize;
        return (LPBYTE)(pBlock + 1);
    }

    // Hash mismatch
    return NULL;
}

// Third method of checking a guarded block; There is 32-bit hash, followed by EKey entry
// The hash covers the EKey entry plus one byte
static LPBYTE CaptureGuardedBlock3(LPBYTE pbFileData, LPBYTE pbFileEnd, size_t EntryLength)
{
    PDWORD PtrEntryHash = (PDWORD)pbFileData;
    DWORD EntryHash;

    // Check the guarded block. There must be enough bytes to contain single entry and the hash
    // Also, the hash must be present
    if((pbFileData + sizeof(DWORD) + EntryLength) >= pbFileEnd)
        return NULL;
    if(PtrEntryHash[0] == 0)
        return NULL;

    EntryHash = hashlittle(pbFileData + sizeof(DWORD), EntryLength+1, 0) | 0x80000000;
    if(EntryHash != PtrEntryHash[0])
        return NULL;

    // Give the output
    return (LPBYTE)(PtrEntryHash + 1);
}

static DWORD LoadIndexItems(TCascStorage * hs, CASC_INDEX_HEADER & InHeader, EKEY_ENTRY_CALLBACK PfnEKeyEntry, LPBYTE pbEKeyEntry, LPBYTE pbEKeyEnd)
{
    size_t EntryLength = InHeader.EntryLength;

    while((pbEKeyEntry + EntryLength) <= pbEKeyEnd)
    {
        // ENCODING for Starcraft II Beta
        // BREAK_ON_XKEY3(pbEKeyEntry, 0x8b, 0x0d, 0x9a);

        if(!PfnEKeyEntry(hs, InHeader, pbEKeyEntry))
            return ERROR_INDEX_PARSING_DONE;

        pbEKeyEntry += EntryLength;
    }

    return ERROR_SUCCESS;
}

static DWORD CaptureIndexHeader_V1(CASC_INDEX_HEADER & InHeader, LPBYTE pbFileData, size_t cbFileData, DWORD BucketIndex)
{
    PFILE_INDEX_HEADER_V1 pIndexHeader = (PFILE_INDEX_HEADER_V1)pbFileData;
    LPBYTE pbKeyEntries;
    LPBYTE pbFileEnd = pbFileData + cbFileData;
    size_t cbKeyEntries;
    DWORD HeaderHash;

    // Check the available size. Note that the index file can be just a header.
    if((pbFileData + sizeof(FILE_INDEX_HEADER_V1)) > pbFileEnd)
        return ERROR_BAD_FORMAT;
    if(pIndexHeader->IndexVersion != 0x05 || pIndexHeader->BucketIndex != (BYTE)BucketIndex || pIndexHeader->field_8 == 0)
        return ERROR_BAD_FORMAT;
    if(pIndexHeader->EncodedSizeLength != 0x04 || pIndexHeader->StorageOffsetLength != 0x05 || pIndexHeader->EKeyLength != 0x09)
        return ERROR_NOT_SUPPORTED;

    // Verify the header hash
    HeaderHash = pIndexHeader->HeaderHash;
    pIndexHeader->HeaderHash = 0;
    if(hashlittle(pbFileData, sizeof(FILE_INDEX_HEADER_V1), 0) != HeaderHash)
        return ERROR_BAD_FORMAT;

    // Return the header hash back
    pIndexHeader->HeaderHash = HeaderHash;

    // Copy the fields
    InHeader.IndexVersion        = pIndexHeader->IndexVersion;
    InHeader.BucketIndex         = pIndexHeader->BucketIndex;
    InHeader.StorageOffsetLength = pIndexHeader->StorageOffsetLength;
    InHeader.EncodedSizeLength   = pIndexHeader->EncodedSizeLength;
    InHeader.EKeyLength          = pIndexHeader->EKeyLength;
    InHeader.FileOffsetBits      = pIndexHeader->FileOffsetBits;
    InHeader.Alignment           = 0;
    InHeader.SegmentSize         = pIndexHeader->SegmentSize;

    // Determine the size of the header
    InHeader.HeaderLength = sizeof(FILE_INDEX_HEADER_V1);
    InHeader.HeaderPadding = 0;
    InHeader.EntryLength = pIndexHeader->EKeyLength + pIndexHeader->StorageOffsetLength + pIndexHeader->EncodedSizeLength;
    InHeader.EKeyCount = pIndexHeader->EKeyCount1 + pIndexHeader->EKeyCount2;

    // Verify the entries hash - 1st block
    pbKeyEntries = pbFileData + InHeader.HeaderLength;
    cbKeyEntries = pIndexHeader->EKeyCount1 * InHeader.EntryLength;
    if((pbKeyEntries + cbKeyEntries) > pbFileEnd)
        return ERROR_FILE_CORRUPT;
    if(hashlittle(pbKeyEntries, cbKeyEntries, 0) != pIndexHeader->KeysHash1)
        return ERROR_FILE_CORRUPT;

    // Verify the entries hash - 2nd block
    pbKeyEntries = pbKeyEntries + cbKeyEntries;
    cbKeyEntries = pIndexHeader->EKeyCount2 * InHeader.EntryLength;
    if((pbKeyEntries + cbKeyEntries) > pbFileEnd)
        return ERROR_FILE_CORRUPT;
    if(hashlittle(pbKeyEntries, cbKeyEntries, 0) != pIndexHeader->KeysHash2)
        return ERROR_FILE_CORRUPT;

    return ERROR_SUCCESS;
}

static DWORD CaptureIndexHeader_V2(CASC_INDEX_HEADER & InHeader, LPBYTE pbFileData, size_t cbFileData, DWORD BucketIndex)
{
    PFILE_INDEX_HEADER_V2 pIndexHeader;
    LPBYTE pbFileEnd = pbFileData + cbFileData;

    // Check for guarded block
    if((pbFileData = CaptureGuardedBlock1(pbFileData, pbFileEnd)) == NULL)
        return ERROR_FILE_CORRUPT;
    pIndexHeader = (PFILE_INDEX_HEADER_V2)pbFileData;

    // Verify the content of the index header
    if(pIndexHeader->IndexVersion != 0x07 || pIndexHeader->BucketIndex != (BYTE)BucketIndex || pIndexHeader->ExtraBytes != 0x00)
        return ERROR_BAD_FORMAT;
    if(pIndexHeader->EncodedSizeLength != 0x04 || pIndexHeader->StorageOffsetLength != 0x05 || pIndexHeader->EKeyLength != 0x09)
        return ERROR_BAD_FORMAT;

    // Capture the values from the index header
    InHeader.IndexVersion        = pIndexHeader->IndexVersion;
    InHeader.BucketIndex         = pIndexHeader->BucketIndex;
    InHeader.StorageOffsetLength = pIndexHeader->StorageOffsetLength;
    InHeader.EncodedSizeLength   = pIndexHeader->EncodedSizeLength;
    InHeader.EKeyLength          = pIndexHeader->EKeyLength;
    InHeader.FileOffsetBits      = pIndexHeader->FileOffsetBits;
    InHeader.Alignment           = 0;
    InHeader.SegmentSize         = pIndexHeader->SegmentSize;

    // Supply the lengths
    InHeader.HeaderLength = sizeof(FILE_INDEX_GUARDED_BLOCK) + sizeof(FILE_INDEX_HEADER_V2);
    InHeader.HeaderPadding = 8;
    InHeader.EntryLength = pIndexHeader->EKeyLength + pIndexHeader->StorageOffsetLength + pIndexHeader->EncodedSizeLength;
    InHeader.EKeyCount = 0;
    return ERROR_SUCCESS;
}    

static DWORD LoadIndexFile_V1(TCascStorage * hs, CASC_INDEX_HEADER & InHeader, EKEY_ENTRY_CALLBACK PfnEKeyEntry, LPBYTE pbFileData, size_t cbFileData)
{
    LPBYTE pbEKeyEntries = pbFileData + InHeader.HeaderLength + InHeader.HeaderPadding;

    // Remember the values from the index header
    SaveFileOffsetBitsAndEKeyLength(hs, InHeader.FileOffsetBits, InHeader.EKeyLength);

    // Load the entries from a continuous array
    return LoadIndexItems(hs, InHeader, PfnEKeyEntry, pbEKeyEntries, pbFileData + cbFileData);
}

static DWORD LoadIndexFile_V2(TCascStorage * hs, CASC_INDEX_HEADER & InHeader, EKEY_ENTRY_CALLBACK PfnEKeyEntry, LPBYTE pbFileData, size_t cbFileData)
{
    LPBYTE pbEKeyEntry;
    LPBYTE pbFileEnd = pbFileData + cbFileData;
    LPBYTE pbFilePtr = pbFileData + InHeader.HeaderLength + InHeader.HeaderPadding;
    size_t EKeyEntriesLength;
    DWORD BlockSize = 0;
    DWORD dwErrCode = ERROR_NOT_SUPPORTED;

    // Remember the values from the index header
    SaveFileOffsetBitsAndEKeyLength(hs, InHeader.FileOffsetBits, InHeader.EKeyLength);

    // Get the pointer to the first block of EKey entries
    if((pbEKeyEntry = CaptureGuardedBlock2(pbFilePtr, pbFileEnd, InHeader.EntryLength, &BlockSize)) != NULL)
    {
        // Supply the number of EKey entries
        InHeader.HeaderPadding += sizeof(FILE_INDEX_GUARDED_BLOCK);

        // Load the continuous array of EKeys
        return LoadIndexItems(hs, InHeader, PfnEKeyEntry, pbEKeyEntry, pbEKeyEntry + BlockSize);
    }

    // Get the pointer to the second block of EKey entries.
    // They are alway at the position aligned to 4096
    EKeyEntriesLength = pbFileEnd - pbFilePtr;
    if(EKeyEntriesLength >= 0x7800)
    {
        LPBYTE pbStartPage = pbFileData + 0x1000;
        LPBYTE pbEndPage = pbStartPage + FILE_INDEX_PAGE_SIZE;
        size_t AlignedLength = ALIGN_TO_SIZE(InHeader.EntryLength, 4);

        // Parse the chunks with the EKey entries
        while(pbStartPage < pbFileEnd)
        {
            pbEKeyEntry = pbStartPage;
            
            while(pbEKeyEntry < pbEndPage)
            {
                // Check the EKey entry protected by 32-bit hash
                if((pbEKeyEntry = CaptureGuardedBlock3(pbEKeyEntry, pbEndPage, InHeader.EntryLength)) == NULL)
                    break;

                // CASC\\0001: Encoding
                //BREAK_ON_XKEY3(pbEKeyEntry, 0xbc, 0xe8, 0x23);

                // Call the EKey entry callback
                if(!PfnEKeyEntry(hs, InHeader, pbEKeyEntry))
                    return ERROR_INDEX_PARSING_DONE;

                pbEKeyEntry += AlignedLength;
            }

            // Move to the next chunk
            pbStartPage += FILE_INDEX_PAGE_SIZE;
        }
        dwErrCode = ERROR_SUCCESS;
    }

    return dwErrCode;
}

static DWORD LoadIndexFile(TCascStorage * hs, EKEY_ENTRY_CALLBACK PfnEKeyEntry, LPBYTE pbFileData, size_t cbFileData, DWORD BucketIndex)
{
    CASC_INDEX_HEADER InHeader;

    // Check for CASC version 2
    if(CaptureIndexHeader_V2(InHeader, pbFileData, cbFileData, BucketIndex) == ERROR_SUCCESS)
        return LoadIndexFile_V2(hs, InHeader, PfnEKeyEntry, pbFileData, cbFileData);

    // Check for CASC index version 1
    if(CaptureIndexHeader_V1(InHeader, pbFileData, cbFileData, BucketIndex) == ERROR_SUCCESS)
        return LoadIndexFile_V1(hs, InHeader, PfnEKeyEntry, pbFileData, cbFileData);

    // Should never happen
    assert(false);
    return ERROR_BAD_FORMAT;
}

// Checks the EKey entry for EKey of the ENCODING manifest
static bool InsertEncodingEKeyToMap(TCascStorage * hs, CASC_INDEX_HEADER &, LPBYTE pbEKeyEntry)
{
    hs->IndexEKeyMap.InsertObject(pbEKeyEntry, pbEKeyEntry);
    return true;
}

static DWORD ProcessLocalIndexFiles(TCascStorage * hs, EKEY_ENTRY_CALLBACK PfnEKeyEntry, DWORD dwIndexCount)
{
    DWORD dwErrCode = ERROR_SUCCESS;

    // Load each index file
    for(DWORD i = 0; i < dwIndexCount; i++)
    {
        CASC_INDEX & IndexFile = hs->IndexFiles[i];

        // Inform the user about what we are doing
        if(InvokeProgressCallback(hs, CascProgressLoadingIndexes, NULL, i, dwIndexCount))
        {
            dwErrCode = ERROR_CANCELLED;
            break;
        }

        // Load the index file
        if((dwErrCode = LoadIndexFile(hs, PfnEKeyEntry, IndexFile.FileData.pbData, IndexFile.FileData.cbData, i)) != ERROR_SUCCESS)
            break;
    }

    // Swallow the "done parsing" error
    if(dwErrCode == ERROR_INDEX_PARSING_DONE)
        dwErrCode = ERROR_SUCCESS;

    // Remember the number of files that are present locally
    hs->LocalFiles = hs->CKeyArray.ItemCount();
    return dwErrCode;
}

static DWORD LoadLocalIndexFiles(TCascStorage * hs)
{
    ULONGLONG TotalSize = 0;
    DWORD dwIndexCount = 0;
    DWORD dwErrCode;

    // Inform the user about what we are doing
    if(InvokeProgressCallback(hs, CascProgressLoadingIndexes, NULL, 0, 0))
        return ERROR_CANCELLED;

    // Perform the directory scan
    if((dwErrCode = ScanDirectory(hs->szIndexPath, NULL, IndexDirectory_OnFileFound, hs)) == ERROR_SUCCESS)
    {
        // If no index file was found, we cannot load anything
        if(hs->szIndexFormat == NULL)
            return ERROR_FILE_NOT_FOUND;

        // Load each index file
        for(DWORD i = 0; i < CASC_INDEX_COUNT; i++)
        {
            CASC_INDEX & IndexFile = hs->IndexFiles[i];

            // Create the file name
            if((IndexFile.szFileName = CreateIndexFileName(hs, i, IndexFile.NewSubIndex)) == NULL)
                return ERROR_NOT_ENOUGH_MEMORY;

            // WoW6 actually reads THE ENTIRE file to memory. Verified on Mac build (x64).
            dwErrCode = LoadFileToMemory(IndexFile.szFileName, IndexFile.FileData);
            if(dwErrCode != ERROR_SUCCESS)
            {
                // Storages downloaded by Blizzget tool don't have all index files present
                if((dwErrCode = GetCascError()) == ERROR_FILE_NOT_FOUND)
                {
                    dwErrCode = ERROR_SUCCESS;
                    break;
                }
                return dwErrCode;
            }

            // Add to the total size of the index files
            TotalSize += IndexFile.FileData.cbData;
            dwIndexCount++;
        }

        // Build the map of EKey -> IndexEKeyEntry
        dwErrCode = hs->IndexEKeyMap.Create((size_t)(TotalSize / sizeof(FILE_EKEY_ENTRY)), CASC_EKEY_SIZE, 0);
        if(dwErrCode == ERROR_SUCCESS)
        {
            dwErrCode = ProcessLocalIndexFiles(hs, InsertEncodingEKeyToMap, dwIndexCount);
        }
    }

    return dwErrCode;
}

//-----------------------------------------------------------------------------
// Online index files
// https://wowdev.wiki/TACT#CDN_File_Organization

static DWORD CaptureArchiveIndexFooter(CASC_ARCINDEX_FOOTER & InFooter, LPBYTE pbIndexFile, size_t cbIndexFile)
{
    FILE_INDEX_FOOTER<0x08> * pFooter08;
    BYTE checksum_data[0x40] = { 0 };
    BYTE md5_hash[MD5_HASH_SIZE];
    DWORD checksum_data_length;

    // Clear the entire structure
    memset(&InFooter, 0, sizeof(CASC_ARCINDEX_FOOTER));

    // Check the variant for checksum == 0x08
    pFooter08 = (FILE_INDEX_FOOTER<0x08> *)(pbIndexFile + cbIndexFile - sizeof(FILE_INDEX_FOOTER<0x08>));
    if(pFooter08->Version == 1 && pFooter08->Reserved[0] == 0 && pFooter08->Reserved[1] == 0 && pFooter08->FooterHashBytes == 8)
    {
        // Copy the entire structure
        memcpy(InFooter.TocHash, pFooter08->TocHash, MD5_HASH_SIZE);
        memcpy(InFooter.FooterHash, pFooter08->FooterHash, pFooter08->FooterHashBytes);
        InFooter.Version = pFooter08->Version;
        InFooter.OffsetBytes = pFooter08->OffsetBytes;
        InFooter.SizeBytes = pFooter08->SizeBytes;
        InFooter.EKeyLength = pFooter08->EKeyLength;
        InFooter.FooterHashBytes = pFooter08->FooterHashBytes;
        InFooter.PageLength = ((size_t)pFooter08->PageSizeKB) << 10;
        InFooter.ItemLength = pFooter08->EKeyLength + pFooter08->OffsetBytes + pFooter08->SizeBytes;
        InFooter.FooterLength = sizeof(FILE_INDEX_FOOTER<0x08>);
        InFooter.ElementCount = ConvertBytesToInteger_4_LE(pFooter08->ElementCount);

        // Verify the hash. FooterHash needs to be cleared in order to calculate footer hash properly
        checksum_data_length = FIELD_OFFSET(FILE_INDEX_FOOTER<0x08>, FooterHash) - FIELD_OFFSET(FILE_INDEX_FOOTER<0x08>, Version);
        memcpy(checksum_data, &pFooter08->Version, checksum_data_length);
        CascHash_MD5(checksum_data, sizeof(FILE_INDEX_FOOTER<0x08>) - MD5_HASH_SIZE, md5_hash);
        if(!memcmp(md5_hash, InFooter.FooterHash, InFooter.FooterHashBytes))
            return ERROR_SUCCESS;
    }
    return ERROR_BAD_FORMAT;
}

static DWORD CaptureIndexEntry(CASC_ARCINDEX_FOOTER  & InFooter, CASC_EKEY_ENTRY & EKeyEntry, LPBYTE pbIndexPage, LPBYTE pbIndexPageEnd, size_t nArchive)
{
    ULONGLONG StorageOffset = nArchive;
    ULONGLONG ArchiveOffset;

    // If there enough bytes for one entry/
    if((pbIndexPage + InFooter.ItemLength) > pbIndexPageEnd)
        return ERROR_BAD_FORMAT;

    // Capture the EKey (variable length)
    pbIndexPage = CaptureEncodedKey(EKeyEntry.EKey, pbIndexPage, InFooter.EKeyLength);

    // Copy the archive offset
    ArchiveOffset = ConvertBytesToInteger_X(pbIndexPage + InFooter.SizeBytes, InFooter.OffsetBytes);
    if(ArchiveOffset >= 0x10000000)
        return ERROR_BAD_FORMAT;

    // Capture the storage offset and encoded size
    EKeyEntry.StorageOffset = (StorageOffset << (InFooter.OffsetBytes * 8)) | ArchiveOffset;
    EKeyEntry.EncodedSize = ConvertBytesToInteger_X(pbIndexPage, InFooter.SizeBytes);
    EKeyEntry.Alignment = 0;

    // Is there a valid hash?
    return CascIsValidMD5(EKeyEntry.EKey) ? ERROR_SUCCESS : ERROR_BAD_FORMAT;
}

static DWORD VerifyIndexSize(CASC_ARCINDEX_FOOTER  & InFooter, LPBYTE pbIndexFile, size_t cbIndexFile, LPBYTE * PtrIndexEnd)
{
    size_t nPageCount;

    // Set the new length (without the footer)
    cbIndexFile = cbIndexFile - InFooter.FooterLength;
    nPageCount = cbIndexFile / (InFooter.PageLength + MD5_HASH_SIZE);

    // There must be equal or more pages
    if(((InFooter.PageLength + MD5_HASH_SIZE) * nPageCount) > cbIndexFile)
        return ERROR_BAD_FORMAT;

    // Return the end-of-index
    nPageCount = cbIndexFile / (InFooter.PageLength + MD5_HASH_SIZE);
    PtrIndexEnd[0] = pbIndexFile + (nPageCount * InFooter.PageLength);
    return ERROR_SUCCESS;
}

static DWORD LoadArchiveIndexPage(TCascStorage * hs, CASC_ARCINDEX_FOOTER & InFooter, LPBYTE pbIndexPage, LPBYTE pbIndexPageEnd, size_t nArchive)
{
    CASC_EKEY_ENTRY EKeyEntry;
    DWORD dwErrCode;

    while (pbIndexPage <= pbIndexPageEnd)
    {
        // Capture the index entry
        dwErrCode = CaptureIndexEntry(InFooter, EKeyEntry, pbIndexPage, pbIndexPageEnd, nArchive);
        if(dwErrCode != ERROR_SUCCESS)
            break;

        // Insert a new entry to the index array
        if((hs->IndexArray.Insert(&EKeyEntry, 1)) == NULL)
            return ERROR_NOT_ENOUGH_MEMORY;

        // Move to the next entry
        pbIndexPage += InFooter.ItemLength;
    }

    return ERROR_SUCCESS;
}

static DWORD LoadArchiveIndexFile(TCascStorage * hs, LPBYTE pbIndexFile, size_t cbIndexFile, size_t nArchive)
{
    CASC_ARCINDEX_FOOTER InFooter;
    LPBYTE pbIndexEnd = NULL;
    DWORD dwErrCode;

    // Validate and capture the footer
    dwErrCode = CaptureArchiveIndexFooter(InFooter, pbIndexFile, cbIndexFile);
    if(dwErrCode != ERROR_SUCCESS)
        return dwErrCode;

    // Remember the file offset and EKey length
    SaveFileOffsetBitsAndEKeyLength(hs, InFooter.OffsetBytes * 8, InFooter.EKeyLength);

    // Verify the size of the index file
    dwErrCode = VerifyIndexSize(InFooter, pbIndexFile, cbIndexFile, &pbIndexEnd);
    if(dwErrCode != ERROR_SUCCESS)
        return dwErrCode;

    // Parse all pages
    while (pbIndexFile < pbIndexEnd)
    {
        // Load the entire page
        dwErrCode = LoadArchiveIndexPage(hs, InFooter, pbIndexFile, pbIndexFile + InFooter.PageLength, nArchive);
        if(dwErrCode != ERROR_SUCCESS)
            break;

        // Move to the next page
        pbIndexFile += InFooter.PageLength;
    }

    return ERROR_SUCCESS;
}

static DWORD BuildMapOfArchiveIndices(TCascStorage * hs)
{
    PCASC_EKEY_ENTRY pEKeyEntry;
    size_t nItemCount = hs->IndexArray.ItemCount();
    DWORD dwErrCode;

    // Create the map
    dwErrCode = hs->IndexMap.Create(nItemCount, MD5_HASH_SIZE, FIELD_OFFSET(CASC_EKEY_ENTRY, EKey));
    if(dwErrCode != ERROR_SUCCESS)
        return dwErrCode;

    // Insert all items
    for(size_t i = 0; i < nItemCount; i++)
    {
        pEKeyEntry = (PCASC_EKEY_ENTRY)hs->IndexArray.ItemAt(i);
        if(pEKeyEntry != NULL)
        {
            if(!hs->IndexMap.InsertObject(pEKeyEntry, pEKeyEntry->EKey))
            {
                return ERROR_NOT_ENOUGH_MEMORY;
            }
        }
    }

    return dwErrCode;
}

static DWORD LoadArchiveIndexFiles(TCascStorage * hs)
{
    CASC_BLOB FileData;
    size_t nArchiveCount = (hs->ArchivesKey.cbData / MD5_HASH_SIZE);
    DWORD dwErrCode = ERROR_SUCCESS;

    // Create the array object for the indices
    dwErrCode = hs->IndexArray.Create(sizeof(CASC_EKEY_ENTRY), 0x10000);
    if(dwErrCode != ERROR_SUCCESS)
        return dwErrCode;

    // Load all the indices
    for(size_t i = 0; i < nArchiveCount; i++)
    {
        CASC_PATH<TCHAR> LocalPath;
        LPBYTE pbIndexHash = hs->ArchivesKey.pbData + (i * MD5_HASH_SIZE);

        // Inform the user about what we are doing
        if(InvokeProgressCallback(hs, CascProgressDownloadingArchiveIndexes, NULL, (DWORD)(i), (DWORD)(nArchiveCount)))
        {
            dwErrCode = ERROR_CANCELLED;
            break;
        }

        // Fetch and parse the archive index
        dwErrCode = FetchCascFile(hs, PathTypeData, pbIndexHash, _T(".index"), LocalPath);
        if(dwErrCode == ERROR_SUCCESS)
        {
            // Load the index file to memory
            if((dwErrCode = LoadFileToMemory(LocalPath, FileData)) == ERROR_SUCCESS)
            {
                dwErrCode = LoadArchiveIndexFile(hs, FileData.pbData, FileData.cbData, i);
            }
        }

        // Break if an error
        if(dwErrCode != ERROR_SUCCESS)
            break;
    }

    // Build map of EKey -> CASC_EKEY_ENTRY
    if(dwErrCode == ERROR_SUCCESS)
        dwErrCode = BuildMapOfArchiveIndices(hs);
    return dwErrCode;
}

//-----------------------------------------------------------------------------
// Public functions

bool CopyEKeyEntry(TCascStorage * hs, PCASC_CKEY_ENTRY pCKeyEntry)
{
    // Don't do this on online storages
    if(!(hs->dwFeatures & CASC_FEATURE_ONLINE))
    {
        LPBYTE pbEKeyEntry;

        // If the file was found, then copy the content to the CKey entry
        pbEKeyEntry = (LPBYTE)hs->IndexEKeyMap.FindObject(pCKeyEntry->EKey);
        if(pbEKeyEntry == NULL)
            return false;

        pCKeyEntry->StorageOffset = ConvertBytesToInteger_5(pbEKeyEntry + hs->EKeyLength);
        pCKeyEntry->EncodedSize = ConvertBytesToInteger_4_LE(pbEKeyEntry + hs->EKeyLength + 5);
        pCKeyEntry->Flags |= CASC_CE_FILE_IS_LOCAL;
    }

    return true;
}

DWORD LoadIndexFiles(TCascStorage * hs)
{
    switch(hs->BuildFileType)
    {
        case CascBuildDb:       // Load the index files from the disk
        case CascBuildInfo:
            return LoadLocalIndexFiles(hs);

        case CascVersions:      // Load the index files from the cache / internet
            return LoadArchiveIndexFiles(hs);

        default:
            assert(false);
            return ERROR_NOT_SUPPORTED;
    }
}

void FreeIndexFiles(TCascStorage * hs)
{
    // Free the map of EKey -> Index Ekey item
    hs->IndexEKeyMap.Free();

    // Free all loaded index files
    for(size_t i = 0; i < CASC_INDEX_COUNT; i++)
    {
        CASC_INDEX & IndexFile = hs->IndexFiles[i];

        // Free the file data
        IndexFile.FileData.Free();

        // Free the file name
        CASC_FREE(IndexFile.szFileName);
    }
}