use std::collections::HashMap;
use std::io::{self, Read, Seek, SeekFrom};
use super::codec::{decode, sector_checksum};
use super::compat::{address, load_layout, read_index, report, EntryLayout, LayoutRequest};
use super::crypto::{crypt, file_key};
use super::extended::read_extended;
use super::format::{
offset_table_size, read_blocks, read_hashes, sector_count, u32_at, validate_name, verify_md5,
HEADER_SIZE,
};
use super::raw;
use super::{BlockEntry, Error, FileFlags, Header, Index};
use super::{ReadMode, RecoveryDiagnostic};
trait ReadSeek: Read + Seek {}
impl<T: Read + Seek> ReadSeek for T {}
#[derive(Clone, Debug)]
pub struct ReadOptions {
pub mode: ReadMode,
pub max_table_entries: u32,
pub max_table_bytes: u64,
pub max_sector_size: u32,
pub max_sector_table_bytes: u32,
pub max_file_size: u32,
pub max_single_unit_bytes: u32,
pub max_header_search_bytes: u64,
pub verify_sector_checksums: bool,
}
impl Default for ReadOptions {
fn default() -> Self {
Self {
mode: ReadMode::Strict,
max_table_entries: 2 << 20,
max_table_bytes: 128 << 20,
max_sector_size: 1 << 20,
max_sector_table_bytes: 16 << 20,
max_file_size: u32::MAX,
max_single_unit_bytes: 64 << 20,
max_header_search_bytes: 16 << 20,
verify_sector_checksums: true,
}
}
}
pub struct Archive<R> {
pub(super) source: R,
source_len: u64,
diagnostics: Vec<RecoveryDiagnostic>,
pub(super) base: u64,
pub(super) index: Index,
pub(super) options: ReadOptions,
}
impl<R: Read + Seek> Archive<R> {
pub fn open(source: R) -> Result<Self, Error> {
Self::with_options(source, ReadOptions::default())
}
pub fn with_options(mut source: R, options: ReadOptions) -> Result<Self, Error> {
let start = source.stream_position()?;
let len = source.seek(SeekFrom::End(0))?;
let last = start
.saturating_add(options.max_header_search_bytes)
.min(len.saturating_sub(4));
let mut discovery_diagnostics: Vec<RecoveryDiagnostic> = Vec::new();
let mut at = start;
while at <= last && at.checked_add(4).is_some_and(|end| end <= len) {
source.seek(SeekFrom::Start(at))?;
let mut magic = [0; 4];
source.read_exact(&mut magic)?;
if &magic == b"MPQ\x1a" {
if options.mode == ReadMode::Strict {
return Self::at(source, at, options);
}
match Archive::at(&mut source as &mut dyn ReadSeek, at, options.clone()) {
Ok(archive) => {
let index = archive.index;
let mut diagnostics = archive.diagnostics;
for diagnostic in discovery_diagnostics {
report(&mut diagnostics, diagnostic.reason, diagnostic.entry);
}
return Ok(Self {
source,
base: at,
index,
options,
source_len: len,
diagnostics,
});
}
Err(Error::InvalidArchive(_)) | Err(Error::UnsupportedVersion(_)) => {
report(&mut discovery_diagnostics, "fake header skipped", None);
}
Err(error) => return Err(error),
}
}
if &magic == b"MPQ\x1b" && options.mode == ReadMode::Permissive && at + 16 <= len {
let mut data = [0; 12];
source.read_exact(&mut data)?;
let relative = u32_at(&data, 4) as u64;
if relative >= 16 {
if let Some(base) = at.checked_add(relative).filter(|base| *base <= last) {
match Archive::at(&mut source as &mut dyn ReadSeek, base, options.clone()) {
Ok(archive) => {
let index = archive.index;
let mut diagnostics = archive.diagnostics;
for diagnostic in discovery_diagnostics {
report(&mut diagnostics, diagnostic.reason, diagnostic.entry);
}
return Ok(Self {
source,
base,
index,
options,
source_len: len,
diagnostics,
});
}
Err(Error::InvalidArchive(_)) | Err(Error::UnsupportedVersion(_)) => {}
Err(error) => return Err(error),
}
}
}
report(
&mut discovery_diagnostics,
"invalid user-data wrapper ignored",
None,
);
}
if &magic == b"MPQ\x1b" && options.mode == ReadMode::Strict {
let mut data = [0; 12];
source.read_exact(&mut data)?;
let relative = u32_at(&data, 4) as u64;
if relative < 16 {
return Err(Error::InvalidArchive("user-data header offset"));
}
let base = at
.checked_add(relative)
.ok_or(Error::InvalidArchive("header offset overflow"))?;
return Self::at(source, base, options);
}
at = match at.checked_add(512) {
Some(next) => next,
None => break,
};
}
Err(Error::InvalidArchive(
"MPQ header not found within search limit",
))
}
pub fn at(mut source: R, base: u64, options: ReadOptions) -> Result<Self, Error> {
let len = source.seek(SeekFrom::End(0))?;
if base
.checked_add(HEADER_SIZE as u64)
.is_none_or(|end| end > len)
{
return Err(Error::InvalidArchive("header outside source"));
}
source.seek(SeekFrom::Start(base))?;
let mut bytes = vec![0; 32];
source.read_exact(&mut bytes)?;
let header_size = u32_at(&bytes, 4) as usize;
let version = u16::from_le_bytes(bytes[12..14].try_into().unwrap());
if options.mode == ReadMode::Permissive && (version != 3 || header_size == 32) {
if version <= 2 && (32..=208).contains(&header_size) {
let mut strict_options = options.clone();
strict_options.mode = ReadMode::Strict;
match Archive::at(&mut source as &mut dyn ReadSeek, base, strict_options) {
Ok(archive) => {
let mut index = archive.index;
let mut diagnostics = Vec::new();
for (slot, entry) in
index.hashes.iter_mut().enumerate().filter(|_| version == 0)
{
if entry.is_file() && entry.platform & 0xff00 != 0 {
entry.platform &= 0xff;
report(
&mut diagnostics,
"hash reserved byte ignored",
Some(slot as u32),
);
}
}
return Ok(Self {
source,
base,
index,
options,
source_len: len,
diagnostics,
});
}
Err(Error::InvalidArchive(_))
| Err(Error::LimitExceeded(_))
| Err(Error::UnsupportedCompression(_))
| Err(Error::UnsupportedFlags(_)) => {}
Err(Error::FeatureDisabled(_))
if [16usize, 20].iter().any(|at| {
let count_at = if *at == 16 { 24 } else { 28 };
address(
base,
u32_at(&bytes, *at) as u64,
u32_at(&bytes, count_at) as u64 * 16,
len,
true,
)
.is_err()
}) => {}
Err(error) => return Err(error),
}
}
source.seek(SeekFrom::Start(base + 32))?;
let mut diagnostics = Vec::new();
let index = read_index(&mut source, base, len, &bytes, &options, &mut diagnostics)?;
return Ok(Self {
source,
base,
index,
options,
source_len: len,
diagnostics,
});
}
if version > 3 {
return Err(Error::UnsupportedVersion(version));
}
if !(32..=208).contains(&header_size) {
return Err(Error::InvalidArchive("header size"));
}
bytes.resize(header_size, 0);
source.read_exact(&mut bytes[32..])?;
let mut header = Header::decode(&bytes)?;
if header.version == 1 || (header.version == 2 && header.shortened_header_size != 0) {
header.archive_size = len - base;
}
let size = header.archive_size;
if size < header_size as u64 || base.checked_add(size).is_none_or(|end| end > len) {
return Err(Error::InvalidArchive("archive size outside source"));
}
let sector_size = header
.sector_size()
.ok_or(Error::InvalidArchive("sector size overflow"))?;
if sector_size > options.max_sector_size {
return Err(Error::LimitExceeded("sector size"));
}
if header.hash_table_entries != 0 && !header.hash_table_entries.is_power_of_two() {
return Err(Error::InvalidArchive(
"hash table size is not a power of two",
));
}
let positions = [
header.hash_table_offset,
header.block_table_offset,
header.hi_block_table_offset,
header.het_table_offset,
header.bet_table_offset,
];
let expected = [
header.hash_table_entries as u64 * 16,
header.block_table_entries as u64 * 16,
header.block_table_entries as u64 * 2,
0,
0,
];
if header.hash_table_entries > options.max_table_entries
|| header.block_table_entries > options.max_table_entries
{
return Err(Error::LimitExceeded("table entries"));
}
for i in 0..5 {
if header.version < 3 {
header.table_sizes[i] = if positions[i] == 0 {
0
} else {
let end = positions
.iter()
.copied()
.filter(|&p| p > positions[i])
.min()
.unwrap_or(size);
let available = end
.checked_sub(positions[i])
.ok_or(Error::InvalidArchive("table offset"))?;
if i < 3 {
available.min(expected[i])
} else {
available
}
};
}
let stored = header.table_sizes[i];
if stored > options.max_table_bytes || expected[i] > options.max_table_bytes {
return Err(Error::LimitExceeded("table bytes"));
}
if (positions[i] == 0 && stored != 0)
|| (positions[i] != 0
&& (positions[i] < header_size as u64
|| positions[i]
.checked_add(stored)
.is_none_or(|end| end > size)))
|| (i < 2 && expected[i] != 0 && stored == 0)
{
return Err(Error::InvalidArchive("index table outside archive"));
}
}
for i in [3, 4] {
if positions[i] != 0 {
raw::verify(
&mut source,
base,
positions[i],
header.table_sizes[i],
&header,
&options,
)?;
}
}
let hashes = read_hashes(
&mut source,
base + header.hash_table_offset,
header.hash_table_entries,
header.table_sizes[0],
&header.md5[1],
)?;
let mut blocks = read_blocks(
&mut source,
base + header.block_table_offset,
header.block_table_entries,
header.table_sizes[1],
&header.md5[0],
)?;
if header.hi_block_table_offset != 0 {
if header.table_sizes[2] != expected[2] {
return Err(Error::InvalidArchive("high block table size"));
}
let mut high = vec![0; expected[2] as usize];
source.seek(SeekFrom::Start(base + header.hi_block_table_offset))?;
source.read_exact(&mut high)?;
verify_md5(&high, &header.md5[2])?;
for (block, word) in blocks.iter_mut().zip(high.as_chunks::<2>().0.iter()) {
block.offset |= (u16::from_le_bytes(*word) as u64) << 32;
}
}
let extended = match (header.het_table_offset != 0, header.bet_table_offset != 0) {
(false, false) => None,
(true, true) => {
let (mut index, extended_blocks) =
read_extended(&mut source, base, &header, &options)?;
if blocks.is_empty() {
blocks = extended_blocks;
} else if blocks != extended_blocks {
let identity =
|b: &BlockEntry| (b.offset, b.stored_size, b.file_size, b.flags.0);
let mut ids: HashMap<_, _> = blocks
.iter()
.enumerate()
.map(|(i, b)| (identity(b), i as u32))
.collect();
let mut remap = Vec::with_capacity(extended_blocks.len());
for block in extended_blocks {
let key = identity(&block);
let id = if let Some(&id) = ids.get(&key) {
id
} else {
if blocks.len() >= options.max_table_entries as usize {
return Err(Error::LimitExceeded("merged table entries"));
}
let id = blocks.len() as u32;
blocks.push(block);
ids.insert(key, id);
id
};
remap.push(id);
}
for (_, id) in index.slots.iter_mut().flatten() {
if *id != u32::MAX {
*id = remap[*id as usize];
}
}
}
Some(index)
}
_ => return Err(Error::InvalidArchive("incomplete HET/BET tables")),
};
for entry in &hashes {
if entry.is_file()
&& (entry.block_index as usize >= blocks.len()
|| !blocks[entry.block_index as usize]
.flags
.contains(FileFlags::EXISTS))
{
return Err(Error::InvalidArchive("hash references missing block"));
}
}
for block in &blocks {
let empty_at_zero = block.offset == 0 && block.file_size == 0 && block.stored_size == 0;
if block.flags.contains(FileFlags::EXISTS)
&& ((block.offset < header_size as u64 && !empty_at_zero)
|| block
.offset
.checked_add(block.stored_size as u64)
.is_none_or(|end| end > size))
{
return Err(Error::InvalidArchive("file block outside archive"));
}
}
Ok(Self {
source,
source_len: len,
diagnostics: Vec::new(),
base,
index: Index {
header,
hashes,
blocks,
extended,
},
options,
})
}
pub fn diagnostics(&self) -> &[RecoveryDiagnostic] {
&self.diagnostics
}
pub fn index(&self) -> &Index {
&self.index
}
pub fn archive_offset(&self) -> u64 {
self.base
}
pub fn into_inner(self) -> R {
self.source
}
pub fn open_file(&mut self, name: impl AsRef<[u8]>) -> Result<EntryReader<'_, R>, Error> {
self.open_file_with_locale(name, 0, 0)
}
pub fn open_file_with_locale(
&mut self,
name: impl AsRef<[u8]>,
locale: u16,
platform: u16,
) -> Result<EntryReader<'_, R>, Error> {
let name = name.as_ref();
validate_name(name)?;
let id = self
.index
.find(name, locale, platform)
.ok_or(Error::FileNotFound)?;
self.open_block(id, Some(name))
}
pub fn open_file_by_index(&mut self, id: u32) -> Result<EntryReader<'_, R>, Error> {
self.open_block(id, None)
}
fn open_block(&mut self, id: u32, name: Option<&[u8]>) -> Result<EntryReader<'_, R>, Error> {
let block = *self
.index
.blocks
.get(id as usize)
.ok_or(Error::FileNotFound)?;
let wrapping = self.options.mode == ReadMode::Permissive && self.index.header.version == 0;
address(self.base, block.offset, 0, self.source_len, wrapping)?;
if !block.flags.contains(FileFlags::EXISTS) {
return Err(Error::FileNotFound);
}
block.flags.validate_codec()?;
if block.file_size > self.options.max_file_size {
return Err(Error::LimitExceeded("file size"));
}
let sector_size = self.index.header.sector_size().unwrap();
if block.flags.contains(FileFlags::SINGLE_UNIT)
&& block.file_size.max(block.stored_size) > self.options.max_single_unit_bytes
{
return Err(Error::LimitExceeded("single-unit buffer"));
}
if !block.flags.compressed() || block.flags.contains(FileFlags::SINGLE_UNIT) {
address(
self.base,
block.offset,
block.stored_size as u64,
self.source_len,
wrapping,
)?;
}
if !block.flags.compressed() && block.file_size != block.stored_size {
return Err(Error::InvalidArchive("stored file size mismatch"));
}
raw::verify(
&mut self.source,
self.base,
block.offset,
block.stored_size as u64,
&self.index.header,
&self.options,
)?;
let key = name
.filter(|_| block.flags.contains(FileFlags::ENCRYPTED))
.map(|name| {
file_key(
name,
block.offset as u32,
block.file_size,
block.flags.contains(FileFlags::FIX_KEY),
)
});
let EntryLayout {
key,
offsets,
checksums,
} = load_layout(
&mut self.source,
LayoutRequest {
base: self.base,
len: self.source_len,
block,
id,
sector_size,
key,
wrapping,
options: &self.options,
},
&mut self.diagnostics,
)?;
Ok(EntryReader {
source: &mut self.source,
source_len: self.source_len,
wrapping,
base: self.base,
block,
sector_size,
key,
offsets,
checksums,
sector: 0,
delivered: 0,
buffer: Vec::new(),
cursor: 0,
failed: false,
})
}
pub fn read_file(&mut self, name: impl AsRef<[u8]>) -> Result<Vec<u8>, Error> {
let mut reader = self.open_file(name)?;
let mut bytes = Vec::new();
reader.read_to_end(&mut bytes)?;
Ok(bytes)
}
pub fn known_names(&mut self) -> Result<Vec<Vec<u8>>, Error> {
let mut entry = match self.open_file(b"(listfile)") {
Ok(entry) => entry,
Err(Error::FileNotFound) => return Ok(Vec::new()),
Err(e) => return Err(e),
};
if entry.block.file_size > 16 << 20 {
return Err(Error::LimitExceeded("listfile size"));
}
let mut bytes = Vec::new();
entry.read_to_end(&mut bytes)?;
Ok(bytes
.split(|b| matches!(b, b'\r' | b'\n'))
.filter(|line| !line.is_empty())
.map(<[u8]>::to_vec)
.collect())
}
pub fn encoded_file(&mut self, block_index: u32) -> Result<impl Read + '_, Error> {
let block = *self
.index
.blocks
.get(block_index as usize)
.filter(|b| b.flags.contains(FileFlags::EXISTS))
.ok_or(Error::FileNotFound)?;
let wrapping = self.options.mode == ReadMode::Permissive && self.index.header.version == 0;
let at = address(
self.base,
block.offset,
block.stored_size as u64,
self.source_len,
wrapping,
)?;
self.source.seek(SeekFrom::Start(at))?;
Ok((&mut self.source).take(block.stored_size as u64))
}
}
#[cfg(test)]
#[path = "reader_tests.rs"]
mod tests;
pub(super) fn load_offsets(
source: &mut (impl Read + Seek),
base: u64,
block: BlockEntry,
sector_size: u32,
key: Option<u32>,
options: &ReadOptions,
) -> Result<Vec<u32>, Error> {
if !block.flags.compressed()
|| block.flags.contains(FileFlags::SINGLE_UNIT)
|| block.file_size == 0
{
return Ok(Vec::new());
}
if block.flags.contains(FileFlags::ENCRYPTED) && key.is_none() {
return Err(Error::InvalidArchive("unknown encryption key"));
}
let size = offset_table_size(
block.file_size,
sector_size,
block.flags.contains(FileFlags::SECTOR_CRC),
)?;
if size > options.max_sector_table_bytes {
return Err(Error::LimitExceeded("sector offset table"));
}
let permissive = options.mode == ReadMode::Permissive;
if !permissive && size > block.stored_size {
return Err(Error::InvalidArchive("truncated sector table"));
}
let mut bytes = vec![0; size as usize];
let len = source.seek(SeekFrom::End(0))?;
let at = address(base, block.offset, size as u64, len, permissive)?;
source.seek(SeekFrom::Start(at))?;
source.read_exact(&mut bytes)?;
if let Some(key) = key {
crypt(&mut bytes, key.wrapping_sub(1), true);
}
let offsets: Vec<u32> = bytes
.as_chunks::<4>()
.0
.iter()
.map(|b| u32_at(b, 0))
.collect();
if !permissive
&& (offsets[0] < size
|| offsets.last().is_none_or(|&n| n > block.stored_size)
|| offsets.windows(2).any(|pair| pair[0] > pair[1]))
{
return Err(Error::InvalidArchive("invalid sector offsets"));
}
let count = sector_count(block.file_size, sector_size) as usize;
for (i, pair) in offsets.windows(2).take(count).enumerate() {
let expected =
(block.file_size as u64 - i as u64 * sector_size as u64).min(sector_size as u64);
let stored = pair[1]
.checked_sub(pair[0])
.ok_or(Error::InvalidArchive("invalid sector offsets"))?;
if permissive {
address(
base,
block.offset + pair[0] as u64,
stored as u64,
len,
true,
)?;
}
if stored == 0 || stored as u64 > expected {
return Err(Error::InvalidArchive("invalid sector length"));
}
}
Ok(offsets)
}
pub struct EntryReader<'a, R> {
source: &'a mut R,
source_len: u64,
wrapping: bool,
base: u64,
block: BlockEntry,
sector_size: u32,
key: Option<u32>,
offsets: Vec<u32>,
checksums: Vec<u32>,
sector: u32,
delivered: u32,
buffer: Vec<u8>,
cursor: usize,
failed: bool,
}
impl<R: Read + Seek> EntryReader<'_, R> {
pub fn metadata(&self) -> &BlockEntry {
&self.block
}
fn fill(&mut self) -> Result<(), Error> {
let single = self.block.flags.contains(FileFlags::SINGLE_UNIT);
let expected = if single {
self.block.file_size
} else {
(self.block.file_size - self.delivered).min(self.sector_size)
};
let (offset, size) = if single {
(0, self.block.stored_size)
} else if self.offsets.is_empty() {
(self.sector * self.sector_size, expected)
} else {
let at = self.sector as usize;
(self.offsets[at], self.offsets[at + 1] - self.offsets[at])
};
if size > expected {
return Err(Error::InvalidArchive("encoded unit exceeds decoded size"));
}
self.buffer.resize(size as usize, 0);
let at = address(
self.base,
self.block.offset + offset as u64,
size as u64,
self.source_len,
self.wrapping,
)?;
self.source.seek(SeekFrom::Start(at))?;
self.source.read_exact(&mut self.buffer)?;
if let Some(key) = self.key {
crypt(
&mut self.buffer,
key.wrapping_add(if single { 0 } else { self.sector }),
true,
);
}
if let Some(&expected) = self.checksums.get(self.sector as usize) {
if expected != 0 && expected != u32::MAX && sector_checksum(&self.buffer) != expected {
return Err(Error::ChecksumMismatch(self.sector));
}
}
if size < expected {
if !self.block.flags.compressed() {
return Err(Error::InvalidArchive("short stored sector"));
}
self.buffer = decode(
&self.buffer,
expected as usize,
self.block.flags.contains(FileFlags::IMPLODE),
)?;
}
self.cursor = 0;
self.sector += 1;
Ok(())
}
}
impl<R: Read + Seek> Read for EntryReader<'_, R> {
fn read(&mut self, output: &mut [u8]) -> io::Result<usize> {
if output.is_empty() {
return Ok(0);
}
if self.failed {
return Err(Error::InvalidArchive("entry reader previously failed").into());
}
if self.delivered == self.block.file_size {
return Ok(0);
}
if self.cursor == self.buffer.len() {
if let Err(e) = self.fill() {
self.failed = true;
return Err(e.into());
}
}
let len = output.len().min(self.buffer.len() - self.cursor);
output[..len].copy_from_slice(&self.buffer[self.cursor..self.cursor + len]);
self.cursor += len;
self.delivered += len as u32;
Ok(len)
}
}