use std::collections::BTreeMap;
use std::fs::{self, File};
use std::io::{self, BufReader, Cursor, Error, ErrorKind, Read, Seek, SeekFrom};
use std::path::{Path, PathBuf};
use crate::config::{Config, WriteOptions, is_rapified};
use crate::error::{ErrorExt, printable};
use crate::pbo::{
EntryFilter, IndexEntry, decompress_entry, prefix_relative, read_index, safe_relative_path,
unpack_relative, warn_leading_separator,
};
use crate::warning::{Warning, WarningSink};
const MAP_FILE: &str = "pbo-map.tsv";
#[derive(Debug, Clone, Default)]
#[non_exhaustive]
pub struct ExtractOptions {
pub force: bool,
pub derapify: bool,
pub exclude: Vec<String>,
pub only: Vec<String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub struct ExtractSummary {
pbos: usize,
files: usize,
}
impl ExtractSummary {
#[must_use]
pub fn pbos(&self) -> usize {
self.pbos
}
#[must_use]
pub fn files(&self) -> usize {
self.files
}
}
struct SourcePbo {
path: PathBuf,
relative: String,
}
struct Staging {
path: PathBuf,
armed: bool,
}
impl Drop for Staging {
fn drop(&mut self) {
if self.armed {
let _ = fs::remove_dir_all(&self.path);
}
}
}
fn is_dir(path: &Path) -> bool {
fs::metadata(path).is_ok_and(|m| m.is_dir())
}
fn find_pbos(source: &Path) -> Result<Vec<SourcePbo>, Error> {
let shown = printable(&source.display().to_string());
let meta = fs::metadata(source).prepend_error(format!("Failed to read source \"{shown}\":"))?;
let mut found = Vec::new();
if meta.is_file() {
let name = source
.file_name()
.map_or_else(String::new, |n| n.to_string_lossy().into_owned());
found.push(SourcePbo {
path: source.to_path_buf(),
relative: name,
});
return Ok(found);
}
let read = |dir: &Path| {
fs::read_dir(dir).prepend_error(format!(
"Failed to read folder \"{}\":",
printable(&dir.display().to_string())
))
};
for folder in read(source)? {
let folder = folder.prepend_error(format!("Failed to read folder \"{shown}\":"))?;
let folder_name = folder.file_name().to_string_lossy().into_owned();
let lower = folder_name.to_lowercase();
if (lower != "dta" && lower != "addons") || !is_dir(&folder.path()) {
continue;
}
for file in read(&folder.path())? {
let file = file.prepend_error("Failed to read folder entry:")?;
let path = file.path();
let is_pbo = path
.extension()
.is_some_and(|e| e.eq_ignore_ascii_case("pbo"));
if is_pbo && fs::metadata(&path).is_ok_and(|m| m.is_file()) {
found.push(SourcePbo {
relative: format!("{folder_name}/{}", file.file_name().to_string_lossy()),
path,
});
}
}
}
found.sort_by(|a, b| a.relative.cmp(&b.relative));
if found.is_empty() {
return Err(error!(
"Found no PBOs in the dta or addons folder of \"{shown}\"."
));
}
Ok(found)
}
fn looks_like_extract_result(folder: &Path) -> bool {
let is_empty = fs::read_dir(folder).is_ok_and(|mut entries| entries.next().is_none());
is_empty || fs::symlink_metadata(folder.join(MAP_FILE)).is_ok_and(|m| m.is_file())
}
fn resolve_for_comparison(path: &Path) -> Option<PathBuf> {
let mut existing = path.to_path_buf();
let mut rest = Vec::new();
let mut resolved = loop {
if existing.as_os_str().is_empty() {
existing = PathBuf::from(".");
}
if let Ok(done) = existing.canonicalize() {
break done;
}
rest.push(existing.file_name()?.to_os_string());
if !existing.pop() {
return None;
}
};
resolved.extend(rest.iter().rev());
Some(resolved)
}
fn check_target(
sources: &[SourcePbo],
source: &Path,
target: &Path,
force: bool,
) -> Result<(), Error> {
let shown = printable(&target.display().to_string());
match fs::symlink_metadata(target) {
Ok(meta) => {
if !meta.is_dir() {
return Err(error!("Target \"{shown}\" exists and is not a folder."));
}
if !force {
return Err(error!(
"Target folder \"{shown}\" already exists. Use --force to replace it."
));
}
let outer = target.canonicalize();
let overlaps = |inner: &Path| match (&outer, inner.canonicalize()) {
(Ok(outer), Ok(inner)) => inner.starts_with(outer),
_ => true,
};
let Ok(resolved_source) = source.canonicalize() else {
return Err(error!(
"Cannot resolve the source folder \"{}\".",
printable(&source.display().to_string())
));
};
let inside = match &outer {
Ok(outer) => outer.starts_with(resolved_source),
Err(_) => true,
};
if inside {
return Err(error!(
"Target folder \"{shown}\" is inside the source; refusing to replace it."
));
}
if overlaps(source) || sources.iter().any(|s| overlaps(&s.path)) {
return Err(error!(
"Target folder \"{shown}\" contains the source; refusing to replace it."
));
}
if !looks_like_extract_result(target) {
return Err(error!(
"Target folder \"{shown}\" is not empty and not the result of an earlier extract (it has no {MAP_FILE}); refusing to replace it."
));
}
Ok(())
}
Err(e) if e.kind() == ErrorKind::NotFound => {
let Some(resolved) = resolve_for_comparison(target) else {
return Err(error!(
"Invalid target folder \"{shown}\": cannot resolve \"..\" after a folder that does not exist."
));
};
let Ok(source) = source.canonicalize() else {
return Err(error!(
"Cannot resolve the source folder \"{}\".",
printable(&source.display().to_string())
));
};
if resolved.starts_with(source) {
return Err(error!(
"Target folder \"{shown}\" is inside the source; refusing to replace it."
));
}
Ok(())
}
Err(e) => Err(e).prepend_error(format!("Failed to read target \"{shown}\":")),
}
}
fn target_name(target: &Path) -> Result<String, Error> {
target
.file_name()
.map(|n| n.to_string_lossy().into_owned())
.ok_or_else(|| {
error!(
"Invalid target folder \"{}\".",
printable(&target.display().to_string())
)
})
}
fn create_staging(target: &Path) -> Result<Staging, Error> {
let name = target_name(target)?;
let parent = parent_of(target);
fs::create_dir_all(&parent).prepend_error("Failed to create the folder of the target:")?;
for attempt in 0..1000_u32 {
let path = parent.join(format!(".{name}.extract-{}-{attempt}", std::process::id()));
match fs::create_dir(&path) {
Ok(()) => return Ok(Staging { path, armed: true }),
Err(e) if e.kind() == ErrorKind::AlreadyExists => {}
Err(e) => return Err(e).prepend_error("Failed to create the staging folder:"),
}
}
Err(error!("Failed to find a free name for the staging folder."))
}
fn parent_of(path: &Path) -> PathBuf {
match path.parent() {
Some(p) if !p.as_os_str().is_empty() => p.to_path_buf(),
_ => PathBuf::from("."),
}
}
fn derapify(data: &[u8]) -> Result<Vec<u8>, Error> {
let config = Config::read_rapified(&mut Cursor::new(data))?;
let mut text = Vec::new();
config.write_with(&mut text, &WriteOptions::new())?;
Ok(text)
}
fn escape(bytes: &[u8], out: &mut Vec<u8>) {
for &b in bytes {
match b {
b'\\' => out.extend_from_slice(b"\\\\"),
b'\t' => out.extend_from_slice(b"\\t"),
b'\r' => out.extend_from_slice(b"\\r"),
b'\n' => out.extend_from_slice(b"\\n"),
_ => out.push(b),
}
}
}
const MAX_IN_MEMORY: u64 = 1 << 29;
enum Content {
Stream {
offset: u64,
size: u64,
},
Bytes(Vec<u8>),
}
struct Run<'a> {
staging: &'a Path,
options: &'a ExtractOptions,
filter: EntryFilter,
names: Vec<String>,
owners: BTreeMap<String, (PathBuf, usize)>,
map: Vec<(Vec<u8>, String)>,
files: usize,
}
impl Run<'_> {
fn clash(&self, relative: &Path, key: &str) -> Option<(PathBuf, usize, bool)> {
for parent in relative.ancestors().skip(1) {
let parent = parent.to_string_lossy().to_lowercase();
if let Some((path, owner)) = self.owners.get(&parent) {
return Some((path.clone(), *owner, true));
}
}
let below = format!("{key}{}", std::path::MAIN_SEPARATOR);
let (found, (path, owner)) = self.owners.range(below.clone()..).next()?;
found
.starts_with(&below)
.then(|| (path.clone(), *owner, false))
}
fn skip_clash(
&self,
index: usize,
relative: &Path,
key: &str,
warnings: &mut dyn WarningSink,
) -> bool {
let Some((other, owner, other_is_file)) = self.clash(relative, key) else {
return false;
};
let pbo_name = |i: usize| self.names.get(i).map_or("?", String::as_str);
let shown = |p: &Path| {
let parts: Vec<_> = p
.components()
.map(|c| c.as_os_str().to_string_lossy())
.collect();
printable(&parts.join("/"))
};
let (pbo, other_pbo) = (printable(pbo_name(index)), printable(pbo_name(owner)));
let message = if other_is_file {
format!(
"Skipping \"{}\" from {pbo}: it needs \"{}\" as a folder, but {other_pbo} has a file there.",
shown(relative),
shown(&other),
)
} else {
format!(
"Skipping \"{}\" from {pbo}: it is a file, but {other_pbo} has a folder there.",
shown(relative),
)
};
warnings.warn(Warning::new(Some("extract-path-clash"), message));
true
}
fn pbo(
&mut self,
index: usize,
source: &SourcePbo,
warnings: &mut dyn WarningSink,
) -> Result<(), Error> {
let shown = printable(&source.relative);
let mut file = BufReader::new(
File::open(&source.path).prepend_error(format!("Failed to open PBO \"{shown}\":"))?,
);
let pbo_index =
read_index(&mut file).prepend_error(format!("Failed to read PBO \"{shown}\":"))?;
let stem = source
.path
.file_stem()
.map_or_else(String::new, |s| s.to_string_lossy().into_owned());
let prefix = pbo_index.prefix.as_deref();
let folder = match prefix_relative(prefix) {
Ok(Some(relative)) => Some(relative),
Ok(None) => safe_relative_path(&stem),
Err(_) => None,
};
let map_prefix = match (prefix_relative(prefix), pbo_index.prefix_bytes) {
(Ok(None), _) => stem.as_bytes().to_vec(),
(_, Some(raw)) => raw,
(_, None) => prefix.unwrap_or_default().as_bytes().to_vec(),
};
let map_prefix: Vec<u8> = map_prefix
.iter()
.map(|&b| if b == b'\\' { b'/' } else { b })
.collect();
self.map.push((map_prefix, source.relative.clone()));
let folder = folder.filter(|f| {
!f.components().next().is_some_and(|c| {
c.as_os_str()
.to_string_lossy()
.eq_ignore_ascii_case(MAP_FILE)
})
});
let Some(folder) = folder else {
warnings.warn(Warning::new(
Some("unsafe-path"),
format!("Skipping PBO \"{shown}\": its prefix is not a safe folder name."),
));
return Ok(());
};
for entry in &pbo_index.entries {
if !self.filter.matches(&entry.name) {
continue;
}
let end = entry.offset.checked_add(entry.size);
if end.is_none_or(|end| end > pbo_index.len) {
return Err(error!(
"PBO \"{shown}\": entry \"{}\" is truncated (its data ends past the end of the file).",
printable(&entry.name)
));
}
self.entry(index, &folder, entry, &mut file, warnings)
.prepend_error(format!("PBO \"{shown}\":"))?;
}
Ok(())
}
fn entry<F: Read + Seek>(
&mut self,
index: usize,
folder: &Path,
entry: &IndexEntry,
file: &mut F,
warnings: &mut dyn WarningSink,
) -> Result<(), Error> {
let name = entry.name.as_str();
if entry.non_utf8 {
warnings.warn(Warning::new(
Some("non-utf8-filename"),
format!("Entry name {name:?} is not valid UTF-8; extracting it as windows-1252."),
));
}
let Some((mut relative, stripped)) = unpack_relative(name) else {
warnings.warn(Warning::new(
Some("unsafe-path"),
format!("Skipping \"{}\": unsafe path.", printable(name)),
));
return Ok(());
};
if stripped {
warn_leading_separator(name, &relative, warnings);
}
let is_bin = relative
.extension()
.is_some_and(|e| e.eq_ignore_ascii_case("bin"));
let fail = |what: &str| format!("{what} \"{}\":", printable(name));
let mut content = Content::Stream {
offset: entry.offset,
size: entry.size,
};
if entry.compressed {
if entry.size > MAX_IN_MEMORY || u64::from(entry.original_size) > MAX_IN_MEMORY {
return Err(error!(
"Compressed entry \"{}\" is too large.",
printable(name)
));
}
let stored =
read_at(file, entry.offset, entry.size).prepend_error(fail("Failed to read"))?;
let data = decompress_entry(&stored, entry.original_size)
.prepend_error(fail("Failed to decompress"))?;
content = Content::Bytes(data);
}
if self.options.derapify && is_bin {
let (derapified, kept) = derapify_entry(entry, content, file, warnings)?;
if derapified {
relative = relative.with_extension("cpp");
}
content = kept;
}
let relative = folder.join(relative);
let key = relative.to_string_lossy().to_lowercase();
if self.skip_clash(index, &relative, &key, warnings) {
return Ok(()); }
if let Some((old, owner)) = self.owners.get(&key).cloned() {
if owner == index {
warnings.warn(Warning::new(
Some("duplicate-entry"),
format!(
"Skipping \"{}\": an entry with the same path was already extracted.",
printable(name)
),
));
return Ok(());
}
let pbo_name = |i: usize| self.names.get(i).map_or("?", String::as_str);
warnings.warn(Warning::new(
Some("duplicate-path"),
format!(
"\"{}\" is in {} and in {}; keeping the one of {}.",
printable(&relative.to_string_lossy()),
printable(pbo_name(owner)),
printable(pbo_name(index)),
printable(pbo_name(index)),
),
));
let _ = fs::remove_file(self.staging.join(old));
}
let path = self.staging.join(&relative);
if let Some(parent) = path.parent() {
fs::create_dir_all(parent).prepend_error(fail("Failed to create the folder of"))?;
}
match content {
Content::Bytes(data) => fs::write(&path, data),
Content::Stream { offset, size } => file
.seek(SeekFrom::Start(offset))
.and_then(|_| File::create(&path))
.and_then(|mut out| {
let copied = io::copy(&mut file.by_ref().take(size), &mut out)?;
if copied == size {
Ok(())
} else {
Err(Error::new(
ErrorKind::UnexpectedEof,
"the entry is truncated",
))
}
}),
}
.prepend_error(fail("Failed to write"))?;
if self.owners.insert(key, (relative, index)).is_none() {
self.files += 1;
}
Ok(())
}
}
fn derapify_entry<F: Read + Seek>(
entry: &IndexEntry,
content: Content,
file: &mut F,
warnings: &mut dyn WarningSink,
) -> Result<(bool, Content), Error> {
let name = entry.name.as_str();
let fail = |what: &str| format!("{what} \"{}\":", printable(name));
let head = match &content {
Content::Bytes(data) => data.get(..4).map(<[u8]>::to_vec),
Content::Stream { offset, size } if *size >= 4 => {
Some(read_at(file, *offset, 4).prepend_error(fail("Failed to read"))?)
}
Content::Stream { .. } => None,
};
if !head.is_some_and(|h| is_rapified(&h)) {
return Ok((false, content));
}
let data = match content {
Content::Bytes(data) => data,
Content::Stream { offset, size } if size <= MAX_IN_MEMORY => {
read_at(file, offset, size).prepend_error(fail("Failed to read"))?
}
Content::Stream { offset, size } => {
warnings.warn(Warning::new(
Some("derapify-failed"),
format!(
"Keeping \"{}\" as it is: the entry is too large to derapify.",
printable(name)
),
));
return Ok((false, Content::Stream { offset, size }));
}
};
match derapify(&data) {
Ok(text) => Ok((true, Content::Bytes(text))),
Err(e) => {
warnings.warn(Warning::new(
Some("derapify-failed"),
format!("Keeping \"{}\" as it is: {e}", printable(name)),
));
Ok((false, Content::Bytes(data)))
}
}
}
fn read_at<F: Read + Seek>(file: &mut F, offset: u64, size: u64) -> Result<Vec<u8>, Error> {
file.seek(SeekFrom::Start(offset))?;
let mut data = Vec::new();
file.by_ref().take(size).read_to_end(&mut data)?;
if u64::try_from(data.len()).ok() != Some(size) {
return Err(Error::new(
ErrorKind::UnexpectedEof,
"the entry is truncated",
));
}
Ok(data)
}
fn swap(staging: &mut Staging, target: &Path, warnings: &mut dyn WarningSink) -> Result<(), Error> {
let shown = printable(&target.display().to_string());
let old = match fs::symlink_metadata(target) {
Ok(_) => {
let mut old = staging.path.clone().into_os_string();
old.push(".old");
let old = PathBuf::from(old);
fs::rename(target, &old)
.prepend_error(format!("Failed to move the old target \"{shown}\" away:"))?;
Some(old)
}
Err(_) => None,
};
if let Err(e) = fs::rename(&staging.path, target) {
if let Some(old) = &old {
if let Err(back) = fs::rename(old, target) {
return Err(error!(
"Failed to move the new tree to \"{shown}\": {e}; the old tree is in \"{}\" ({back}).",
printable(&old.display().to_string())
));
}
}
return Err(e).prepend_error(format!("Failed to move the new tree to \"{shown}\":"));
}
staging.armed = false;
if let Some(old) = old
&& let Err(e) = fs::remove_dir_all(&old)
{
warnings.warn(Warning::new(
Some("old-tree-left"),
format!(
"Could not remove the old tree \"{}\": {e}",
printable(&old.display().to_string())
),
));
}
Ok(())
}
pub fn extract(
source: &Path,
target: &Path,
options: &ExtractOptions,
warnings: &mut dyn WarningSink,
) -> Result<ExtractSummary, Error> {
let sources = find_pbos(source)?;
target_name(target)?;
check_target(&sources, source, target, options.force)?;
let mut staging = create_staging(target)?;
let mut run = Run {
staging: &staging.path.clone(),
options,
names: sources.iter().map(|s| s.relative.clone()).collect(),
filter: EntryFilter::new(options.only.clone(), options.exclude.clone()),
owners: BTreeMap::new(),
map: Vec::new(),
files: 0,
};
for (index, source) in sources.iter().enumerate() {
run.pbo(index, source, warnings)?;
}
run.map.sort();
let mut text = Vec::new();
for (prefix, path) in &run.map {
escape(prefix, &mut text);
text.push(b'\t');
escape(path.as_bytes(), &mut text);
text.push(b'\n');
}
fs::write(staging.path.join(MAP_FILE), &text).prepend_error("Failed to write the PBO map:")?;
let summary = ExtractSummary {
pbos: sources.len(),
files: run.files,
};
swap(&mut staging, target, warnings)?;
Ok(summary)
}