use std::collections::HashSet;
use std::ffi::OsStr;
use std::fs::{self, File, OpenOptions};
use std::io::{self, Read, Seek, Write};
use std::path::{Component, Path, PathBuf};
use std::thread;
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use blake3::Hasher as Blake3;
use fs2::FileExt;
use reqwest::header::{CONTENT_LENGTH, CONTENT_TYPE, HeaderValue};
use same_file::Handle;
use sha1::{Digest, Sha1};
use crate::commons::{
error_chain, same_nonvolatile_metadata, trusted_download_url, validate_summary,
};
use crate::image::{PhotoFormat, validate_image_file};
use crate::{
ArchiveFailure, ArchiveOptions, ArchiveProgress, ArchiveProgressPhase, ArchiveSummary,
ArchivedPhoto, MAX_RETAINED_IMAGE_BYTES, PhotoForge, PhotoForgeError, PhotoIntegrity,
PhotoManifest, PhotoProvider, PhotoSummary, Result,
};
const MANIFEST_NAME: &str = "photoforge-photo.json";
const MANIFEST_OWNER: &str = "photoforge";
const MANIFEST_VERSION: &str = "1";
const MAX_MANIFEST_BYTES: u64 = 2 * 1024 * 1024;
const MAX_ERROR_RESPONSE_BYTES: u64 = 8 * 1024;
const PROGRESS_GRANULARITY: u64 = 1024 * 1024;
const LOCK_TIMEOUT: Duration = Duration::from_secs(5);
const LOCK_POLL_INTERVAL: Duration = Duration::from_millis(25);
const MAX_TRANSACTION_LOG_BYTES: u64 = 64 * 1024;
const TRANSACTION_COMPACT_BYTES: u64 = 60 * 1024;
const LOCK_HEADER: &str = "photoforge-lock-v1";
const STAGING_MARKER: &str = ".photoforge-staging";
pub(crate) fn archive(
forge: &PhotoForge,
photos: &[PhotoSummary],
options: &ArchiveOptions,
mut progress: Option<&mut dyn FnMut(ArchiveProgress)>,
) -> Result<ArchiveSummary> {
if options.max_download_bytes == 0 {
return Err(PhotoForgeError::InvalidOption(
"max_download_bytes must be nonzero".to_string(),
));
}
if options.max_download_bytes > MAX_RETAINED_IMAGE_BYTES {
return Err(PhotoForgeError::InvalidOption(format!(
"max_download_bytes must not exceed the {MAX_RETAINED_IMAGE_BYTES}-byte hard retained-image limit"
)));
}
let output = prepare_output(&options.output)?;
let selected = unique_photos(photos);
let mut summary = ArchiveSummary {
selected: selected.len(),
selected_ids: selected.iter().map(|photo| photo.id).collect(),
..ArchiveSummary::default()
};
emit_progress(
&mut progress,
progress_snapshot(ArchiveProgressPhase::Starting, &summary, None),
);
for selected_photo in selected {
let destination =
(selected_photo.id > 0).then(|| output.join(selected_photo.id.to_string()));
let result = archive_one(
forge,
selected_photo,
options,
&output,
&summary,
&mut progress,
);
match result {
Ok(ArchiveOutcome::Archived(photo)) => {
summary.archived += 1;
summary.bytes_written = summary.bytes_written.saturating_add(photo.bytes_written);
summary.photos.push(photo);
}
Ok(ArchiveOutcome::Skipped(photo)) => {
record_skipped(&mut summary, photo, &mut progress);
}
Err(error) => {
summary.failed += 1;
summary.failures.push(ArchiveFailure {
photo_id: selected_photo.id,
output: destination,
message: bounded_diagnostic(&error.to_string()),
});
emit_progress(
&mut progress,
progress_snapshot(ArchiveProgressPhase::Failed, &summary, Some(selected_photo)),
);
}
}
}
debug_assert_eq!(
summary.selected,
summary.archived + summary.skipped + summary.failed
);
emit_progress(
&mut progress,
progress_snapshot(ArchiveProgressPhase::Completed, &summary, None),
);
Ok(summary)
}
fn unique_photos(photos: &[PhotoSummary]) -> Vec<&PhotoSummary> {
let mut seen = HashSet::new();
photos
.iter()
.filter(|photo| seen.insert(photo.id))
.collect()
}
fn archive_one(
forge: &PhotoForge,
selected: &PhotoSummary,
options: &ArchiveOptions,
output: &Path,
aggregate: &ArchiveSummary,
progress: &mut Option<&mut dyn FnMut(ArchiveProgress)>,
) -> Result<ArchiveOutcome> {
if selected.provider != forge.provider || selected.id == 0 {
return Err(PhotoForgeError::VerificationFailed {
path: selected.id.to_string(),
message: "selected photo identity or provider is invalid".to_string(),
});
}
let destination = output.join(selected.id.to_string());
ensure_contained(output, &destination)?;
let mut transaction = DestinationTransaction::acquire(output, selected.id)?;
transaction.recover(output, &destination, selected.id)?;
let existing = recognized_manifest(&destination, selected.id)?;
let mut current = forge.fetch_exact(&selected.title)?;
validate_fresh_summary(selected, ¤t)?;
if options.skip_existing {
if let Some(skipped) = try_verified_skip(
&destination,
¤t,
existing.as_ref(),
progress,
aggregate,
)? {
return Ok(ArchiveOutcome::Skipped(skipped));
}
if existing.is_some() {
current = forge.fetch_exact(&selected.title)?;
validate_fresh_summary(selected, ¤t)?;
}
}
if current.file.size > options.max_download_bytes {
return Err(PhotoForgeError::DownloadLimitExceeded {
size: current.file.size,
limit: options.max_download_bytes,
});
}
let format = PhotoFormat::from_mime(¤t.file.mime)?;
let relative = format.relative_name();
transaction.mark_preparing()?;
let stage = match create_stage(output, current.id) {
Ok(stage) => stage,
Err(error) => {
let idle = transaction.mark_idle();
return combine_transaction_error(&destination, error, None, idle.err());
}
};
let build_result = (|| -> Result<PhotoManifest> {
let image_path = stage.join(&relative);
let downloaded = download_file(
forge,
¤t,
&image_path,
options.max_download_bytes,
|active_bytes| {
emit_progress(
progress,
active_progress(
ArchiveProgressPhase::Downloading,
aggregate,
¤t,
&relative,
active_bytes,
),
);
},
)?;
let manifest = PhotoManifest {
format_version: MANIFEST_VERSION.to_string(),
owner: MANIFEST_OWNER.to_string(),
photo: current.clone(),
integrity: PhotoIntegrity {
path: relative.clone(),
bytes: downloaded.bytes,
blake3: downloaded.blake3,
commons_sha1: current.file.sha1.clone(),
},
complete: true,
};
write_manifest(&stage, &manifest)?;
remove_staging_marker(&stage)?;
sync_directory(&stage)?;
emit_progress(
progress,
active_progress(
ArchiveProgressPhase::Verifying,
aggregate,
¤t,
&relative,
0,
),
);
let verified = verify_directory(&stage, false)?;
if verified != manifest {
return Err(PhotoForgeError::VerificationFailed {
path: stage.display().to_string(),
message: "staged manifest changed during verification".to_string(),
});
}
Ok(manifest)
})();
let manifest = match build_result {
Ok(manifest) => manifest,
Err(error) => {
let cleanup = cleanup_stage(&stage, current.id).err();
let idle = transaction.mark_idle().err();
return combine_transaction_error(&destination, error, cleanup, idle);
}
};
emit_progress(
progress,
active_progress(
ArchiveProgressPhase::Installing,
aggregate,
¤t,
&relative,
0,
),
);
install_stage(
&stage,
&destination,
output,
current.id,
existing.as_ref(),
&mut transaction,
)?;
let verified = verify_archive(&destination)?;
if verified != manifest {
return Err(PhotoForgeError::ArchiveTransaction {
path: destination,
message: "installed manifest changed after staging verification".to_string(),
});
}
transaction.mark_idle()?;
Ok(ArchiveOutcome::Archived(archived_photo(
&destination,
verified,
)?))
}
fn try_verified_skip(
destination: &Path,
current: &PhotoSummary,
existing: Option<&PhotoManifest>,
progress: &mut Option<&mut dyn FnMut(ArchiveProgress)>,
aggregate: &ArchiveSummary,
) -> Result<Option<ArchivedPhoto>> {
let Some(existing) = existing else {
return Ok(None);
};
if !same_nonvolatile_metadata(&existing.photo, current) {
return Ok(None);
}
emit_progress(
progress,
active_progress(
ArchiveProgressPhase::Verifying,
aggregate,
current,
&existing.integrity.path,
0,
),
);
let Ok(manifest) = verify_archive(destination) else {
return Ok(None);
};
if !same_nonvolatile_metadata(&manifest.photo, current) {
return Ok(None);
}
Ok(Some(archived_photo(destination, manifest)?))
}
fn record_skipped(
summary: &mut ArchiveSummary,
photo: ArchivedPhoto,
progress: &mut Option<&mut dyn FnMut(ArchiveProgress)>,
) {
summary.skipped += 1;
summary.skipped_photos.push(photo);
let archived = summary
.skipped_photos
.last()
.expect("skipped photo was just appended");
emit_progress(
progress,
active_progress(
ArchiveProgressPhase::Skipped,
summary,
&archived.manifest.photo,
&archived.manifest.integrity.path,
0,
),
);
}
fn validate_fresh_summary(selected: &PhotoSummary, current: &PhotoSummary) -> Result<()> {
if current.id != selected.id {
return Err(PhotoForgeError::VerificationFailed {
path: selected.id.to_string(),
message: "Commons page identity changed during metadata revalidation".to_string(),
});
}
validate_summary(current)
}
struct DownloadedFile {
bytes: u64,
blake3: String,
}
fn download_file<F>(
forge: &PhotoForge,
photo: &PhotoSummary,
output: &Path,
max_download_bytes: u64,
mut progress: F,
) -> Result<DownloadedFile>
where
F: FnMut(u64),
{
if max_download_bytes == 0 || max_download_bytes > MAX_RETAINED_IMAGE_BYTES {
return Err(PhotoForgeError::InvalidOption(
"max_download_bytes must be nonzero and no greater than the hard retained-image limit"
.to_string(),
));
}
if photo.file.size > max_download_bytes {
return Err(PhotoForgeError::DownloadLimitExceeded {
size: photo.file.size,
limit: max_download_bytes,
});
}
let url = trusted_download_url(&photo.file.url)?;
let mut response = forge.client.get(url.clone()).send()?;
if response.url() != &url {
return Err(PhotoForgeError::UntrustedUrl(response.url().to_string()));
}
if !response.status().is_success() {
let status = response.status().as_u16();
return Err(PhotoForgeError::ProviderApi {
provider: photo.provider.slug(),
status,
body: bounded_error_body(response),
});
}
validate_content_length(
response.headers().get(CONTENT_LENGTH),
photo.file.size,
photo.id,
)?;
if let Some(content_type) = response
.headers()
.get(CONTENT_TYPE)
.and_then(|value| value.to_str().ok())
.and_then(|value| value.split(';').next())
.map(str::trim)
{
if !content_type.eq_ignore_ascii_case(&photo.file.mime) {
return Err(PhotoForgeError::VerificationFailed {
path: photo.id.to_string(),
message: format!(
"server Content-Type '{content_type}' differs from Commons MIME '{}'",
photo.file.mime
),
});
}
}
let mut file = OpenOptions::new()
.write(true)
.create_new(true)
.open(output)?;
let mut sha1 = Sha1::new();
let mut blake3 = Blake3::new();
let mut total = 0_u64;
let mut last_progress = 0_u64;
let mut buffer = [0_u8; 64 * 1024];
loop {
let count = response
.read(&mut buffer)
.map_err(|error| PhotoForgeError::Request(error_chain(&error)))?;
if count == 0 {
break;
}
total = total.saturating_add(count as u64);
if total > photo.file.size || total > max_download_bytes {
return Err(PhotoForgeError::DownloadLimitExceeded {
size: total,
limit: photo.file.size.min(max_download_bytes),
});
}
file.write_all(&buffer[..count])?;
sha1.update(&buffer[..count]);
blake3.update(&buffer[..count]);
if total.saturating_sub(last_progress) >= PROGRESS_GRANULARITY {
progress(total);
last_progress = total;
}
}
if total != photo.file.size {
return Err(PhotoForgeError::VerificationFailed {
path: photo.id.to_string(),
message: format!(
"expected {} bytes from Commons but received {total}",
photo.file.size
),
});
}
let provider_sha1 = format!("{:x}", sha1.finalize());
if !provider_sha1.eq_ignore_ascii_case(&photo.file.sha1) {
return Err(PhotoForgeError::VerificationFailed {
path: photo.id.to_string(),
message: format!(
"expected Commons SHA-1 {} but received {provider_sha1}",
photo.file.sha1
),
});
}
file.flush()?;
file.sync_all()?;
drop(file);
validate_image_file(
output,
&photo.file.mime,
photo.file.width,
photo.file.height,
)?;
if last_progress != total {
progress(total);
}
Ok(DownloadedFile {
bytes: total,
blake3: blake3.finalize().to_hex().to_string(),
})
}
fn validate_content_length(
value: Option<&HeaderValue>,
expected: u64,
photo_id: u64,
) -> Result<()> {
let Some(value) = value else {
return Ok(());
};
let length = value
.to_str()
.ok()
.and_then(|value| value.parse::<u64>().ok())
.ok_or_else(|| PhotoForgeError::VerificationFailed {
path: photo_id.to_string(),
message: "server supplied an invalid Content-Length".to_string(),
})?;
if length != expected {
return Err(PhotoForgeError::VerificationFailed {
path: photo_id.to_string(),
message: "server Content-Length differs from current Commons metadata".to_string(),
});
}
Ok(())
}
pub fn inspect_archive(directory: impl AsRef<Path>) -> Result<PhotoManifest> {
verify_archive(directory)
}
pub fn verify_archive(directory: impl AsRef<Path>) -> Result<PhotoManifest> {
let directory = directory.as_ref();
validate_existing_path_components(directory)?;
let canonical = fs::canonicalize(directory)?;
verify_directory(&canonical, true)
}
fn verify_directory(directory: &Path, enforce_directory_name: bool) -> Result<PhotoManifest> {
let directory_metadata = fs::symlink_metadata(directory)?;
if directory_metadata.file_type().is_symlink() || !directory_metadata.is_dir() {
return Err(PhotoForgeError::UnsafePath(directory.display().to_string()));
}
let manifest = read_manifest(directory)?;
validate_manifest_structure(&manifest, directory)?;
if enforce_directory_name
&& directory.file_name() != Some(OsStr::new(&manifest.photo.id.to_string()))
{
return Err(PhotoForgeError::VerificationFailed {
path: directory.display().to_string(),
message: "archive directory name does not match the Commons page ID".to_string(),
});
}
require_exact_archive_files(directory, &manifest.integrity.path)?;
let format = PhotoFormat::from_mime(&manifest.photo.file.mime)?;
let expected_path = format.relative_name();
if manifest.integrity.path != expected_path
|| manifest.integrity.bytes != manifest.photo.file.size
|| !manifest
.integrity
.commons_sha1
.eq_ignore_ascii_case(&manifest.photo.file.sha1)
{
return Err(PhotoForgeError::VerificationFailed {
path: directory.display().to_string(),
message: "manifest image path, size, or Commons digest is inconsistent".to_string(),
});
}
let image_path = directory.join(&manifest.integrity.path);
let image_metadata = fs::symlink_metadata(&image_path)?;
if image_metadata.file_type().is_symlink()
|| !image_metadata.is_file()
|| image_metadata.len() != manifest.integrity.bytes
{
return Err(PhotoForgeError::VerificationFailed {
path: image_path.display().to_string(),
message: "archived image is not a regular file of the manifest size".to_string(),
});
}
validate_image_file(
&image_path,
&manifest.photo.file.mime,
manifest.photo.file.width,
manifest.photo.file.height,
)?;
let digests = hash_file(&image_path, manifest.integrity.bytes)?;
if !digests
.sha1
.eq_ignore_ascii_case(&manifest.integrity.commons_sha1)
|| !digests
.blake3
.eq_ignore_ascii_case(&manifest.integrity.blake3)
{
return Err(PhotoForgeError::VerificationFailed {
path: image_path.display().to_string(),
message: "archived SHA-1 or BLAKE3 does not match the manifest".to_string(),
});
}
Ok(manifest)
}
fn validate_manifest_structure(manifest: &PhotoManifest, directory: &Path) -> Result<()> {
if manifest.format_version != MANIFEST_VERSION
|| manifest.owner != MANIFEST_OWNER
|| !manifest.complete
|| manifest.integrity.blake3.len() != 64
|| !manifest
.integrity
.blake3
.bytes()
.all(|byte| byte.is_ascii_hexdigit())
|| manifest.integrity.commons_sha1.len() != 40
|| !manifest
.integrity
.commons_sha1
.bytes()
.all(|byte| byte.is_ascii_hexdigit())
|| !matches!(manifest.integrity.path.as_str(), "photo.jpg" | "photo.png")
{
return Err(PhotoForgeError::VerificationFailed {
path: directory.display().to_string(),
message: "manifest ownership, version, completion, path, or digests are invalid"
.to_string(),
});
}
validate_summary(&manifest.photo)
}
fn require_exact_archive_files(directory: &Path, image_name: &str) -> Result<()> {
let mut saw_manifest = false;
let mut saw_image = false;
let mut entries = 0_u8;
for entry in fs::read_dir(directory)? {
let entry = entry?;
entries = entries.saturating_add(1);
if entries > 2 {
return Err(PhotoForgeError::VerificationFailed {
path: directory.display().to_string(),
message: "archive contains more than its photo and PhotoForge manifest".to_string(),
});
}
let metadata = fs::symlink_metadata(entry.path())?;
if metadata.file_type().is_symlink() || !metadata.is_file() {
return Err(PhotoForgeError::UnsafePath(format!(
"archive entry '{}' is not a regular non-symlink file",
entry.path().display()
)));
}
let name = entry.file_name();
if name == OsStr::new(MANIFEST_NAME) {
if saw_manifest {
return Err(PhotoForgeError::VerificationFailed {
path: directory.display().to_string(),
message: "archive contains duplicate manifest entries".to_string(),
});
}
saw_manifest = true;
} else if name == OsStr::new(image_name) {
if saw_image {
return Err(PhotoForgeError::VerificationFailed {
path: directory.display().to_string(),
message: "archive contains duplicate image entries".to_string(),
});
}
saw_image = true;
} else {
return Err(PhotoForgeError::VerificationFailed {
path: directory.display().to_string(),
message: format!(
"archive contains unknown entry '{}'",
entry.path().display()
),
});
}
}
if entries != 2 || !saw_manifest || !saw_image {
return Err(PhotoForgeError::VerificationFailed {
path: directory.display().to_string(),
message: "archive must contain exactly its photo and PhotoForge manifest".to_string(),
});
}
Ok(())
}
struct FileDigests {
sha1: String,
blake3: String,
}
fn hash_file(path: &Path, expected_bytes: u64) -> Result<FileDigests> {
let mut file = File::open(path)?.take(expected_bytes.saturating_add(1));
let mut sha1 = Sha1::new();
let mut blake3 = Blake3::new();
let mut total = 0_u64;
let mut buffer = [0_u8; 64 * 1024];
loop {
let count = file.read(&mut buffer)?;
if count == 0 {
break;
}
total = total.saturating_add(count as u64);
sha1.update(&buffer[..count]);
blake3.update(&buffer[..count]);
}
if total != expected_bytes {
return Err(PhotoForgeError::VerificationFailed {
path: path.display().to_string(),
message: "archived image length changed during hashing".to_string(),
});
}
Ok(FileDigests {
sha1: format!("{:x}", sha1.finalize()),
blake3: blake3.finalize().to_hex().to_string(),
})
}
fn write_manifest(directory: &Path, manifest: &PhotoManifest) -> Result<()> {
let encoded = serde_json::to_vec_pretty(manifest)?;
if encoded.len() as u64 > MAX_MANIFEST_BYTES {
return Err(PhotoForgeError::ResponseLimitExceeded {
label: MANIFEST_NAME.to_string(),
size: encoded.len() as u64,
limit: MAX_MANIFEST_BYTES,
});
}
let path = directory.join(MANIFEST_NAME);
let mut file = OpenOptions::new().write(true).create_new(true).open(path)?;
file.write_all(&encoded)?;
file.write_all(b"\n")?;
file.sync_all()?;
sync_directory(directory)
}
fn read_manifest(directory: &Path) -> Result<PhotoManifest> {
let path = directory.join(MANIFEST_NAME);
let metadata = fs::symlink_metadata(&path)?;
if metadata.file_type().is_symlink() || !metadata.is_file() {
return Err(PhotoForgeError::UnsafePath(path.display().to_string()));
}
if metadata.len() > MAX_MANIFEST_BYTES {
return Err(PhotoForgeError::ResponseLimitExceeded {
label: MANIFEST_NAME.to_string(),
size: metadata.len(),
limit: MAX_MANIFEST_BYTES,
});
}
let mut bytes = Vec::with_capacity(metadata.len() as usize);
File::open(&path)?
.take(MAX_MANIFEST_BYTES.saturating_add(1))
.read_to_end(&mut bytes)?;
if bytes.len() as u64 > MAX_MANIFEST_BYTES {
return Err(PhotoForgeError::ResponseLimitExceeded {
label: MANIFEST_NAME.to_string(),
size: bytes.len() as u64,
limit: MAX_MANIFEST_BYTES,
});
}
Ok(serde_json::from_slice(&bytes)?)
}
fn recognized_manifest(directory: &Path, id: u64) -> Result<Option<PhotoManifest>> {
match fs::symlink_metadata(directory) {
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None),
Err(error) => return Err(error.into()),
Ok(metadata) if metadata.file_type().is_symlink() || !metadata.is_dir() => {
return Err(PhotoForgeError::UnrecognizedExistingOutput {
path: directory.to_path_buf(),
reason: "destination is not a regular non-symlink directory".to_string(),
});
}
Ok(_) => {}
}
let manifest =
read_manifest(directory).map_err(|error| PhotoForgeError::UnrecognizedExistingOutput {
path: directory.to_path_buf(),
reason: format!("no readable bounded PhotoForge manifest: {error}"),
})?;
let recognized = manifest.owner == MANIFEST_OWNER
&& manifest.format_version == MANIFEST_VERSION
&& manifest.photo.provider == PhotoProvider::WikimediaCommons
&& manifest.photo.id == id
&& manifest.photo.title.starts_with("File:")
&& matches!(manifest.integrity.path.as_str(), "photo.jpg" | "photo.png")
&& directory.file_name() == Some(OsStr::new(&id.to_string()));
if !recognized {
return Err(PhotoForgeError::UnrecognizedExistingOutput {
path: directory.to_path_buf(),
reason: "manifest ownership or page identity does not match the destination"
.to_string(),
});
}
Ok(Some(manifest))
}
fn archived_photo(directory: &Path, manifest: PhotoManifest) -> Result<ArchivedPhoto> {
let output = fs::canonicalize(directory)?;
Ok(ArchivedPhoto {
photo_id: manifest.photo.id,
output,
bytes_written: manifest.integrity.bytes,
manifest,
})
}
fn prepare_output(output: &Path) -> Result<PathBuf> {
if output.as_os_str().is_empty() {
return Err(PhotoForgeError::UnsafePath("empty output root".to_string()));
}
ensure_safe_directory(output)?;
let canonical = fs::canonicalize(output)?;
let metadata = fs::symlink_metadata(&canonical)?;
if metadata.file_type().is_symlink() || !metadata.is_dir() {
return Err(PhotoForgeError::OutputNotDirectory(output.to_path_buf()));
}
Ok(canonical)
}
fn ensure_safe_directory(path: &Path) -> Result<()> {
let absolute = if path.is_absolute() {
path.to_path_buf()
} else {
std::env::current_dir()?.join(path)
};
let mut current = PathBuf::new();
for component in absolute.components() {
match component {
Component::Prefix(prefix) => current.push(prefix.as_os_str()),
Component::RootDir => current.push(component.as_os_str()),
Component::CurDir => {}
Component::ParentDir => {
return Err(PhotoForgeError::UnsafePath(
"output root must not contain parent traversal".to_string(),
));
}
Component::Normal(part) => {
current.push(part);
match fs::symlink_metadata(¤t) {
Ok(metadata) => {
if metadata.file_type().is_symlink() || !metadata.is_dir() {
return Err(PhotoForgeError::UnsafePath(format!(
"output component '{}' is not a regular directory",
current.display()
)));
}
}
Err(error) if error.kind() == io::ErrorKind::NotFound => {
fs::create_dir(¤t)?;
sync_parent(¤t)?;
let metadata = fs::symlink_metadata(¤t)?;
if metadata.file_type().is_symlink() || !metadata.is_dir() {
return Err(PhotoForgeError::UnsafePath(format!(
"new output component '{}' is unsafe",
current.display()
)));
}
}
Err(error) => return Err(error.into()),
}
}
}
}
Ok(())
}
fn validate_existing_path_components(path: &Path) -> Result<()> {
let absolute = if path.is_absolute() {
path.to_path_buf()
} else {
std::env::current_dir()?.join(path)
};
let mut current = PathBuf::new();
for component in absolute.components() {
match component {
Component::Prefix(prefix) => current.push(prefix.as_os_str()),
Component::RootDir => current.push(component.as_os_str()),
Component::CurDir => {}
Component::ParentDir => {
return Err(PhotoForgeError::UnsafePath(
"archive path must not contain parent traversal".to_string(),
));
}
Component::Normal(part) => {
current.push(part);
let metadata = fs::symlink_metadata(¤t)?;
if metadata.file_type().is_symlink() || !metadata.is_dir() {
return Err(PhotoForgeError::UnsafePath(format!(
"archive path component '{}' is not a regular directory",
current.display()
)));
}
}
}
}
Ok(())
}
fn ensure_contained(root: &Path, path: &Path) -> Result<()> {
let parent = path
.parent()
.ok_or_else(|| PhotoForgeError::UnsafePath(path.display().to_string()))?;
if fs::canonicalize(parent)? != root {
return Err(PhotoForgeError::UnsafePath(path.display().to_string()));
}
if path.exists() {
let canonical = fs::canonicalize(path)?;
if canonical.parent() != Some(root) {
return Err(PhotoForgeError::UnsafePath(path.display().to_string()));
}
}
Ok(())
}
fn sync_parent(path: &Path) -> Result<()> {
if let Some(parent) = path.parent() {
sync_directory(parent)?;
}
Ok(())
}
fn create_stage(output: &Path, id: u64) -> Result<PathBuf> {
let nonce = unix_nanos()?;
for sequence in 0..100_u32 {
let stage = output.join(format!(
".photoforge-{id}-stage-{}-{nonce}-{sequence}",
std::process::id()
));
match fs::create_dir(&stage) {
Ok(()) => {
let marker_path = stage.join(STAGING_MARKER);
let mut marker = OpenOptions::new()
.write(true)
.create_new(true)
.open(marker_path)?;
writeln!(marker, "{id}")?;
marker.sync_all()?;
sync_directory(&stage)?;
sync_directory(output)?;
return Ok(stage);
}
Err(error) if error.kind() == io::ErrorKind::AlreadyExists => continue,
Err(error) => return Err(error.into()),
}
}
Err(PhotoForgeError::ArchiveTransaction {
path: output.to_path_buf(),
message: "could not allocate a unique staging directory".to_string(),
})
}
fn remove_staging_marker(stage: &Path) -> Result<()> {
let marker = stage.join(STAGING_MARKER);
let metadata = fs::symlink_metadata(&marker)?;
if metadata.file_type().is_symlink() || !metadata.is_file() {
return Err(PhotoForgeError::UnsafePath(marker.display().to_string()));
}
fs::remove_file(marker)?;
sync_directory(stage)
}
fn cleanup_stage(stage: &Path, id: u64) -> Result<()> {
if !stage.exists() {
return Ok(());
}
if !recognizable_stage(stage, id)? {
return Err(PhotoForgeError::ArchiveTransaction {
path: stage.to_path_buf(),
message: "staging directory is no longer recognizable; preserving it".to_string(),
});
}
fs::remove_dir_all(stage)?;
sync_parent(stage)
}
fn recognizable_stage(stage: &Path, id: u64) -> Result<bool> {
let metadata = fs::symlink_metadata(stage)?;
if metadata.file_type().is_symlink() || !metadata.is_dir() {
return Ok(false);
}
let marker = stage.join(STAGING_MARKER);
if marker.exists() {
let metadata = fs::symlink_metadata(&marker)?;
if metadata.file_type().is_symlink() || !metadata.is_file() || metadata.len() > 32 {
return Ok(false);
}
if fs::read_to_string(&marker)?.trim() != id.to_string() {
return Ok(false);
}
for entry in fs::read_dir(stage)? {
let entry = entry?;
let metadata = fs::symlink_metadata(entry.path())?;
if metadata.file_type().is_symlink() || !metadata.is_file() {
return Ok(false);
}
let name = entry.file_name();
if name != OsStr::new(STAGING_MARKER)
&& name != OsStr::new(MANIFEST_NAME)
&& name != OsStr::new("photo.jpg")
&& name != OsStr::new("photo.png")
{
return Ok(false);
}
}
return Ok(true);
}
Ok(verify_directory(stage, false).is_ok_and(|manifest| manifest.photo.id == id))
}
fn revalidate_destination(
destination: &Path,
id: u64,
expected: Option<&PhotoManifest>,
) -> Result<()> {
match expected {
None => match fs::symlink_metadata(destination) {
Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()),
Ok(_) => Err(PhotoForgeError::UnrecognizedExistingOutput {
path: destination.to_path_buf(),
reason: "destination appeared while the photo was staged".to_string(),
}),
Err(error) => Err(error.into()),
},
Some(expected) => {
let current = recognized_manifest(destination, id)?.ok_or_else(|| {
PhotoForgeError::UnrecognizedExistingOutput {
path: destination.to_path_buf(),
reason: "destination disappeared while the photo was staged".to_string(),
}
})?;
if ¤t != expected {
return Err(PhotoForgeError::UnrecognizedExistingOutput {
path: destination.to_path_buf(),
reason: "destination manifest changed while the photo was staged".to_string(),
});
}
Ok(())
}
}
}
fn install_stage(
stage: &Path,
destination: &Path,
output: &Path,
id: u64,
expected_existing: Option<&PhotoManifest>,
transaction: &mut DestinationTransaction,
) -> Result<()> {
verify_directory(stage, false)?;
revalidate_destination(destination, id, expected_existing)?;
if expected_existing.is_none() {
transaction.mark_installing()?;
if let Err(error) = fs::rename(stage, destination) {
return Err(PhotoForgeError::ArchiveTransaction {
path: destination.to_path_buf(),
message: format!("could not install staged archive: {error}"),
});
}
if let Err(error) =
sync_directory(output).and_then(|()| verify_directory(destination, true).map(|_| ()))
{
let cleanup = if recognized_manifest(destination, id).is_ok() {
fs::remove_dir_all(destination).err()
} else {
None
};
let cleanup_sync = sync_directory(output).err();
return Err(PhotoForgeError::ArchiveTransaction {
path: destination.to_path_buf(),
message: format!(
"new archive installation was not durable and verified: {error}; cleanup={cleanup:?}; cleanup sync={cleanup_sync:?}"
),
});
}
return Ok(());
}
let backup = create_backup_container(output, id)?;
let backup_archive = backup.join("archive");
if let Err(error) = fs::rename(destination, &backup_archive) {
let cleanup = fs::remove_dir(&backup).err();
return Err(PhotoForgeError::ArchiveTransaction {
path: destination.to_path_buf(),
message: format!(
"could not move old archive into backup: {error}; cleanup={cleanup:?}"
),
});
}
if let Err(error) = sync_directory(&backup).and_then(|()| sync_directory(output)) {
let rollback = fs::rename(&backup_archive, destination).err();
let cleanup = fs::remove_dir(&backup).err();
return Err(PhotoForgeError::ArchiveTransaction {
path: destination.to_path_buf(),
message: format!(
"could not durably back up old archive: {error}; rollback={rollback:?}; cleanup={cleanup:?}"
),
});
}
transaction.mark_installing()?;
if let Err(error) = fs::rename(stage, destination) {
let rollback = fs::rename(&backup_archive, destination).err();
let rollback_sync = sync_directory(output).err();
return Err(PhotoForgeError::ArchiveTransaction {
path: destination.to_path_buf(),
message: format!(
"replacement install failed: {error}; rollback={rollback:?}; rollback sync={rollback_sync:?}"
),
});
}
if let Err(error) =
sync_directory(output).and_then(|()| verify_directory(destination, true).map(|_| ()))
{
let failed_new = backup.join("failed-new");
let move_new = fs::rename(destination, &failed_new).err();
let restore = if move_new.is_none() {
fs::rename(&backup_archive, destination).err()
} else {
Some(io::Error::other("new archive could not be moved aside"))
};
let rollback_sync = sync_directory(output).err();
return Err(PhotoForgeError::ArchiveTransaction {
path: destination.to_path_buf(),
message: format!(
"replacement was not durable and verified: {error}; move new={move_new:?}; restore old={restore:?}; rollback sync={rollback_sync:?}; state preserved at '{}'",
backup.display()
),
});
}
if let Err(error) = fs::remove_dir_all(&backup) {
return Err(PhotoForgeError::ArchiveTransaction {
path: destination.to_path_buf(),
message: format!(
"replacement is verified but backup '{}' could not be removed: {error}",
backup.display()
),
});
}
sync_directory(output)
}
fn create_backup_container(output: &Path, id: u64) -> Result<PathBuf> {
let nonce = unix_nanos()?;
for sequence in 0..100_u32 {
let path = output.join(format!(
".photoforge-{id}-backup-{}-{nonce}-{sequence}",
std::process::id()
));
match fs::create_dir(&path) {
Ok(()) => {
sync_directory(output)?;
return Ok(path);
}
Err(error) if error.kind() == io::ErrorKind::AlreadyExists => continue,
Err(error) => return Err(error.into()),
}
}
Err(PhotoForgeError::ArchiveTransaction {
path: output.to_path_buf(),
message: "could not allocate a unique backup directory".to_string(),
})
}
struct DestinationTransaction {
file: File,
output: PathBuf,
path: PathBuf,
interrupted: bool,
}
impl DestinationTransaction {
fn acquire(output: &Path, id: u64) -> Result<Self> {
let path = output.join(format!(".photoforge-{id}.lock"));
if let Ok(metadata) = fs::symlink_metadata(&path) {
if metadata.file_type().is_symlink() || !metadata.is_file() {
return Err(PhotoForgeError::UnsafePath(format!(
"transaction lock '{}' is not a regular file",
path.display()
)));
}
}
let (mut file, created) = match OpenOptions::new()
.read(true)
.write(true)
.create_new(true)
.open(&path)
{
Ok(file) => (file, true),
Err(error) if error.kind() == io::ErrorKind::AlreadyExists => (
OpenOptions::new().read(true).write(true).open(&path)?,
false,
),
Err(error) => return Err(error.into()),
};
let metadata = fs::symlink_metadata(&path)?;
if metadata.file_type().is_symlink() || !metadata.is_file() || !file.metadata()?.is_file() {
return Err(PhotoForgeError::UnsafePath(path.display().to_string()));
}
let started = Instant::now();
loop {
match file.try_lock_exclusive() {
Ok(()) => break,
Err(error) if error.kind() == io::ErrorKind::WouldBlock => {
if started.elapsed() >= LOCK_TIMEOUT {
return Err(PhotoForgeError::ArchiveTransaction {
path,
message: "timed out waiting for the per-photo destination lock"
.to_string(),
});
}
thread::sleep(LOCK_POLL_INTERVAL);
}
Err(error) => return Err(error.into()),
}
}
ensure_lock_identity(&file, &path)?;
ensure_single_link(&file, &path)?;
let length = file.metadata()?.len();
if length > MAX_TRANSACTION_LOG_BYTES {
return Err(PhotoForgeError::ArchiveTransaction {
path: path.clone(),
message: "transaction state log exceeds 64 KiB".to_string(),
});
}
if created {
if length != 0 {
return Err(PhotoForgeError::ArchiveTransaction {
path: path.clone(),
message: "new transaction lock unexpectedly contains data".to_string(),
});
}
writeln!(file, "{LOCK_HEADER}")?;
writeln!(file, "idle")?;
file.sync_all()?;
sync_directory(output)?;
}
file.rewind()?;
let mut state_log = String::new();
Read::by_ref(&mut file)
.take(MAX_TRANSACTION_LOG_BYTES.saturating_add(1))
.read_to_string(&mut state_log)?;
let complete_log = if state_log.ends_with('\n') {
state_log.as_str()
} else {
state_log
.rsplit_once('\n')
.map_or("", |(complete, _)| complete)
};
let mut lines = complete_log.lines();
if lines.next() != Some(LOCK_HEADER) {
return Err(PhotoForgeError::ArchiveTransaction {
path: path.clone(),
message: "preexisting transaction lock is blank, foreign, or has an unsupported ownership header".to_string(),
});
}
let mut state = None;
for line in lines {
let line = line.trim();
if !matches!(line, "idle" | "preparing" | "installing") {
return Err(PhotoForgeError::ArchiveTransaction {
path: path.clone(),
message: "transaction state log contains an unknown state".to_string(),
});
}
state = Some(line);
}
let state = state.ok_or_else(|| PhotoForgeError::ArchiveTransaction {
path: path.clone(),
message: "transaction lock has no complete state record".to_string(),
})?;
Ok(Self {
file,
output: output.to_path_buf(),
path,
interrupted: matches!(state, "preparing" | "installing"),
})
}
fn recover(&mut self, output: &Path, destination: &Path, id: u64) -> Result<()> {
let stages = transaction_directories(output, &format!(".photoforge-{id}-stage-"))?;
let backups = transaction_directories(output, &format!(".photoforge-{id}-backup-"))?;
if !self.interrupted {
if !stages.is_empty() || !backups.is_empty() {
return Err(PhotoForgeError::ArchiveTransaction {
path: output.to_path_buf(),
message: "transaction directories exist while the state log is idle; operator review is required".to_string(),
});
}
return Ok(());
}
if backups.len() > 1 {
return Err(PhotoForgeError::ArchiveTransaction {
path: output.to_path_buf(),
message: "multiple interrupted backups exist; operator review is required"
.to_string(),
});
}
for stage in &stages {
if !recognizable_stage(stage, id)? {
return Err(PhotoForgeError::ArchiveTransaction {
path: stage.clone(),
message: "interrupted staging is not safely recognizable".to_string(),
});
}
}
if verify_archive(destination).is_ok() {
if let Some(backup) = backups.first() {
recognize_backup(backup, id)?;
fs::remove_dir_all(backup)?;
}
for stage in stages {
fs::remove_dir_all(stage)?;
}
sync_directory(output)?;
self.mark_idle()?;
return Ok(());
}
if let Some(backup) = backups.first() {
recognize_backup(backup, id)?;
if destination.exists() {
recognized_manifest(destination, id)?;
fs::remove_dir_all(destination)?;
}
fs::rename(backup.join("archive"), destination)?;
fs::remove_dir(backup)?;
} else if destination.exists() {
recognized_manifest(destination, id)?;
}
for stage in stages {
fs::remove_dir_all(stage)?;
}
sync_directory(output)?;
self.mark_idle()
}
fn mark_preparing(&mut self) -> Result<()> {
self.write_state("preparing")?;
self.interrupted = true;
Ok(())
}
fn mark_installing(&mut self) -> Result<()> {
self.write_state("installing")?;
self.interrupted = true;
Ok(())
}
fn mark_idle(&mut self) -> Result<()> {
self.write_state("idle")?;
self.interrupted = false;
Ok(())
}
fn write_state(&mut self, state: &str) -> Result<()> {
ensure_lock_identity(&self.file, &self.path)?;
ensure_single_link(&self.file, &self.path)?;
if self
.file
.metadata()?
.len()
.saturating_add(state.len() as u64 + 1)
> TRANSACTION_COMPACT_BYTES
{
self.file.set_len(0)?;
self.file.rewind()?;
writeln!(self.file, "{LOCK_HEADER}")?;
}
self.file.seek(io::SeekFrom::End(0))?;
writeln!(self.file, "{state}")?;
self.file.sync_all()?;
ensure_lock_identity(&self.file, &self.path)?;
sync_directory(&self.output)
}
}
fn ensure_lock_identity(file: &File, path: &Path) -> Result<()> {
let opened = Handle::from_file(file.try_clone()?)?;
let current = Handle::from_path(path)?;
if opened != current {
return Err(PhotoForgeError::ArchiveTransaction {
path: path.to_path_buf(),
message: "transaction lock path was replaced after opening".to_string(),
});
}
Ok(())
}
#[cfg(unix)]
fn ensure_single_link(file: &File, path: &Path) -> Result<()> {
use std::os::unix::fs::MetadataExt;
if file.metadata()?.nlink() != 1 {
return Err(PhotoForgeError::ArchiveTransaction {
path: path.to_path_buf(),
message: "transaction lock is hard-linked and will not be modified".to_string(),
});
}
Ok(())
}
#[cfg(windows)]
fn ensure_single_link(file: &File, path: &Path) -> Result<()> {
use cap_primitives::fs::{_WindowsByHandle, Metadata};
if Metadata::from_file(file)?.number_of_links() != Some(1) {
return Err(PhotoForgeError::ArchiveTransaction {
path: path.to_path_buf(),
message: "transaction lock is hard-linked or its link count is unavailable and it will not be modified".to_string(),
});
}
Ok(())
}
#[cfg(target_os = "wasi")]
fn ensure_single_link(file: &File, path: &Path) -> Result<()> {
use std::os::wasi::fs::MetadataExt;
if file.metadata()?.nlink() != 1 {
return Err(PhotoForgeError::ArchiveTransaction {
path: path.to_path_buf(),
message: "transaction lock is hard-linked and will not be modified".to_string(),
});
}
Ok(())
}
#[cfg(not(any(unix, windows, target_os = "wasi")))]
fn ensure_single_link(_file: &File, path: &Path) -> Result<()> {
Err(PhotoForgeError::ArchiveTransaction {
path: path.to_path_buf(),
message: "this platform cannot establish transaction-lock link count safely".to_string(),
})
}
impl Drop for DestinationTransaction {
fn drop(&mut self) {
let _ = FileExt::unlock(&self.file);
}
}
fn transaction_directories(output: &Path, prefix: &str) -> Result<Vec<PathBuf>> {
let mut directories = Vec::new();
for entry in fs::read_dir(output)? {
let entry = entry?;
if entry
.file_name()
.to_str()
.is_some_and(|name| name.starts_with(prefix))
{
let metadata = fs::symlink_metadata(entry.path())?;
if metadata.file_type().is_symlink() || !metadata.is_dir() {
return Err(PhotoForgeError::UnsafePath(
entry.path().display().to_string(),
));
}
directories.push(entry.path());
}
}
directories.sort();
Ok(directories)
}
fn recognize_backup(backup: &Path, id: u64) -> Result<()> {
let mut archive = None;
for entry in fs::read_dir(backup)? {
let entry = entry?;
if archive.is_some() || entry.file_name() != OsStr::new("archive") {
return Err(PhotoForgeError::ArchiveTransaction {
path: backup.to_path_buf(),
message: "backup does not contain exactly one archived destination".to_string(),
});
}
archive = Some(entry.path());
}
let archive = archive.ok_or_else(|| PhotoForgeError::ArchiveTransaction {
path: backup.to_path_buf(),
message: "backup is empty".to_string(),
})?;
let manifest =
verify_directory(&archive, false).map_err(|error| PhotoForgeError::ArchiveTransaction {
path: backup.to_path_buf(),
message: format!("backup failed full verification and was preserved: {error}"),
})?;
if manifest.photo.id != id {
return Err(PhotoForgeError::ArchiveTransaction {
path: backup.to_path_buf(),
message: "backup archive page identity does not match the transaction".to_string(),
});
}
Ok(())
}
fn combine_transaction_error<T>(
destination: &Path,
primary: PhotoForgeError,
cleanup: Option<PhotoForgeError>,
state: Option<PhotoForgeError>,
) -> Result<T> {
if cleanup.is_none() && state.is_none() {
return Err(primary);
}
Err(PhotoForgeError::ArchiveTransaction {
path: destination.to_path_buf(),
message: format!(
"{primary}; staging cleanup={}; state cleanup={}",
cleanup
.map(|error| error.to_string())
.unwrap_or_else(|| "ok".to_string()),
state
.map(|error| error.to_string())
.unwrap_or_else(|| "ok".to_string())
),
})
}
fn progress_snapshot(
phase: ArchiveProgressPhase,
summary: &ArchiveSummary,
active: Option<&PhotoSummary>,
) -> ArchiveProgress {
ArchiveProgress {
phase,
total_photos: summary.selected,
completed_photos: summary.archived + summary.skipped + summary.failed,
archived_photos: summary.archived,
skipped_photos: summary.skipped,
failed_photos: summary.failed,
bytes_written: summary.bytes_written,
active_photo_id: active.map(|photo| photo.id),
active_title: active.map(|photo| bounded_title(&photo.title)),
active_file: None,
active_bytes_written: 0,
active_total_bytes: active.map(|photo| photo.file.size),
}
}
fn active_progress(
phase: ArchiveProgressPhase,
summary: &ArchiveSummary,
photo: &PhotoSummary,
relative: &str,
active_bytes: u64,
) -> ArchiveProgress {
ArchiveProgress {
phase,
total_photos: summary.selected,
completed_photos: summary.archived + summary.skipped + summary.failed,
archived_photos: summary.archived,
skipped_photos: summary.skipped,
failed_photos: summary.failed,
bytes_written: summary.bytes_written.saturating_add(active_bytes),
active_photo_id: Some(photo.id),
active_title: Some(bounded_title(&photo.title)),
active_file: Some(relative.to_string()),
active_bytes_written: active_bytes,
active_total_bytes: Some(photo.file.size),
}
}
fn emit_progress(
progress: &mut Option<&mut dyn FnMut(ArchiveProgress)>,
snapshot: ArchiveProgress,
) {
if let Some(progress) = progress.as_deref_mut() {
progress(snapshot);
}
}
fn bounded_title(value: &str) -> String {
truncate_utf8(value, 512).to_string()
}
fn bounded_diagnostic(value: &str) -> String {
truncate_utf8(value, 4096).to_string()
}
fn truncate_utf8(value: &str, max: usize) -> &str {
if value.len() <= max {
return value;
}
let mut boundary = max;
while boundary > 0 && !value.is_char_boundary(boundary) {
boundary -= 1;
}
&value[..boundary]
}
fn bounded_error_body(mut response: reqwest::blocking::Response) -> String {
let mut bytes = Vec::new();
match response
.by_ref()
.take(MAX_ERROR_RESPONSE_BYTES.saturating_add(1))
.read_to_end(&mut bytes)
{
Ok(_) if bytes.len() as u64 <= MAX_ERROR_RESPONSE_BYTES => {
bounded_diagnostic(&String::from_utf8_lossy(&bytes))
}
Ok(_) => "<error response exceeded safety limit>".to_string(),
Err(_) => "<unable to read error response>".to_string(),
}
}
fn unix_nanos() -> Result<u128> {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|duration| duration.as_nanos())
.map_err(|error| PhotoForgeError::ArchiveTransaction {
path: PathBuf::new(),
message: error.to_string(),
})
}
#[cfg(unix)]
fn sync_directory(path: &Path) -> Result<()> {
File::open(path)?.sync_all()?;
Ok(())
}
#[cfg(windows)]
fn sync_directory(path: &Path) -> Result<()> {
use std::os::windows::fs::OpenOptionsExt;
const FILE_FLAG_BACKUP_SEMANTICS: u32 = 0x0200_0000;
OpenOptions::new()
.read(true)
.custom_flags(FILE_FLAG_BACKUP_SEMANTICS)
.open(path)?
.sync_all()?;
Ok(())
}
#[cfg(not(any(unix, windows)))]
fn sync_directory(_path: &Path) -> Result<()> {
Ok(())
}
enum ArchiveOutcome {
Archived(ArchivedPhoto),
Skipped(ArchivedPhoto),
}
#[cfg(test)]
mod tests {
use super::*;
use crate::image::{test_jpeg, test_png};
use crate::{COMMONS_REUSE_URL, PhotoFile, PhotoRights};
fn fixture_summary(id: u64, mime: &str, bytes: &[u8], width: u32, height: u32) -> PhotoSummary {
PhotoSummary {
provider: PhotoProvider::WikimediaCommons,
id,
title: format!(
"File:Fixture-{id}.{}",
if mime == "image/png" { "png" } else { "jpg" }
),
object_name: "Fixture".to_string(),
description: "Generated test image".to_string(),
categories: vec!["Fixtures".to_string()],
assessments: Vec::new(),
usage_count: 0,
usage_count_complete: true,
source_url: format!(
"https://commons.wikimedia.org/wiki/File:Fixture-{id}.{}",
if mime == "image/png" { "png" } else { "jpg" }
),
rights: PhotoRights {
license: "CC0-1.0".to_string(),
license_url: "https://creativecommons.org/publicdomain/zero/1.0/".to_string(),
raw_license_label: "CC0".to_string(),
creator: "Fixture Creator".to_string(),
custom_attribution: String::new(),
attribution: "Fixture Creator; Wikimedia Commons".to_string(),
source_credit: "Fixture source".to_string(),
permission: String::new(),
usage_terms: "CC0 1.0".to_string(),
raw_attribution_required: "false".to_string(),
attribution_required: false,
restrictions: String::new(),
copyrighted: "false".to_string(),
non_free: "false".to_string(),
deletion_reason: String::new(),
metadata_retrieved_at: 1,
provider_guidance_url: COMMONS_REUSE_URL.to_string(),
},
file: PhotoFile {
url: format!(
"https://upload.wikimedia.org/wikipedia/commons/a/ab/Fixture-{id}.{}",
if mime == "image/png" { "png" } else { "jpg" }
),
size: bytes.len() as u64,
sha1: format!("{:x}", Sha1::digest(bytes)),
mime: mime.to_string(),
media_type: "BITMAP".to_string(),
width,
height,
revision_id: Some(id + 100),
revision_timestamp: Some("2026-01-01T00:00:00Z".to_string()),
},
}
}
fn create_archive(parent: &Path, id: u64, mime: &str) -> PathBuf {
let bytes = if mime == "image/png" {
test_png(2, 1)
} else {
test_jpeg(2, 1)
};
let photo = fixture_summary(id, mime, &bytes, 2, 1);
let directory = parent.join(id.to_string());
fs::create_dir(&directory).unwrap();
let relative = PhotoFormat::from_mime(mime).unwrap().relative_name();
fs::write(directory.join(&relative), &bytes).unwrap();
let manifest = PhotoManifest {
format_version: MANIFEST_VERSION.to_string(),
owner: MANIFEST_OWNER.to_string(),
photo,
integrity: PhotoIntegrity {
path: relative,
bytes: bytes.len() as u64,
blake3: blake3::hash(&bytes).to_hex().to_string(),
commons_sha1: format!("{:x}", Sha1::digest(&bytes)),
},
complete: true,
};
write_manifest(&directory, &manifest).unwrap();
directory
}
#[test]
fn verifies_exact_jpeg_and_png_archives() {
let temp = tempfile::tempdir().unwrap();
let png = create_archive(temp.path(), 1, "image/png");
let jpeg = create_archive(temp.path(), 2, "image/jpeg");
assert_eq!(verify_archive(&png).unwrap().photo.id, 1);
assert_eq!(inspect_archive(&jpeg).unwrap().photo.id, 2);
}
#[test]
fn verification_deliberately_rejects_external_sidecars() {
let temp = tempfile::tempdir().unwrap();
let directory = create_archive(temp.path(), 3, "image/png");
fs::write(directory.join("thesa-sidecar.json"), b"{}").unwrap();
let error = verify_archive(directory).unwrap_err().to_string();
assert!(error.contains("more than its photo") || error.contains("unknown entry"));
}
#[test]
fn verification_rejects_corruption_wrong_dimensions_and_manifest_traversal() {
let temp = tempfile::tempdir().unwrap();
let directory = create_archive(temp.path(), 4, "image/png");
fs::write(directory.join("photo.png"), test_png(1, 1)).unwrap();
assert!(verify_archive(&directory).is_err());
let directory = create_archive(temp.path(), 5, "image/png");
let mut manifest = read_manifest(&directory).unwrap();
manifest.photo.file.width = 9;
fs::remove_file(directory.join(MANIFEST_NAME)).unwrap();
write_manifest(&directory, &manifest).unwrap();
assert!(verify_archive(&directory).is_err());
let directory = create_archive(temp.path(), 6, "image/jpeg");
let mut manifest = read_manifest(&directory).unwrap();
manifest.integrity.path = "../photo.jpg".to_string();
fs::remove_file(directory.join(MANIFEST_NAME)).unwrap();
write_manifest(&directory, &manifest).unwrap();
assert!(verify_archive(&directory).is_err());
}
#[cfg(unix)]
#[test]
fn output_and_archive_symlinks_are_rejected() {
use std::os::unix::fs::symlink;
let temp = tempfile::tempdir().unwrap();
let real = temp.path().join("real");
fs::create_dir(&real).unwrap();
let link = temp.path().join("link");
symlink(&real, &link).unwrap();
assert!(prepare_output(&link).is_err());
let directory = create_archive(&real, 7, "image/png");
let image = directory.join("photo.png");
let outside = temp.path().join("outside.png");
fs::rename(&image, &outside).unwrap();
symlink(&outside, &image).unwrap();
assert!(verify_archive(directory).is_err());
}
#[test]
fn foreign_destination_is_never_recognized_or_removed() {
let temp = tempfile::tempdir().unwrap();
let directory = temp.path().join("8");
fs::create_dir(&directory).unwrap();
fs::write(directory.join("foreign.txt"), b"keep").unwrap();
assert!(recognized_manifest(&directory, 8).is_err());
assert_eq!(fs::read(directory.join("foreign.txt")).unwrap(), b"keep");
}
#[test]
fn recognized_corrupt_output_can_be_repaired_transactionally() {
let temp = tempfile::tempdir().unwrap();
let output = prepare_output(temp.path()).unwrap();
let destination = create_archive(&output, 9, "image/png");
fs::write(destination.join("photo.png"), b"corrupt").unwrap();
let existing = recognized_manifest(&destination, 9).unwrap().unwrap();
assert!(verify_archive(&destination).is_err());
let stage = create_stage(&output, 9).unwrap();
let replacement_bytes = test_png(2, 1);
fs::write(stage.join("photo.png"), &replacement_bytes).unwrap();
let photo = fixture_summary(9, "image/png", &replacement_bytes, 2, 1);
let replacement = PhotoManifest {
format_version: MANIFEST_VERSION.to_string(),
owner: MANIFEST_OWNER.to_string(),
photo,
integrity: PhotoIntegrity {
path: "photo.png".to_string(),
bytes: replacement_bytes.len() as u64,
blake3: blake3::hash(&replacement_bytes).to_hex().to_string(),
commons_sha1: format!("{:x}", Sha1::digest(&replacement_bytes)),
},
complete: true,
};
write_manifest(&stage, &replacement).unwrap();
remove_staging_marker(&stage).unwrap();
let mut transaction = DestinationTransaction::acquire(&output, 9).unwrap();
transaction.mark_preparing().unwrap();
install_stage(
&stage,
&destination,
&output,
9,
Some(&existing),
&mut transaction,
)
.unwrap();
transaction.mark_idle().unwrap();
assert_eq!(verify_archive(destination).unwrap(), replacement);
}
#[test]
fn verified_skip_returns_a_full_verified_record() {
let temp = tempfile::tempdir().unwrap();
let directory = create_archive(temp.path(), 10, "image/jpeg");
let manifest = read_manifest(&directory).unwrap();
let mut current = manifest.photo.clone();
current.rights.metadata_retrieved_at = 2;
current.usage_count = 50;
current.usage_count_complete = false;
let summary = ArchiveSummary {
selected: 1,
selected_ids: vec![10],
..ArchiveSummary::default()
};
let mut no_progress = None;
let skipped = try_verified_skip(
&directory,
¤t,
Some(&manifest),
&mut no_progress,
&summary,
)
.unwrap()
.unwrap();
assert_eq!(skipped.photo_id, 10);
assert_eq!(skipped.manifest, manifest);
assert!(skipped.output.is_absolute());
}
#[test]
fn zero_cap_is_rejected_before_network_or_output_work() {
let forge =
PhotoForge::with_user_agent("photoforge-test/0.1 (test@example.invalid)").unwrap();
let options = ArchiveOptions {
output: PathBuf::new(),
skip_existing: false,
max_download_bytes: 0,
};
assert!(matches!(
archive(&forge, &[], &options, None),
Err(PhotoForgeError::InvalidOption(_))
));
}
#[test]
fn supplied_content_length_must_be_numeric_and_exact() {
assert!(validate_content_length(None, 10, 1).is_ok());
assert!(validate_content_length(Some(&HeaderValue::from_static("10")), 10, 1).is_ok());
assert!(validate_content_length(Some(&HeaderValue::from_static("9")), 10, 1).is_err());
assert!(
validate_content_length(Some(&HeaderValue::from_static("not-a-number")), 10, 1)
.is_err()
);
}
#[test]
fn empty_archive_emits_bounded_start_and_completed_snapshots() {
let temp = tempfile::tempdir().unwrap();
let forge =
PhotoForge::with_user_agent("photoforge-test/0.1 (test@example.invalid)").unwrap();
let options = ArchiveOptions {
output: temp.path().join("photos"),
..ArchiveOptions::default()
};
let mut phases = Vec::new();
let summary = forge
.archive_with_progress(&[], &options, |snapshot| phases.push(snapshot.phase))
.unwrap();
assert_eq!(
phases,
[
ArchiveProgressPhase::Starting,
ArchiveProgressPhase::Completed
]
);
assert_eq!(summary.selected, 0);
assert!(summary.is_success());
}
#[test]
fn selection_deduplicates_ids_in_first_selected_order() {
let png = test_png(1, 1);
let first = fixture_summary(11, "image/png", &png, 1, 1);
let duplicate = fixture_summary(11, "image/png", &png, 1, 1);
let second = fixture_summary(12, "image/png", &png, 1, 1);
let selected = [first, duplicate, second];
let unique = unique_photos(&selected);
assert_eq!(
unique.iter().map(|photo| photo.id).collect::<Vec<_>>(),
[11, 12]
);
}
#[test]
fn interrupted_preparation_removes_only_recognizable_stage() {
let temp = tempfile::tempdir().unwrap();
let output = prepare_output(temp.path()).unwrap();
let destination = output.join("13");
let mut transaction = DestinationTransaction::acquire(&output, 13).unwrap();
transaction.mark_preparing().unwrap();
let stage = create_stage(&output, 13).unwrap();
drop(transaction);
let mut recovered = DestinationTransaction::acquire(&output, 13).unwrap();
recovered.recover(&output, &destination, 13).unwrap();
assert!(!stage.exists());
}
#[test]
fn interrupted_replacement_restores_recognized_backup() {
let temp = tempfile::tempdir().unwrap();
let output = prepare_output(temp.path()).unwrap();
let destination = create_archive(&output, 14, "image/png");
let backup = create_backup_container(&output, 14).unwrap();
let mut transaction = DestinationTransaction::acquire(&output, 14).unwrap();
transaction.mark_installing().unwrap();
fs::rename(&destination, backup.join("archive")).unwrap();
sync_directory(&output).unwrap();
drop(transaction);
let mut recovered = DestinationTransaction::acquire(&output, 14).unwrap();
recovered.recover(&output, &destination, 14).unwrap();
verify_archive(&destination).unwrap();
assert!(!backup.exists());
}
#[test]
fn corrupt_interrupted_backup_is_preserved_and_state_stays_interrupted() {
let temp = tempfile::tempdir().unwrap();
let output = prepare_output(temp.path()).unwrap();
let destination = create_archive(&output, 140, "image/png");
let backup = create_backup_container(&output, 140).unwrap();
let mut transaction = DestinationTransaction::acquire(&output, 140).unwrap();
transaction.mark_installing().unwrap();
fs::rename(&destination, backup.join("archive")).unwrap();
fs::write(backup.join("archive/photo.png"), b"tampered").unwrap();
drop(transaction);
let mut recovered = DestinationTransaction::acquire(&output, 140).unwrap();
assert!(recovered.interrupted);
assert!(recovered.recover(&output, &destination, 140).is_err());
assert!(recovered.interrupted);
assert!(backup.join("archive/photo.png").exists());
assert!(!destination.exists());
}
#[test]
fn interrupted_state_preserves_unrecognized_destination() {
let temp = tempfile::tempdir().unwrap();
let output = prepare_output(temp.path()).unwrap();
let destination = output.join("15");
let mut transaction = DestinationTransaction::acquire(&output, 15).unwrap();
transaction.mark_installing().unwrap();
fs::create_dir(&destination).unwrap();
fs::write(destination.join("foreign"), b"keep").unwrap();
drop(transaction);
let mut recovered = DestinationTransaction::acquire(&output, 15).unwrap();
assert!(recovered.recover(&output, &destination, 15).is_err());
assert_eq!(fs::read(destination.join("foreign")).unwrap(), b"keep");
}
#[test]
fn oversized_manifest_is_rejected_without_deserialization() {
let temp = tempfile::tempdir().unwrap();
let directory = temp.path().join("16");
fs::create_dir(&directory).unwrap();
let file = File::create(directory.join(MANIFEST_NAME)).unwrap();
file.set_len(MAX_MANIFEST_BYTES + 1).unwrap();
assert!(matches!(
verify_archive(directory),
Err(PhotoForgeError::ResponseLimitExceeded { .. })
));
}
#[test]
fn hard_retained_limit_rejects_options_and_manifest_claims() {
let forge =
PhotoForge::with_user_agent("photoforge-test/0.1 (test@example.invalid)").unwrap();
let options = ArchiveOptions {
output: PathBuf::from("unused"),
skip_existing: false,
max_download_bytes: MAX_RETAINED_IMAGE_BYTES + 1,
};
assert!(matches!(
archive(&forge, &[], &options, None),
Err(PhotoForgeError::InvalidOption(_))
));
let temp = tempfile::tempdir().unwrap();
let directory = create_archive(temp.path(), 160, "image/png");
let mut manifest = read_manifest(&directory).unwrap();
manifest.photo.file.size = MAX_RETAINED_IMAGE_BYTES + 1;
manifest.integrity.bytes = manifest.photo.file.size;
fs::remove_file(directory.join(MANIFEST_NAME)).unwrap();
write_manifest(&directory, &manifest).unwrap();
assert!(verify_archive(directory).is_err());
}
#[test]
fn skipped_progress_observes_updated_summary_and_full_record() {
let temp = tempfile::tempdir().unwrap();
let directory = create_archive(temp.path(), 161, "image/png");
let manifest = verify_archive(&directory).unwrap();
let archived = archived_photo(&directory, manifest).unwrap();
let mut summary = ArchiveSummary {
selected: 1,
selected_ids: vec![161],
..ArchiveSummary::default()
};
let mut snapshots = Vec::new();
let mut callback = |snapshot| snapshots.push(snapshot);
let mut progress: Option<&mut dyn FnMut(ArchiveProgress)> = Some(&mut callback);
record_skipped(&mut summary, archived, &mut progress);
assert_eq!(summary.skipped, 1);
assert_eq!(summary.skipped_photos.len(), 1);
let snapshot = snapshots.last().unwrap();
assert_eq!(snapshot.phase, ArchiveProgressPhase::Skipped);
assert_eq!(snapshot.completed_photos, 1);
assert_eq!(snapshot.skipped_photos, 1);
}
#[test]
fn manifest_json_shape_and_round_trip_are_stable() {
let temp = tempfile::tempdir().unwrap();
let directory = create_archive(temp.path(), 162, "image/jpeg");
let manifest = verify_archive(directory).unwrap();
let value = serde_json::to_value(&manifest).unwrap();
let object = value.as_object().unwrap();
let keys = object
.keys()
.map(String::as_str)
.collect::<std::collections::BTreeSet<_>>();
assert_eq!(
keys,
["complete", "format_version", "integrity", "owner", "photo"]
.into_iter()
.collect()
);
let integrity = object["integrity"].as_object().unwrap();
let integrity_keys = integrity
.keys()
.map(String::as_str)
.collect::<std::collections::BTreeSet<_>>();
assert_eq!(
integrity_keys,
["blake3", "bytes", "commons_sha1", "path"]
.into_iter()
.collect()
);
assert_eq!(
serde_json::from_value::<PhotoManifest>(value).unwrap(),
manifest
);
}
#[test]
fn foreign_blank_lock_is_refused_without_modification() {
let temp = tempfile::tempdir().unwrap();
let output = prepare_output(temp.path()).unwrap();
let blank = output.join(".photoforge-169.lock");
fs::write(&blank, b"").unwrap();
assert!(DestinationTransaction::acquire(&output, 169).is_err());
assert!(fs::read(blank).unwrap().is_empty());
let path = output.join(".photoforge-170.lock");
fs::write(&path, b"foreign lock bytes").unwrap();
assert!(DestinationTransaction::acquire(&output, 170).is_err());
assert_eq!(fs::read(path).unwrap(), b"foreign lock bytes");
}
#[test]
fn transaction_compaction_preserves_ownership_header_and_latest_state() {
let temp = tempfile::tempdir().unwrap();
let output = prepare_output(temp.path()).unwrap();
let path = output.join(".photoforge-173.lock");
let mut transaction = DestinationTransaction::acquire(&output, 173).unwrap();
transaction.file.seek(io::SeekFrom::End(0)).unwrap();
while transaction.file.metadata().unwrap().len() <= TRANSACTION_COMPACT_BYTES {
transaction.file.write_all(b"idle\n").unwrap();
}
transaction.file.sync_all().unwrap();
transaction.mark_preparing().unwrap();
let state = fs::read_to_string(path).unwrap();
assert!(state.starts_with(&format!("{LOCK_HEADER}\n")));
assert!(state.ends_with("preparing\n"));
assert!(state.len() as u64 <= TRANSACTION_COMPACT_BYTES);
}
#[cfg(unix)]
#[test]
fn replaced_or_hard_linked_lock_is_never_modified() {
let temp = tempfile::tempdir().unwrap();
let output = prepare_output(temp.path()).unwrap();
let path = output.join(".photoforge-171.lock");
let moved = output.join("owned-lock-moved");
let mut transaction = DestinationTransaction::acquire(&output, 171).unwrap();
fs::rename(&path, &moved).unwrap();
fs::write(&path, b"foreign replacement").unwrap();
assert!(transaction.mark_preparing().is_err());
assert_eq!(fs::read(&path).unwrap(), b"foreign replacement");
assert!(!fs::read_to_string(&moved).unwrap().contains("preparing"));
drop(transaction);
let transaction = DestinationTransaction::acquire(&output, 172).unwrap();
drop(transaction);
let path = output.join(".photoforge-172.lock");
let alias = output.join("hard-link-alias");
fs::hard_link(&path, &alias).unwrap();
let before = fs::read(&path).unwrap();
assert!(DestinationTransaction::acquire(&output, 172).is_err());
assert_eq!(fs::read(&path).unwrap(), before);
assert_eq!(fs::read(alias).unwrap(), before);
}
#[test]
#[ignore = "requires bounded live Wikimedia Commons API and image access"]
fn live_discover_archive_verify_and_verified_skip() {
let forge = PhotoForge::with_user_agent(
"photoforge-live-test/0.1 (https://github.com/Tknott95/PhotoForge; automated bounded test)",
)
.unwrap()
.with_discovery_limit(5);
let discovered = forge
.discover(&crate::PhotoTarget::File(
"Bee in Loto in Botanical Garden of São Paulo.jpg".to_string(),
))
.unwrap();
let selected = discovered
.into_iter()
.find(|photo| photo.file.size <= 8 * 1024 * 1024)
.expect("known Commons file should remain a small allowlisted JPEG");
let temp = tempfile::tempdir().unwrap();
let options = ArchiveOptions {
output: temp.path().join("photos"),
skip_existing: false,
max_download_bytes: 8 * 1024 * 1024,
};
let first = forge
.archive(std::slice::from_ref(&selected), &options)
.unwrap();
assert!(first.is_success(), "failures: {:?}", first.failures);
assert_eq!(first.archived, 1);
verify_archive(&first.photos[0].output).unwrap();
let skipped = forge
.archive(
&[selected],
&ArchiveOptions {
skip_existing: true,
..options
},
)
.unwrap();
assert!(skipped.is_success(), "failures: {:?}", skipped.failures);
assert_eq!(skipped.skipped, 1);
assert_eq!(skipped.skipped_photos.len(), 1);
verify_archive(&skipped.skipped_photos[0].output).unwrap();
}
}