#![forbid(unsafe_code)]
#![warn(missing_docs)]
use std::fmt;
use std::io;
use std::path::{Path, PathBuf};
use std::time::Duration;
use reqwest::blocking::Client;
use reqwest::redirect::Policy;
use serde::{Deserialize, Serialize};
mod archive;
mod commons;
mod image;
pub use archive::{inspect_archive, verify_archive};
const DEFAULT_DISCOVERY_LIMIT: usize = 20;
const MAX_DISCOVERY_LIMIT: usize = 50;
const DEFAULT_MAX_DOWNLOAD_BYTES: u64 = 256 * 1024 * 1024;
pub const MAX_RETAINED_IMAGE_BYTES: u64 = 512 * 1024 * 1024;
pub const COMMONS_REUSE_URL: &str =
"https://commons.wikimedia.org/wiki/Commons:Reusing_content_outside_Wikimedia";
pub const AGENT_GUIDE: &str = include_str!("../AGENT_GUIDE.md");
pub type Result<T> = std::result::Result<T, PhotoForgeError>;
#[derive(Debug, thiserror::Error)]
#[non_exhaustive]
pub enum PhotoForgeError {
#[error("invalid photo target: {0}")]
InvalidTarget(String),
#[error("invalid PhotoForge option: {0}")]
InvalidOption(String),
#[error("{provider} API returned HTTP {status}: {body}")]
ProviderApi {
provider: &'static str,
status: u16,
body: String,
},
#[error("{provider} API error {code}: {info}")]
ProviderApiError {
provider: &'static str,
code: String,
info: String,
},
#[error("{provider} file '{photo}' was not found or is not eligible")]
NotFound {
provider: &'static str,
photo: String,
},
#[error("refusing untrusted photo URL '{0}'")]
UntrustedUrl(String),
#[error("refusing unsafe photo path '{0}'")]
UnsafePath(String),
#[error("photo download is {size} bytes, exceeding the {limit} byte limit")]
DownloadLimitExceeded {
size: u64,
limit: u64,
},
#[error("{label} is {size} bytes, exceeding the {limit} byte limit")]
ResponseLimitExceeded {
label: String,
size: u64,
limit: u64,
},
#[error("photo verification failed for '{path}': {message}")]
VerificationFailed {
path: String,
message: String,
},
#[error("refusing to replace unrecognized output '{}': {reason}", path.display())]
UnrecognizedExistingOutput {
path: PathBuf,
reason: String,
},
#[error("archive transaction failed for '{}': {message}", path.display())]
ArchiveTransaction {
path: PathBuf,
message: String,
},
#[error("output path '{}' is not a directory", .0.display())]
OutputNotDirectory(PathBuf),
#[error("I/O error: {0}")]
Io(#[from] io::Error),
#[error("request error: {0}")]
Request(String),
#[error("JSON error: {0}")]
Json(#[from] serde_json::Error),
}
impl From<reqwest::Error> for PhotoForgeError {
fn from(error: reqwest::Error) -> Self {
Self::Request(commons::error_chain(&error))
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum PhotoProvider {
WikimediaCommons,
}
impl PhotoProvider {
pub const ALL: [Self; 1] = [Self::WikimediaCommons];
#[must_use]
pub const fn slug(self) -> &'static str {
match self {
Self::WikimediaCommons => "wikimedia-commons",
}
}
#[must_use]
pub const fn label(self) -> &'static str {
match self {
Self::WikimediaCommons => "Wikimedia Commons",
}
}
}
impl fmt::Display for PhotoProvider {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str(self.slug())
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case", tag = "kind", content = "value")]
pub enum PhotoTarget {
Search(String),
Category(String),
File(String),
Top,
Latest,
}
impl PhotoTarget {
#[must_use]
pub const fn is_collection(&self) -> bool {
!matches!(self, Self::File(_))
}
}
pub fn parse_photo_target(input: &str) -> Result<PhotoTarget> {
commons::parse_target(input)
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct PhotoRights {
pub license: String,
pub license_url: String,
pub raw_license_label: String,
pub creator: String,
pub custom_attribution: String,
pub attribution: String,
pub source_credit: String,
pub permission: String,
pub usage_terms: String,
pub raw_attribution_required: String,
pub attribution_required: bool,
pub restrictions: String,
pub copyrighted: String,
pub non_free: String,
pub deletion_reason: String,
pub metadata_retrieved_at: u64,
pub provider_guidance_url: String,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct PhotoFile {
pub url: String,
pub size: u64,
pub sha1: String,
pub mime: String,
pub media_type: String,
pub width: u32,
pub height: u32,
pub revision_id: Option<u64>,
pub revision_timestamp: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct PhotoSummary {
pub provider: PhotoProvider,
pub id: u64,
pub title: String,
pub object_name: String,
pub description: String,
pub categories: Vec<String>,
pub assessments: Vec<String>,
pub usage_count: u64,
pub usage_count_complete: bool,
pub source_url: String,
pub rights: PhotoRights,
pub file: PhotoFile,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct DiscoveryResult {
pub photos: Vec<PhotoSummary>,
pub candidate_count: usize,
pub eligible_candidate_count: usize,
pub candidate_budget: usize,
pub candidate_cap_reached: bool,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ArchiveOptions {
pub output: PathBuf,
pub skip_existing: bool,
pub max_download_bytes: u64,
}
impl Default for ArchiveOptions {
fn default() -> Self {
Self {
output: PathBuf::from("photos"),
skip_existing: false,
max_download_bytes: DEFAULT_MAX_DOWNLOAD_BYTES,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct PhotoIntegrity {
pub path: String,
pub bytes: u64,
pub blake3: String,
pub commons_sha1: String,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct PhotoManifest {
pub format_version: String,
pub owner: String,
pub photo: PhotoSummary,
pub integrity: PhotoIntegrity,
pub complete: bool,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ArchivedPhoto {
pub photo_id: u64,
pub output: PathBuf,
pub bytes_written: u64,
pub manifest: PhotoManifest,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ArchiveFailure {
pub photo_id: u64,
pub output: Option<PathBuf>,
pub message: String,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct ArchiveSummary {
pub selected: usize,
pub selected_ids: Vec<u64>,
pub archived: usize,
pub skipped: usize,
pub failed: usize,
pub bytes_written: u64,
pub photos: Vec<ArchivedPhoto>,
pub skipped_photos: Vec<ArchivedPhoto>,
pub failures: Vec<ArchiveFailure>,
}
impl ArchiveSummary {
#[must_use]
pub fn is_success(&self) -> bool {
self.failures.is_empty()
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ArchiveProgressPhase {
Starting,
Downloading,
Verifying,
Installing,
Skipped,
Failed,
Completed,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ArchiveProgress {
pub phase: ArchiveProgressPhase,
pub total_photos: usize,
pub completed_photos: usize,
pub archived_photos: usize,
pub skipped_photos: usize,
pub failed_photos: usize,
pub bytes_written: u64,
pub active_photo_id: Option<u64>,
pub active_title: Option<String>,
pub active_file: Option<String>,
pub active_bytes_written: u64,
pub active_total_bytes: Option<u64>,
}
#[derive(Clone)]
pub struct PhotoForge {
pub(crate) provider: PhotoProvider,
pub(crate) client: Client,
pub(crate) discovery_limit: usize,
}
impl PhotoForge {
pub fn with_user_agent(user_agent: &str) -> Result<Self> {
let user_agent = user_agent.trim();
if user_agent.is_empty() {
return Err(PhotoForgeError::InvalidOption(
"User-Agent must not be blank".to_string(),
));
}
if user_agent.len() > 512 || user_agent.chars().any(char::is_control) {
return Err(PhotoForgeError::InvalidOption(
"User-Agent must contain at most 512 bytes and no control characters".to_string(),
));
}
let client = Client::builder()
.user_agent(user_agent)
.redirect(Policy::none())
.no_proxy()
.connect_timeout(Duration::from_secs(20))
.timeout(Duration::from_secs(90))
.build()?;
Ok(Self {
provider: PhotoProvider::WikimediaCommons,
client,
discovery_limit: DEFAULT_DISCOVERY_LIMIT,
})
}
#[must_use]
pub fn with_discovery_limit(mut self, limit: usize) -> Self {
self.discovery_limit = limit.clamp(1, MAX_DISCOVERY_LIMIT);
self
}
#[must_use]
pub const fn provider(&self) -> PhotoProvider {
self.provider
}
pub fn discover(&self, target: &PhotoTarget) -> Result<Vec<PhotoSummary>> {
Ok(self.discover_detailed(target)?.photos)
}
pub fn discover_detailed(&self, target: &PhotoTarget) -> Result<DiscoveryResult> {
commons::discover(self, target)
}
pub fn archive(
&self,
photos: &[PhotoSummary],
options: &ArchiveOptions,
) -> Result<ArchiveSummary> {
archive::archive(self, photos, options, None)
}
pub fn archive_with_progress<F>(
&self,
photos: &[PhotoSummary],
options: &ArchiveOptions,
mut progress: F,
) -> Result<ArchiveSummary>
where
F: FnMut(ArchiveProgress),
{
archive::archive(self, photos, options, Some(&mut progress))
}
pub fn inspect_archive(&self, directory: impl AsRef<Path>) -> Result<PhotoManifest> {
inspect_archive(directory)
}
pub fn verify_archive(&self, directory: impl AsRef<Path>) -> Result<PhotoManifest> {
verify_archive(directory)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn client_requires_a_bounded_clean_user_agent_and_clamps_limit() {
assert!(PhotoForge::with_user_agent("").is_err());
assert!(PhotoForge::with_user_agent("bad\nagent").is_err());
let forge = PhotoForge::with_user_agent("photoforge-test/0.1 (test@example.invalid)")
.unwrap()
.with_discovery_limit(0);
assert_eq!(forge.discovery_limit, 1);
assert_eq!(forge.provider(), PhotoProvider::WikimediaCommons);
assert_eq!(PhotoProvider::ALL, [PhotoProvider::WikimediaCommons]);
}
#[test]
fn archive_defaults_are_bounded_and_photo_native() {
let options = ArchiveOptions::default();
assert_eq!(options.output, PathBuf::from("photos"));
assert!(!options.skip_existing);
assert_eq!(options.max_download_bytes, 256 * 1024 * 1024);
}
#[test]
fn embedded_guide_is_exact() {
assert_eq!(AGENT_GUIDE.as_bytes(), include_bytes!("../AGENT_GUIDE.md"));
}
#[test]
fn public_provider_target_and_progress_json_names_are_stable() {
assert_eq!(
serde_json::to_string(&PhotoProvider::WikimediaCommons).unwrap(),
"\"wikimedia-commons\""
);
assert_eq!(
serde_json::to_value(PhotoTarget::Search("blue flower".to_string())).unwrap(),
serde_json::json!({"kind":"search","value":"blue flower"})
);
assert_eq!(
serde_json::to_value(PhotoTarget::Top).unwrap(),
serde_json::json!({"kind":"top"})
);
assert_eq!(
serde_json::from_value::<PhotoTarget>(serde_json::json!({
"kind":"category",
"value":"Flowers"
}))
.unwrap(),
PhotoTarget::Category("Flowers".to_string())
);
for (phase, expected) in [
(ArchiveProgressPhase::Starting, "starting"),
(ArchiveProgressPhase::Downloading, "downloading"),
(ArchiveProgressPhase::Verifying, "verifying"),
(ArchiveProgressPhase::Installing, "installing"),
(ArchiveProgressPhase::Skipped, "skipped"),
(ArchiveProgressPhase::Failed, "failed"),
(ArchiveProgressPhase::Completed, "completed"),
] {
assert_eq!(serde_json::to_value(phase).unwrap(), expected);
assert_eq!(
serde_json::from_value::<ArchiveProgressPhase>(serde_json::json!(expected))
.unwrap(),
phase
);
}
}
}