use std::collections::{BTreeMap, BTreeSet, HashSet};
use std::error::Error as StdError;
use std::io::Read;
use std::thread;
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use reqwest::Url;
use reqwest::blocking::Response;
use serde::Deserialize;
use crate::{
COMMONS_REUSE_URL, DiscoveryResult, MAX_RETAINED_IMAGE_BYTES, PhotoFile, PhotoForge,
PhotoForgeError, PhotoProvider, PhotoRights, PhotoSummary, PhotoTarget, Result,
};
const COMMONS_API: &str = "https://commons.wikimedia.org/w/api.php";
const COMMONS_HOST: &str = "commons.wikimedia.org";
const COMMONS_DOWNLOAD_HOST: &str = "upload.wikimedia.org";
const MAX_API_RESPONSE_BYTES: u64 = 16 * 1024 * 1024;
const MAX_ERROR_RESPONSE_BYTES: u64 = 8 * 1024;
const MAX_API_ATTEMPTS: usize = 3;
const MAX_API_RETRY_WAIT: Duration = Duration::from_secs(5);
const MAX_API_REQUESTS: usize = 10;
const MAX_TARGET_BYTES: usize = 1024;
const MAX_TEXT_INPUT_BYTES: usize = 64 * 1024;
const MAX_TEXT_BYTES: usize = 16 * 1024;
const MAX_DIMENSION: u64 = 32_768;
const MAX_PIXELS: u64 = 100_000_000;
const METADATA_FILTER: &str = "ObjectName|ImageDescription|Artist|Credit|Attribution|AttributionRequired|License|LicenseShortName|LicenseUrl|UsageTerms|Copyrighted|NonFree|Restrictions|Permission|DeletionReason|Assessments";
pub(crate) fn parse_target(input: &str) -> Result<PhotoTarget> {
let input = input.trim();
validate_target_text(input, "target")?;
match input.to_ascii_lowercase().as_str() {
"top" | "popular" | "trending" => return Ok(PhotoTarget::Top),
"latest" | "newest" | "new" => return Ok(PhotoTarget::Latest),
_ => {}
}
for (prefix, constructor) in [
("search:", PhotoTarget::Search as fn(String) -> PhotoTarget),
("category:", PhotoTarget::Category),
("file:", PhotoTarget::File),
] {
if input
.get(..prefix.len())
.is_some_and(|value| value.eq_ignore_ascii_case(prefix))
{
let value = input[prefix.len()..].trim();
validate_target_text(value, prefix.trim_end_matches(':'))?;
validate_typed_value(prefix, value)?;
return Ok(constructor(value.to_string()));
}
}
if looks_like_url(input) {
return parse_commons_file_url(input);
}
Ok(PhotoTarget::Search(input.to_string()))
}
fn looks_like_url(value: &str) -> bool {
let lower = value.to_ascii_lowercase();
lower.contains("://")
|| lower.starts_with("http:")
|| lower.starts_with("https:")
|| Url::parse(value).is_ok_and(|url| url.has_host())
}
fn parse_commons_file_url(input: &str) -> Result<PhotoTarget> {
let url = Url::parse(input).map_err(|_| PhotoForgeError::InvalidTarget(input.to_string()))?;
if !secure_https_url(&url)
|| url.host_str() != Some(COMMONS_HOST)
|| url.query().is_some()
|| url.fragment().is_some()
{
return Err(PhotoForgeError::InvalidTarget(input.to_string()));
}
let segments = url
.path_segments()
.map(|segments| segments.collect::<Vec<_>>())
.unwrap_or_default();
if segments.len() != 2 || segments[0] != "wiki" {
return Err(PhotoForgeError::InvalidTarget(input.to_string()));
}
let title = urlencoding::decode(segments[1])
.map_err(|_| PhotoForgeError::InvalidTarget(input.to_string()))?
.into_owned();
if !title.starts_with("File:") || title.len() == 5 {
return Err(PhotoForgeError::InvalidTarget(input.to_string()));
}
validate_target_text(&title, "file")?;
Ok(PhotoTarget::File(title))
}
fn validate_target_text(value: &str, label: &str) -> Result<()> {
if value.is_empty() {
return Err(PhotoForgeError::InvalidTarget(format!(
"{label} must not be empty"
)));
}
if value.len() > MAX_TARGET_BYTES {
return Err(PhotoForgeError::InvalidTarget(format!(
"{label} exceeds the {MAX_TARGET_BYTES}-byte limit"
)));
}
if value.chars().any(char::is_control) {
return Err(PhotoForgeError::InvalidTarget(format!(
"{label} contains control characters"
)));
}
Ok(())
}
fn validate_target(target: &PhotoTarget) -> Result<()> {
match target {
PhotoTarget::Search(value) => {
validate_target_text(value.trim(), "search")?;
if looks_like_url(value.trim()) {
return Err(PhotoForgeError::InvalidTarget(
"raw URLs are not search targets".to_string(),
));
}
Ok(())
}
PhotoTarget::Category(value) => {
validate_target_text(value.trim(), "category")?;
validate_typed_value("category:", value.trim())
}
PhotoTarget::File(value) => {
validate_target_text(value.trim(), "file")?;
validate_typed_value("file:", value.trim())
}
PhotoTarget::Top | PhotoTarget::Latest => Ok(()),
}
}
fn validate_typed_value(prefix: &str, value: &str) -> Result<()> {
if prefix == "search:" {
if looks_like_url(value) {
return Err(PhotoForgeError::InvalidTarget(
"raw URLs are not search targets".to_string(),
));
}
return Ok(());
}
let namespace = match prefix {
"file:" => "File:",
"category:" => "Category:",
_ => return Ok(()),
};
let unprefixed = value
.get(..namespace.len())
.filter(|candidate| candidate.eq_ignore_ascii_case(namespace))
.map_or(value, |_| &value[namespace.len()..])
.trim();
if unprefixed.is_empty() || looks_like_url(value) {
return Err(PhotoForgeError::InvalidTarget(format!(
"{} target has no valid title",
prefix.trim_end_matches(':')
)));
}
Ok(())
}
pub(crate) fn discover(forge: &PhotoForge, target: &PhotoTarget) -> Result<DiscoveryResult> {
validate_target(target)?;
if let PhotoTarget::File(title) = target {
let photo = forge.fetch_exact(title)?;
return Ok(DiscoveryResult {
photos: vec![photo],
candidate_count: 1,
eligible_candidate_count: 1,
candidate_budget: 1,
candidate_cap_reached: false,
});
}
let candidate_budget = collection_candidate_budget(forge.discovery_limit);
let mut operation = ApiOperation::new();
let candidate_url = candidate_url(target, candidate_budget)?;
let response: CommonsResponse = forge.get_json(&candidate_url, &mut operation)?;
let provider_continues = has_candidate_continuation(target, &response.continuation);
let mut candidates = response.query.map_or_else(Vec::new, |query| query.pages);
let (ids, provider_over_budget) = bounded_candidate_ids(&mut candidates, candidate_budget);
let candidate_count = ids.len();
let retrieved_at = unix_timestamp()?;
let metadata = if ids.is_empty() {
CommonsPages {
pages: Vec::new(),
usage_count_complete: true,
}
} else {
let url = metadata_url_for_ids(&ids);
forge.get_metadata_pages(&url, &mut operation)?
};
let mut by_id = metadata
.pages
.into_iter()
.filter_map(|page| page.pageid.map(|id| (id, page)))
.collect::<BTreeMap<_, _>>();
let mut photos = Vec::new();
for id in ids {
if let Some(page) = by_id.remove(&id) {
if let Some(photo) = commons_summary(page, metadata.usage_count_complete, retrieved_at)?
{
photos.push(photo);
}
}
}
let eligible_candidate_count = photos.len();
rank_photos(&mut photos, target);
photos.truncate(forge.discovery_limit);
Ok(DiscoveryResult {
photos,
candidate_count,
eligible_candidate_count,
candidate_budget,
candidate_cap_reached: provider_continues || provider_over_budget,
})
}
fn collection_candidate_budget(limit: usize) -> usize {
limit.saturating_mul(3).min(50)
}
impl PhotoForge {
pub(crate) fn fetch_exact(&self, title: &str) -> Result<PhotoSummary> {
validate_target_text(title.trim(), "file")?;
let title = normalized_file_title(title);
let url = metadata_url_for_title(&title);
let mut operation = ApiOperation::new();
let response = self.get_metadata_pages(&url, &mut operation)?;
let retrieved_at = unix_timestamp()?;
let photo = response
.pages
.into_iter()
.find_map(|page| {
commons_summary(page, response.usage_count_complete, retrieved_at).transpose()
})
.transpose()?;
photo.ok_or_else(|| PhotoForgeError::NotFound {
provider: self.provider.slug(),
photo: title,
})
}
fn get_metadata_pages(
&self,
base_url: &Url,
operation: &mut ApiOperation,
) -> Result<CommonsPages> {
let mut pages = Vec::new();
let mut continuation = BTreeMap::new();
let mut usage_count_complete = true;
let mut prior_continuations = BTreeSet::new();
let mut made_request = false;
loop {
if !operation.has_send_capacity() {
if !made_request {
return Err(PhotoForgeError::ProviderApiError {
provider: self.provider.slug(),
code: "request-budget-exhausted".to_string(),
info: "Commons request retries exhausted the operation budget before metadata retrieval".to_string(),
});
}
usage_count_complete = !continuation.contains_key("gucontinue");
return Ok(CommonsPages {
pages,
usage_count_complete,
});
}
let mut url = base_url.clone();
append_continuation(&mut url, &continuation)?;
let response: CommonsResponse = self.get_json(&url, operation)?;
made_request = true;
if let Some(query) = response.query {
merge_pages(&mut pages, query.pages);
}
let metadata_continuation = filtered_metadata_continuation(response.continuation)?;
if metadata_continuation.is_empty() {
return Ok(CommonsPages {
pages,
usage_count_complete,
});
}
let fingerprint = serde_json::to_string(&metadata_continuation)?;
if !prior_continuations.insert(fingerprint) {
return Err(PhotoForgeError::ProviderApiError {
provider: self.provider.slug(),
code: "repeated-continuation".to_string(),
info: "Commons repeated a metadata continuation token".to_string(),
});
}
continuation = metadata_continuation;
}
}
fn get_json<T>(&self, url: &Url, operation: &mut ApiOperation) -> Result<T>
where
T: for<'de> Deserialize<'de>,
{
trusted_api_url(url)?;
for attempt in 0..MAX_API_ATTEMPTS {
operation.reserve_send()?;
let response = self.client.get(url.clone()).send()?;
if response.url() != url {
return Err(PhotoForgeError::UntrustedUrl(response.url().to_string()));
}
let retry_after = numeric_retry_after(&response);
if !response.status().is_success() {
let status = response.status().as_u16();
if retryable_http_status(status)
&& attempt + 1 < MAX_API_ATTEMPTS
&& operation.wait_before_retry(retry_delay(attempt, retry_after, None))
{
continue;
}
let body = bounded_error_body(response);
if let Ok(Some(error)) = api_envelope_error(body.as_bytes()) {
return Err(PhotoForgeError::ProviderApiError {
provider: self.provider.slug(),
code: error.code,
info: error.info,
});
}
return Err(PhotoForgeError::ProviderApi {
provider: self.provider.slug(),
status,
body,
});
}
let bytes =
read_response_limited(response, MAX_API_RESPONSE_BYTES, "Commons API response")?;
if let Some(error) = api_envelope_error(&bytes)? {
if error.code.eq_ignore_ascii_case("maxlag")
&& attempt + 1 < MAX_API_ATTEMPTS
&& operation.wait_before_retry(retry_delay(
attempt,
retry_after,
error.lag.map(duration_from_lag),
))
{
continue;
}
return Err(PhotoForgeError::ProviderApiError {
provider: self.provider.slug(),
code: error.code,
info: error.info,
});
}
return Ok(serde_json::from_slice(&bytes)?);
}
unreachable!("bounded Commons API attempt loop always returns")
}
}
#[derive(Debug, Deserialize)]
struct CommonsResponse {
query: Option<CommonsQuery>,
#[serde(default, rename = "continue")]
continuation: BTreeMap<String, serde_json::Value>,
}
#[derive(Debug, Deserialize)]
struct CommonsQuery {
#[serde(default)]
pages: Vec<CommonsPage>,
}
#[derive(Debug, Deserialize)]
struct CommonsPage {
pageid: Option<u64>,
#[serde(default)]
title: String,
#[serde(default)]
index: Option<u64>,
#[serde(default)]
missing: bool,
#[serde(default)]
lastrevid: Option<u64>,
#[serde(default)]
imageinfo: Vec<CommonsImageInfo>,
#[serde(default)]
categories: Vec<CommonsCategory>,
#[serde(default)]
globalusage: Vec<CommonsUsage>,
}
#[derive(Debug, Deserialize)]
struct CommonsImageInfo {
#[serde(default)]
size: u64,
#[serde(default)]
width: u64,
#[serde(default)]
height: u64,
#[serde(default)]
url: String,
#[serde(default)]
descriptionurl: String,
#[serde(default)]
sha1: String,
#[serde(default)]
mime: String,
#[serde(default)]
mediatype: String,
#[serde(default)]
timestamp: Option<String>,
#[serde(default)]
extmetadata: BTreeMap<String, CommonsMetadataValue>,
}
#[derive(Debug, Deserialize)]
struct CommonsMetadataValue {
#[serde(default)]
value: String,
}
#[derive(Debug, Deserialize)]
struct CommonsCategory {
title: String,
}
#[derive(Debug, Deserialize)]
struct CommonsUsage {
#[serde(default)]
title: String,
#[serde(default)]
wiki: String,
}
struct CommonsPages {
pages: Vec<CommonsPage>,
usage_count_complete: bool,
}
struct ApiEnvelopeError {
code: String,
info: String,
lag: Option<f64>,
}
struct ApiOperation {
sends: usize,
started: Instant,
retry_sleep: Duration,
}
impl ApiOperation {
fn new() -> Self {
Self {
sends: 0,
started: Instant::now(),
retry_sleep: Duration::ZERO,
}
}
fn has_send_capacity(&self) -> bool {
self.sends < MAX_API_REQUESTS
}
fn reserve_send(&mut self) -> Result<()> {
if !self.has_send_capacity() {
return Err(PhotoForgeError::ProviderApiError {
provider: PhotoProvider::WikimediaCommons.slug(),
code: "request-budget-exhausted".to_string(),
info: format!(
"Commons operation exceeded its {MAX_API_REQUESTS} actual HTTP-send limit"
),
});
}
self.sends += 1;
Ok(())
}
fn wait_before_retry(&mut self, delay: Duration) -> bool {
if !self.has_send_capacity() {
return false;
}
let elapsed_remaining = MAX_API_RETRY_WAIT.saturating_sub(self.started.elapsed());
let sleep_remaining = MAX_API_RETRY_WAIT.saturating_sub(self.retry_sleep);
let allowed = elapsed_remaining.min(sleep_remaining);
if allowed.is_zero() || delay > allowed {
return false;
}
thread::sleep(delay);
self.retry_sleep = self.retry_sleep.saturating_add(delay);
true
}
}
struct AllowedLicense {
id: &'static str,
url: String,
attribution_required: bool,
}
fn commons_api_url() -> Url {
Url::parse(COMMONS_API).expect("static Commons API URL is valid")
}
fn append_api_basics(url: &mut Url) {
url.query_pairs_mut()
.append_pair("action", "query")
.append_pair("format", "json")
.append_pair("formatversion", "2")
.append_pair("maxlag", "5");
}
fn append_metadata_query(url: &mut Url) {
url.query_pairs_mut()
.append_pair("prop", "info|imageinfo|categories|globalusage")
.append_pair(
"iiprop",
"url|size|sha1|mime|mediatype|extmetadata|timestamp",
)
.append_pair("iiextmetadatalanguage", "en")
.append_pair("iiextmetadatafilter", METADATA_FILTER)
.append_pair("iilimit", "1")
.append_pair("cllimit", "max")
.append_pair("gulimit", "max");
}
fn candidate_url(target: &PhotoTarget, budget: usize) -> Result<Url> {
let mut url = commons_api_url();
append_api_basics(&mut url);
match target {
PhotoTarget::Search(value) => {
let query = format!("{} filetype:bitmap", literal_search_query(value));
url.query_pairs_mut()
.append_pair("generator", "search")
.append_pair("gsrsearch", &query)
.append_pair("gsrnamespace", "6")
.append_pair("gsrlimit", &budget.to_string())
.append_pair("gsrsort", "relevance");
}
PhotoTarget::Category(value) => {
let category = normalized_category_title(value);
url.query_pairs_mut()
.append_pair("generator", "categorymembers")
.append_pair("gcmtitle", &category)
.append_pair("gcmnamespace", "6")
.append_pair("gcmtype", "file")
.append_pair("gcmlimit", &budget.to_string());
}
PhotoTarget::Top => {
url.query_pairs_mut()
.append_pair("generator", "categorymembers")
.append_pair(
"gcmtitle",
"Category:Featured pictures on Wikimedia Commons",
)
.append_pair("gcmnamespace", "6")
.append_pair("gcmtype", "file")
.append_pair("gcmlimit", &budget.to_string());
}
PhotoTarget::Latest => {
url.query_pairs_mut()
.append_pair("generator", "allimages")
.append_pair("gaisort", "timestamp")
.append_pair("gaidir", "descending")
.append_pair("gailimit", &budget.to_string());
}
PhotoTarget::File(_) => {
return Err(PhotoForgeError::InvalidTarget(
"exact files do not use a candidate query".to_string(),
));
}
}
Ok(url)
}
fn bounded_candidate_ids(pages: &mut Vec<CommonsPage>, budget: usize) -> (Vec<u64>, bool) {
pages.sort_by_key(|page| page.index.unwrap_or(u64::MAX));
let provider_over_budget = pages.len() > budget;
pages.truncate(budget);
let mut seen = HashSet::new();
let ids = pages
.iter()
.filter_map(|page| page.pageid)
.filter(|id| *id > 0 && seen.insert(*id))
.collect::<Vec<_>>();
debug_assert!(ids.len() <= budget);
(ids, provider_over_budget)
}
fn metadata_url_for_ids(ids: &[u64]) -> Url {
let mut url = commons_api_url();
append_api_basics(&mut url);
append_metadata_query(&mut url);
let ids = ids.iter().map(u64::to_string).collect::<Vec<_>>().join("|");
url.query_pairs_mut().append_pair("pageids", &ids);
url
}
fn metadata_url_for_title(title: &str) -> Url {
let mut url = commons_api_url();
append_api_basics(&mut url);
append_metadata_query(&mut url);
url.query_pairs_mut()
.append_pair("titles", title)
.append_pair("redirects", "1");
url
}
fn has_candidate_continuation(
target: &PhotoTarget,
continuation: &BTreeMap<String, serde_json::Value>,
) -> bool {
let key = match target {
PhotoTarget::Search(_) => "gsroffset",
PhotoTarget::Category(_) | PhotoTarget::Top => "gcmcontinue",
PhotoTarget::Latest => "gaicontinue",
PhotoTarget::File(_) => return false,
};
continuation.contains_key(key)
}
fn filtered_metadata_continuation(
continuation: BTreeMap<String, serde_json::Value>,
) -> Result<BTreeMap<String, serde_json::Value>> {
let mut filtered = BTreeMap::new();
let has_metadata = continuation
.keys()
.any(|key| matches!(key.as_str(), "clcontinue" | "gucontinue"));
if !has_metadata {
return Ok(filtered);
}
for (key, value) in continuation {
if matches!(key.as_str(), "continue" | "clcontinue" | "gucontinue") {
validate_continuation_value(&key, &value)?;
filtered.insert(key, value);
}
}
Ok(filtered)
}
fn validate_continuation_value(key: &str, value: &serde_json::Value) -> Result<()> {
let valid = value
.as_str()
.is_some_and(|value| value.len() <= 2048 && !value.chars().any(char::is_control))
|| value.is_number()
|| value.is_boolean();
if !valid {
return Err(PhotoForgeError::ProviderApiError {
provider: PhotoProvider::WikimediaCommons.slug(),
code: "invalid-continuation".to_string(),
info: format!("Commons returned an invalid continuation value for '{key}'"),
});
}
Ok(())
}
fn append_continuation(
url: &mut Url,
continuation: &BTreeMap<String, serde_json::Value>,
) -> Result<()> {
for (key, value) in continuation {
validate_continuation_value(key, value)?;
let rendered;
let value = if let Some(value) = value.as_str() {
value
} else {
rendered = value.to_string();
&rendered
};
url.query_pairs_mut().append_pair(key, value);
}
Ok(())
}
fn merge_pages(pages: &mut Vec<CommonsPage>, incoming: Vec<CommonsPage>) {
for mut page in incoming {
let existing = pages.iter_mut().find(|existing| {
existing.pageid == page.pageid
&& (existing.pageid.is_some() || existing.title == page.title)
});
let Some(existing) = existing else {
pages.push(page);
continue;
};
if existing.title.is_empty() {
existing.title = page.title;
}
if existing.index.is_none() {
existing.index = page.index;
}
if existing.lastrevid.is_none() {
existing.lastrevid = page.lastrevid;
}
if existing.imageinfo.is_empty() {
existing.imageinfo = page.imageinfo;
}
existing.missing |= page.missing;
for category in page.categories.drain(..) {
if !existing
.categories
.iter()
.any(|item| item.title == category.title)
{
existing.categories.push(category);
}
}
for usage in page.globalusage.drain(..) {
if !existing
.globalusage
.iter()
.any(|item| item.wiki == usage.wiki && item.title == usage.title)
{
existing.globalusage.push(usage);
}
}
}
}
fn commons_summary(
page: CommonsPage,
usage_count_complete: bool,
metadata_retrieved_at: u64,
) -> Result<Option<PhotoSummary>> {
if page.missing || page.pageid.is_none_or(|id| id == 0) || !page.title.starts_with("File:") {
return Ok(None);
}
let Some(info) = page.imageinfo.into_iter().next() else {
return Ok(None);
};
if info.mediatype != "BITMAP"
|| !supported_photo_mime(&info.mime)
|| !valid_dimensions(info.width, info.height)
|| info.size == 0
|| info.size > MAX_RETAINED_IMAGE_BYTES
{
return Ok(None);
}
trusted_download_url(&info.url)?;
trusted_source_url(&info.descriptionurl)?;
if info.sha1.len() != 40 || !info.sha1.bytes().all(|byte| byte.is_ascii_hexdigit()) {
return Err(PhotoForgeError::VerificationFailed {
path: page.title,
message: "Commons SHA-1 is not 40 hexadecimal characters".to_string(),
});
}
let short_license = metadata_value(&info.extmetadata, "LicenseShortName");
let full_license = metadata_value(&info.extmetadata, "License");
let license_url = metadata_value(&info.extmetadata, "LicenseUrl");
let Some(license) = allowed_license(&short_license, &full_license, &license_url) else {
return Ok(None);
};
let creator = metadata_text(&info.extmetadata, "Artist");
let custom_attribution = metadata_text(&info.extmetadata, "Attribution");
let source_credit = metadata_text(&info.extmetadata, "Credit");
let usage_terms = metadata_text(&info.extmetadata, "UsageTerms");
let raw_attribution_required = metadata_text(&info.extmetadata, "AttributionRequired");
let permission = metadata_text(&info.extmetadata, "Permission");
let restrictions = metadata_text(&info.extmetadata, "Restrictions");
let deletion_reason = metadata_text(&info.extmetadata, "DeletionReason");
let copyrighted = metadata_text(&info.extmetadata, "Copyrighted");
let non_free = metadata_text(&info.extmetadata, "NonFree");
if !permission.is_empty()
|| !restrictions.is_empty()
|| !deletion_reason.is_empty()
|| explicit_true_or_unknown(&non_free)
|| (license.attribution_required && explicit_false(©righted))
|| !usage_terms_match_license(license.id, &usage_terms)
|| !attribution_requirement_matches(license.id, &raw_attribution_required)
{
return Ok(None);
}
if license.attribution_required
&& !usable_attribution_text(&creator)
&& !usable_attribution_text(&custom_attribution)
{
return Ok(None);
}
let attribution = selected_attribution(
&creator,
&custom_attribution,
&source_credit,
&info.descriptionurl,
license.attribution_required,
);
if attribution.is_empty() {
return Ok(None);
}
let categories = page
.categories
.into_iter()
.map(|category| {
bounded_text(
category
.title
.strip_prefix("Category:")
.unwrap_or(&category.title),
)
})
.filter(|category| !category.is_empty())
.collect::<Vec<_>>();
let assessments = assessment_labels(&metadata_text(&info.extmetadata, "Assessments"));
let width = u32::try_from(info.width).map_err(|_| PhotoForgeError::VerificationFailed {
path: page.title.clone(),
message: "image width does not fit the public model".to_string(),
})?;
let height = u32::try_from(info.height).map_err(|_| PhotoForgeError::VerificationFailed {
path: page.title.clone(),
message: "image height does not fit the public model".to_string(),
})?;
let summary = PhotoSummary {
provider: PhotoProvider::WikimediaCommons,
id: page.pageid.expect("positive page ID checked"),
title: bounded_text(&page.title),
object_name: metadata_text(&info.extmetadata, "ObjectName"),
description: metadata_text(&info.extmetadata, "ImageDescription"),
categories,
assessments,
usage_count: page.globalusage.len() as u64,
usage_count_complete,
source_url: info.descriptionurl,
rights: PhotoRights {
license: license.id.to_string(),
license_url: license.url,
raw_license_label: bounded_text(&html_to_text(&short_license)),
creator,
custom_attribution,
attribution,
source_credit,
permission,
usage_terms,
raw_attribution_required,
attribution_required: license.attribution_required,
restrictions,
copyrighted,
non_free,
deletion_reason,
metadata_retrieved_at,
provider_guidance_url: COMMONS_REUSE_URL.to_string(),
},
file: PhotoFile {
url: info.url,
size: info.size,
sha1: info.sha1.to_ascii_lowercase(),
mime: info.mime,
media_type: info.mediatype,
width,
height,
revision_id: page.lastrevid,
revision_timestamp: info.timestamp.map(|value| bounded_text(&value)),
},
};
validate_summary(&summary)?;
Ok(Some(summary))
}
fn metadata_value(metadata: &BTreeMap<String, CommonsMetadataValue>, key: &str) -> String {
metadata
.get(key)
.map(|value| value.value.clone())
.unwrap_or_default()
}
fn metadata_text(metadata: &BTreeMap<String, CommonsMetadataValue>, key: &str) -> String {
html_to_text(&metadata_value(metadata, key))
}
fn html_to_text(value: &str) -> String {
let input = truncate_utf8(value, MAX_TEXT_INPUT_BYTES);
let text = html2text::from_read(input.as_bytes(), 10_000)
.split_whitespace()
.collect::<Vec<_>>()
.join(" ");
bounded_text(&text)
}
fn bounded_text(value: &str) -> String {
truncate_utf8(value.trim(), MAX_TEXT_BYTES).to_string()
}
fn truncate_utf8(value: &str, max: usize) -> &str {
if value.len() <= max {
return value;
}
let mut boundary = max;
while boundary > 0 && !value.is_char_boundary(boundary) {
boundary -= 1;
}
&value[..boundary]
}
fn allowed_license(short: &str, full: &str, url: &str) -> Option<AllowedLicense> {
let short_id = license_label_id(&html_to_text(short))?;
if !full.trim().is_empty() && license_label_id(&html_to_text(full)) != Some(short_id) {
return None;
}
let (path, attribution_required) = match short_id {
"CC0-1.0" => ("/publicdomain/zero/1.0/", false),
"CC-BY-4.0" => ("/licenses/by/4.0/", true),
"CC-BY-SA-4.0" => ("/licenses/by-sa/4.0/", true),
_ => return None,
};
Some(AllowedLicense {
id: short_id,
url: canonical_cc_url(url, path)?,
attribution_required,
})
}
fn license_label_id(value: &str) -> Option<&'static str> {
let value = value.trim().to_ascii_lowercase();
match value.as_str() {
"cc0" | "cc0 1.0" | "cc0-1.0" | "cc-zero" => Some("CC0-1.0"),
"cc by 4.0" | "cc-by-4.0" => Some("CC-BY-4.0"),
"cc by-sa 4.0" | "cc-by-sa-4.0" => Some("CC-BY-SA-4.0"),
_ => None,
}
}
fn canonical_cc_url(value: &str, expected_path: &str) -> Option<String> {
let plain = html_to_text(value);
if plain.is_empty() {
return None;
}
let url = Url::parse(&plain).ok()?;
if !matches!(url.scheme(), "http" | "https")
|| !url.username().is_empty()
|| url.password().is_some()
|| url.port().is_some_and(|port| {
(url.scheme() == "http" && port != 80) || (url.scheme() == "https" && port != 443)
})
|| url.host_str() != Some("creativecommons.org")
|| url.path().trim_end_matches('/') != expected_path.trim_end_matches('/')
|| url.query().is_some()
|| url.fragment().is_some()
{
return None;
}
Some(format!("https://creativecommons.org{expected_path}"))
}
fn allowed_license_id(id: &str, url: &str) -> bool {
let path = match id {
"CC0-1.0" => "/publicdomain/zero/1.0/",
"CC-BY-4.0" => "/licenses/by/4.0/",
"CC-BY-SA-4.0" => "/licenses/by-sa/4.0/",
_ => return false,
};
canonical_cc_url(url, path).is_some_and(|canonical| canonical == url)
}
fn usage_terms_match_license(id: &str, value: &str) -> bool {
if value.is_empty() {
return true;
}
let value = value
.split_whitespace()
.collect::<Vec<_>>()
.join(" ")
.to_ascii_lowercase();
match id {
"CC0-1.0" => matches!(
value.as_str(),
"cc0 1.0"
| "cc0 1.0 universal"
| "creative commons cc0 1.0 universal"
| "cc0 1.0 universal public domain dedication"
| "creative commons cc0 1.0 universal public domain dedication"
| "public domain dedication (cc0 1.0)"
),
"CC-BY-4.0" => matches!(
value.as_str(),
"cc by 4.0"
| "cc by 4.0 international"
| "creative commons attribution 4.0"
| "creative commons attribution 4.0 international"
),
"CC-BY-SA-4.0" => matches!(
value.as_str(),
"cc by-sa 4.0"
| "cc by-sa 4.0 international"
| "creative commons attribution-share alike 4.0"
| "creative commons attribution-share alike 4.0 international"
),
_ => false,
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum StrictMetadataBool {
Empty,
True,
False,
Invalid,
}
fn strict_metadata_bool(value: &str) -> StrictMetadataBool {
match value.trim().to_ascii_lowercase().as_str() {
"" => StrictMetadataBool::Empty,
"true" | "yes" | "1" => StrictMetadataBool::True,
"false" | "no" | "0" => StrictMetadataBool::False,
_ => StrictMetadataBool::Invalid,
}
}
fn attribution_requirement_matches(id: &str, value: &str) -> bool {
matches!(
(id, strict_metadata_bool(value)),
(_, StrictMetadataBool::Empty)
| ("CC0-1.0", StrictMetadataBool::False)
| ("CC-BY-4.0" | "CC-BY-SA-4.0", StrictMetadataBool::True)
)
}
fn explicit_false(value: &str) -> bool {
matches!(
normalized_text(value).as_str(),
"false" | "no" | "0" | "not copyrighted"
)
}
fn explicit_true_or_unknown(value: &str) -> bool {
let value = normalized_text(value);
!value.is_empty() && !matches!(value.as_str(), "false" | "no" | "0")
}
fn usable_attribution_text(value: &str) -> bool {
let value = normalized_text(value);
let unusable_phrases = [
"unknown",
"unidentified",
"anonymous",
"own work",
"uploader",
"uploaded by",
"see description",
"see file description",
"not provided",
"not specified",
"no author",
"no creator",
"author unavailable",
"creator unavailable",
];
!value.is_empty()
&& !matches!(value.as_str(), "none" | "n a" | "self" | "various")
&& !unusable_phrases
.iter()
.any(|phrase| contains_phrase(&value, phrase))
}
fn selected_attribution(
creator: &str,
custom: &str,
credit: &str,
source_url: &str,
required: bool,
) -> String {
if usable_attribution_text(custom) {
return custom.to_string();
}
if usable_attribution_text(creator) {
if usable_source_credit(credit) {
return bounded_text(&format!("{creator}; {credit}; {source_url}"));
}
return bounded_text(&format!("{creator}; {source_url}"));
}
if required {
String::new()
} else {
format!("Wikimedia Commons source: {source_url}")
}
}
fn usable_source_credit(value: &str) -> bool {
let value = normalized_text(value);
!value.is_empty()
&& !matches!(value.as_str(), "self" | "none" | "n a")
&& !contains_phrase(&value, "own work")
}
fn assessment_labels(value: &str) -> Vec<String> {
if value.is_empty() {
Vec::new()
} else {
value
.split([',', ';', '|'])
.map(bounded_text)
.filter(|value| !value.is_empty())
.take(64)
.collect()
}
}
fn valid_dimensions(width: u64, height: u64) -> bool {
width > 0
&& height > 0
&& width <= MAX_DIMENSION
&& height <= MAX_DIMENSION
&& width
.checked_mul(height)
.is_some_and(|pixels| pixels <= MAX_PIXELS)
}
pub(crate) fn supported_photo_mime(mime: &str) -> bool {
matches!(mime, "image/jpeg" | "image/png")
}
pub(crate) fn validate_summary(photo: &PhotoSummary) -> Result<()> {
let dimensions_valid =
valid_dimensions(u64::from(photo.file.width), u64::from(photo.file.height));
let license_requires_attribution =
matches!(photo.rights.license.as_str(), "CC-BY-4.0" | "CC-BY-SA-4.0");
let creator_valid = !photo.rights.attribution_required
|| usable_attribution_text(&photo.rights.creator)
|| usable_attribution_text(&photo.rights.custom_attribution);
if photo.provider != PhotoProvider::WikimediaCommons
|| photo.id == 0
|| !photo.title.starts_with("File:")
|| photo.title.len() <= 5
|| photo.rights.raw_license_label.is_empty()
|| license_label_id(&photo.rights.raw_license_label) != Some(photo.rights.license.as_str())
|| photo.rights.attribution.is_empty()
|| photo.rights.attribution_required != license_requires_attribution
|| !creator_valid
|| !photo.rights.permission.is_empty()
|| !photo.rights.restrictions.is_empty()
|| !photo.rights.deletion_reason.is_empty()
|| explicit_true_or_unknown(&photo.rights.non_free)
|| (photo.rights.attribution_required && explicit_false(&photo.rights.copyrighted))
|| !allowed_license_id(&photo.rights.license, &photo.rights.license_url)
|| photo.rights.metadata_retrieved_at == 0
|| photo.rights.provider_guidance_url != COMMONS_REUSE_URL
|| photo.file.size == 0
|| photo.file.size > MAX_RETAINED_IMAGE_BYTES
|| photo.file.sha1.len() != 40
|| !photo.file.sha1.bytes().all(|byte| byte.is_ascii_hexdigit())
|| !supported_photo_mime(&photo.file.mime)
|| photo.file.media_type != "BITMAP"
|| !dimensions_valid
|| !usage_terms_match_license(&photo.rights.license, &photo.rights.usage_terms)
|| !attribution_requirement_matches(
&photo.rights.license,
&photo.rights.raw_attribution_required,
)
{
return Err(PhotoForgeError::VerificationFailed {
path: photo.id.to_string(),
message: "photo identity, rights, format, or dimensions are invalid".to_string(),
});
}
let source = trusted_source_url(&photo.source_url)?;
let source_title = decoded_source_title(&source);
let expected_title = photo.title.replace(' ', "_");
if source_title.as_deref() != Some(expected_title.as_str()) {
return Err(PhotoForgeError::VerificationFailed {
path: photo.id.to_string(),
message: "canonical source URL does not match the Commons file title".to_string(),
});
}
let download = trusted_download_url(&photo.file.url)?;
let expected_filename = photo
.title
.strip_prefix("File:")
.expect("validated file namespace")
.replace(' ', "_");
if decoded_download_filename(&download).as_deref() != Some(expected_filename.as_str()) {
return Err(PhotoForgeError::VerificationFailed {
path: photo.id.to_string(),
message: "original upload filename does not match the Commons file title".to_string(),
});
}
Ok(())
}
pub(crate) fn same_nonvolatile_metadata(archived: &PhotoSummary, current: &PhotoSummary) -> bool {
let mut archived = archived.clone();
archived.rights.metadata_retrieved_at = current.rights.metadata_retrieved_at;
archived.usage_count = current.usage_count;
archived.usage_count_complete = current.usage_count_complete;
archived == *current
}
fn rank_photos(photos: &mut Vec<PhotoSummary>, target: &PhotoTarget) {
match target {
PhotoTarget::Search(query) => {
photos.retain(|photo| search_score(photo, query).is_some());
photos.sort_by(|left, right| {
search_score(right, query)
.cmp(&search_score(left, query))
.then_with(|| assessment_score(right).cmp(&assessment_score(left)))
.then_with(|| right.usage_count.cmp(&left.usage_count))
.then_with(|| left.id.cmp(&right.id))
});
}
PhotoTarget::Category(category) => photos.sort_by(|left, right| {
category_title_score(right, category)
.cmp(&category_title_score(left, category))
.then_with(|| assessment_score(right).cmp(&assessment_score(left)))
.then_with(|| right.usage_count.cmp(&left.usage_count))
.then_with(|| left.id.cmp(&right.id))
}),
PhotoTarget::Top => photos.sort_by(|left, right| {
assessment_score(right)
.cmp(&assessment_score(left))
.then_with(|| right.usage_count.cmp(&left.usage_count))
.then_with(|| left.id.cmp(&right.id))
}),
PhotoTarget::Latest | PhotoTarget::File(_) => {}
}
}
fn search_score(photo: &PhotoSummary, query: &str) -> Option<i64> {
let query = normalized_text(query);
let terms = query.split_whitespace().collect::<BTreeSet<_>>();
if terms.is_empty() {
return None;
}
let title = normalized_photo_title(&photo.title);
let object_name = normalized_text(&photo.object_name);
let categories = normalized_text(&photo.categories.join(" "));
let description = normalized_text(&photo.description);
let all = format!("{title} {object_name} {categories} {description}");
let all_terms = all.split_whitespace().collect::<BTreeSet<_>>();
if !terms.iter().all(|term| all_terms.contains(term)) {
return None;
}
let title_terms = title.split_whitespace().collect::<BTreeSet<_>>();
let score = if title == query || object_name == query {
600
} else if contains_phrase(&title, &query) || contains_phrase(&object_name, &query) {
500
} else if terms.iter().all(|term| title_terms.contains(term)) {
450
} else if terms
.iter()
.all(|term| title_terms.contains(term) || categories.split_whitespace().any(|v| v == *term))
{
400
} else if terms.iter().any(|term| title_terms.contains(term)) {
300
} else {
100
};
Some(score)
}
fn category_title_score(photo: &PhotoSummary, category: &str) -> i64 {
let category = normalized_text(category.trim_start_matches("Category:"));
let title = normalized_photo_title(&photo.title);
if title == category {
20
} else if contains_phrase(&title, &category) {
10
} else {
0
}
}
fn assessment_score(photo: &PhotoSummary) -> i64 {
let evidence = normalized_text(&format!(
"{} {}",
photo.assessments.join(" "),
photo.categories.join(" ")
));
let featured = contains_phrase(&evidence, "featured picture")
|| contains_phrase(&evidence, "featured pictures")
|| contains_phrase(&evidence, "featured image");
let quality =
contains_phrase(&evidence, "quality image") || contains_phrase(&evidence, "quality images");
i64::from(featured) * 20 + i64::from(quality) * 10
}
fn normalized_photo_title(value: &str) -> String {
let value = value.strip_prefix("File:").unwrap_or(value);
let value = value
.rsplit_once('.')
.filter(|(_, extension)| {
matches!(
extension.to_ascii_lowercase().as_str(),
"jpg" | "jpeg" | "png"
)
})
.map_or(value, |(stem, _)| stem);
normalized_text(value)
}
fn contains_phrase(haystack: &str, phrase: &str) -> bool {
!phrase.is_empty() && format!(" {haystack} ").contains(&format!(" {phrase} "))
}
fn normalized_text(value: &str) -> String {
value
.chars()
.map(|character| {
if character.is_alphanumeric() {
character.to_ascii_lowercase()
} else {
' '
}
})
.collect::<String>()
.split_whitespace()
.collect::<Vec<_>>()
.join(" ")
}
pub(crate) fn literal_search_query(value: &str) -> String {
value
.split_whitespace()
.map(|token| {
let escaped = token.replace('\\', "\\\\").replace('"', "\\\"");
format!("\"{escaped}\"")
})
.collect::<Vec<_>>()
.join(" ")
}
fn normalized_file_title(value: &str) -> String {
let value = value.trim().replace(' ', "_");
if value
.get(..5)
.is_some_and(|prefix| prefix.eq_ignore_ascii_case("File:"))
{
format!("File:{}", &value[5..])
} else {
format!("File:{value}")
}
}
fn normalized_category_title(value: &str) -> String {
let value = value.trim();
if value
.get(..9)
.is_some_and(|prefix| prefix.eq_ignore_ascii_case("Category:"))
{
format!("Category:{}", &value[9..])
} else {
format!("Category:{value}")
}
}
pub(crate) fn trusted_api_url(url: &Url) -> Result<()> {
if !secure_https_url(url)
|| url.host_str() != Some(COMMONS_HOST)
|| url.path() != "/w/api.php"
|| url.fragment().is_some()
{
return Err(PhotoForgeError::UntrustedUrl(url.to_string()));
}
Ok(())
}
pub(crate) fn trusted_source_url(value: &str) -> Result<Url> {
let url = Url::parse(value).map_err(|_| PhotoForgeError::UntrustedUrl(value.to_string()))?;
let valid_path = url
.path()
.strip_prefix("/wiki/File:")
.is_some_and(|name| !name.is_empty() && !name.contains('/'));
if !secure_https_url(&url)
|| url.host_str() != Some(COMMONS_HOST)
|| !valid_path
|| url.query().is_some()
|| url.fragment().is_some()
{
return Err(PhotoForgeError::UntrustedUrl(value.to_string()));
}
Ok(url)
}
pub(crate) fn trusted_download_url(value: &str) -> Result<Url> {
let url = Url::parse(value).map_err(|_| PhotoForgeError::UntrustedUrl(value.to_string()))?;
let segments = url
.path_segments()
.map(|segments| segments.collect::<Vec<_>>())
.unwrap_or_default();
let valid_path = segments.len() == 5
&& segments[0] == "wikipedia"
&& segments[1] == "commons"
&& segments[2].len() == 1
&& segments[2].bytes().all(|byte| byte.is_ascii_hexdigit())
&& segments[3].len() == 2
&& segments[3].bytes().all(|byte| byte.is_ascii_hexdigit())
&& !segments[4].is_empty();
if !secure_https_url(&url)
|| url.host_str() != Some(COMMONS_DOWNLOAD_HOST)
|| !valid_path
|| url.query().is_some()
|| url.fragment().is_some()
{
return Err(PhotoForgeError::UntrustedUrl(value.to_string()));
}
Ok(url)
}
fn decoded_source_title(url: &Url) -> Option<String> {
let encoded = url.path().strip_prefix("/wiki/")?;
let decoded = urlencoding::decode(encoded).ok()?;
(!decoded.contains(['/', '\\']) && !decoded.chars().any(char::is_control))
.then(|| decoded.replace(' ', "_"))
}
fn decoded_download_filename(url: &Url) -> Option<String> {
let encoded = url.path_segments()?.next_back()?;
let decoded = urlencoding::decode(encoded).ok()?;
(!decoded.contains(['/', '\\']) && !decoded.chars().any(char::is_control))
.then(|| decoded.replace(' ', "_"))
}
fn secure_https_url(url: &Url) -> bool {
url.scheme() == "https"
&& url.username().is_empty()
&& url.password().is_none()
&& url.port().is_none_or(|port| port == 443)
}
fn read_response_limited(mut response: Response, limit: u64, label: &str) -> Result<Vec<u8>> {
if response
.content_length()
.is_some_and(|length| length > limit)
{
return Err(PhotoForgeError::ResponseLimitExceeded {
label: label.to_string(),
size: response.content_length().unwrap_or_default(),
limit,
});
}
let mut bytes = Vec::new();
response
.by_ref()
.take(limit.saturating_add(1))
.read_to_end(&mut bytes)?;
if bytes.len() as u64 > limit {
return Err(PhotoForgeError::ResponseLimitExceeded {
label: label.to_string(),
size: bytes.len() as u64,
limit,
});
}
Ok(bytes)
}
fn bounded_error_body(response: Response) -> String {
match read_response_limited(response, MAX_ERROR_RESPONSE_BYTES, "Commons error response") {
Ok(bytes) => bounded_text(&String::from_utf8_lossy(&bytes)),
Err(PhotoForgeError::ResponseLimitExceeded { .. }) => {
"<error response exceeded safety limit>".to_string()
}
Err(_) => "<unable to read error response>".to_string(),
}
}
fn api_envelope_error(bytes: &[u8]) -> Result<Option<ApiEnvelopeError>> {
let value: serde_json::Value = serde_json::from_slice(bytes)?;
let Some(error) = value.get("error") else {
return Ok(None);
};
let code = error
.get("code")
.and_then(serde_json::Value::as_str)
.map(bounded_text)
.unwrap_or_else(|| "unknown".to_string());
let info = error
.get("info")
.and_then(serde_json::Value::as_str)
.map(bounded_text)
.unwrap_or_else(|| "provider supplied no error information".to_string());
let lag = error.get("lag").and_then(|lag| {
lag.as_f64()
.or_else(|| lag.as_str().and_then(|value| value.parse::<f64>().ok()))
.filter(|value| value.is_finite() && *value >= 0.0)
});
Ok(Some(ApiEnvelopeError { code, info, lag }))
}
pub(crate) fn retryable_http_status(status: u16) -> bool {
status == 429 || (500..=599).contains(&status)
}
fn numeric_retry_after(response: &Response) -> Option<Duration> {
response
.headers()
.get(reqwest::header::RETRY_AFTER)
.and_then(|value| value.to_str().ok())
.and_then(|value| value.trim().parse::<u64>().ok())
.map(Duration::from_secs)
}
pub(crate) fn retry_delay(
attempt: usize,
retry_after: Option<Duration>,
provider_lag: Option<Duration>,
) -> Duration {
retry_after
.or(provider_lag)
.unwrap_or_else(|| Duration::from_secs(1 << attempt.min(2)))
.min(MAX_API_RETRY_WAIT)
}
fn duration_from_lag(lag: f64) -> Duration {
Duration::from_secs_f64(lag.min(MAX_API_RETRY_WAIT.as_secs_f64()))
}
fn unix_timestamp() -> Result<u64> {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|duration| duration.as_secs())
.map_err(|error| PhotoForgeError::InvalidOption(error.to_string()))
}
pub(crate) fn error_chain(error: &(dyn StdError + 'static)) -> String {
let mut message = error.to_string();
let mut source = error.source();
while let Some(error) = source {
message.push_str("; caused by: ");
message.push_str(&error.to_string());
source = error.source();
}
bounded_text(&message)
}
#[cfg(test)]
mod tests {
use super::*;
fn metadata(values: &[(&str, &str)]) -> BTreeMap<String, CommonsMetadataValue> {
values
.iter()
.map(|(key, value)| {
(
(*key).to_string(),
CommonsMetadataValue {
value: (*value).to_string(),
},
)
})
.collect()
}
fn fixture_page(license: &str, full_license: &str, url: &str) -> CommonsPage {
let (usage_terms, attribution_required) = match license {
"CC0" | "CC0 1.0" => ("Creative Commons CC0 1.0 Universal", "false"),
"CC BY 4.0" => ("Creative Commons Attribution 4.0 International", "true"),
"CC BY-SA 4.0" => (
"Creative Commons Attribution-Share Alike 4.0 International",
"true",
),
_ => (license, "true"),
};
CommonsPage {
pageid: Some(42),
title: "File:Blue flower.jpg".to_string(),
index: Some(1),
missing: false,
lastrevid: Some(99),
imageinfo: vec![CommonsImageInfo {
size: 100,
width: 10,
height: 10,
url: "https://upload.wikimedia.org/wikipedia/commons/a/ab/Blue_flower.jpg"
.to_string(),
descriptionurl: "https://commons.wikimedia.org/wiki/File:Blue_flower.jpg"
.to_string(),
sha1: "a".repeat(40),
mime: "image/jpeg".to_string(),
mediatype: "BITMAP".to_string(),
timestamp: Some("2026-01-02T03:04:05Z".to_string()),
extmetadata: metadata(&[
("LicenseShortName", license),
("License", full_license),
("LicenseUrl", url),
("Artist", "<b>Fixture Creator</b>"),
("Credit", "Own work"),
("AttributionRequired", attribution_required),
("ObjectName", "Blue flower"),
("ImageDescription", "<p>A blue & white flower.</p>"),
("UsageTerms", usage_terms),
("Copyrighted", "true"),
("NonFree", "false"),
("Assessments", "Featured picture; Quality image"),
]),
}],
categories: vec![CommonsCategory {
title: "Category:Blue flowers".to_string(),
}],
globalusage: vec![CommonsUsage {
title: "Flower".to_string(),
wiki: "enwiki".to_string(),
}],
}
}
#[test]
fn parses_target_grammar_and_aliases() {
assert_eq!(
parse_target("blue flower").unwrap(),
PhotoTarget::Search("blue flower".to_string())
);
assert_eq!(
parse_target("search:blue flower").unwrap(),
PhotoTarget::Search("blue flower".to_string())
);
assert_eq!(
parse_target("category:Featured pictures").unwrap(),
PhotoTarget::Category("Featured pictures".to_string())
);
assert_eq!(
parse_target("file:Example.jpg").unwrap(),
PhotoTarget::File("Example.jpg".to_string())
);
assert_eq!(parse_target("popular").unwrap(), PhotoTarget::Top);
assert_eq!(parse_target("newest").unwrap(), PhotoTarget::Latest);
assert!(!PhotoTarget::File("x".to_string()).is_collection());
assert!(PhotoTarget::Top.is_collection());
}
#[test]
fn canonical_commons_url_is_the_only_accepted_url_target() {
let parsed =
parse_target("https://commons.wikimedia.org/wiki/File:Blue_flower_%28detail%29.jpg")
.unwrap();
assert_eq!(
parsed,
PhotoTarget::File("File:Blue_flower_(detail).jpg".to_string())
);
for value in [
"",
"file:File:",
"category:Category:",
"search:https://example.invalid/image.jpg",
"http://commons.wikimedia.org/wiki/File:X.jpg",
"https://user@commons.wikimedia.org/wiki/File:X.jpg",
"https://commons.wikimedia.org:444/wiki/File:X.jpg",
"https://commons.wikimedia.org/wiki/File:X.jpg#history",
"https://commons.wikimedia.org/wiki/File:X.jpg?download=1",
"https://commons.wikimedia.org/wiki/Category:X",
"https://commons.wikimedia.org/w/index.php?title=File:X.jpg",
"https://example.invalid/wiki/File:X.jpg",
"ftp://commons.wikimedia.org/wiki/File:X.jpg",
] {
assert!(
parse_target(value).is_err(),
"unexpectedly accepted {value}"
);
}
assert!(
validate_target(&PhotoTarget::File("https://example.invalid/x.jpg".into())).is_err()
);
}
#[test]
fn literal_query_quotes_every_user_token_before_bitmap_operator() {
assert_eq!(
literal_search_query("flower incategory:Hidden -unsafe a\\b c\"d"),
"\"flower\" \"incategory:Hidden\" \"-unsafe\" \"a\\\\b\" \"c\\\"d\""
);
let url = candidate_url(&PhotoTarget::Search("-cat".to_string()), 12).unwrap();
let search = url
.query_pairs()
.find(|(key, _)| key == "gsrsearch")
.unwrap()
.1;
assert_eq!(search, "\"-cat\" filetype:bitmap");
}
#[test]
fn category_top_and_latest_use_direct_provider_modules() {
let category = candidate_url(&PhotoTarget::Category("Flowers".to_string()), 10).unwrap();
assert!(
category
.query_pairs()
.any(|(k, v)| k == "generator" && v == "categorymembers")
);
assert!(
category
.query_pairs()
.any(|(k, v)| k == "gcmtitle" && v == "Category:Flowers")
);
let top = candidate_url(&PhotoTarget::Top, 10).unwrap();
assert!(top.query_pairs().any(|(k, v)| {
k == "gcmtitle" && v == "Category:Featured pictures on Wikimedia Commons"
}));
let latest = candidate_url(&PhotoTarget::Latest, 10).unwrap();
assert!(
latest
.query_pairs()
.any(|(k, v)| k == "generator" && v == "allimages")
);
assert!(
latest
.query_pairs()
.any(|(k, v)| k == "gaidir" && v == "descending")
);
}
#[test]
fn provider_fixture_maps_complete_photo_evidence() {
let page = fixture_page(
"CC BY 4.0",
"cc-by-4.0",
"http://creativecommons.org/licenses/by/4.0/",
);
let photo = commons_summary(page, true, 123).unwrap().unwrap();
assert_eq!(photo.id, 42);
assert_eq!(photo.file.revision_id, Some(99));
assert_eq!(photo.file.width, 10);
assert_eq!(photo.rights.license, "CC-BY-4.0");
assert_eq!(
photo.rights.license_url,
"https://creativecommons.org/licenses/by/4.0/"
);
assert_eq!(photo.rights.creator, "Fixture Creator");
assert!(photo.rights.attribution.contains("Fixture Creator"));
assert!(photo.rights.attribution.contains(photo.source_url.as_str()));
assert!(!photo.rights.attribution.contains("Own work;"));
assert_eq!(photo.description, "A blue & white flower.");
assert_eq!(photo.usage_count, 1);
assert!(photo.usage_count_complete);
validate_summary(&photo).unwrap();
}
#[test]
fn strict_license_allowlist_requires_corresponding_url_and_version() {
assert!(
allowed_license(
"CC0",
"cc-zero",
"https://creativecommons.org/publicdomain/zero/1.0/"
)
.is_some()
);
assert!(
allowed_license(
"CC BY-SA 4.0",
"cc-by-sa-4.0",
"https://creativecommons.org/licenses/by-sa/4.0/"
)
.is_some()
);
for (label, full, url) in [
(
"CC BY 3.0",
"cc-by-3.0",
"https://creativecommons.org/licenses/by/3.0/",
),
(
"CC BY-NC 4.0",
"cc-by-nc-4.0",
"https://creativecommons.org/licenses/by-nc/4.0/",
),
(
"CC BY-ND 4.0",
"cc-by-nd-4.0",
"https://creativecommons.org/licenses/by-nd/4.0/",
),
(
"Public domain",
"pd",
"https://creativecommons.org/publicdomain/mark/1.0/",
),
(
"PDM 1.0",
"pdm",
"https://creativecommons.org/publicdomain/mark/1.0/",
),
("FAL", "fal", "https://artlibre.org/licence/lal/en/"),
("GFDL", "gfdl", "https://www.gnu.org/licenses/fdl.html"),
("CC BY 4.0", "cc-by-4.0", ""),
(
"CC BY 4.0",
"cc-by-sa-4.0",
"https://creativecommons.org/licenses/by/4.0/",
),
(
"CC BY 4.0 / GFDL",
"cc-by-4.0",
"https://creativecommons.org/licenses/by/4.0/",
),
(
"CC BY 4.0",
"cc-by-4.0",
"https://evil.invalid/licenses/by/4.0/",
),
] {
assert!(
allowed_license(label, full, url).is_none(),
"accepted {label}"
);
}
}
#[test]
fn by_requires_creator_or_custom_attribution_and_rejects_restrictions() {
let mut page = fixture_page(
"CC BY-SA 4.0",
"cc-by-sa-4.0",
"https://creativecommons.org/licenses/by-sa/4.0/",
);
let metadata = &mut page.imageinfo[0].extmetadata;
metadata.insert(
"Artist".to_string(),
CommonsMetadataValue {
value: "Own work".to_string(),
},
);
assert!(commons_summary(page, true, 1).unwrap().is_none());
let mut page = fixture_page(
"CC BY 4.0",
"cc-by-4.0",
"https://creativecommons.org/licenses/by/4.0/",
);
page.imageinfo[0].extmetadata.insert(
"Artist".to_string(),
CommonsMetadataValue {
value: String::new(),
},
);
page.imageinfo[0].extmetadata.insert(
"Attribution".to_string(),
CommonsMetadataValue {
value: "Photo by Jane Example".to_string(),
},
);
assert!(commons_summary(page, true, 1).unwrap().is_some());
for field in ["Restrictions", "Permission", "DeletionReason"] {
let mut page = fixture_page(
"CC0",
"cc-zero",
"https://creativecommons.org/publicdomain/zero/1.0/",
);
page.imageinfo[0].extmetadata.insert(
field.to_string(),
CommonsMetadataValue {
value: "review required".to_string(),
},
);
assert!(commons_summary(page, true, 1).unwrap().is_none());
}
}
#[test]
fn rights_contradictions_are_conservatively_filtered() {
let mut nonfree = fixture_page(
"CC0",
"cc-zero",
"https://creativecommons.org/publicdomain/zero/1.0/",
);
nonfree.imageinfo[0].extmetadata.insert(
"NonFree".to_string(),
CommonsMetadataValue {
value: "true".to_string(),
},
);
assert!(commons_summary(nonfree, true, 1).unwrap().is_none());
let mut contradiction = fixture_page(
"CC BY 4.0",
"cc-by-4.0",
"https://creativecommons.org/licenses/by/4.0/",
);
contradiction.imageinfo[0].extmetadata.insert(
"Copyrighted".to_string(),
CommonsMetadataValue {
value: "false".to_string(),
},
);
assert!(commons_summary(contradiction, true, 1).unwrap().is_none());
}
#[test]
fn usage_terms_and_attribution_required_must_match_the_exact_license() {
let cases = [
(
"UsageTerms",
"Creative Commons Attribution-NonCommercial 4.0",
),
("UsageTerms", "Creative Commons Attribution-Share Alike 4.0"),
("UsageTerms", "CC BY 4.0 or later"),
("AttributionRequired", "false"),
("AttributionRequired", "sometimes"),
];
for (field, value) in cases {
let mut page = fixture_page(
"CC BY 4.0",
"cc-by-4.0",
"https://creativecommons.org/licenses/by/4.0/",
);
page.imageinfo[0].extmetadata.insert(
field.to_string(),
CommonsMetadataValue {
value: value.to_string(),
},
);
assert!(
commons_summary(page, true, 1).unwrap().is_none(),
"accepted contradictory {field}={value}"
);
}
let mut cc0 = fixture_page(
"CC0",
"cc-zero",
"https://creativecommons.org/publicdomain/zero/1.0/",
);
cc0.imageinfo[0].extmetadata.insert(
"AttributionRequired".to_string(),
CommonsMetadataValue {
value: "true".to_string(),
},
);
assert!(commons_summary(cc0, true, 1).unwrap().is_none());
assert!(usage_terms_match_license(
"CC0-1.0",
"CC0 1.0 Universal Public Domain Dedication"
));
assert!(usage_terms_match_license(
"CC-BY-4.0",
"Creative Commons Attribution 4.0"
));
assert!(usage_terms_match_license(
"CC-BY-SA-4.0",
"Creative Commons Attribution-Share Alike 4.0 International"
));
assert!(!usage_terms_match_license(
"CC-BY-SA-4.0",
"Creative Commons Attribution-NoDerivatives 4.0"
));
let page = fixture_page(
"CC BY 4.0",
"cc-by-4.0",
"https://creativecommons.org/licenses/by/4.0/",
);
let mut photo = commons_summary(page, true, 1).unwrap().unwrap();
photo.rights.usage_terms = "Creative Commons Attribution-NonCommercial 4.0".into();
assert!(validate_summary(&photo).is_err());
photo.rights.usage_terms = "Creative Commons Attribution 4.0".into();
photo.rights.raw_attribution_required = "false".into();
assert!(validate_summary(&photo).is_err());
}
#[test]
fn vague_creator_or_custom_attribution_phrases_are_unusable() {
for value in [
"Unknown photographer",
"Anonymous contributor",
"Own work by uploader",
"Uploader of this file",
"See description for author",
"Creator not provided",
"Unidentified artist",
] {
assert!(!usable_attribution_text(value), "accepted '{value}'");
}
assert!(usable_attribution_text("Photo by Jane Example"));
let mut page = fixture_page(
"CC BY 4.0",
"cc-by-4.0",
"https://creativecommons.org/licenses/by/4.0/",
);
page.imageinfo[0].extmetadata.insert(
"Artist".to_string(),
CommonsMetadataValue {
value: "See description for creator".to_string(),
},
);
assert!(commons_summary(page, true, 1).unwrap().is_none());
}
#[test]
fn unsupported_formats_and_unsafe_dimensions_are_ineligible() {
for mime in [
"image/gif",
"image/svg+xml",
"image/webp",
"image/tiff",
"image/avif",
] {
let mut page = fixture_page(
"CC0",
"cc-zero",
"https://creativecommons.org/publicdomain/zero/1.0/",
);
page.imageinfo[0].mime = mime.to_string();
assert!(commons_summary(page, true, 1).unwrap().is_none());
}
assert!(!valid_dimensions(0, 1));
assert!(!valid_dimensions(32_769, 1));
assert!(!valid_dimensions(20_000, 20_000));
assert!(valid_dimensions(10_000, 10_000));
}
#[test]
fn host_path_port_and_query_boundaries_are_exact() {
assert!(trusted_api_url(&Url::parse(COMMONS_API).unwrap()).is_ok());
assert!(
trusted_api_url(&Url::parse("https://commons.wikimedia.org/wiki/X").unwrap()).is_err()
);
assert!(trusted_source_url("https://commons.wikimedia.org/wiki/File:X.jpg").is_ok());
assert!(trusted_source_url("https://commons.wikimedia.org/wiki/File:X.jpg?x=1").is_err());
assert!(
trusted_download_url("https://upload.wikimedia.org/wikipedia/commons/a/ab/X.jpg")
.is_ok()
);
assert!(
trusted_download_url("https://upload.wikimedia.org/wikipedia/thumb/a/ab/X.jpg")
.is_err()
);
assert!(
trusted_download_url("https://upload.wikimedia.org/wikipedia/commons/a/ab/X.jpg?x=1")
.is_err()
);
for value in [
"https://upload.wikimedia.org/wikipedia/commons/thumb/a/ab/X.jpg/200px-X.jpg",
"https://upload.wikimedia.org/wikipedia/archive/a/ab/X.jpg",
"https://upload.wikimedia.org/wikipedia/commons/temp/a/ab/X.jpg",
"https://upload.wikimedia.org/wikipedia/commons/gg/ab/X.jpg",
"https://upload.wikimedia.org/wikipedia/commons/a/b/X.jpg",
"https://upload.wikimedia.org/wikipedia/commons/a/ab/sub/X.jpg",
] {
assert!(trusted_download_url(value).is_err(), "accepted {value}");
}
let page = fixture_page(
"CC0",
"cc-zero",
"https://creativecommons.org/publicdomain/zero/1.0/",
);
let mut photo = commons_summary(page, true, 1).unwrap().unwrap();
photo.source_url = "https://commons.wikimedia.org/wiki/File:Other.jpg".to_string();
assert!(validate_summary(&photo).is_err());
photo.source_url = "https://commons.wikimedia.org/wiki/File:Blue_flower.jpg".to_string();
photo.file.url =
"https://upload.wikimedia.org/wikipedia/commons/a/ab/Other.jpg".to_string();
assert!(validate_summary(&photo).is_err());
}
#[test]
fn continuation_merge_retains_usage_and_completeness_evidence() {
let mut pages = vec![fixture_page(
"CC0",
"cc-zero",
"https://creativecommons.org/publicdomain/zero/1.0/",
)];
let mut later = fixture_page(
"CC0",
"cc-zero",
"https://creativecommons.org/publicdomain/zero/1.0/",
);
later.imageinfo.clear();
later.globalusage.push(CommonsUsage {
wiki: "dewiki".to_string(),
title: "Blume".to_string(),
});
merge_pages(&mut pages, vec![later]);
assert_eq!(pages.len(), 1);
assert_eq!(pages[0].globalusage.len(), 2);
let continuation = filtered_metadata_continuation(BTreeMap::from([
("continue".to_string(), serde_json::json!("||")),
("gucontinue".to_string(), serde_json::json!("42|0")),
("gsroffset".to_string(), serde_json::json!(50)),
]))
.unwrap();
assert!(continuation.contains_key("gucontinue"));
assert!(!continuation.contains_key("gsroffset"));
}
#[test]
fn action_api_error_and_retry_decisions_are_explicit() {
let error =
api_envelope_error(br#"{"error":{"code":"maxlag","info":"Waiting","lag":"2.5"}}"#)
.unwrap()
.unwrap();
assert_eq!(error.code, "maxlag");
assert_eq!(error.lag, Some(2.5));
assert!(retryable_http_status(429));
assert!(retryable_http_status(500));
assert!(!retryable_http_status(404));
assert_eq!(
retry_delay(0, Some(Duration::from_secs(30)), None),
Duration::from_secs(5)
);
assert_eq!(retry_delay(1, None, None), Duration::from_secs(2));
}
#[test]
fn ranking_requires_whole_token_coverage_and_prefers_title_and_assessments() {
let make = |id, title: &str, categories: &[&str], usage| {
let mut page = fixture_page(
"CC0",
"cc-zero",
"https://creativecommons.org/publicdomain/zero/1.0/",
);
page.pageid = Some(id);
page.title = title.to_string();
let filename = title.strip_prefix("File:").unwrap().replace(' ', "_");
page.imageinfo[0].descriptionurl =
format!("https://commons.wikimedia.org/wiki/File:{filename}");
page.imageinfo[0].url =
format!("https://upload.wikimedia.org/wikipedia/commons/a/ab/{filename}");
page.categories = categories
.iter()
.map(|value| CommonsCategory {
title: (*value).to_string(),
})
.collect();
page.globalusage = (0..usage)
.map(|index| CommonsUsage {
wiki: format!("wiki{index}"),
title: title.to_string(),
})
.collect();
let mut photo = commons_summary(page, true, 1).unwrap().unwrap();
photo.object_name.clear();
photo.description.clear();
photo
};
let mut photos = vec![
make(1, "File:Blue flower.jpg", &[], 1),
make(2, "File:Blue object.jpg", &["Category:Flower"], 100),
make(3, "File:Blue flowering.jpg", &[], 1000),
];
rank_photos(&mut photos, &PhotoTarget::Search("blue flower".to_string()));
assert_eq!(
photos.iter().map(|photo| photo.id).collect::<Vec<_>>(),
[1, 2]
);
assert!(search_score(&photos[0], "flow").is_none());
}
#[test]
fn latest_keeps_provider_order() {
let mut photos = [3_u64, 1, 2]
.into_iter()
.map(|id| {
let mut page = fixture_page(
"CC0",
"cc-zero",
"https://creativecommons.org/publicdomain/zero/1.0/",
);
page.pageid = Some(id);
commons_summary(page, true, 1).unwrap().unwrap()
})
.collect::<Vec<_>>();
rank_photos(&mut photos, &PhotoTarget::Latest);
assert_eq!(
photos.iter().map(|photo| photo.id).collect::<Vec<_>>(),
[3, 1, 2]
);
}
#[test]
fn nonvolatile_match_ignores_only_retrieval_and_usage_fields() {
let page = fixture_page(
"CC0",
"cc-zero",
"https://creativecommons.org/publicdomain/zero/1.0/",
);
let archived = commons_summary(page, true, 1).unwrap().unwrap();
let mut current = archived.clone();
current.rights.metadata_retrieved_at = 2;
current.usage_count = 100;
current.usage_count_complete = false;
assert!(same_nonvolatile_metadata(&archived, ¤t));
current.description.push_str(" changed");
assert!(!same_nonvolatile_metadata(&archived, ¤t));
}
#[test]
fn json_fixture_preserves_generator_index_and_page_fields() {
let response: CommonsResponse = serde_json::from_str(
r#"{"continue":{"gaicontinue":"20260101000000|Example.jpg","continue":"-||"},"query":{"pages":[{"pageid":7,"title":"File:Example.jpg","index":2,"lastrevid":9}]}}"#,
)
.unwrap();
assert!(has_candidate_continuation(
&PhotoTarget::Latest,
&response.continuation
));
let page = &response.query.unwrap().pages[0];
assert_eq!(page.pageid, Some(7));
assert_eq!(page.index, Some(2));
assert_eq!(page.lastrevid, Some(9));
}
#[test]
fn candidate_window_is_sorted_truncated_and_deduplicated_before_metadata() {
let page = |id: Option<u64>, index: u64| CommonsPage {
pageid: id,
title: format!("File:{index}.jpg"),
index: Some(index),
missing: false,
lastrevid: None,
imageinfo: Vec::new(),
categories: Vec::new(),
globalusage: Vec::new(),
};
let mut pages = vec![
page(Some(3), 3),
page(Some(1), 1),
page(Some(1), 2),
page(Some(4), 4),
];
let (ids, over_budget) = bounded_candidate_ids(&mut pages, 3);
assert_eq!(ids, [1, 3]);
assert!(over_budget);
assert_eq!(pages.len(), 3);
assert_eq!(collection_candidate_budget(20), 50);
assert_eq!(collection_candidate_budget(5), 15);
let latest = candidate_url(&PhotoTarget::Latest, collection_candidate_budget(5)).unwrap();
assert!(
latest
.query_pairs()
.any(|(key, value)| { key == "gailimit" && value == "15" })
);
}
#[test]
fn api_operation_counts_every_send_and_shares_retry_deadline() {
let mut operation = ApiOperation::new();
for expected in 1..=MAX_API_REQUESTS {
operation.reserve_send().unwrap();
assert_eq!(operation.sends, expected);
}
assert!(operation.reserve_send().is_err());
assert!(!operation.wait_before_retry(Duration::from_secs(1)));
let mut expired = ApiOperation::new();
expired.started = Instant::now()
.checked_sub(MAX_API_RETRY_WAIT + Duration::from_millis(1))
.unwrap();
assert!(!expired.wait_before_retry(Duration::from_secs(1)));
assert_eq!(expired.retry_sleep, Duration::ZERO);
}
#[test]
#[ignore = "requires bounded live Wikimedia Commons API access"]
fn live_bounded_search_category_top_and_latest_probes() {
let forge = PhotoForge::with_user_agent(
"photoforge-live-discovery-test/0.1 (https://github.com/Tknott95/PhotoForge; bounded automated probe)",
)
.unwrap()
.with_discovery_limit(2);
for target in [
PhotoTarget::Search("Bee Loto Botanical Garden São Paulo".to_string()),
PhotoTarget::Category("Featured pictures on Wikimedia Commons".to_string()),
PhotoTarget::Top,
PhotoTarget::Latest,
] {
let result = forge.discover_detailed(&target).unwrap();
assert_eq!(result.candidate_budget, 6);
assert!(result.candidate_count <= result.candidate_budget);
assert!(result.photos.len() <= 2);
}
}
}