soundforge 0.2.2

//! License-aware discovery and verified archival of reusable audio.

use std::collections::{BTreeMap, BTreeSet};
use std::error::Error as StdError;
use std::fmt;
use std::fs::{self, File};
use std::io::{self, Read, Write};
use std::path::{Component, Path, PathBuf};
use std::thread;
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};

use blake3::Hasher as Blake3;
use cap_std::ambient_authority;
use cap_std::fs::Dir as CapDir;
use reqwest::blocking::{Client, Response};
use reqwest::redirect::Policy;
use serde::{Deserialize, Serialize};
use sha1::{Digest, Sha1};

const COMMONS_API: &str = "https://commons.wikimedia.org/w/api.php";
const COMMONS_HOST: &str = "commons.wikimedia.org";
const COMMONS_DOWNLOAD_HOST: &str = "upload.wikimedia.org";
const DEFAULT_DISCOVERY_LIMIT: usize = 20;
const MAX_DISCOVERY_LIMIT: usize = 50;
const DEFAULT_MAX_DOWNLOAD_BYTES: u64 = 512 * 1024 * 1024;
const MAX_API_RESPONSE_BYTES: u64 = 16 * 1024 * 1024;
const MAX_ERROR_RESPONSE_BYTES: u64 = 8 * 1024;
const MAX_MANIFEST_BYTES: u64 = 2 * 1024 * 1024;
const PROGRESS_GRANULARITY: u64 = 1024 * 1024;
const MAX_API_ATTEMPTS: usize = 3;
const MAX_API_RETRY_WAIT: Duration = Duration::from_secs(5);
const MAX_API_CONTINUATIONS: usize = 10;
const MAX_MAGIC_BYTES: usize = 256 * 1024;
const MANIFEST_NAME: &str = "soundforge-audio.json";

/// Wikimedia Commons reuse guidance.
pub const COMMONS_REUSE_URL: &str =
    "https://commons.wikimedia.org/wiki/Commons:Reusing_content_outside_Wikimedia";

/// Package-included guide for agents integrating SoundForge.
pub const AGENT_GUIDE: &str = include_str!("../AGENT_GUIDE.md");

/// Result type returned by SoundForge.
pub type Result<T> = std::result::Result<T, SoundForgeError>;

/// SoundForge failure.
#[derive(Debug, thiserror::Error)]
pub enum SoundForgeError {
    /// Invalid discovery target.
    #[error("invalid audio target '{0}'")]
    InvalidTarget(String),
    /// Invalid option.
    #[error("invalid audio option '{0}'")]
    InvalidOption(String),
    /// Provider API failure.
    #[error("{provider} API returned {status}: {body}")]
    ProviderApi {
        /// Provider slug.
        provider: &'static str,
        /// HTTP status.
        status: u16,
        /// Bounded response body.
        body: String,
    },
    /// Provider Action API error envelope.
    #[error("{provider} API error {code}: {info}")]
    ProviderApiError {
        /// Provider slug.
        provider: &'static str,
        /// Stable provider error code.
        code: String,
        /// Provider diagnostic.
        info: String,
    },
    /// Exact file was not found or was not acceptable audio.
    #[error("{provider} audio '{audio}' was not found with an allowed license")]
    NotFound {
        /// Provider slug.
        provider: &'static str,
        /// Requested audio identifier.
        audio: String,
    },
    /// Download URL was outside the provider policy.
    #[error("refusing untrusted audio URL '{0}'")]
    UntrustedUrl(String),
    /// Unsafe local path.
    #[error("refusing unsafe audio path '{0}'")]
    UnsafePath(String),
    /// Download exceeded its cap.
    #[error("audio download is {size} bytes, exceeding the {limit} byte limit")]
    DownloadLimitExceeded {
        /// Observed bytes.
        size: u64,
        /// Configured cap.
        limit: u64,
    },
    /// Response exceeded an internal safety limit.
    #[error("{label} is {size} bytes, exceeding the {limit} byte limit")]
    ResponseLimitExceeded {
        /// Response label.
        label: String,
        /// Observed bytes.
        size: u64,
        /// Configured cap.
        limit: u64,
    },
    /// Downloaded or archived data failed integrity checks.
    #[error("audio verification failed for '{path}': {message}")]
    VerificationFailed {
        /// Relative path.
        path: String,
        /// Failure details.
        message: String,
    },
    /// Existing output was not created by SoundForge.
    #[error("refusing to replace unrecognized output '{}': {reason}", path.display())]
    UnrecognizedExistingOutput {
        /// Existing path.
        path: PathBuf,
        /// Validation failure.
        reason: String,
    },
    /// Archive staging, replacement, or rollback failure.
    #[error("archive transaction failed for '{}': {message}", path.display())]
    ArchiveTransaction {
        /// Destination path.
        path: PathBuf,
        /// Transaction and recovery details.
        message: String,
    },
    /// Output exists and is not a directory.
    #[error("output path '{}' is not a directory", .0.display())]
    OutputNotDirectory(PathBuf),
    /// Filesystem failure.
    #[error("I/O error: {0}")]
    Io(#[from] io::Error),
    /// HTTP failure.
    #[error("request error: {0}")]
    Request(String),
    /// JSON failure.
    #[error("JSON error: {0}")]
    Json(#[from] serde_json::Error),
}

impl From<reqwest::Error> for SoundForgeError {
    fn from(error: reqwest::Error) -> Self {
        Self::Request(error_chain(&error))
    }
}

/// Supported audio providers.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum SoundProvider {
    /// Wikimedia Commons through the Action API.
    WikimediaCommons,
}

impl SoundProvider {
    /// Stable provider list.
    pub const ALL: [Self; 1] = [Self::WikimediaCommons];

    /// Provider slug.
    pub fn slug(self) -> &'static str {
        match self {
            Self::WikimediaCommons => "wikimedia-commons",
        }
    }

    /// Human label.
    pub fn label(self) -> &'static str {
        match self {
            Self::WikimediaCommons => "Wikimedia Commons",
        }
    }
}

impl fmt::Display for SoundProvider {
    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
        formatter.write_str(self.slug())
    }
}

/// Intended audio use, used to tune searches and ranking.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum SoundKind {
    /// Songs, instrumental music, and musical performances.
    Music,
    /// Sound effects, ambience, UI sounds, and other game-oriented audio.
    GameAudio,
    /// Any reusable audio.
    Any,
}

impl SoundKind {
    /// Stable slug.
    pub fn slug(self) -> &'static str {
        match self {
            Self::Music => "music",
            Self::GameAudio => "game-audio",
            Self::Any => "any",
        }
    }
}

impl fmt::Display for SoundKind {
    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
        formatter.write_str(self.slug())
    }
}

/// Audio discovery target.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum SoundTarget {
    /// Exact Commons file title.
    File(String),
    /// Free-text search.
    Search(String),
    /// Exact Commons category name without the `Category:` prefix.
    Category(String),
    /// Widely reused audio.
    Top,
    /// Recently uploaded audio.
    Latest,
}

impl SoundTarget {
    /// Whether this target can return multiple files.
    pub fn is_collection(&self) -> bool {
        !matches!(self, Self::File(_))
    }
}

/// Parse a SoundForge target.
pub fn parse_sound_target(input: &str) -> Result<SoundTarget> {
    let input = input.trim();
    if input.is_empty() {
        return Err(SoundForgeError::InvalidTarget("empty target".to_string()));
    }
    match input.to_ascii_lowercase().as_str() {
        "top" | "popular" | "trending" => return Ok(SoundTarget::Top),
        "latest" | "newest" | "new" => return Ok(SoundTarget::Latest),
        _ => {}
    }
    for (prefix, constructor) in [
        ("file:", SoundTarget::File as fn(String) -> SoundTarget),
        ("search:", SoundTarget::Search),
        ("category:", SoundTarget::Category),
    ] {
        if input
            .get(..prefix.len())
            .is_some_and(|value| value.eq_ignore_ascii_case(prefix))
        {
            return non_empty_target(&input[prefix.len()..], constructor);
        }
    }
    if input.contains("://") {
        return parse_commons_url(input);
    }
    Ok(SoundTarget::Search(input.to_string()))
}

/// License and attribution evidence.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct SoundRights {
    /// Normalized license identifier.
    pub license: String,
    /// License URL.
    pub license_url: String,
    /// Sanitized creator attribution.
    pub attribution: String,
    /// Sanitized artist metadata as supplied by the provider.
    #[serde(default)]
    pub artist: String,
    /// Sanitized provider credit line.
    #[serde(default)]
    pub credit: String,
    /// HTTPS URL retained from provider credit markup, when available.
    #[serde(default)]
    pub credit_url: Option<String>,
    /// Sanitized provider usage terms.
    #[serde(default)]
    pub usage_terms: String,
    /// Raw provider license label, converted to plain text.
    #[serde(default)]
    pub raw_license_label: String,
    /// Whether attribution is legally required.
    pub attribution_required: bool,
    /// Raw provider attribution-required metadata.
    #[serde(default)]
    pub raw_attribution_required: String,
    /// UNIX timestamp when this license metadata was retrieved.
    #[serde(default)]
    pub metadata_retrieved_at: u64,
    /// Provider reuse guidance.
    pub provider_terms_url: String,
}

/// Original provider audio file.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct SoundFile {
    /// Original download URL.
    pub url: String,
    /// Exact provider byte size.
    pub size: u64,
    /// Provider SHA-1 digest.
    pub sha1: String,
    /// MIME type.
    pub mime: String,
    /// Duration in milliseconds.
    pub duration_ms: u64,
}

/// Search result and archival metadata.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct SoundSummary {
    /// Provider.
    pub provider: SoundProvider,
    /// Stable provider page id.
    pub id: String,
    /// Commons file title.
    pub title: String,
    /// Sanitized description.
    pub description: String,
    /// Commons categories.
    pub categories: Vec<String>,
    /// Cross-wiki usage count used as a popularity signal.
    pub usage_count: u64,
    /// Whether pagination completed for the cross-wiki usage count.
    #[serde(default)]
    pub usage_count_complete: bool,
    /// Human-facing source page.
    pub source_url: String,
    /// License evidence.
    pub rights: SoundRights,
    /// Original audio file.
    pub file: SoundFile,
}

/// Archive options.
#[derive(Debug, Clone)]
pub struct ArchiveOptions {
    /// Root directory containing one directory per audio file.
    pub output: PathBuf,
    /// Verify and skip matching complete archives.
    pub skip_existing: bool,
    /// Per-file byte cap. Zero disables the cap.
    pub max_download_bytes: u64,
}

impl Default for ArchiveOptions {
    fn default() -> Self {
        Self {
            output: PathBuf::from("audio"),
            skip_existing: false,
            max_download_bytes: DEFAULT_MAX_DOWNLOAD_BYTES,
        }
    }
}

/// Per-file archive failure.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ArchiveFailure {
    /// Provider page id.
    pub sound_id: String,
    /// Diagnostic.
    pub message: String,
}

/// Successful archive record.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ArchivedSound {
    /// Provider page id.
    pub sound_id: String,
    /// Local output directory.
    pub output: PathBuf,
    /// Downloaded bytes.
    pub bytes_written: u64,
}

/// Aggregate archive result.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct ArchiveSummary {
    /// Selected files.
    pub selected: usize,
    /// Newly archived files.
    pub archived: usize,
    /// Verified existing files.
    pub skipped: usize,
    /// Downloaded bytes.
    pub bytes_written: u64,
    /// Successful records.
    pub sounds: Vec<ArchivedSound>,
    /// Failures.
    pub failed: Vec<ArchiveFailure>,
}

impl ArchiveSummary {
    /// Whether every selected sound succeeded or was skipped.
    pub fn is_success(&self) -> bool {
        self.failed.is_empty()
    }
}

/// Progress phase.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ArchiveProgressPhase {
    /// Preparing output.
    Starting,
    /// Streaming an audio file.
    Downloading,
    /// All selected files were attempted.
    Completed,
}

/// Byte-level archive progress.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ArchiveProgress {
    /// Phase.
    pub phase: ArchiveProgressPhase,
    /// Selected files.
    pub total_sounds: usize,
    /// Completed files.
    pub completed_sounds: usize,
    /// Successful files.
    pub succeeded_sounds: usize,
    /// Failed files.
    pub failed_sounds: usize,
    /// Total retained and active bytes.
    pub bytes_written: u64,
    /// Active sound title.
    pub active_sound: Option<String>,
    /// Active relative file.
    pub active_file: Option<String>,
    /// Active downloaded bytes.
    pub active_bytes_written: u64,
    /// Expected active bytes.
    pub active_total_bytes: Option<u64>,
}

/// Per-audio archive manifest.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct SoundManifest {
    /// Manifest format version.
    pub format_version: String,
    /// Archived metadata.
    pub sound: SoundSummary,
    /// Local relative file path.
    pub path: String,
    /// Actual bytes.
    pub bytes_written: u64,
    /// Local BLAKE3 digest.
    pub blake3: String,
    /// Complete marker.
    pub complete: bool,
}

/// Blocking SoundForge client.
#[derive(Clone)]
pub struct SoundForge {
    provider: SoundProvider,
    client: Client,
    discovery_limit: usize,
}

impl SoundForge {
    /// Build a Wikimedia-compatible client with a descriptive User-Agent.
    pub fn with_user_agent(user_agent: &str) -> Result<Self> {
        let user_agent = user_agent.trim();
        if user_agent.is_empty() {
            return Err(SoundForgeError::InvalidOption(
                "User-Agent must not be empty".to_string(),
            ));
        }
        let client = Client::builder()
            .user_agent(user_agent)
            .redirect(Policy::none())
            .timeout(Duration::from_secs(90))
            .build()?;
        Ok(Self {
            provider: SoundProvider::WikimediaCommons,
            client,
            discovery_limit: DEFAULT_DISCOVERY_LIMIT,
        })
    }

    /// Set result limit from 1 through 50.
    pub fn with_discovery_limit(mut self, limit: usize) -> Self {
        self.discovery_limit = limit.clamp(1, MAX_DISCOVERY_LIMIT);
        self
    }

    /// Discover reusable audio.
    pub fn discover(&self, target: &SoundTarget, kind: SoundKind) -> Result<Vec<SoundSummary>> {
        match target {
            SoundTarget::File(title) => Ok(vec![self.commons_file(title)?]),
            _ => self.commons_search(target, kind),
        }
    }

    /// Archive selected audio.
    pub fn archive(
        &self,
        sounds: &[SoundSummary],
        options: &ArchiveOptions,
    ) -> Result<ArchiveSummary> {
        self.archive_inner(sounds, options, None)
    }

    /// Archive selected audio with byte-level progress.
    pub fn archive_with_progress<F>(
        &self,
        sounds: &[SoundSummary],
        options: &ArchiveOptions,
        mut progress: F,
    ) -> Result<ArchiveSummary>
    where
        F: FnMut(ArchiveProgress),
    {
        self.archive_inner(sounds, options, Some(&mut progress))
    }

    fn commons_search(&self, target: &SoundTarget, kind: SoundKind) -> Result<Vec<SoundSummary>> {
        let query = match target {
            SoundTarget::Search(value) => {
                format!("{} filetype:audio", literal_search_query(value))
            }
            SoundTarget::Category(value) => {
                format!(
                    "incategory:\"{}\" filetype:audio",
                    escape_search_phrase(value)
                )
            }
            SoundTarget::Top => match kind {
                SoundKind::Music => "incategory:\"Music\" filetype:audio".to_string(),
                SoundKind::GameAudio => "incategory:\"Sound effects\" filetype:audio".to_string(),
                SoundKind::Any => "filetype:audio".to_string(),
            },
            SoundTarget::Latest => "filetype:audio".to_string(),
            SoundTarget::File(_) => unreachable!(),
        };
        let candidate_limit = match target {
            SoundTarget::Search(_) => self.discovery_limit.saturating_mul(3).min(50),
            _ => self.discovery_limit,
        };
        let sort = match target {
            SoundTarget::Top => "incoming_links_desc",
            SoundTarget::Latest => "create_timestamp_desc",
            _ => "relevance",
        };
        let mut url = commons_api_url();
        append_common_query(&mut url);
        url.query_pairs_mut()
            .append_pair("generator", "search")
            .append_pair("gsrsearch", &query)
            .append_pair("gsrnamespace", "6")
            .append_pair("gsrlimit", &candidate_limit.to_string())
            .append_pair("gsrsort", sort);
        let response = self.get_commons_pages(&url)?;
        let mut pages = response.pages;
        sort_generator_pages(&mut pages);
        let retrieved_at = unix_timestamp()?;
        let mut sounds = pages
            .into_iter()
            .filter_map(|page| {
                commons_summary(page, response.usage_count_complete, retrieved_at).transpose()
            })
            .collect::<Result<Vec<_>>>()?;
        rank_sounds(&mut sounds, target, kind);
        sounds.truncate(self.discovery_limit);
        Ok(sounds)
    }

    fn commons_file(&self, title: &str) -> Result<SoundSummary> {
        let title = normalized_file_title(title);
        let mut url = commons_api_url();
        append_common_query(&mut url);
        url.query_pairs_mut()
            .append_pair("titles", &title)
            .append_pair("redirects", "1");
        let response = self.get_commons_pages(&url)?;
        let page = response
            .pages
            .into_iter()
            .next()
            .ok_or_else(|| SoundForgeError::NotFound {
                provider: self.provider.slug(),
                audio: title.clone(),
            })?;
        commons_summary(page, response.usage_count_complete, unix_timestamp()?)?.ok_or_else(|| {
            SoundForgeError::NotFound {
                provider: self.provider.slug(),
                audio: title,
            }
        })
    }

    fn get_commons_pages(&self, base_url: &reqwest::Url) -> Result<CommonsPages> {
        let mut pages = Vec::new();
        let mut continuation: BTreeMap<String, serde_json::Value> = BTreeMap::new();
        let mut usage_count_complete = true;

        for request_index in 0..MAX_API_CONTINUATIONS {
            let mut url = base_url.clone();
            for (key, value) in &continuation {
                let rendered;
                let value = if let Some(value) = value.as_str() {
                    value
                } else if value.is_number() || value.is_boolean() {
                    rendered = value.to_string();
                    &rendered
                } else {
                    return Err(SoundForgeError::ProviderApiError {
                        provider: self.provider.slug(),
                        code: "invalid-continuation".to_string(),
                        info: format!("unsupported continuation value for '{key}'"),
                    });
                };
                url.query_pairs_mut().append_pair(key, value);
            }
            let response: CommonsResponse = self.get_json(url.as_str())?;
            if let Some(query) = response.query {
                merge_commons_pages(&mut pages, query.pages);
            }
            if response.continuation.is_empty()
                || !has_metadata_continuation(&response.continuation)
            {
                return Ok(CommonsPages {
                    pages,
                    usage_count_complete,
                });
            }
            continuation = response.continuation;
            if request_index + 1 == MAX_API_CONTINUATIONS {
                usage_count_complete = !continuation.contains_key("gucontinue");
            }
        }

        Ok(CommonsPages {
            pages,
            usage_count_complete,
        })
    }

    fn archive_inner(
        &self,
        sounds: &[SoundSummary],
        options: &ArchiveOptions,
        mut progress: Option<&mut dyn FnMut(ArchiveProgress)>,
    ) -> Result<ArchiveSummary> {
        prepare_output(&options.output)?;
        let mut summary = ArchiveSummary {
            selected: sounds.len(),
            ..ArchiveSummary::default()
        };
        emit_progress(
            &mut progress,
            progress_snapshot(ArchiveProgressPhase::Starting, sounds.len(), &summary),
        );

        for sound in sounds {
            let result = self.archive_one(sound, options, sounds.len(), &summary, &mut progress);
            match result {
                Ok(Some(archived)) => {
                    summary.archived += 1;
                    summary.bytes_written =
                        summary.bytes_written.saturating_add(archived.bytes_written);
                    summary.sounds.push(archived);
                }
                Ok(None) => summary.skipped += 1,
                Err(error) => summary.failed.push(ArchiveFailure {
                    sound_id: sound.id.clone(),
                    message: error.to_string(),
                }),
            }
            emit_progress(
                &mut progress,
                progress_snapshot(ArchiveProgressPhase::Downloading, sounds.len(), &summary),
            );
        }
        emit_progress(
            &mut progress,
            progress_snapshot(ArchiveProgressPhase::Completed, sounds.len(), &summary),
        );
        Ok(summary)
    }

    fn archive_one(
        &self,
        sound: &SoundSummary,
        options: &ArchiveOptions,
        total_sounds: usize,
        summary: &ArchiveSummary,
        progress: &mut Option<&mut dyn FnMut(ArchiveProgress)>,
    ) -> Result<Option<ArchivedSound>> {
        if sound.provider != self.provider {
            return Err(SoundForgeError::VerificationFailed {
                path: sound.id.clone(),
                message: "sound metadata belongs to another provider".to_string(),
            });
        }
        let authoritative = self.commons_file(&sound.title)?;
        if authoritative.id != sound.id {
            return Err(SoundForgeError::VerificationFailed {
                path: sound.id.clone(),
                message: "Commons page identity changed during license revalidation".to_string(),
            });
        }
        let sound = &authoritative;
        validate_summary(sound)?;
        if options.max_download_bytes > 0 && sound.file.size > options.max_download_bytes {
            return Err(SoundForgeError::DownloadLimitExceeded {
                size: sound.file.size,
                limit: options.max_download_bytes,
            });
        }
        let sound_dir = options.output.join(safe_id(&sound.id)?);
        let mut recognized_existing = false;
        if options.skip_existing && sound_dir.join(MANIFEST_NAME).exists() {
            let manifest = read_manifest(&sound_dir).map_err(|error| {
                SoundForgeError::UnrecognizedExistingOutput {
                    path: sound_dir.clone(),
                    reason: error.to_string(),
                }
            })?;
            recognize_manifest(&manifest, &sound.id).map_err(|error| {
                SoundForgeError::UnrecognizedExistingOutput {
                    path: sound_dir.clone(),
                    reason: error.to_string(),
                }
            })?;
            recognized_existing = true;
            if same_archive_metadata(&manifest.sound, sound)
                && validate_manifest(&manifest, &sound.id).is_ok()
                && verify_manifest_file(&sound_dir, &manifest).is_ok()
            {
                return Ok(None);
            }
        }
        if !recognized_existing {
            ensure_replaceable(&sound_dir, &sound.id)?;
        }
        let staging = staging_path(&options.output, &sound.id)?;
        fs::create_dir(&staging)?;
        let archive_result = (|| -> Result<ArchivedSound> {
            let extension = extension_for(&sound.file.mime, &sound.file.url)?;
            let relative = format!("audio.{extension}");
            let output = staging.join(&relative);
            let downloaded = download_file(
                &self.client,
                sound,
                &output,
                options.max_download_bytes,
                |active_bytes| {
                    emit_progress(
                        progress,
                        ArchiveProgress {
                            phase: ArchiveProgressPhase::Downloading,
                            total_sounds,
                            completed_sounds: summary.archived
                                + summary.skipped
                                + summary.failed.len(),
                            succeeded_sounds: summary.archived,
                            failed_sounds: summary.failed.len(),
                            bytes_written: summary.bytes_written.saturating_add(active_bytes),
                            active_sound: Some(sound.title.clone()),
                            active_file: Some(relative.clone()),
                            active_bytes_written: active_bytes,
                            active_total_bytes: Some(sound.file.size),
                        },
                    );
                },
            )?;
            let manifest = SoundManifest {
                format_version: "1".to_string(),
                sound: sound.clone(),
                path: relative,
                bytes_written: downloaded.bytes,
                blake3: downloaded.blake3,
                complete: true,
            };
            write_manifest(&staging, &manifest)?;
            validate_manifest(&manifest, &sound.id)?;
            verify_manifest_file(&staging, &manifest)?;
            Ok(ArchivedSound {
                sound_id: sound.id.clone(),
                output: sound_dir.clone(),
                bytes_written: downloaded.bytes,
            })
        })();
        match archive_result {
            Ok(archived) => {
                commit_staging(&staging, &sound_dir)?;
                Ok(Some(archived))
            }
            Err(error) => {
                if let Err(cleanup) = fs::remove_dir_all(&staging) {
                    return Err(SoundForgeError::ArchiveTransaction {
                        path: sound_dir,
                        message: format!(
                            "{error}; additionally failed to clean staging '{}': {cleanup}",
                            staging.display()
                        ),
                    });
                }
                Err(error)
            }
        }
    }

    fn get_json<T>(&self, url: &str) -> Result<T>
    where
        T: for<'de> Deserialize<'de>,
    {
        trusted_api_url(url)?;
        let started = Instant::now();
        for attempt in 0..MAX_API_ATTEMPTS {
            let response = self.client.get(url).send()?;
            let retry_after = numeric_retry_after(&response);
            if !response.status().is_success() {
                let status = response.status().as_u16();
                if retryable_http_status(status) && attempt + 1 < MAX_API_ATTEMPTS {
                    wait_before_retry(started, retry_delay(attempt, retry_after, None));
                    continue;
                }
                let body = bounded_error_body(response);
                if let Ok(Some(error)) = api_envelope_error(body.as_bytes()) {
                    return Err(SoundForgeError::ProviderApiError {
                        provider: self.provider.slug(),
                        code: error.code,
                        info: error.info,
                    });
                }
                return Err(SoundForgeError::ProviderApi {
                    provider: self.provider.slug(),
                    status,
                    body,
                });
            }
            let bytes =
                read_response_limited(response, MAX_API_RESPONSE_BYTES, "provider response")?;
            if let Some(error) = api_envelope_error(&bytes)? {
                if error.code.eq_ignore_ascii_case("maxlag") && attempt + 1 < MAX_API_ATTEMPTS {
                    wait_before_retry(
                        started,
                        retry_delay(attempt, retry_after, error.lag.map(duration_from_lag)),
                    );
                    continue;
                }
                return Err(SoundForgeError::ProviderApiError {
                    provider: self.provider.slug(),
                    code: error.code,
                    info: error.info,
                });
            }
            return Ok(serde_json::from_slice(&bytes)?);
        }
        unreachable!("bounded API attempt loop always returns")
    }
}

#[derive(Debug, Deserialize)]
struct CommonsResponse {
    query: Option<CommonsQuery>,
    #[serde(default, rename = "continue")]
    continuation: BTreeMap<String, serde_json::Value>,
}

#[derive(Debug, Deserialize)]
struct CommonsQuery {
    #[serde(default)]
    pages: Vec<CommonsPage>,
    #[serde(default)]
    #[allow(dead_code)]
    redirects: Vec<CommonsRedirect>,
}

#[derive(Debug, Deserialize)]
struct CommonsRedirect {
    #[allow(dead_code)]
    from: String,
    #[allow(dead_code)]
    to: String,
}

#[derive(Debug, Deserialize)]
struct CommonsPage {
    pageid: Option<u64>,
    title: String,
    #[serde(default)]
    index: Option<u64>,
    #[serde(default)]
    missing: bool,
    #[serde(default)]
    imageinfo: Vec<CommonsImageInfo>,
    #[serde(default)]
    categories: Vec<CommonsCategory>,
    #[serde(default)]
    globalusage: Vec<CommonsUsage>,
}

#[derive(Debug, Deserialize)]
struct CommonsCategory {
    title: String,
}

#[derive(Debug, Deserialize)]
struct CommonsUsage {
    title: String,
    #[serde(default)]
    wiki: String,
}

#[derive(Debug, Deserialize)]
struct CommonsImageInfo {
    size: u64,
    #[serde(default)]
    duration: f64,
    url: String,
    descriptionurl: String,
    sha1: String,
    mime: String,
    mediatype: String,
    #[serde(default)]
    extmetadata: BTreeMap<String, CommonsMetadataValue>,
}

#[derive(Debug, Deserialize)]
struct CommonsMetadataValue {
    #[serde(default)]
    value: String,
}

struct CommonsPages {
    pages: Vec<CommonsPage>,
    usage_count_complete: bool,
}

struct ApiEnvelopeError {
    code: String,
    info: String,
    lag: Option<f64>,
}

struct AllowedLicense {
    id: String,
    url: String,
    attribution_required: bool,
}

fn merge_commons_pages(pages: &mut Vec<CommonsPage>, incoming: Vec<CommonsPage>) {
    for mut page in incoming {
        let existing = pages.iter_mut().find(|existing| {
            existing.pageid == page.pageid
                && (existing.pageid.is_some() || existing.title == page.title)
        });
        let Some(existing) = existing else {
            pages.push(page);
            continue;
        };
        if existing.imageinfo.is_empty() {
            existing.imageinfo = page.imageinfo;
        }
        if existing.index.is_none() {
            existing.index = page.index;
        }
        for category in page.categories.drain(..) {
            if !existing
                .categories
                .iter()
                .any(|item| item.title == category.title)
            {
                existing.categories.push(category);
            }
        }
        for usage in page.globalusage.drain(..) {
            if !existing
                .globalusage
                .iter()
                .any(|item| item.wiki == usage.wiki && item.title == usage.title)
            {
                existing.globalusage.push(usage);
            }
        }
    }
}

fn has_metadata_continuation(continuation: &BTreeMap<String, serde_json::Value>) -> bool {
    continuation.contains_key("clcontinue") || continuation.contains_key("gucontinue")
}

fn sort_generator_pages(pages: &mut [CommonsPage]) {
    pages.sort_by_key(|page| page.index.unwrap_or(u64::MAX));
}

fn same_archive_metadata(archived: &SoundSummary, current: &SoundSummary) -> bool {
    let mut archived = archived.clone();
    archived.rights.metadata_retrieved_at = current.rights.metadata_retrieved_at;
    archived.usage_count = current.usage_count;
    archived.usage_count_complete = current.usage_count_complete;
    archived == *current
}

fn commons_summary(
    page: CommonsPage,
    usage_count_complete: bool,
    metadata_retrieved_at: u64,
) -> Result<Option<SoundSummary>> {
    if page.missing {
        return Ok(None);
    }
    let Some(pageid) = page.pageid else {
        return Ok(None);
    };
    let Some(info) = page.imageinfo.into_iter().next() else {
        return Ok(None);
    };
    if info.mediatype != "AUDIO" || info.size == 0 || !supported_audio_mime(&info.mime) {
        return Ok(None);
    }
    trusted_download_url(&info.url)?;
    let source_url = reqwest::Url::parse(&info.descriptionurl).map_err(|_| {
        SoundForgeError::VerificationFailed {
            path: page.title.clone(),
            message: "Commons source URL is invalid".to_string(),
        }
    })?;
    if !secure_https_url(&source_url) || source_url.host_str() != Some(COMMONS_HOST) {
        return Err(SoundForgeError::VerificationFailed {
            path: page.title.clone(),
            message: "Commons source URL is outside the trusted origin".to_string(),
        });
    }
    let license_name = metadata_value(&info.extmetadata, "LicenseShortName");
    let license_url = metadata_value(&info.extmetadata, "LicenseUrl");
    let Some(license) = allowed_license(&license_name, &license_url) else {
        return Ok(None);
    };
    if info.sha1.len() != 40 || !info.sha1.bytes().all(|byte| byte.is_ascii_hexdigit()) {
        return Err(SoundForgeError::VerificationFailed {
            path: page.title,
            message: "Commons SHA-1 is not 40 hexadecimal characters".to_string(),
        });
    }
    let raw_artist = metadata_value(&info.extmetadata, "Artist");
    let raw_credit = metadata_value(&info.extmetadata, "Credit");
    let artist = html_to_text(&raw_artist);
    let credit = html_to_text(&raw_credit);
    let raw_attribution_required =
        html_to_text(&metadata_value(&info.extmetadata, "AttributionRequired"));
    let attribution_required = license.attribution_required
        || matches!(
            raw_attribution_required.to_ascii_lowercase().as_str(),
            "true" | "yes" | "1"
        );
    if attribution_required
        && !usable_attribution_text(&artist)
        && !usable_attribution_text(&credit)
    {
        return Ok(None);
    }
    let attribution = if usable_attribution_text(&credit) {
        credit.clone()
    } else if usable_attribution_text(&artist) {
        format!("{artist} via Wikimedia Commons")
    } else {
        "Wikimedia Commons public-domain source".to_string()
    };
    let description = html_to_text(&metadata_value(&info.extmetadata, "ImageDescription"));
    let usage_terms = html_to_text(&metadata_value(&info.extmetadata, "UsageTerms"));
    let categories = page
        .categories
        .into_iter()
        .map(|category| {
            category
                .title
                .strip_prefix("Category:")
                .unwrap_or(&category.title)
                .to_string()
        })
        .collect::<Vec<_>>();
    Ok(Some(SoundSummary {
        provider: SoundProvider::WikimediaCommons,
        id: pageid.to_string(),
        title: page.title,
        description,
        categories,
        usage_count: page.globalusage.len() as u64,
        usage_count_complete,
        source_url: source_url.to_string(),
        rights: SoundRights {
            license: license.id,
            license_url: license.url,
            attribution,
            artist,
            credit,
            credit_url: extract_credit_url(&raw_credit),
            usage_terms,
            raw_license_label: html_to_text(&license_name),
            attribution_required,
            raw_attribution_required,
            metadata_retrieved_at,
            provider_terms_url: COMMONS_REUSE_URL.to_string(),
        },
        file: SoundFile {
            url: info.url,
            size: info.size,
            sha1: info.sha1.to_ascii_lowercase(),
            mime: info.mime,
            duration_ms: (info.duration.max(0.0) * 1000.0).round() as u64,
        },
    }))
}

fn allowed_license(name: &str, url: &str) -> Option<AllowedLicense> {
    let normalized = html_to_text(name).to_ascii_lowercase();
    if normalized == "cc0" || normalized.starts_with("cc0 ") {
        return Some(AllowedLicense {
            id: "CC0-1.0".to_string(),
            url: canonical_cc_url(url, "/publicdomain/zero/1.0/")?,
            attribution_required: false,
        });
    }
    if matches!(
        normalized.as_str(),
        "public domain mark 1.0" | "pdm 1.0" | "pdm-1.0"
    ) {
        return Some(AllowedLicense {
            id: "PDM-1.0".to_string(),
            url: canonical_cc_url(url, "/publicdomain/mark/1.0/")?,
            attribution_required: false,
        });
    }
    if normalized == "public domain" {
        return Some(AllowedLicense {
            id: "Public-Domain".to_string(),
            url: canonical_public_domain_url(url)?,
            attribution_required: false,
        });
    }
    for (prefix, id) in [("cc by-sa ", "CC-BY-SA"), ("cc by ", "CC-BY")] {
        if let Some(version) = normalized.strip_prefix(prefix) {
            if matches!(version, "2.0" | "2.5" | "3.0" | "4.0") {
                let family = if id == "CC-BY-SA" { "by-sa" } else { "by" };
                return Some(AllowedLicense {
                    id: format!("{id}-{version}"),
                    url: canonical_cc_url(url, &format!("/licenses/{family}/{version}/"))?,
                    attribution_required: true,
                });
            }
        }
    }
    None
}

fn canonical_cc_url(value: &str, expected_path: &str) -> Option<String> {
    if value.trim().is_empty() {
        return Some(format!("https://creativecommons.org{expected_path}"));
    }
    let plain = html_to_text(value);
    let url = reqwest::Url::parse(&plain).ok()?;
    if !matches!(url.scheme(), "http" | "https")
        || !url.username().is_empty()
        || url.password().is_some()
        || url.port().is_some_and(|port| {
            (url.scheme() == "http" && port != 80) || (url.scheme() == "https" && port != 443)
        })
        || !matches!(
            url.host_str(),
            Some("creativecommons.org" | "www.creativecommons.org")
        )
        || url.path().trim_end_matches('/') != expected_path.trim_end_matches('/')
        || url.query().is_some()
        || url.fragment().is_some()
    {
        return None;
    }
    Some(format!("https://creativecommons.org{expected_path}"))
}

fn canonical_public_domain_url(value: &str) -> Option<String> {
    if value.trim().is_empty() {
        return Some(String::new());
    }
    let plain = html_to_text(value);
    if let Some(url) = canonical_cc_url(&plain, "/publicdomain/mark/1.0/") {
        return Some(url);
    }
    let url = reqwest::Url::parse(&plain).ok()?;
    secure_https_url(&url).then_some(plain)
}

fn rank_sounds(sounds: &mut Vec<SoundSummary>, target: &SoundTarget, kind: SoundKind) {
    match target {
        SoundTarget::Search(query) => {
            sounds.retain(|sound| relevance_score(sound, query, kind).is_some());
            sounds.sort_by(|left, right| {
                relevance_score(right, query, kind)
                    .cmp(&relevance_score(left, query, kind))
                    .then_with(|| right.usage_count.cmp(&left.usage_count))
            });
        }
        SoundTarget::Category(_) | SoundTarget::Top => sounds.sort_by(|left, right| {
            kind_signal_score(right, kind)
                .cmp(&kind_signal_score(left, kind))
                .then_with(|| right.usage_count.cmp(&left.usage_count))
        }),
        SoundTarget::Latest | SoundTarget::File(_) => {}
    }
}

fn relevance_score(sound: &SoundSummary, query: &str, kind: SoundKind) -> Option<i64> {
    let query = normalized_text(query);
    let terms = query.split_whitespace().collect::<BTreeSet<_>>();
    if terms.is_empty() {
        return None;
    }
    let normalized_title = normalized_text(&sound.title);
    let title = normalized_title
        .strip_prefix("file ")
        .unwrap_or(&normalized_title);
    let categories = normalized_text(&sound.categories.join(" "));
    let description = normalized_text(&sound.description);
    let all = format!("{title} {categories} {description}");
    let all_terms = all.split_whitespace().collect::<BTreeSet<_>>();
    let title_terms = title.split_whitespace().collect::<BTreeSet<_>>();
    let category_terms = categories.split_whitespace().collect::<BTreeSet<_>>();
    if !terms.iter().all(|term| all_terms.contains(term)) {
        return None;
    }
    let mut score: i64 = if title == query {
        600
    } else if contains_phrase(title, &query) {
        500
    } else if terms.iter().all(|term| title_terms.contains(term)) {
        450
    } else if terms
        .iter()
        .all(|term| title_terms.contains(term) || category_terms.contains(term))
    {
        400
    } else if terms.iter().any(|term| title_terms.contains(term)) {
        300
    } else {
        100
    };
    score += kind_signal_score(sound, kind);
    Some(score)
}

fn kind_signal_score(sound: &SoundSummary, kind: SoundKind) -> i64 {
    let title = normalized_text(&sound.title);
    let categories = normalized_text(&sound.categories.join(" "));
    let description = normalized_text(&sound.description);
    let title_and_categories = format!("{title} {categories}");
    let all = format!("{title_and_categories} {description}");
    match kind {
        SoundKind::Music => {
            if [
                "music",
                "song",
                "instrumental",
                "musical performance",
                "orchestra",
                "symphony",
            ]
            .iter()
            .any(|signal| contains_phrase(&title_and_categories, signal))
            {
                40
            } else {
                0
            }
        }
        SoundKind::GameAudio => {
            let positive = [
                "sound effect",
                "sound effects",
                "video game",
                "game sound",
                "ambience",
                "ambient sound",
                "ui sound",
                "gravity sound",
            ]
            .iter()
            .any(|signal| contains_phrase(&title_and_categories, signal));
            let negative = [
                "science",
                "scientific",
                "speech",
                "pronunciation",
                "audiobook",
                "lecture",
                "spoken word",
                "news",
                "radio",
                "interview",
                "podcast",
            ]
            .iter()
            .any(|signal| contains_phrase(&all, signal));
            let long_form = sound.file.duration_ms > 120_000;
            i64::from(positive) * 75 - i64::from(negative) * 100 - i64::from(long_form) * 50
        }
        SoundKind::Any => 0,
    }
}

fn contains_phrase(haystack: &str, phrase: &str) -> bool {
    let haystack = format!(" {haystack} ");
    let phrase = format!(" {phrase} ");
    haystack.contains(&phrase)
}

fn commons_api_url() -> reqwest::Url {
    reqwest::Url::parse(COMMONS_API).expect("static Commons API URL")
}

fn append_common_query(url: &mut reqwest::Url) {
    url.query_pairs_mut()
        .append_pair("action", "query")
        .append_pair("prop", "imageinfo|categories|globalusage")
        .append_pair(
            "iiprop",
            "url|size|sha1|mime|mediatype|extmetadata|commonmetadata",
        )
        .append_pair(
            "iiextmetadatafilter",
            "Artist|LicenseShortName|LicenseUrl|UsageTerms|AttributionRequired|Categories|ImageDescription|Credit",
        )
        .append_pair("cllimit", "max")
        .append_pair("gulimit", "max")
        .append_pair("format", "json")
        .append_pair("formatversion", "2")
        .append_pair("maxlag", "5");
}

fn parse_commons_url(input: &str) -> Result<SoundTarget> {
    let url = reqwest::Url::parse(input)
        .map_err(|_| SoundForgeError::InvalidTarget(input.to_string()))?;
    if !secure_https_url(&url) || url.host_str() != Some(COMMONS_HOST) {
        return Err(SoundForgeError::InvalidTarget(input.to_string()));
    }
    let segments = url
        .path_segments()
        .map(|segments| segments.filter(|part| !part.is_empty()).collect::<Vec<_>>())
        .unwrap_or_default();
    if segments.len() != 2 || segments[0] != "wiki" {
        return Err(SoundForgeError::InvalidTarget(input.to_string()));
    }
    let title = urlencoding::decode(segments[1])
        .map_err(|_| SoundForgeError::InvalidTarget(input.to_string()))?
        .into_owned();
    if !title.starts_with("File:") {
        return Err(SoundForgeError::InvalidTarget(input.to_string()));
    }
    non_empty_target(&title, SoundTarget::File)
}

fn non_empty_target(value: &str, constructor: fn(String) -> SoundTarget) -> Result<SoundTarget> {
    let value = value.trim();
    if value.is_empty() {
        Err(SoundForgeError::InvalidTarget(value.to_string()))
    } else {
        Ok(constructor(value.to_string()))
    }
}

fn normalized_file_title(value: &str) -> String {
    let value = value.trim().replace(' ', "_");
    if value.to_ascii_lowercase().starts_with("file:") {
        format!("File:{}", &value[5..])
    } else {
        format!("File:{value}")
    }
}

fn escape_search_phrase(value: &str) -> String {
    value.replace(['"', '\\'], " ")
}

fn literal_search_query(value: &str) -> String {
    value
        .split_whitespace()
        .map(|term| {
            let escaped = term.replace('\\', "\\\\").replace('"', "\\\"");
            format!("\"{escaped}\"")
        })
        .collect::<Vec<_>>()
        .join(" ")
}

fn metadata_value(metadata: &BTreeMap<String, CommonsMetadataValue>, key: &str) -> String {
    metadata
        .get(key)
        .map(|value| value.value.clone())
        .unwrap_or_default()
}

fn html_to_text(value: &str) -> String {
    html2text::from_read(value.as_bytes(), 10_000)
        .split_whitespace()
        .collect::<Vec<_>>()
        .join(" ")
}

fn usable_attribution_text(value: &str) -> bool {
    let normalized = normalized_text(value);
    !normalized.is_empty()
        && !matches!(
            normalized.as_str(),
            "unknown" | "unknown author" | "unknown creator" | "none" | "n a" | "not provided"
        )
}

fn extract_credit_url(value: &str) -> Option<String> {
    for scheme in ["https://", "http://"] {
        let Some(start) = value.find(scheme) else {
            continue;
        };
        let candidate = &value[start..];
        let end = candidate
            .find(|character: char| {
                character.is_whitespace() || matches!(character, '"' | '\'' | '<' | '>')
            })
            .unwrap_or(candidate.len());
        let plain = html_to_text(&candidate[..end]);
        let Ok(url) = reqwest::Url::parse(&plain) else {
            continue;
        };
        if secure_https_url(&url) {
            return Some(url.to_string());
        }
    }
    None
}

fn normalized_text(value: &str) -> String {
    value
        .chars()
        .map(|character| {
            if character.is_alphanumeric() {
                character.to_ascii_lowercase()
            } else {
                ' '
            }
        })
        .collect::<String>()
        .split_whitespace()
        .collect::<Vec<_>>()
        .join(" ")
}

fn validate_summary(sound: &SoundSummary) -> Result<()> {
    let source_url = reqwest::Url::parse(&sound.source_url).ok();
    let credit_url_valid = sound
        .rights
        .credit_url
        .as_ref()
        .is_none_or(|value| reqwest::Url::parse(value).is_ok_and(|url| secure_https_url(&url)));
    if sound.provider != SoundProvider::WikimediaCommons
        || sound.id.parse::<u64>().is_err()
        || source_url
            .as_ref()
            .is_none_or(|url| !secure_https_url(url) || url.host_str() != Some(COMMONS_HOST))
        || sound.rights.attribution.trim().is_empty()
        || sound.rights.raw_license_label.trim().is_empty()
        || sound.rights.metadata_retrieved_at == 0
        || (sound.rights.attribution_required
            && !usable_attribution_text(&sound.rights.artist)
            && !usable_attribution_text(&sound.rights.credit))
        || !credit_url_valid
        || sound.rights.provider_terms_url != COMMONS_REUSE_URL
        || sound.file.size == 0
        || sound.file.sha1.len() != 40
        || !sound.file.sha1.bytes().all(|byte| byte.is_ascii_hexdigit())
        || !allowed_license_id(&sound.rights.license, &sound.rights.license_url)
        || !supported_audio_mime(&sound.file.mime)
    {
        return Err(SoundForgeError::VerificationFailed {
            path: sound.id.clone(),
            message: "sound identity, license, or file metadata is invalid".to_string(),
        });
    }
    trusted_download_url(&sound.file.url)?;
    Ok(())
}

fn allowed_license_id(id: &str, license_url: &str) -> bool {
    let expected_path = match id {
        "CC0-1.0" => Some("/publicdomain/zero/1.0/".to_string()),
        "PDM-1.0" => Some("/publicdomain/mark/1.0/".to_string()),
        "Public-Domain" => {
            return license_url.is_empty()
                || reqwest::Url::parse(license_url).is_ok_and(|url| secure_https_url(&url));
        }
        _ => [("CC-BY-SA-", "by-sa"), ("CC-BY-", "by")]
            .into_iter()
            .find_map(|(prefix, family)| {
                id.strip_prefix(prefix).and_then(|version| {
                    matches!(version, "2.0" | "2.5" | "3.0" | "4.0")
                        .then(|| format!("/licenses/{family}/{version}/"))
                })
            }),
    };
    expected_path.is_some_and(|path| {
        canonical_cc_url(license_url, &path).is_some_and(|canonical| canonical == license_url)
    })
}

fn trusted_api_url(url: &str) -> Result<reqwest::Url> {
    let url =
        reqwest::Url::parse(url).map_err(|_| SoundForgeError::UntrustedUrl(url.to_string()))?;
    if !secure_https_url(&url) || url.host_str() != Some(COMMONS_HOST) || url.path() != "/w/api.php"
    {
        return Err(SoundForgeError::UntrustedUrl(url.to_string()));
    }
    Ok(url)
}

fn trusted_download_url(url: &str) -> Result<reqwest::Url> {
    let url =
        reqwest::Url::parse(url).map_err(|_| SoundForgeError::UntrustedUrl(url.to_string()))?;
    if !secure_https_url(&url) || url.host_str() != Some(COMMONS_DOWNLOAD_HOST) {
        return Err(SoundForgeError::UntrustedUrl(url.to_string()));
    }
    Ok(url)
}

fn secure_https_url(url: &reqwest::Url) -> bool {
    url.scheme() == "https"
        && url.port().is_none_or(|port| port == 443)
        && url.username().is_empty()
        && url.password().is_none()
}

struct DownloadedFile {
    bytes: u64,
    blake3: String,
}

fn download_file<F>(
    client: &Client,
    sound: &SoundSummary,
    output: &Path,
    max_download_bytes: u64,
    mut progress: F,
) -> Result<DownloadedFile>
where
    F: FnMut(u64),
{
    trusted_download_url(&sound.file.url)?;
    if !supported_audio_mime(&sound.file.mime) {
        return Err(SoundForgeError::VerificationFailed {
            path: sound.id.clone(),
            message: format!("unsupported audio MIME type '{}'", sound.file.mime),
        });
    }
    let mut response = client.get(&sound.file.url).send()?;
    trusted_download_url(response.url().as_str())?;
    if !response.status().is_success() {
        let status = response.status().as_u16();
        return Err(SoundForgeError::ProviderApi {
            provider: sound.provider.slug(),
            status,
            body: bounded_error_body(response),
        });
    }
    if response
        .content_length()
        .is_some_and(|length| length != sound.file.size)
    {
        return Err(SoundForgeError::VerificationFailed {
            path: sound.id.clone(),
            message: "server Content-Length differs from Commons metadata".to_string(),
        });
    }
    let partial = partial_path(output);
    let mut file = File::create(&partial)?;
    let mut sha1 = Sha1::new();
    let mut blake3 = Blake3::new();
    let mut bytes_written = 0u64;
    let mut last_emit = 0u64;
    let mut magic = Vec::with_capacity(
        usize::try_from(sound.file.size)
            .unwrap_or(MAX_MAGIC_BYTES)
            .min(MAX_MAGIC_BYTES),
    );
    let mut buffer = [0u8; 64 * 1024];
    loop {
        let read = match response.read(&mut buffer) {
            Ok(read) => read,
            Err(error) => {
                drop(file);
                let _ = fs::remove_file(&partial);
                return Err(SoundForgeError::Request(error_chain(&error)));
            }
        };
        if read == 0 {
            break;
        }
        file.write_all(&buffer[..read])?;
        if magic.len() < MAX_MAGIC_BYTES {
            let retained = (MAX_MAGIC_BYTES - magic.len()).min(read);
            magic.extend_from_slice(&buffer[..retained]);
        }
        sha1.update(&buffer[..read]);
        blake3.update(&buffer[..read]);
        bytes_written = bytes_written.saturating_add(read as u64);
        if bytes_written > sound.file.size
            || (max_download_bytes > 0 && bytes_written > max_download_bytes)
        {
            drop(file);
            let _ = fs::remove_file(&partial);
            return Err(SoundForgeError::DownloadLimitExceeded {
                size: bytes_written,
                limit: if max_download_bytes == 0 {
                    sound.file.size
                } else {
                    max_download_bytes.min(sound.file.size)
                },
            });
        }
        if bytes_written.saturating_sub(last_emit) >= PROGRESS_GRANULARITY {
            progress(bytes_written);
            last_emit = bytes_written;
        }
    }
    if bytes_written != sound.file.size {
        drop(file);
        let _ = fs::remove_file(&partial);
        return Err(SoundForgeError::VerificationFailed {
            path: sound.id.clone(),
            message: format!(
                "expected {} bytes but received {bytes_written}",
                sound.file.size
            ),
        });
    }
    if !valid_audio_magic(&sound.file.mime, &magic) {
        drop(file);
        let _ = fs::remove_file(&partial);
        return Err(SoundForgeError::VerificationFailed {
            path: sound.id.clone(),
            message: format!(
                "downloaded bytes do not match audio MIME type '{}'",
                sound.file.mime
            ),
        });
    }
    let provider_sha1 = format!("{:x}", sha1.finalize());
    if !provider_sha1.eq_ignore_ascii_case(&sound.file.sha1) {
        drop(file);
        let _ = fs::remove_file(&partial);
        return Err(SoundForgeError::VerificationFailed {
            path: sound.id.clone(),
            message: format!(
                "expected Commons SHA-1 {} but received {provider_sha1}",
                sound.file.sha1
            ),
        });
    }
    file.flush()?;
    file.sync_all()?;
    drop(file);
    if last_emit != bytes_written {
        progress(bytes_written);
    }
    fs::rename(&partial, output)?;
    Ok(DownloadedFile {
        bytes: bytes_written,
        blake3: blake3.finalize().to_hex().to_string(),
    })
}

fn valid_audio_magic(mime: &str, bytes: &[u8]) -> bool {
    match mime {
        "audio/mpeg" => valid_mp3_frames(bytes),
        "audio/ogg" | "application/ogg" => bytes.starts_with(b"OggS"),
        "audio/flac" | "audio/x-flac" => bytes.starts_with(b"fLaC"),
        "audio/wav" | "audio/x-wav" | "audio/wave" => {
            bytes.starts_with(b"RIFF") && bytes.get(8..12) == Some(b"WAVE")
        }
        "audio/webm" => bytes.starts_with(&[0x1a, 0x45, 0xdf, 0xa3]),
        "audio/mp4" | "audio/x-m4a" => bytes.get(4..8) == Some(b"ftyp"),
        _ => false,
    }
}

fn supported_audio_mime(mime: &str) -> bool {
    matches!(
        mime,
        "audio/mpeg"
            | "audio/ogg"
            | "application/ogg"
            | "audio/flac"
            | "audio/x-flac"
            | "audio/wav"
            | "audio/x-wav"
            | "audio/wave"
            | "audio/webm"
            | "audio/mp4"
            | "audio/x-m4a"
    )
}

#[derive(Clone, Copy, PartialEq, Eq)]
struct Mp3FrameHeader {
    signature: (u8, u8, u8),
    length: usize,
}

fn valid_mp3_frames(bytes: &[u8]) -> bool {
    let offset = if bytes.starts_with(b"ID3") {
        let Some(header) = bytes.get(..10) else {
            return false;
        };
        if header[3] == 0xff
            || header[4] == 0xff
            || header[6..10].iter().any(|byte| byte & 0x80 != 0)
        {
            return false;
        }
        let tag_size = header[6..10]
            .iter()
            .fold(0usize, |size, byte| (size << 7) | usize::from(*byte));
        10usize
            .saturating_add(tag_size)
            .saturating_add(usize::from(header[5] & 0x10 != 0) * 10)
    } else {
        0
    };
    let Some(first_bytes) = bytes.get(offset..) else {
        return false;
    };
    let Some(first) = mp3_frame_header(first_bytes) else {
        return false;
    };
    let second_offset = offset.saturating_add(first.length);
    let Some(second_bytes) = bytes.get(second_offset..) else {
        return false;
    };
    mp3_frame_header(second_bytes)
        .is_some_and(|second| first.signature == second.signature && second.length > 0)
}

fn mp3_frame_header(bytes: &[u8]) -> Option<Mp3FrameHeader> {
    let header = bytes.get(..4)?;
    if header[0] != 0xff || header[1] & 0xe0 != 0xe0 {
        return None;
    }
    let version = (header[1] >> 3) & 0x03;
    let layer = (header[1] >> 1) & 0x03;
    let bitrate_index = (header[2] >> 4) as usize;
    let sample_rate_index = (header[2] >> 2) & 0x03;
    if version == 1
        || layer == 0
        || bitrate_index == 0
        || bitrate_index == 15
        || sample_rate_index == 3
        || header[3] & 0x03 == 2
    {
        return None;
    }
    const MPEG1_LAYER1: [u32; 15] = [
        0, 32, 64, 96, 128, 160, 192, 224, 256, 288, 320, 352, 384, 416, 448,
    ];
    const MPEG1_LAYER2: [u32; 15] = [
        0, 32, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320, 384,
    ];
    const MPEG1_LAYER3: [u32; 15] = [
        0, 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320,
    ];
    const MPEG2_LAYER1: [u32; 15] = [
        0, 32, 48, 56, 64, 80, 96, 112, 128, 144, 160, 176, 192, 224, 256,
    ];
    const MPEG2_LAYER23: [u32; 15] = [0, 8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 144, 160];
    let bitrate_kbps = match (version, layer) {
        (3, 3) => MPEG1_LAYER1[bitrate_index],
        (3, 2) => MPEG1_LAYER2[bitrate_index],
        (3, 1) => MPEG1_LAYER3[bitrate_index],
        (_, 3) => MPEG2_LAYER1[bitrate_index],
        _ => MPEG2_LAYER23[bitrate_index],
    };
    let base_sample_rate = [44_100u32, 48_000, 32_000][sample_rate_index as usize];
    let sample_rate = match version {
        3 => base_sample_rate,
        2 => base_sample_rate / 2,
        0 => base_sample_rate / 4,
        _ => return None,
    };
    let bitrate = bitrate_kbps * 1_000;
    let padding = u32::from(header[2] & 0x02 != 0);
    let length = if layer == 3 {
        ((12 * bitrate / sample_rate) + padding) * 4
    } else if layer == 1 && version != 3 {
        72 * bitrate / sample_rate + padding
    } else {
        144 * bitrate / sample_rate + padding
    };
    Some(Mp3FrameHeader {
        signature: (version, layer, sample_rate_index),
        length: length as usize,
    })
}

fn extension_for(mime: &str, _url: &str) -> Result<&'static str> {
    match mime {
        "audio/mpeg" => Ok("mp3"),
        "audio/ogg" | "application/ogg" => Ok("ogg"),
        "audio/flac" | "audio/x-flac" => Ok("flac"),
        "audio/wav" | "audio/x-wav" | "audio/wave" => Ok("wav"),
        "audio/webm" => Ok("webm"),
        "audio/mp4" | "audio/x-m4a" => Ok("m4a"),
        _ => Err(SoundForgeError::InvalidOption(format!(
            "unsupported audio MIME type '{mime}'"
        ))),
    }
}

fn prepare_output(output: &Path) -> Result<()> {
    if output.exists() && !output.is_dir() {
        return Err(SoundForgeError::OutputNotDirectory(output.to_path_buf()));
    }
    fs::create_dir_all(output)?;
    Ok(())
}

fn safe_id(value: &str) -> Result<String> {
    if value.is_empty() || !value.bytes().all(|byte| byte.is_ascii_digit()) {
        return Err(SoundForgeError::UnsafePath(value.to_string()));
    }
    Ok(value.to_string())
}

fn safe_relative_path(value: &str) -> Result<PathBuf> {
    let path = Path::new(value);
    if path.is_absolute() || value.len() > 256 {
        return Err(SoundForgeError::UnsafePath(value.to_string()));
    }
    let mut safe = PathBuf::new();
    for component in path.components() {
        match component {
            Component::Normal(part) if !part.is_empty() => safe.push(part),
            _ => return Err(SoundForgeError::UnsafePath(value.to_string())),
        }
    }
    if safe.as_os_str().is_empty() {
        return Err(SoundForgeError::UnsafePath(value.to_string()));
    }
    Ok(safe)
}

fn partial_path(path: &Path) -> PathBuf {
    path.with_extension(format!(
        "{}.soundforge-part",
        path.extension()
            .and_then(|value| value.to_str())
            .unwrap_or("audio")
    ))
}

fn staging_path(output: &Path, id: &str) -> Result<PathBuf> {
    let nonce = SystemTime::now()
        .duration_since(UNIX_EPOCH)
        .map_err(|error| SoundForgeError::InvalidOption(error.to_string()))?
        .as_nanos();
    Ok(output.join(format!(
        ".soundforge-{}-{}-{nonce}.staging",
        safe_id(id)?,
        std::process::id()
    )))
}

fn write_manifest(directory: &Path, manifest: &SoundManifest) -> Result<()> {
    let payload = serde_json::to_vec_pretty(manifest)?;
    if payload.len() as u64 > MAX_MANIFEST_BYTES {
        return Err(SoundForgeError::ResponseLimitExceeded {
            label: MANIFEST_NAME.to_string(),
            size: payload.len() as u64,
            limit: MAX_MANIFEST_BYTES,
        });
    }
    let temporary = directory.join(format!("{MANIFEST_NAME}.tmp"));
    let mut file = File::create(&temporary)?;
    file.write_all(&payload)?;
    file.flush()?;
    file.sync_all()?;
    drop(file);
    fs::rename(temporary, directory.join(MANIFEST_NAME))?;
    Ok(())
}

fn read_manifest(directory: &Path) -> Result<SoundManifest> {
    let directory = open_archive_directory(directory)?;
    let metadata = directory.symlink_metadata(MANIFEST_NAME)?;
    if !metadata.is_file() || metadata.file_type().is_symlink() {
        return Err(SoundForgeError::UnsafePath(MANIFEST_NAME.to_string()));
    }
    if metadata.len() > MAX_MANIFEST_BYTES {
        return Err(SoundForgeError::ResponseLimitExceeded {
            label: MANIFEST_NAME.to_string(),
            size: metadata.len(),
            limit: MAX_MANIFEST_BYTES,
        });
    }
    let mut contents = Vec::with_capacity(metadata.len() as usize);
    directory
        .open(MANIFEST_NAME)?
        .take(MAX_MANIFEST_BYTES.saturating_add(1))
        .read_to_end(&mut contents)?;
    Ok(serde_json::from_slice(&contents)?)
}

fn validate_manifest(manifest: &SoundManifest, id: &str) -> Result<()> {
    validate_summary(&manifest.sound)?;
    let valid = manifest.format_version == "1"
        && manifest.complete
        && manifest.sound.id == id
        && manifest.bytes_written == manifest.sound.file.size
        && manifest.blake3.len() == 64
        && manifest.blake3.bytes().all(|byte| byte.is_ascii_hexdigit())
        && safe_relative_path(&manifest.path).is_ok()
        && !manifest.path.eq_ignore_ascii_case(MANIFEST_NAME);
    if !valid {
        return Err(SoundForgeError::VerificationFailed {
            path: MANIFEST_NAME.to_string(),
            message: "manifest identity or integrity metadata is inconsistent".to_string(),
        });
    }
    Ok(())
}

fn recognize_manifest(manifest: &SoundManifest, id: &str) -> Result<()> {
    let source_url = reqwest::Url::parse(&manifest.sound.source_url).ok();
    let recognized = manifest.format_version == "1"
        && manifest.complete
        && manifest.sound.provider == SoundProvider::WikimediaCommons
        && manifest.sound.id == id
        && manifest.sound.id.parse::<u64>().is_ok()
        && manifest.sound.title.starts_with("File:")
        && source_url
            .as_ref()
            .is_some_and(|url| secure_https_url(url) && url.host_str() == Some(COMMONS_HOST))
        && !manifest.sound.rights.attribution.trim().is_empty()
        && manifest.sound.rights.provider_terms_url == COMMONS_REUSE_URL
        && allowed_license_id(
            &manifest.sound.rights.license,
            &manifest.sound.rights.license_url,
        )
        && manifest.sound.file.size > 0
        && manifest.sound.file.sha1.len() == 40
        && manifest
            .sound
            .file
            .sha1
            .bytes()
            .all(|byte| byte.is_ascii_hexdigit())
        && supported_audio_mime(&manifest.sound.file.mime)
        && trusted_download_url(&manifest.sound.file.url).is_ok()
        && manifest.bytes_written == manifest.sound.file.size
        && manifest.blake3.len() == 64
        && manifest.blake3.bytes().all(|byte| byte.is_ascii_hexdigit())
        && safe_relative_path(&manifest.path).is_ok()
        && !manifest.path.eq_ignore_ascii_case(MANIFEST_NAME);
    if !recognized {
        return Err(SoundForgeError::VerificationFailed {
            path: MANIFEST_NAME.to_string(),
            message: "manifest does not safely identify a SoundForge archive".to_string(),
        });
    }
    Ok(())
}

fn verify_manifest_file(directory: &Path, manifest: &SoundManifest) -> Result<()> {
    let directory = open_archive_directory(directory)?;
    let relative = safe_relative_path(&manifest.path)?;
    let metadata = directory.symlink_metadata(&relative)?;
    if !metadata.is_file() || metadata.file_type().is_symlink() {
        return Err(SoundForgeError::UnsafePath(manifest.path.clone()));
    }
    let opened = directory.open(&relative)?;
    let mut opened = opened.take(manifest.bytes_written.saturating_add(1));
    let mut sha1 = Sha1::new();
    let mut blake3 = Blake3::new();
    let mut bytes = 0u64;
    let mut buffer = [0u8; 64 * 1024];
    loop {
        let read = opened.read(&mut buffer)?;
        if read == 0 {
            break;
        }
        bytes = bytes.saturating_add(read as u64);
        sha1.update(&buffer[..read]);
        blake3.update(&buffer[..read]);
    }
    let provider_sha1 = format!("{:x}", sha1.finalize());
    let local_blake3 = blake3.finalize().to_hex().to_string();
    if bytes != manifest.bytes_written
        || !provider_sha1.eq_ignore_ascii_case(&manifest.sound.file.sha1)
        || !local_blake3.eq_ignore_ascii_case(&manifest.blake3)
    {
        return Err(SoundForgeError::VerificationFailed {
            path: manifest.path.clone(),
            message: "archived bytes or checksums do not match the manifest".to_string(),
        });
    }
    Ok(())
}

fn ensure_replaceable(directory: &Path, id: &str) -> Result<()> {
    if !directory.exists() {
        return Ok(());
    }
    let manifest =
        read_manifest(directory).map_err(|error| SoundForgeError::UnrecognizedExistingOutput {
            path: directory.to_path_buf(),
            reason: error.to_string(),
        })?;
    validate_manifest(&manifest, id).map_err(|error| SoundForgeError::UnrecognizedExistingOutput {
        path: directory.to_path_buf(),
        reason: error.to_string(),
    })
}

fn open_archive_directory(path: &Path) -> Result<CapDir> {
    let parent_path = path.parent().unwrap_or_else(|| Path::new("."));
    let name = path
        .file_name()
        .ok_or_else(|| SoundForgeError::UnsafePath(path.display().to_string()))?;
    let parent = CapDir::open_ambient_dir(parent_path, ambient_authority())?;
    let directory = parent.open_dir(name)?;
    if !directory.dir_metadata()?.is_dir() {
        return Err(SoundForgeError::UnsafePath(path.display().to_string()));
    }
    Ok(directory)
}

fn commit_staging(staging: &Path, destination: &Path) -> Result<()> {
    if !destination.exists() {
        return fs::rename(staging, destination).map_err(|error| {
            let cleanup = fs::remove_dir_all(staging).err();
            SoundForgeError::ArchiveTransaction {
                path: destination.to_path_buf(),
                message: transaction_error("failed to install staging", error, cleanup, None),
            }
        });
    }
    let name = destination
        .file_name()
        .and_then(|name| name.to_str())
        .ok_or_else(|| SoundForgeError::UnsafePath(destination.display().to_string()))?;
    let backup =
        destination.with_file_name(format!(".soundforge-{name}-{}-backup", std::process::id()));
    if backup.exists() {
        let cleanup = fs::remove_dir_all(staging).err();
        return Err(SoundForgeError::UnrecognizedExistingOutput {
            path: backup,
            reason: match cleanup {
                Some(error) => format!(
                    "stale replacement backup exists; failed to clean staging '{}': {error}",
                    staging.display()
                ),
                None => "stale replacement backup exists".to_string(),
            },
        });
    }
    if let Err(error) = fs::rename(destination, &backup) {
        let cleanup = fs::remove_dir_all(staging).err();
        return Err(SoundForgeError::ArchiveTransaction {
            path: destination.to_path_buf(),
            message: transaction_error("failed to create replacement backup", error, cleanup, None),
        });
    }
    if let Err(error) = fs::rename(staging, destination) {
        let rollback = fs::rename(&backup, destination).err();
        let cleanup = fs::remove_dir_all(staging).err();
        return Err(SoundForgeError::ArchiveTransaction {
            path: destination.to_path_buf(),
            message: transaction_error("failed to install staging", error, cleanup, rollback),
        });
    }
    fs::remove_dir_all(&backup).map_err(|error| SoundForgeError::ArchiveTransaction {
        path: destination.to_path_buf(),
        message: format!(
            "replacement succeeded but backup '{}' could not be removed: {error}",
            backup.display()
        ),
    })?;
    Ok(())
}

fn transaction_error(
    action: &str,
    error: io::Error,
    cleanup: Option<io::Error>,
    rollback: Option<io::Error>,
) -> String {
    let mut message = format!("{action}: {error}");
    if let Some(error) = rollback {
        message.push_str(&format!("; rollback also failed: {error}"));
    }
    if let Some(error) = cleanup {
        message.push_str(&format!("; staging cleanup also failed: {error}"));
    }
    message
}

fn progress_snapshot(
    phase: ArchiveProgressPhase,
    total: usize,
    summary: &ArchiveSummary,
) -> ArchiveProgress {
    ArchiveProgress {
        phase,
        total_sounds: total,
        completed_sounds: summary.archived + summary.skipped + summary.failed.len(),
        succeeded_sounds: summary.archived,
        failed_sounds: summary.failed.len(),
        bytes_written: summary.bytes_written,
        active_sound: None,
        active_file: None,
        active_bytes_written: 0,
        active_total_bytes: None,
    }
}

fn emit_progress(
    progress: &mut Option<&mut dyn FnMut(ArchiveProgress)>,
    snapshot: ArchiveProgress,
) {
    if let Some(progress) = progress.as_deref_mut() {
        progress(snapshot);
    }
}

fn read_response_limited(mut response: Response, limit: u64, label: &str) -> Result<Vec<u8>> {
    if response
        .content_length()
        .is_some_and(|length| length > limit)
    {
        return Err(SoundForgeError::ResponseLimitExceeded {
            label: label.to_string(),
            size: response.content_length().unwrap_or_default(),
            limit,
        });
    }
    let mut bytes = Vec::new();
    response
        .by_ref()
        .take(limit.saturating_add(1))
        .read_to_end(&mut bytes)?;
    if bytes.len() as u64 > limit {
        return Err(SoundForgeError::ResponseLimitExceeded {
            label: label.to_string(),
            size: bytes.len() as u64,
            limit,
        });
    }
    Ok(bytes)
}

fn api_envelope_error(bytes: &[u8]) -> Result<Option<ApiEnvelopeError>> {
    let value: serde_json::Value = serde_json::from_slice(bytes)?;
    let Some(error) = value.get("error") else {
        return Ok(None);
    };
    let code = error
        .get("code")
        .and_then(serde_json::Value::as_str)
        .unwrap_or("unknown")
        .to_string();
    let info = error
        .get("info")
        .and_then(serde_json::Value::as_str)
        .unwrap_or("provider supplied no error information")
        .to_string();
    let lag = error.get("lag").and_then(|lag| {
        lag.as_f64()
            .or_else(|| lag.as_str().and_then(|value| value.parse::<f64>().ok()))
            .filter(|value| value.is_finite() && *value >= 0.0)
    });
    Ok(Some(ApiEnvelopeError { code, info, lag }))
}

fn retryable_http_status(status: u16) -> bool {
    status == 429 || (500..=599).contains(&status)
}

fn numeric_retry_after(response: &Response) -> Option<Duration> {
    response
        .headers()
        .get(reqwest::header::RETRY_AFTER)
        .and_then(|value| value.to_str().ok())
        .and_then(|value| value.trim().parse::<u64>().ok())
        .map(Duration::from_secs)
}

fn retry_delay(
    attempt: usize,
    retry_after: Option<Duration>,
    provider_lag: Option<Duration>,
) -> Duration {
    retry_after
        .or(provider_lag)
        .unwrap_or_else(|| Duration::from_secs(1 << attempt.min(2)))
        .min(MAX_API_RETRY_WAIT)
}

fn duration_from_lag(lag: f64) -> Duration {
    Duration::from_secs_f64(lag.min(MAX_API_RETRY_WAIT.as_secs_f64()))
}

fn wait_before_retry(started: Instant, delay: Duration) {
    let remaining = MAX_API_RETRY_WAIT.saturating_sub(started.elapsed());
    thread::sleep(delay.min(remaining));
}

fn unix_timestamp() -> Result<u64> {
    SystemTime::now()
        .duration_since(UNIX_EPOCH)
        .map(|duration| duration.as_secs())
        .map_err(|error| SoundForgeError::InvalidOption(error.to_string()))
}

fn bounded_error_body(response: Response) -> String {
    match read_response_limited(
        response,
        MAX_ERROR_RESPONSE_BYTES,
        "provider error response",
    ) {
        Ok(bytes) => String::from_utf8_lossy(&bytes).into_owned(),
        Err(SoundForgeError::ResponseLimitExceeded { .. }) => {
            "<error response exceeded safety limit>".to_string()
        }
        Err(_) => "<unable to read body>".to_string(),
    }
}

fn error_chain(error: &(dyn StdError + 'static)) -> String {
    let mut message = error.to_string();
    let mut source = error.source();
    while let Some(error) = source {
        message.push_str("; caused by: ");
        message.push_str(&error.to_string());
        source = error.source();
    }
    message
}

#[cfg(test)]
mod tests {
    use super::*;
    use tempfile::tempdir;

    fn fixture_sound(
        id: &str,
        title: &str,
        description: &str,
        categories: &[&str],
        usage_count: u64,
    ) -> SoundSummary {
        SoundSummary {
            provider: SoundProvider::WikimediaCommons,
            id: id.to_string(),
            title: title.to_string(),
            description: description.to_string(),
            categories: categories
                .iter()
                .map(|value| (*value).to_string())
                .collect(),
            usage_count,
            usage_count_complete: true,
            source_url: format!("https://commons.wikimedia.org/wiki/{title}"),
            rights: SoundRights {
                license: "CC0-1.0".to_string(),
                license_url: "https://creativecommons.org/publicdomain/zero/1.0/".to_string(),
                attribution: "Fixture creator via Wikimedia Commons".to_string(),
                artist: "Fixture creator".to_string(),
                credit: String::new(),
                credit_url: None,
                usage_terms: "CC0 1.0".to_string(),
                raw_license_label: "CC0 1.0".to_string(),
                attribution_required: false,
                raw_attribution_required: "false".to_string(),
                metadata_retrieved_at: 1,
                provider_terms_url: COMMONS_REUSE_URL.to_string(),
            },
            file: SoundFile {
                url: format!("https://upload.wikimedia.org/{id}.ogg"),
                size: 4,
                sha1: "a".repeat(40),
                mime: "application/ogg".to_string(),
                duration_ms: 1_000,
            },
        }
    }

    fn fixture_page(
        license: &str,
        license_url: &str,
        artist: &str,
        credit: &str,
        mime: &str,
    ) -> CommonsPage {
        let mut extmetadata = BTreeMap::new();
        for (key, value) in [
            ("LicenseShortName", license),
            ("LicenseUrl", license_url),
            ("Artist", artist),
            ("Credit", credit),
            ("UsageTerms", license),
            ("AttributionRequired", "true"),
            ("ImageDescription", "<b>Fixture</b> &amp; description"),
        ] {
            extmetadata.insert(
                key.to_string(),
                CommonsMetadataValue {
                    value: value.to_string(),
                },
            );
        }
        CommonsPage {
            pageid: Some(7),
            title: "File:Canonical.ogg".to_string(),
            index: Some(1),
            missing: false,
            imageinfo: vec![CommonsImageInfo {
                size: 4,
                duration: 1.0,
                url: "https://upload.wikimedia.org/fixture.ogg".to_string(),
                descriptionurl: "https://commons.wikimedia.org/wiki/File:Canonical.ogg".to_string(),
                sha1: "a".repeat(40),
                mime: mime.to_string(),
                mediatype: "AUDIO".to_string(),
                extmetadata,
            }],
            categories: Vec::new(),
            globalusage: Vec::new(),
        }
    }

    #[test]
    fn parses_search_category_file_and_commons_url_targets() {
        assert_eq!(
            parse_sound_target("laser").unwrap(),
            SoundTarget::Search("laser".to_string())
        );
        assert_eq!(
            parse_sound_target("category:Sound effects").unwrap(),
            SoundTarget::Category("Sound effects".to_string())
        );
        assert_eq!(
            parse_sound_target("file:Laser (Gravity Sound).mp3").unwrap(),
            SoundTarget::File("Laser (Gravity Sound).mp3".to_string())
        );
        assert_eq!(
            parse_sound_target(
                "https://commons.wikimedia.org/wiki/File:Laser_%28Gravity_Sound%29.mp3"
            )
            .unwrap(),
            SoundTarget::File("File:Laser_(Gravity_Sound).mp3".to_string())
        );
        assert!(parse_sound_target("https://example.com/audio.mp3").is_err());
    }

    #[test]
    fn accepts_only_commercial_capable_license_families() {
        assert!(allowed_license("CC0 1.0", "").is_some());
        assert!(allowed_license("Public domain", "").is_some());
        assert!(allowed_license("CC BY 4.0", "").is_some());
        assert!(allowed_license("CC BY-SA 3.0", "").is_some());
        assert!(allowed_license("CC BY-NC 4.0", "").is_none());
        assert!(allowed_license("CC BY-ND 4.0", "").is_none());
        assert!(allowed_license("Sampling+", "").is_none());
    }

    #[test]
    fn parses_action_api_errors_without_treating_them_as_empty_queries() {
        let error =
            api_envelope_error(br#"{"error":{"code":"badtitle","info":"The title is invalid"}}"#)
                .unwrap()
                .unwrap();
        assert_eq!(error.code, "badtitle");
        assert_eq!(error.info, "The title is invalid");

        let maxlag = api_envelope_error(
            br#"{"error":{"code":"maxlag","info":"Waiting for replicas","lag":"2.5"}}"#,
        )
        .unwrap()
        .unwrap();
        assert_eq!(maxlag.code, "maxlag");
        assert_eq!(maxlag.lag, Some(2.5));
        assert!(
            api_envelope_error(br#"{"query":{"pages":[]}}"#)
                .unwrap()
                .is_none()
        );
        assert!(retryable_http_status(429));
        assert!(retryable_http_status(503));
        assert!(!retryable_http_status(404));
        assert_eq!(
            retry_delay(0, Some(Duration::from_secs(30)), None),
            MAX_API_RETRY_WAIT
        );
    }

    #[test]
    fn canonicalizes_only_corresponding_creative_commons_urls() {
        let by = allowed_license(
            "CC BY 4.0",
            "http://www.creativecommons.org/licenses/by/4.0/",
        )
        .unwrap();
        assert_eq!(by.id, "CC-BY-4.0");
        assert_eq!(by.url, "https://creativecommons.org/licenses/by/4.0/");
        assert!(
            allowed_license(
                "CC BY 4.0",
                "http://creativecommons.org/licenses/by-sa/4.0/"
            )
            .is_none()
        );

        let public_domain = allowed_license("Public domain", "").unwrap();
        assert_eq!(public_domain.id, "Public-Domain");
        assert_eq!(public_domain.url, "");
        assert_eq!(
            allowed_license("Public domain mark 1.0", "").unwrap().id,
            "PDM-1.0"
        );
    }

    #[test]
    fn exact_lookup_requests_redirects_and_parses_canonical_page_shape() {
        let mut url = commons_api_url();
        append_common_query(&mut url);
        url.query_pairs_mut()
            .append_pair("titles", "File:Alias.ogg")
            .append_pair("redirects", "1");
        assert!(
            url.query_pairs()
                .any(|(key, value)| { key == "redirects" && value == "1" })
        );

        let response: CommonsResponse = serde_json::from_str(
            r#"{
                "continue": {"gsroffset":15,"continue":"-||"},
                "query": {
                    "redirects": [{"from":"File:Alias.ogg","to":"File:Canonical.ogg"}],
                    "pages": [{"pageid":7,"title":"File:Canonical.ogg","index":2}]
                }
            }"#,
        )
        .unwrap();
        assert_eq!(response.continuation["gsroffset"], 15);
        assert!(!has_metadata_continuation(&response.continuation));
        let query = response.query.unwrap();
        assert_eq!(query.redirects[0].to, "File:Canonical.ogg");
        assert_eq!(query.pages[0].title, "File:Canonical.ogg");
        assert_eq!(query.pages[0].pageid, Some(7));
    }

    #[test]
    fn latest_pages_follow_ascending_generator_index() {
        let page = |title: &str, index: Option<u64>| CommonsPage {
            pageid: Some(index.unwrap_or(99)),
            title: title.to_string(),
            index,
            missing: false,
            imageinfo: Vec::new(),
            categories: Vec::new(),
            globalusage: Vec::new(),
        };
        let mut pages = vec![
            page("File:Third.ogg", Some(3)),
            page("File:First.ogg", Some(1)),
            page("File:Unknown.ogg", None),
            page("File:Second.ogg", Some(2)),
        ];
        sort_generator_pages(&mut pages);
        assert_eq!(
            pages
                .iter()
                .map(|page| page.title.as_str())
                .collect::<Vec<_>>(),
            [
                "File:First.ogg",
                "File:Second.ogg",
                "File:Third.ogg",
                "File:Unknown.ogg"
            ]
        );
    }

    #[test]
    fn continuation_pages_merge_without_losing_generator_order_or_usage() {
        assert!(has_metadata_continuation(&BTreeMap::from([(
            "gucontinue".to_string(),
            serde_json::Value::String("7|0".to_string()),
        )])));
        let page = |usages: &[(&str, &str)], categories: &[&str], with_info: bool| CommonsPage {
            pageid: Some(7),
            title: "File:Canonical.ogg".to_string(),
            index: Some(2),
            missing: false,
            imageinfo: if with_info {
                fixture_page("CC0 1.0", "", "Creator", "", "audio/ogg").imageinfo
            } else {
                Vec::new()
            },
            categories: categories
                .iter()
                .map(|title| CommonsCategory {
                    title: (*title).to_string(),
                })
                .collect(),
            globalusage: usages
                .iter()
                .map(|(wiki, title)| CommonsUsage {
                    title: (*title).to_string(),
                    wiki: (*wiki).to_string(),
                })
                .collect(),
        };
        let mut pages = vec![page(
            &[("enwiki", "Laser")],
            &["Category:Sound effects"],
            true,
        )];
        merge_commons_pages(
            &mut pages,
            vec![page(
                &[("enwiki", "Laser"), ("dewiki", "Laser")],
                &["Category:Sound effects", "Category:Games"],
                false,
            )],
        );
        assert_eq!(pages.len(), 1);
        assert_eq!(pages[0].globalusage.len(), 2);
        assert_eq!(pages[0].categories.len(), 2);
        assert_eq!(pages[0].index, Some(2));
        assert_eq!(pages[0].imageinfo.len(), 1);
    }

    #[test]
    fn rejects_attribution_licenses_without_usable_artist_or_credit() {
        let page = fixture_page("CC BY 4.0", "", "Unknown creator", "", "audio/ogg");
        assert!(commons_summary(page, true, 1).unwrap().is_none());

        let page = fixture_page(
            "CC BY-SA 4.0",
            "",
            "",
            "<a href=\"https://example.com/credit\">Fixture credit</a>",
            "audio/ogg",
        );
        let summary = commons_summary(page, true, 1).unwrap().unwrap();
        assert!(summary.rights.attribution_required);
        assert!(summary.rights.credit.contains("Fixture credit"));
        assert_eq!(
            summary.rights.credit_url.as_deref(),
            Some("https://example.com/credit")
        );
    }

    #[test]
    fn free_text_search_is_literal_and_matching_uses_whole_tokens() {
        assert_eq!(
            literal_search_query("laser incategory:Music -speech"),
            "\"laser\" \"incategory:Music\" \"-speech\""
        );
        let sound = fixture_sound("1", "File:Laser.ogg", "", &[], 0);
        assert!(relevance_score(&sound, "laser", SoundKind::Any).is_some());
        assert!(relevance_score(&sound, "las", SoundKind::Any).is_none());
    }

    #[test]
    fn deterministic_kind_ranking_covers_game_science_speech_and_music() {
        let mut game = vec![
            fixture_sound(
                "1",
                "File:Laser scientific recording.ogg",
                "science",
                &[],
                10_000,
            ),
            fixture_sound("2", "File:Laser speech.ogg", "spoken lecture", &[], 20_000),
            fixture_sound("3", "File:Laser blast.ogg", "", &["Sound effects"], 10),
            fixture_sound("4", "File:Laser symphony.ogg", "", &["Music"], 100),
        ];
        rank_sounds(
            &mut game,
            &SoundTarget::Search("laser".to_string()),
            SoundKind::GameAudio,
        );
        assert_eq!(game[0].id, "3");
        assert_eq!(game[2].id, "2");
        assert_eq!(game[3].id, "1");

        let mut music = game.clone();
        rank_sounds(&mut music, &SoundTarget::Top, SoundKind::Music);
        assert_eq!(music[0].id, "4");

        let mut category = vec![
            fixture_sound("5", "File:Lecture.ogg", "speech", &[], 100_000),
            fixture_sound(
                "6",
                "File:Click.ogg",
                "",
                &["UI sounds", "Sound effects"],
                1,
            ),
        ];
        rank_sounds(
            &mut category,
            &SoundTarget::Category("Recordings".to_string()),
            SoundKind::GameAudio,
        );
        assert_eq!(category[0].id, "6");

        let mut radio = fixture_sound(
            "7",
            "File:Laser radio report.ogg",
            "news interview",
            &[],
            100_000,
        );
        radio.file.duration_ms = 180_000;
        let mut effects = vec![
            radio,
            fixture_sound("8", "File:Laser (Gravity Sound).wav", "", &[], 0),
        ];
        rank_sounds(
            &mut effects,
            &SoundTarget::Search("laser".to_string()),
            SoundKind::GameAudio,
        );
        assert_eq!(effects[0].id, "8");
    }

    #[test]
    fn rejects_unsupported_mime_and_requires_strong_audio_magic() {
        assert!(!supported_audio_mime("audio/aac"));
        assert!(extension_for("audio/aac", "https://upload.wikimedia.org/file.mp3").is_err());
        assert!(!valid_audio_magic("audio/aac", b"anything"));
        assert!(valid_audio_magic("audio/ogg", b"OggS"));
        assert!(!valid_audio_magic("audio/ogg", b"RIFF....WAVE"));
        assert!(!valid_audio_magic("audio/mpeg", b"ID3"));

        let header = [0xff, 0xfb, 0x90, 0x00];
        let frame_length = mp3_frame_header(&header).unwrap().length;
        let mut mp3 = vec![0; frame_length * 2];
        mp3[..4].copy_from_slice(&header);
        mp3[frame_length..frame_length + 4].copy_from_slice(&header);
        assert!(valid_audio_magic("audio/mpeg", &mp3));
        assert!(!valid_audio_magic("audio/mpeg", &header));
    }

    #[test]
    fn html_metadata_conversion_decodes_entities_without_merging_tags() {
        assert_eq!(html_to_text("<span>A &amp; <b>B</b></span>"), "A & B");
    }

    #[test]
    fn relevance_beats_usage_count() {
        let sound = |id: &str, title: &str, description: &str, usage_count: u64| SoundSummary {
            provider: SoundProvider::WikimediaCommons,
            id: id.to_string(),
            title: title.to_string(),
            description: description.to_string(),
            categories: Vec::new(),
            usage_count,
            usage_count_complete: true,
            source_url: format!("https://commons.wikimedia.org/wiki/File:{title}"),
            rights: SoundRights {
                license: "CC0-1.0".to_string(),
                license_url: "https://creativecommons.org/publicdomain/zero/1.0/".to_string(),
                attribution: "Creator via Wikimedia Commons".to_string(),
                artist: "Creator".to_string(),
                credit: String::new(),
                credit_url: None,
                usage_terms: "CC0 1.0".to_string(),
                raw_license_label: "CC0 1.0".to_string(),
                attribution_required: false,
                raw_attribution_required: "false".to_string(),
                metadata_retrieved_at: 1,
                provider_terms_url: COMMONS_REUSE_URL.to_string(),
            },
            file: SoundFile {
                url: format!("https://upload.wikimedia.org/{id}.ogg"),
                size: 1,
                sha1: "a".repeat(40),
                mime: "application/ogg".to_string(),
                duration_ms: 1000,
            },
        };
        let mut sounds = vec![
            sound("1", "Laser scientific recording", "science", 10_000),
            sound("2", "Laser sound effect", "game sound", 10),
        ];
        rank_sounds(
            &mut sounds,
            &SoundTarget::Search("laser".to_string()),
            SoundKind::GameAudio,
        );
        assert_eq!(sounds[0].title, "Laser sound effect");
    }

    #[test]
    fn recognized_corrupt_archive_can_be_repaired_transactionally() {
        let temp = tempdir().unwrap();
        let directory = temp.path().join("123");
        fs::create_dir(&directory).unwrap();
        let bytes = b"audio";
        fs::write(directory.join("audio.ogg"), bytes).unwrap();
        let sha1 = format!("{:x}", Sha1::digest(bytes));
        let blake3 = blake3::hash(bytes).to_hex().to_string();
        let manifest = SoundManifest {
            format_version: "1".to_string(),
            sound: SoundSummary {
                provider: SoundProvider::WikimediaCommons,
                id: "123".to_string(),
                title: "File:Audio.ogg".to_string(),
                description: String::new(),
                categories: Vec::new(),
                usage_count: 0,
                usage_count_complete: true,
                source_url: "https://commons.wikimedia.org/wiki/File:Audio.ogg".to_string(),
                rights: SoundRights {
                    license: "CC0-1.0".to_string(),
                    license_url: "https://creativecommons.org/publicdomain/zero/1.0/".to_string(),
                    attribution: "Creator via Wikimedia Commons".to_string(),
                    artist: "Creator".to_string(),
                    credit: String::new(),
                    credit_url: None,
                    usage_terms: "CC0 1.0".to_string(),
                    raw_license_label: "CC0 1.0".to_string(),
                    attribution_required: false,
                    raw_attribution_required: "false".to_string(),
                    metadata_retrieved_at: 1,
                    provider_terms_url: COMMONS_REUSE_URL.to_string(),
                },
                file: SoundFile {
                    url: "https://upload.wikimedia.org/audio.ogg".to_string(),
                    size: bytes.len() as u64,
                    sha1,
                    mime: "application/ogg".to_string(),
                    duration_ms: 1000,
                },
            },
            path: "audio.ogg".to_string(),
            bytes_written: bytes.len() as u64,
            blake3,
            complete: true,
        };
        write_manifest(&directory, &manifest).unwrap();
        verify_manifest_file(&directory, &manifest).unwrap();
        fs::write(directory.join("audio.ogg"), b"evil!").unwrap();
        assert!(verify_manifest_file(&directory, &manifest).is_err());
        recognize_manifest(&manifest, "123").unwrap();

        let staging = temp.path().join("staging");
        fs::create_dir(&staging).unwrap();
        fs::write(staging.join("replacement"), b"repaired").unwrap();
        commit_staging(&staging, &directory).unwrap();
        assert_eq!(
            fs::read(directory.join("replacement")).unwrap(),
            b"repaired"
        );
        assert!(!staging.exists());
    }

    #[test]
    #[ignore = "requires Wikimedia Commons access and downloads one small CC BY sound"]
    fn live_commons_game_audio_archive_and_progress() {
        let forge = SoundForge::with_user_agent(
            "soundforge-live-test/0.1 (https://github.com/Tknott95/SoundForge)",
        )
        .unwrap()
        .with_discovery_limit(5);
        let sounds = forge
            .discover(
                &SoundTarget::Search("laser gravity sound".to_string()),
                SoundKind::GameAudio,
            )
            .unwrap();
        assert!(!sounds.is_empty());
        let music = forge
            .discover(
                &SoundTarget::Search("Mozart symphony".to_string()),
                SoundKind::Music,
            )
            .unwrap();
        assert!(!music.is_empty());
        let selected = sounds
            .iter()
            .find(|sound| sound.title.contains("Gravity Sound"))
            .cloned()
            .unwrap_or_else(|| sounds[0].clone());
        let temp = tempdir().unwrap();
        let options = ArchiveOptions {
            output: temp.path().to_path_buf(),
            skip_existing: false,
            max_download_bytes: 16 * 1024 * 1024,
        };
        let mut saw_bytes = false;
        let summary = forge
            .archive_with_progress(&[selected], &options, |progress| {
                saw_bytes |= progress.active_bytes_written > 0;
            })
            .unwrap();
        assert!(summary.is_success(), "failures: {:?}", summary.failed);
        assert!(saw_bytes);
        let manifest = read_manifest(&summary.sounds[0].output).unwrap();
        assert_eq!(manifest.sound.rights.license, "CC-BY-4.0");
        assert_eq!(manifest.blake3.len(), 64);
    }
}