paperforge 0.1.0

//! Rights-aware discovery and verified local archival of scholarly papers.

use std::cmp::Ordering;
use std::collections::{BTreeSet, HashSet};
use std::fmt;
use std::fs::{self, File};
use std::io::{self, BufReader, Read, Seek, SeekFrom, Write};
use std::path::{Component, Path, PathBuf};
use std::str::FromStr;
use std::sync::{Mutex, MutexGuard, OnceLock};
use std::thread;
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};

use blake3::Hasher as Blake3;
use quick_xml::Reader;
use quick_xml::events::Event;
use reqwest::blocking::{Client, RequestBuilder, Response};
use reqwest::header::{CONTENT_LENGTH, CONTENT_TYPE, RETRY_AFTER};
use reqwest::redirect::Policy;
use serde::{Deserialize, Serialize};
use unicode_normalization::UnicodeNormalization;

const ARXIV_API: &str = "https://export.arxiv.org/api/query";
const EUROPE_PMC_API: &str = "https://www.ebi.ac.uk/europepmc/webservices/rest/search";
const NASA_SEARCH_API: &str = "https://ntrs.nasa.gov/api/citations/search";
const DEFAULT_DISCOVERY_LIMIT: usize = 20;
const MAX_DISCOVERY_LIMIT: usize = 100;
const PROVIDER_CANDIDATE_BUDGET: usize = 100;
const MAX_QUERY_BYTES: usize = 8 * 1024;
const MAX_QUERY_CLAUSES: usize = 64;
const MAX_QUERY_IDENTITIES: usize = 20;
const MAX_QUERY_VALUE_BYTES: usize = 512;
const DEFAULT_MAX_DOWNLOAD_BYTES: u64 = 256 * 1024 * 1024;
const MAX_API_RESPONSE_BYTES: u64 = 16 * 1024 * 1024;
const MAX_ERROR_RESPONSE_BYTES: u64 = 8 * 1024;
const MAX_MANIFEST_BYTES: u64 = 2 * 1024 * 1024;
const MAX_API_ATTEMPTS: usize = 3;
const MAX_RETRY_WAIT: Duration = Duration::from_secs(5);
const ARXIV_INTERVAL: Duration = Duration::from_secs(3);
const PROGRESS_GRANULARITY: u64 = 1024 * 1024;
const MAX_XML_ATTRIBUTES_PER_ELEMENT: usize = 256;
const MAX_JATS_LICENSE_RECORDS: usize = 256;
const MAX_JATS_PERMISSIONS: usize = 256;
const MAX_JATS_EVIDENCE_VALUE_BYTES: usize = 16 * 1024;
const MIN_PDF_BYTES: u64 = 128;
const PDF_TRAILER_WINDOW: u64 = 64 * 1024;
const MANIFEST_NAME: &str = "paperforge-paper.json";

#[derive(Default)]
struct ArxivRequestGate {
    last_start: Option<Instant>,
}

static ARXIV_REQUEST_GATE: OnceLock<Mutex<ArxivRequestGate>> = OnceLock::new();

/// Autonomous-agent operating instructions shipped with the crate.
pub const AGENT_GUIDE: &str = include_str!("../AGENT_GUIDE.md");

/// Result type returned by PaperForge.
pub type Result<T> = std::result::Result<T, PaperForgeError>;

/// PaperForge failure.
#[derive(Debug, thiserror::Error)]
pub enum PaperForgeError {
    /// Query text was invalid or unsafe.
    #[error("invalid paper query: {0}")]
    InvalidQuery(String),
    /// A client option was invalid.
    #[error("invalid option: {0}")]
    InvalidOption(String),
    /// A provider rejected or failed a request.
    #[error("{provider} API returned {status}: {body}")]
    ProviderApi {
        /// Provider slug.
        provider: &'static str,
        /// HTTP status.
        status: u16,
        /// Bounded diagnostic.
        body: String,
    },
    /// A provider returned an explicit error envelope.
    #[error("{provider} API error: {message}")]
    ProviderApiError {
        /// Provider slug.
        provider: &'static str,
        /// Provider diagnostic.
        message: String,
    },
    /// An exact paper or eligible document was not found.
    #[error("{provider} paper '{id}' was not found or is not eligible")]
    NotFound {
        /// Provider slug.
        provider: &'static str,
        /// Requested identity.
        id: String,
    },
    /// A URL was outside a provider's allowlist.
    #[error("refusing untrusted paper URL '{0}'")]
    UntrustedUrl(String),
    /// A local path was unsafe.
    #[error("refusing unsafe paper path '{0}'")]
    UnsafePath(String),
    /// A response exceeded its configured cap.
    #[error("{label} is {size} bytes, exceeding the {limit} byte limit")]
    ResponseLimitExceeded {
        /// Response label.
        label: String,
        /// Observed bytes.
        size: u64,
        /// Allowed bytes.
        limit: u64,
    },
    /// Downloaded or archived bytes failed validation.
    #[error("paper verification failed for '{path}': {message}")]
    VerificationFailed {
        /// Paper identity or path.
        path: String,
        /// Validation detail.
        message: String,
    },
    /// Existing output was not safely attributable to PaperForge.
    #[error("refusing to replace unrecognized output '{}': {reason}", path.display())]
    UnrecognizedExistingOutput {
        /// Existing path.
        path: PathBuf,
        /// Recognition failure.
        reason: String,
    },
    /// Staging, replacement, or rollback failed.
    #[error("archive transaction failed for '{}': {message}", path.display())]
    ArchiveTransaction {
        /// Destination path.
        path: PathBuf,
        /// Transaction detail.
        message: String,
    },
    /// Output exists but is not a directory.
    #[error("output path '{}' is not a directory", .0.display())]
    OutputNotDirectory(PathBuf),
    /// Filesystem failure.
    #[error("I/O error: {0}")]
    Io(#[from] io::Error),
    /// HTTP transport failure.
    #[error("request error: {0}")]
    Request(String),
    /// JSON decoding failure.
    #[error("JSON error: {0}")]
    Json(#[from] serde_json::Error),
    /// XML decoding failure.
    #[error("XML error: {0}")]
    Xml(String),
}

impl From<reqwest::Error> for PaperForgeError {
    fn from(error: reqwest::Error) -> Self {
        Self::Request(error_chain(&error))
    }
}

/// Official paper providers supported by this release.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum PaperProvider {
    /// arXiv Atom API and paper PDFs.
    Arxiv,
    /// Europe PMC core JSON and JATS XML.
    EuropePmc,
    /// NASA Technical Reports Server.
    NasaNtrs,
}

impl PaperProvider {
    /// Stable provider list.
    pub const ALL: [Self; 3] = [Self::Arxiv, Self::EuropePmc, Self::NasaNtrs];

    /// Stable machine slug.
    pub const fn slug(self) -> &'static str {
        match self {
            Self::Arxiv => "arxiv",
            Self::EuropePmc => "europe-pmc",
            Self::NasaNtrs => "nasa-ntrs",
        }
    }

    /// Human-readable label.
    pub const fn label(self) -> &'static str {
        match self {
            Self::Arxiv => "arXiv",
            Self::EuropePmc => "Europe PMC",
            Self::NasaNtrs => "NASA Technical Reports Server",
        }
    }

    /// Compact display label.
    pub const fn short_label(self) -> &'static str {
        match self {
            Self::Arxiv => "arXiv",
            Self::EuropePmc => "Europe PMC",
            Self::NasaNtrs => "NASA NTRS",
        }
    }

    /// Parse a stable provider slug.
    pub fn from_slug(value: &str) -> Option<Self> {
        match value.trim().to_ascii_lowercase().as_str() {
            "arxiv" => Some(Self::Arxiv),
            "europe-pmc" | "europepmc" => Some(Self::EuropePmc),
            "nasa-ntrs" | "ntrs" | "nasa" => Some(Self::NasaNtrs),
            _ => None,
        }
    }
}

impl fmt::Display for PaperProvider {
    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
        formatter.write_str(self.slug())
    }
}

/// Result ordering.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum PaperSort {
    /// Local relevance tiers and weighted score.
    Relevance,
    /// Newest publication date first.
    Date,
    /// Highest real provider citation count first.
    Citations,
}

impl PaperSort {
    /// Stable machine slug.
    pub const fn slug(self) -> &'static str {
        match self {
            Self::Relevance => "relevance",
            Self::Date => "date",
            Self::Citations => "citations",
        }
    }

    /// Human-readable label.
    pub const fn display(self) -> &'static str {
        match self {
            Self::Relevance => "Relevance",
            Self::Date => "Newest",
            Self::Citations => "Most cited",
        }
    }
}

impl FromStr for PaperSort {
    type Err = PaperForgeError;

    fn from_str(value: &str) -> Result<Self> {
        match value.trim().to_ascii_lowercase().as_str() {
            "relevance" | "relevant" => Ok(Self::Relevance),
            "date" | "newest" | "recent" => Ok(Self::Date),
            "citations" | "cited" | "top" => Ok(Self::Citations),
            _ => Err(PaperForgeError::InvalidQuery(format!(
                "unknown sort '{value}'"
            ))),
        }
    }
}

impl fmt::Display for PaperSort {
    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
        formatter.write_str(self.slug())
    }
}

/// Search mode used when no strict sort is supplied.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum PaperMode {
    /// Relevance-first discovery.
    #[default]
    Standard,
    /// Relevance tier followed by date.
    Recent,
    /// Relevance tier followed by citations.
    Top,
}

/// Text field addressed by a clause.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum TextField {
    /// All searchable text.
    Any,
    /// Paper title.
    Title,
    /// Abstract text.
    Abstract,
    /// Author names.
    Author,
    /// Provider category or topic.
    Topic,
    /// Journal or venue.
    Journal,
}

/// Clause inclusion behavior.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum ClausePolarity {
    /// Optional positive evidence.
    Optional,
    /// Required positive evidence.
    Required,
    /// Excluded evidence.
    Excluded,
}

/// Typed text query clause.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct TextClause {
    /// Addressed metadata field.
    pub field: TextField,
    /// Inclusion behavior.
    pub polarity: ClausePolarity,
    /// Unescaped user value.
    pub value: String,
    /// Whether the value was quoted and must be contiguous.
    pub phrase: bool,
}

/// Exact identity kind.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum IdentityKind {
    /// Version-preserving arXiv identifier.
    Arxiv,
    /// PubMed numeric identifier.
    Pmid,
    /// PubMed Central identifier.
    Pmcid,
    /// NASA NTRS citation identifier.
    Ntrs,
    /// Digital Object Identifier.
    Doi,
}

/// Exact paper identity.
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub struct PaperIdentity {
    /// Identity namespace.
    pub kind: IdentityKind,
    /// Canonical value within that namespace.
    pub value: String,
}

/// Parsed, provider-independent paper query.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct PaperQuery {
    /// Typed text clauses.
    pub clauses: Vec<TextClause>,
    /// Exact identities, matched as alternatives.
    pub identities: Vec<PaperIdentity>,
    /// Inclusive lower publication-date bound (`YYYY-MM-DD`).
    pub after: Option<String>,
    /// Inclusive upper publication-date bound (`YYYY-MM-DD`).
    pub before: Option<String>,
    /// Relevance/date/citation mode.
    pub mode: PaperMode,
    /// Strict explicit sort override.
    pub sort: Option<PaperSort>,
    /// Optional result limit from 1 through 100.
    pub limit: Option<usize>,
    /// Optional provider restriction.
    pub provider: Option<PaperProvider>,
    /// Original query text retained as evidence.
    pub original: String,
}

#[derive(Debug)]
struct Lexeme {
    text: String,
    quoted: bool,
}

/// Parse the PaperForge query language into typed clauses and directives.
///
/// # Examples
///
/// ```
/// let query = paperforge::parse_paper_query("graph networks @limit:10")?;
/// assert_eq!(query.limit, Some(10));
/// # Ok::<(), paperforge::PaperForgeError>(())
/// ```
pub fn parse_paper_query(input: &str) -> Result<PaperQuery> {
    let input = input.trim();
    if input.is_empty() {
        return Err(PaperForgeError::InvalidQuery("query is empty".to_string()));
    }
    if input.len() > MAX_QUERY_BYTES {
        return Err(PaperForgeError::InvalidQuery(format!(
            "query exceeds the {MAX_QUERY_BYTES} byte limit"
        )));
    }
    let mut query = PaperQuery {
        original: input.to_string(),
        ..PaperQuery::default()
    };
    for lexeme in lex_query(input)? {
        parse_lexeme(&mut query, lexeme)?;
    }
    if let (Some(after), Some(before)) = (&query.after, &query.before) {
        if after > before {
            return Err(PaperForgeError::InvalidQuery(format!(
                "after date {after} is later than before date {before}"
            )));
        }
    }
    let has_positive = query
        .clauses
        .iter()
        .any(|clause| clause.polarity != ClausePolarity::Excluded);
    let has_negative = query
        .clauses
        .iter()
        .any(|clause| clause.polarity == ClausePolarity::Excluded);
    let bounded = has_positive
        || !query.identities.is_empty()
        || query.after.is_some()
        || query.before.is_some()
        || query.mode != PaperMode::Standard;
    if has_negative && !bounded {
        return Err(PaperForgeError::InvalidQuery(
            "excluded clauses cannot be the only search constraint".to_string(),
        ));
    }
    if query.clauses.is_empty()
        && query.identities.is_empty()
        && query.after.is_none()
        && query.before.is_none()
        && query.mode == PaperMode::Standard
    {
        return Err(PaperForgeError::InvalidQuery(
            "query has no discovery constraint".to_string(),
        ));
    }
    validate_query_structure(&query)?;
    Ok(query)
}

fn lex_query(input: &str) -> Result<Vec<Lexeme>> {
    let mut result = Vec::new();
    let mut text = String::new();
    let mut in_quote = false;
    let mut quoted = false;
    let mut escaped = false;
    for character in input.chars() {
        if escaped {
            text.push(character);
            escaped = false;
            continue;
        }
        if in_quote && character == '\\' {
            escaped = true;
            continue;
        }
        if character == '"' {
            in_quote = !in_quote;
            quoted = true;
            continue;
        }
        if character.is_whitespace() && !in_quote {
            if !text.is_empty() {
                result.push(Lexeme {
                    text: std::mem::take(&mut text),
                    quoted,
                });
                quoted = false;
            }
        } else {
            text.push(character);
        }
    }
    if in_quote || escaped {
        return Err(PaperForgeError::InvalidQuery(
            "unterminated quoted phrase".to_string(),
        ));
    }
    if !text.is_empty() {
        result.push(Lexeme { text, quoted });
    }
    Ok(result)
}

fn parse_lexeme(query: &mut PaperQuery, lexeme: Lexeme) -> Result<()> {
    if lexeme.text.starts_with('@') {
        if lexeme.quoted {
            return Err(PaperForgeError::InvalidQuery(
                "directives cannot be quoted".to_string(),
            ));
        }
        return parse_directive(query, &lexeme.text);
    }
    let (polarity, token) = match lexeme.text.as_bytes().first() {
        Some(b'+') => (ClausePolarity::Required, &lexeme.text[1..]),
        Some(b'-') => (ClausePolarity::Excluded, &lexeme.text[1..]),
        _ => (ClausePolarity::Optional, lexeme.text.as_str()),
    };
    if token.is_empty() {
        return Err(PaperForgeError::InvalidQuery(
            "empty signed clause".to_string(),
        ));
    }
    if token.contains("://") {
        if polarity == ClausePolarity::Excluded {
            return Err(PaperForgeError::InvalidQuery(
                "exact URLs cannot be excluded".to_string(),
            ));
        }
        query.identities.push(parse_exact_url(token)?);
        return Ok(());
    }
    if let Some((prefix, value)) = token.split_once(':') {
        if value.trim().is_empty() {
            return Err(PaperForgeError::InvalidQuery(format!(
                "empty '{prefix}' value"
            )));
        }
        let prefix = prefix.to_ascii_lowercase();
        match prefix.as_str() {
            "title" | "abstract" | "author" | "category" | "topic" | "journal" => {
                let field = match prefix.as_str() {
                    "title" => TextField::Title,
                    "abstract" => TextField::Abstract,
                    "author" => TextField::Author,
                    "category" | "topic" => TextField::Topic,
                    "journal" => TextField::Journal,
                    _ => unreachable!(),
                };
                query.clauses.push(TextClause {
                    field,
                    polarity,
                    value: value.to_string(),
                    phrase: lexeme.quoted,
                });
            }
            "after" | "before" => {
                if polarity != ClausePolarity::Optional || lexeme.quoted {
                    return Err(PaperForgeError::InvalidQuery(format!(
                        "'{prefix}' is a date bound, not a signed text clause"
                    )));
                }
                let date = parse_date_bound(value, prefix == "before")?;
                let target = if prefix == "after" {
                    &mut query.after
                } else {
                    &mut query.before
                };
                if target.replace(date).is_some() {
                    return Err(PaperForgeError::InvalidQuery(format!(
                        "duplicate '{prefix}' bound"
                    )));
                }
            }
            "id" | "arxiv" | "pmid" | "pmcid" | "ntrs" | "doi" => {
                if polarity == ClausePolarity::Excluded || lexeme.quoted {
                    return Err(PaperForgeError::InvalidQuery(format!(
                        "'{prefix}' must be a positive unquoted exact identity"
                    )));
                }
                query.identities.push(parse_identity(&prefix, value)?);
            }
            _ => {
                return Err(PaperForgeError::InvalidQuery(format!(
                    "unknown field '{prefix}'"
                )));
            }
        }
    } else {
        query.clauses.push(TextClause {
            field: TextField::Any,
            polarity,
            value: token.to_string(),
            phrase: lexeme.quoted,
        });
    }
    Ok(())
}

fn parse_directive(query: &mut PaperQuery, directive: &str) -> Result<()> {
    match directive.to_ascii_lowercase().as_str() {
        "@recent" => {
            if query.mode != PaperMode::Standard {
                return Err(PaperForgeError::InvalidQuery(
                    "conflicting query modes".to_string(),
                ));
            }
            query.mode = PaperMode::Recent;
            return Ok(());
        }
        "@top" => {
            if query.mode != PaperMode::Standard {
                return Err(PaperForgeError::InvalidQuery(
                    "conflicting query modes".to_string(),
                ));
            }
            query.mode = PaperMode::Top;
            return Ok(());
        }
        _ => {}
    }
    let Some((name, value)) = directive.split_once(':') else {
        return Err(PaperForgeError::InvalidQuery(format!(
            "unknown directive '{directive}'"
        )));
    };
    match name.to_ascii_lowercase().as_str() {
        "@sort" => {
            if query.sort.replace(value.parse()?).is_some() {
                return Err(PaperForgeError::InvalidQuery(
                    "duplicate @sort directive".to_string(),
                ));
            }
        }
        "@limit" => {
            let limit = value.parse::<usize>().map_err(|_| {
                PaperForgeError::InvalidQuery("@limit must be an integer".to_string())
            })?;
            if !(1..=MAX_DISCOVERY_LIMIT).contains(&limit) {
                return Err(PaperForgeError::InvalidQuery(
                    "@limit must be from 1 through 100".to_string(),
                ));
            }
            if query.limit.replace(limit).is_some() {
                return Err(PaperForgeError::InvalidQuery(
                    "duplicate @limit directive".to_string(),
                ));
            }
        }
        "@provider" => {
            let provider = PaperProvider::from_slug(value).ok_or_else(|| {
                PaperForgeError::InvalidQuery(format!("unknown provider '{value}'"))
            })?;
            if query.provider.replace(provider).is_some() {
                return Err(PaperForgeError::InvalidQuery(
                    "duplicate @provider directive".to_string(),
                ));
            }
        }
        _ => {
            return Err(PaperForgeError::InvalidQuery(format!(
                "unknown directive '{name}'"
            )));
        }
    }
    Ok(())
}

fn parse_date_bound(value: &str, upper: bool) -> Result<String> {
    let parts = value.split('-').collect::<Vec<_>>();
    if parts.is_empty() || parts.len() > 3 || parts.iter().any(|part| part.is_empty()) {
        return Err(PaperForgeError::InvalidQuery(format!(
            "invalid date '{value}'"
        )));
    }
    let year = parse_date_part(parts[0], 4, "year", value)?;
    if !(1..=9999).contains(&year) {
        return Err(PaperForgeError::InvalidQuery(format!(
            "invalid date '{value}'"
        )));
    }
    let month = if parts.len() >= 2 {
        parse_date_part(parts[1], 2, "month", value)?
    } else if upper {
        12
    } else {
        1
    };
    if !(1..=12).contains(&month) {
        return Err(PaperForgeError::InvalidQuery(format!(
            "invalid date '{value}'"
        )));
    }
    let max_day = days_in_month(year, month);
    let day = if parts.len() == 3 {
        parse_date_part(parts[2], 2, "day", value)?
    } else if upper {
        max_day
    } else {
        1
    };
    if !(1..=max_day).contains(&day) {
        return Err(PaperForgeError::InvalidQuery(format!(
            "invalid date '{value}'"
        )));
    }
    Ok(format!("{year:04}-{month:02}-{day:02}"))
}

fn parse_date_part(value: &str, width: usize, label: &str, complete: &str) -> Result<u32> {
    if value.len() != width || !value.bytes().all(|byte| byte.is_ascii_digit()) {
        return Err(PaperForgeError::InvalidQuery(format!(
            "invalid {label} in date '{complete}'"
        )));
    }
    value
        .parse()
        .map_err(|_| PaperForgeError::InvalidQuery(format!("invalid date '{complete}'")))
}

fn days_in_month(year: u32, month: u32) -> u32 {
    match month {
        4 | 6 | 9 | 11 => 30,
        2 if year % 400 == 0 || (year % 4 == 0 && year % 100 != 0) => 29,
        2 => 28,
        _ => 31,
    }
}

fn parse_identity(prefix: &str, value: &str) -> Result<PaperIdentity> {
    if prefix == "id" {
        for candidate in ["arxiv", "pmcid", "pmid", "ntrs", "doi"] {
            if let Ok(identity) = parse_identity(candidate, value) {
                return Ok(identity);
            }
        }
        return Err(PaperForgeError::InvalidQuery(format!(
            "unrecognized exact identity '{value}'"
        )));
    }
    let (kind, value) = match prefix {
        "arxiv" if valid_arxiv_id(value) => (IdentityKind::Arxiv, value.to_string()),
        "pmid" if value.len() <= 10 && ascii_digits(value) => {
            (IdentityKind::Pmid, value.to_string())
        }
        "pmcid" => {
            let canonical = value.to_ascii_uppercase();
            if canonical
                .strip_prefix("PMC")
                .is_none_or(|digits| digits.is_empty() || !ascii_digits(digits))
            {
                return Err(PaperForgeError::InvalidQuery(format!(
                    "invalid PMCID '{value}'"
                )));
            }
            (IdentityKind::Pmcid, canonical)
        }
        "ntrs" if (8..=20).contains(&value.len()) && ascii_digits(value) => {
            (IdentityKind::Ntrs, value.to_string())
        }
        "doi" if valid_doi(value) => (
            IdentityKind::Doi,
            value.trim().to_ascii_lowercase().to_string(),
        ),
        _ => {
            return Err(PaperForgeError::InvalidQuery(format!(
                "invalid {prefix} identity '{value}'"
            )));
        }
    };
    Ok(PaperIdentity { kind, value })
}

fn parse_exact_url(input: &str) -> Result<PaperIdentity> {
    let url = reqwest::Url::parse(input)
        .map_err(|_| PaperForgeError::InvalidQuery(format!("invalid exact URL '{input}'")))?;
    if !secure_url(&url) || url.query().is_some() {
        return Err(PaperForgeError::InvalidQuery(format!(
            "unsupported or insecure URL '{input}'"
        )));
    }
    let host = url.host_str().unwrap_or_default();
    let segments = url
        .path_segments()
        .map(|parts| parts.filter(|part| !part.is_empty()).collect::<Vec<_>>())
        .unwrap_or_default();
    let parsed = match host {
        "arxiv.org" | "export.arxiv.org"
            if segments.len() == 2 && matches!(segments[0], "abs" | "pdf") =>
        {
            let value = segments[1].strip_suffix(".pdf").unwrap_or(segments[1]);
            parse_identity("arxiv", value)
        }
        "europepmc.org" | "www.europepmc.org"
            if segments.len() == 2 && segments[0] == "articles" =>
        {
            parse_identity("pmcid", segments[1])
        }
        "europepmc.org" | "www.europepmc.org"
            if segments.len() == 3 && segments[0] == "article" =>
        {
            match segments[1].to_ascii_uppercase().as_str() {
                "PMC" => parse_identity("pmcid", segments[2]),
                "MED" => parse_identity("pmid", segments[2]),
                _ => Err(PaperForgeError::InvalidQuery(format!(
                    "unsupported Europe PMC URL '{input}'"
                ))),
            }
        }
        "www.ebi.ac.uk"
            if segments.len() == 5
                && segments[..3] == ["europepmc", "webservices", "rest"]
                && segments[4] == "fullTextXML" =>
        {
            parse_identity("pmcid", segments[3])
        }
        "ntrs.nasa.gov" if segments.len() == 2 && segments[0] == "citations" => {
            parse_identity("ntrs", segments[1])
        }
        "doi.org" | "dx.doi.org" if !url.path().trim_matches('/').is_empty() => {
            parse_identity("doi", url.path().trim_start_matches('/'))
        }
        _ => Err(PaperForgeError::InvalidQuery(format!(
            "unsupported URL host or path '{input}'"
        ))),
    }?;
    Ok(parsed)
}

fn valid_arxiv_id(value: &str) -> bool {
    let (base, version_valid) = if let Some(index) = value.rfind('v') {
        let (base, version) = value.split_at(index);
        if version[1..].is_empty() || !ascii_digits(&version[1..]) {
            return false;
        }
        (base, true)
    } else {
        (value, true)
    };
    version_valid
        && (((9..=10).contains(&base.len())
            && base.as_bytes().get(4) == Some(&b'.')
            && ascii_digits(&base[..4])
            && (4..=5).contains(&base[5..].len())
            && ascii_digits(&base[5..]))
            || base.split_once('/').is_some_and(|(archive, number)| {
                !archive.is_empty()
                    && archive
                        .bytes()
                        .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'-'))
                    && (7..=8).contains(&number.len())
                    && ascii_digits(number)
            }))
}

fn valid_doi(value: &str) -> bool {
    let value = value.trim();
    let Some((registrant, suffix)) = value.split_once('/') else {
        return false;
    };
    registrant
        .strip_prefix("10.")
        .is_some_and(|digits| (4..=9).contains(&digits.len()) && ascii_digits(digits))
        && !suffix.is_empty()
        && !suffix.chars().any(char::is_whitespace)
        && value.len() <= 512
}

fn ascii_digits(value: &str) -> bool {
    !value.is_empty() && value.bytes().all(|byte| byte.is_ascii_digit())
}

/// Document access and license evidence, separated from metadata licensing.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct PaperRights {
    /// License or terms applying to provider metadata.
    pub metadata_license: String,
    /// Explicit document license, or a truthful provider distribution status.
    pub document_license: String,
    /// License reported by provider search metadata, before document inspection.
    #[serde(default)]
    pub search_license: Option<String>,
    /// License or controlling permissions extracted from archived JATS.
    #[serde(default)]
    pub jats_license: Option<String>,
    /// Every bounded JATS license record retained as raw evidence.
    #[serde(default)]
    pub jats_licenses: Vec<String>,
    /// Plain-text JATS permissions retained separately from search metadata.
    #[serde(default)]
    pub jats_permissions: Vec<String>,
    /// Whether search and JATS license families conflict.
    #[serde(default)]
    pub license_conflict: bool,
    /// Provider access classification.
    pub access_status: String,
    /// Creator or provider attribution evidence.
    pub attribution: String,
    /// License or provider-use terms URL when supplied.
    pub terms_url: Option<String>,
    /// Plain-text provider and document permissions evidence.
    #[serde(default)]
    pub evidence: Vec<String>,
    /// Warning against inferring redistribution permission.
    pub redistribution_warning: String,
}

/// Archived document representation.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum PaperDocumentFormat {
    /// Portable Document Format.
    Pdf,
    /// Journal Article Tag Suite XML.
    JatsXml,
}

impl PaperDocumentFormat {
    fn extension(self) -> &'static str {
        match self {
            Self::Pdf => "pdf",
            Self::JatsXml => "xml",
        }
    }
}

/// Provider document selected for archival.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct PaperDocument {
    /// First-party HTTPS download URL.
    pub url: String,
    /// Expected bytes when supplied by the provider.
    pub size: Option<u64>,
    /// Expected MIME type.
    pub mime: String,
    /// Exact archived representation.
    pub format: PaperDocumentFormat,
    /// Provider revision or version evidence.
    pub provider_revision: String,
}

/// Local match classification.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum RelevanceTier {
    /// Date/provider filters without positive text.
    FilterOnly = 1,
    /// Required clauses and at least one optional clause.
    RequiredPlusOptional = 2,
    /// Every positive clause matched.
    AllPositive = 3,
    /// Every positive matched with title or phrase evidence.
    AllPositiveTitleOrPhrase = 4,
    /// A quoted clause exactly equals the title.
    ExactTitlePhrase = 5,
    /// An exact identity matched.
    ExactIdentity = 6,
}

/// Local query-match evidence retained with a result.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct QueryEvidence {
    /// Local tier.
    pub tier: RelevanceTier,
    /// Weighted local score.
    pub score: i64,
    /// Canonicalized matching clauses or identities.
    pub matched: Vec<String>,
}

/// Search result and archival metadata.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct PaperSummary {
    /// Provider that owns this record.
    pub provider: PaperProvider,
    /// Provider-qualified canonical identity.
    pub id: String,
    /// Native provider identity, preserving revisions where relevant.
    pub native_id: String,
    /// Paper title.
    pub title: String,
    /// Abstract or provider summary.
    pub abstract_text: String,
    /// Ordered author display names.
    pub authors: Vec<String>,
    /// Provider topics or categories.
    pub topics: Vec<String>,
    /// DOI when supplied.
    pub doi: Option<String>,
    /// Journal or venue when supplied.
    pub journal: Option<String>,
    /// Publication date (`YYYY-MM-DD` when available).
    pub published: Option<String>,
    /// Provider update date (`YYYY-MM-DD` when available).
    pub updated: Option<String>,
    /// Real provider citation count. Never synthesized.
    pub citation_count: Option<u64>,
    /// Canonical provider source or landing URL.
    pub source_url: String,
    /// Separate metadata and paper rights evidence.
    pub rights: PaperRights,
    /// First-party archive representation.
    pub document: PaperDocument,
    /// Local relevance evidence.
    pub relevance: QueryEvidence,
}

/// Bounded provider-candidate discovery result.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct DiscoveryResult {
    /// Locally filtered and ranked output papers.
    pub papers: Vec<PaperSummary>,
    /// Raw provider records examined before eligibility and local filtering.
    pub candidate_count: usize,
    /// Eligible, provider-local summaries available for local query filtering.
    pub eligible_candidate_count: usize,
    /// Whether the provider candidate budget may have truncated matching records.
    pub candidate_cap_reached: bool,
    /// Maximum provider candidates gathered for this call.
    pub candidate_budget: usize,
}

/// Archive configuration.
#[derive(Debug, Clone)]
pub struct ArchiveOptions {
    /// Root containing one directory per paper.
    pub output: PathBuf,
    /// Verify and skip matching complete archives.
    pub skip_existing: bool,
    /// Per-paper byte cap. Zero disables the caller cap; internal response caps remain.
    pub max_download_bytes: u64,
}

impl Default for ArchiveOptions {
    fn default() -> Self {
        Self {
            output: PathBuf::from("papers"),
            skip_existing: false,
            max_download_bytes: DEFAULT_MAX_DOWNLOAD_BYTES,
        }
    }
}

/// One successful archive record.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ArchivedPaper {
    /// Canonical paper identity.
    pub paper_id: String,
    /// Local output directory.
    pub output: PathBuf,
    /// Retained document bytes.
    pub bytes_written: u64,
    /// Number of retained document files.
    pub files_written: usize,
    /// Final verified manifest, including post-download JATS rights evidence.
    pub manifest: PaperManifest,
}

/// One per-paper archive failure.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ArchiveFailure {
    /// Canonical paper identity.
    pub paper_id: String,
    /// Intended local output directory, when the identity was path-safe.
    pub output: Option<PathBuf>,
    /// Bounded diagnostic.
    pub message: String,
}

/// Aggregate archive outcome.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct ArchiveSummary {
    /// Number selected by the caller.
    pub selected: usize,
    /// Selected canonical identities.
    pub selected_ids: Vec<String>,
    /// Number newly archived or repaired.
    pub archived: usize,
    /// Number of verified existing archives skipped.
    pub skipped: usize,
    /// Number that failed.
    pub failed_count: usize,
    /// Downloaded document bytes.
    pub bytes_written: u64,
    /// Newly retained document files.
    pub files_written: usize,
    /// Successful records.
    pub papers: Vec<ArchivedPaper>,
    /// Verified existing records, including their retained bytes and path.
    pub skipped_papers: Vec<ArchivedPaper>,
    /// Per-paper failures.
    pub failed: Vec<ArchiveFailure>,
}

impl ArchiveSummary {
    /// Whether every selected paper archived or verified successfully.
    pub fn is_success(&self) -> bool {
        self.failed.is_empty()
    }
}

/// Archive progress phase.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ArchiveProgressPhase {
    /// Output validation and setup.
    Starting,
    /// A document is active.
    Downloading,
    /// Every selected document was attempted.
    Completed,
}

/// Aggregate and active-document progress.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ArchiveProgress {
    /// Current phase.
    pub phase: ArchiveProgressPhase,
    /// Selected papers.
    pub total_papers: usize,
    /// Completed attempts.
    pub completed_papers: usize,
    /// Successful archive or skip count.
    pub succeeded_papers: usize,
    /// Failed paper count.
    pub failed_papers: usize,
    /// Aggregate new bytes including the active transfer.
    pub bytes_written: u64,
    /// Active paper identity.
    pub active_paper: Option<String>,
    /// Active relative file.
    pub active_file: Option<String>,
    /// Active downloaded bytes.
    pub active_bytes_written: u64,
    /// Expected active bytes, if known.
    pub active_total_bytes: Option<u64>,
}

/// Per-paper integrity record.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct PaperIntegrity {
    /// Relative document path.
    pub path: String,
    /// Actual retained bytes.
    pub bytes: u64,
    /// Lowercase BLAKE3 digest.
    pub blake3: String,
}

/// Per-paper archive manifest.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct PaperManifest {
    /// Manifest schema version.
    pub format_version: String,
    /// Full source, query, rights, and document evidence.
    pub paper: PaperSummary,
    /// Local integrity evidence.
    pub integrity: PaperIntegrity,
    /// Completion marker written only after verification.
    pub complete: bool,
}

/// Blocking provider client.
#[derive(Clone)]
pub struct PaperForge {
    provider: PaperProvider,
    client: Client,
    discovery_limit: usize,
}

impl PaperForge {
    /// Build a provider client with a descriptive User-Agent.
    pub fn with_user_agent(provider: PaperProvider, user_agent: &str) -> Result<Self> {
        let user_agent = user_agent.trim();
        if user_agent.is_empty() {
            return Err(PaperForgeError::InvalidOption(
                "User-Agent must not be empty".to_string(),
            ));
        }
        let client = Client::builder()
            .user_agent(user_agent)
            .redirect(Policy::none())
            .connect_timeout(Duration::from_secs(20))
            .timeout(Duration::from_secs(90))
            .build()?;
        Ok(Self {
            provider,
            client,
            discovery_limit: DEFAULT_DISCOVERY_LIMIT,
        })
    }

    /// Set the client result cap, clamped to 1 through 100.
    #[must_use]
    pub fn with_discovery_limit(mut self, limit: usize) -> Self {
        self.discovery_limit = limit.clamp(1, MAX_DISCOVERY_LIMIT);
        self
    }

    /// Client provider.
    pub const fn provider(&self) -> PaperProvider {
        self.provider
    }

    /// Discover papers, locally reapply every query constraint, and rank deterministically.
    pub fn discover(&self, query: &PaperQuery) -> Result<Vec<PaperSummary>> {
        Ok(self.discover_detailed(query)?.papers)
    }

    /// Discover papers with explicit bounded-provider-candidate evidence.
    pub fn discover_detailed(&self, query: &PaperQuery) -> Result<DiscoveryResult> {
        validate_query_for_provider(query, self.provider)?;
        let limit = query
            .limit
            .unwrap_or(self.discovery_limit)
            .min(self.discovery_limit);
        let candidates = self.gather_provider_candidates(query, PROVIDER_CANDIDATE_BUDGET)?;
        Ok(finalize_discovery(
            candidates.papers,
            query,
            limit,
            PROVIDER_CANDIDATE_BUDGET,
            candidates.raw_count,
            candidates.cap_reached,
        ))
    }

    /// Archive selected papers.
    pub fn archive(
        &self,
        papers: &[PaperSummary],
        options: &ArchiveOptions,
    ) -> Result<ArchiveSummary> {
        self.archive_inner(papers, options, None)
    }

    /// Archive selected papers with byte-level progress callbacks.
    pub fn archive_with_progress<F>(
        &self,
        papers: &[PaperSummary],
        options: &ArchiveOptions,
        mut progress: F,
    ) -> Result<ArchiveSummary>
    where
        F: FnMut(ArchiveProgress),
    {
        self.archive_inner(papers, options, Some(&mut progress))
    }
}

fn validate_query_for_provider(query: &PaperQuery, provider: PaperProvider) -> Result<()> {
    validate_query_structure(query)?;
    if query.provider.is_some_and(|selected| selected != provider) {
        return Err(PaperForgeError::InvalidQuery(format!(
            "query provider does not match the {} client",
            provider.slug()
        )));
    }
    for identity in &query.identities {
        let supported = match provider {
            PaperProvider::Arxiv => {
                matches!(identity.kind, IdentityKind::Arxiv | IdentityKind::Doi)
            }
            PaperProvider::EuropePmc => matches!(
                identity.kind,
                IdentityKind::Pmid | IdentityKind::Pmcid | IdentityKind::Doi
            ),
            PaperProvider::NasaNtrs => identity.kind == IdentityKind::Ntrs,
        };
        if !supported {
            return Err(PaperForgeError::InvalidQuery(format!(
                "{} identities are not supported by {}",
                identity_kind_slug(identity.kind),
                provider.slug()
            )));
        }
    }
    if matches!(provider, PaperProvider::Arxiv | PaperProvider::NasaNtrs)
        && (query.mode == PaperMode::Top || query.sort == Some(PaperSort::Citations))
    {
        return Err(PaperForgeError::InvalidQuery(format!(
            "{} does not provide citation counts; @top and citation sorting are unsupported",
            provider.slug()
        )));
    }
    if provider == PaperProvider::Arxiv
        && (query
            .clauses
            .iter()
            .any(|clause| clause.value.contains('"') || clause.value.contains('\\'))
            || query
                .identities
                .iter()
                .any(|identity| identity.value.contains('"') || identity.value.contains('\\')))
    {
        return Err(PaperForgeError::InvalidQuery(
            "arXiv query values cannot contain quotes or backslashes".to_string(),
        ));
    }
    Ok(())
}

fn validate_query_structure(query: &PaperQuery) -> Result<()> {
    if query.original.trim().is_empty() || query.original.len() > MAX_QUERY_BYTES {
        return Err(PaperForgeError::InvalidQuery(format!(
            "original query must contain 1 through {MAX_QUERY_BYTES} bytes"
        )));
    }
    if query.clauses.len() > MAX_QUERY_CLAUSES {
        return Err(PaperForgeError::InvalidQuery(format!(
            "query exceeds the {MAX_QUERY_CLAUSES} clause limit"
        )));
    }
    if query.identities.len() > MAX_QUERY_IDENTITIES {
        return Err(PaperForgeError::InvalidQuery(format!(
            "query exceeds the {MAX_QUERY_IDENTITIES} identity limit"
        )));
    }
    if query
        .limit
        .is_some_and(|limit| !(1..=MAX_DISCOVERY_LIMIT).contains(&limit))
    {
        return Err(PaperForgeError::InvalidQuery(
            "result limit must be from 1 through 100".to_string(),
        ));
    }
    for (label, value) in [("after", &query.after), ("before", &query.before)] {
        if let Some(value) = value {
            if parse_date_bound(value, label == "before").ok().as_deref() != Some(value) {
                return Err(PaperForgeError::InvalidQuery(format!(
                    "{label} must be a normalized YYYY-MM-DD date"
                )));
            }
        }
    }
    if query
        .after
        .as_ref()
        .zip(query.before.as_ref())
        .is_some_and(|(after, before)| after > before)
    {
        return Err(PaperForgeError::InvalidQuery(
            "after date is later than before date".to_string(),
        ));
    }
    for clause in &query.clauses {
        if clause.value.trim().is_empty()
            || clause.value.len() > MAX_QUERY_VALUE_BYTES
            || clause.value.chars().any(|character| character.is_control())
            || (!clause.phrase && clause.value.chars().any(char::is_whitespace))
        {
            return Err(PaperForgeError::InvalidQuery(format!(
                "clause values must contain 1 through {MAX_QUERY_VALUE_BYTES} bytes, no control characters, and no whitespace unless phrase is true"
            )));
        }
    }
    for identity in &query.identities {
        if identity.value.len() > MAX_QUERY_VALUE_BYTES
            || identity
                .value
                .chars()
                .any(|character| character.is_control())
        {
            return Err(PaperForgeError::InvalidQuery(
                "identity value is too long or contains control characters".to_string(),
            ));
        }
        let parsed = parse_identity(identity_kind_slug(identity.kind), &identity.value)?;
        if parsed != *identity {
            return Err(PaperForgeError::InvalidQuery(format!(
                "identity '{}:{}' is not canonical",
                identity_kind_slug(identity.kind),
                identity.value
            )));
        }
    }
    let has_positive = query
        .clauses
        .iter()
        .any(|clause| clause.polarity != ClausePolarity::Excluded);
    let has_negative = query
        .clauses
        .iter()
        .any(|clause| clause.polarity == ClausePolarity::Excluded);
    let bounded = has_positive
        || !query.identities.is_empty()
        || query.after.is_some()
        || query.before.is_some()
        || query.mode != PaperMode::Standard;
    if has_negative && !bounded {
        return Err(PaperForgeError::InvalidQuery(
            "excluded clauses cannot be the only search constraint".to_string(),
        ));
    }
    if query.clauses.is_empty()
        && query.identities.is_empty()
        && query.after.is_none()
        && query.before.is_none()
        && query.mode == PaperMode::Standard
    {
        return Err(PaperForgeError::InvalidQuery(
            "query has no discovery constraint".to_string(),
        ));
    }
    Ok(())
}

fn identity_kind_slug(kind: IdentityKind) -> &'static str {
    match kind {
        IdentityKind::Arxiv => "arxiv",
        IdentityKind::Pmid => "pmid",
        IdentityKind::Pmcid => "pmcid",
        IdentityKind::Ntrs => "ntrs",
        IdentityKind::Doi => "doi",
    }
}

struct ProviderCandidates {
    papers: Vec<PaperSummary>,
    raw_count: usize,
    cap_reached: bool,
}

impl PaperForge {
    fn gather_provider_candidates(
        &self,
        query: &PaperQuery,
        budget: usize,
    ) -> Result<ProviderCandidates> {
        let mut papers = Vec::new();
        let mut raw_count = 0;
        let mut cap_reached = false;
        if query.identities.is_empty() {
            return Ok(match self.provider {
                PaperProvider::Arxiv => self.discover_arxiv(query, budget)?,
                PaperProvider::EuropePmc => self.discover_europe_pmc(query, budget)?,
                PaperProvider::NasaNtrs => self.discover_nasa(query, budget)?,
            });
        } else {
            for (index, identity) in query.identities.iter().enumerate() {
                let remaining = budget.saturating_sub(raw_count);
                if remaining == 0 {
                    cap_reached = true;
                    break;
                }
                let mut exact = match self.provider {
                    PaperProvider::Arxiv => self.discover_arxiv_identity(identity, remaining)?,
                    PaperProvider::EuropePmc => {
                        self.discover_europe_pmc_identity(identity, remaining)?
                    }
                    PaperProvider::NasaNtrs => match self.fetch_nasa_exact(&identity.value) {
                        Ok(paper) => ProviderCandidates {
                            raw_count: usize::from(paper.is_some()),
                            papers: paper.into_iter().collect(),
                            cap_reached: false,
                        },
                        Err(PaperForgeError::ProviderApi { status: 404, .. }) => {
                            ProviderCandidates {
                                papers: Vec::new(),
                                raw_count: 0,
                                cap_reached: false,
                            }
                        }
                        Err(error) => return Err(error),
                    },
                };
                raw_count = raw_count.saturating_add(exact.raw_count);
                cap_reached |= exact.cap_reached;
                papers.append(&mut exact.papers);
                dedupe_provider(&mut papers);
                if raw_count >= budget && index + 1 < query.identities.len() {
                    cap_reached = true;
                }
            }
        }
        Ok(ProviderCandidates {
            papers,
            raw_count,
            cap_reached,
        })
    }

    fn discover_arxiv(&self, query: &PaperQuery, limit: usize) -> Result<ProviderCandidates> {
        let url = arxiv_request_url(query, limit)?;
        trusted_provider_url(PaperProvider::Arxiv, &url, UrlPurpose::Api)?;
        let bytes = self.api_request(|| self.client.get(url.clone()), "arXiv Atom response")?;
        let papers = parse_arxiv_atom(&bytes)?;
        Ok(ProviderCandidates {
            raw_count: papers.len(),
            cap_reached: papers.len() >= limit,
            papers,
        })
    }

    fn discover_arxiv_identity(
        &self,
        identity: &PaperIdentity,
        limit: usize,
    ) -> Result<ProviderCandidates> {
        let url = arxiv_identity_request_url(identity, limit)?;
        trusted_provider_url(PaperProvider::Arxiv, &url, UrlPurpose::Api)?;
        let bytes = self.api_request(|| self.client.get(url.clone()), "arXiv Atom response")?;
        let papers = parse_arxiv_atom(&bytes)?;
        Ok(ProviderCandidates {
            raw_count: papers.len(),
            cap_reached: papers.len() >= limit,
            papers,
        })
    }

    fn discover_europe_pmc(&self, query: &PaperQuery, limit: usize) -> Result<ProviderCandidates> {
        let url = europe_pmc_request_url(query, limit)?;
        trusted_provider_url(PaperProvider::EuropePmc, &url, UrlPurpose::Api)?;
        let bytes =
            self.api_request(|| self.client.get(url.clone()), "Europe PMC JSON response")?;
        let mut candidates = parse_europe_pmc_page(&bytes)?;
        candidates.cap_reached = candidates.raw_count >= limit;
        Ok(candidates)
    }

    fn discover_europe_pmc_identity(
        &self,
        identity: &PaperIdentity,
        limit: usize,
    ) -> Result<ProviderCandidates> {
        let query = PaperQuery {
            identities: vec![identity.clone()],
            original: format!("{}:{}", identity_kind_slug(identity.kind), identity.value),
            ..PaperQuery::default()
        };
        self.discover_europe_pmc(&query, limit)
    }

    fn discover_nasa(&self, query: &PaperQuery, limit: usize) -> Result<ProviderCandidates> {
        let url = reqwest::Url::parse(NASA_SEARCH_API).expect("static NASA URL");
        trusted_provider_url(PaperProvider::NasaNtrs, &url, UrlPurpose::Api)?;
        let body = nasa_request_body(query, limit);
        let bytes = self.api_request(
            || self.client.post(url.clone()).json(&body),
            "NASA NTRS JSON response",
        )?;
        let value: serde_json::Value = serde_json::from_slice(&bytes)?;
        parse_nasa_candidates(&value, limit)
    }

    fn api_request<F>(&self, make_request: F, label: &str) -> Result<Vec<u8>>
    where
        F: Fn() -> RequestBuilder,
    {
        for attempt in 0..MAX_API_ATTEMPTS {
            let arxiv_guard = self.acquire_provider_slot()?;
            let response = make_request().send()?;
            let retry_after = retry_after(&response);
            if !response.status().is_success() {
                let status = response.status().as_u16();
                if retryable_status(status) && attempt + 1 < MAX_API_ATTEMPTS {
                    drop(arxiv_guard);
                    thread::sleep(retry_delay(attempt, retry_after));
                    continue;
                }
                return Err(PaperForgeError::ProviderApi {
                    provider: self.provider.slug(),
                    status,
                    body: bounded_error_body(response),
                });
            }
            let bytes = read_response_limited(response, MAX_API_RESPONSE_BYTES, label)?;
            if let Some(message) = provider_error_envelope(&bytes) {
                return Err(PaperForgeError::ProviderApiError {
                    provider: self.provider.slug(),
                    message,
                });
            }
            return Ok(bytes);
        }
        unreachable!("bounded API request loop always returns")
    }

    fn acquire_provider_slot(&self) -> Result<Option<MutexGuard<'static, ArxivRequestGate>>> {
        if self.provider != PaperProvider::Arxiv {
            return Ok(None);
        }
        let mut gate = ARXIV_REQUEST_GATE
            .get_or_init(|| Mutex::new(ArxivRequestGate::default()))
            .lock()
            .map_err(|_| {
                PaperForgeError::Request("arXiv rate-limit state was poisoned".to_string())
            })?;
        if let Some(previous) = gate.last_start {
            let elapsed = previous.elapsed();
            if elapsed < ARXIV_INTERVAL {
                thread::sleep(ARXIV_INTERVAL - elapsed);
            }
        }
        gate.last_start = Some(Instant::now());
        Ok(Some(gate))
    }
}

fn arxiv_request_url(query: &PaperQuery, limit: usize) -> Result<reqwest::Url> {
    let mut url = reqwest::Url::parse(ARXIV_API).expect("static arXiv URL");
    let expression = arxiv_expression(query);
    if !expression.is_empty() {
        url.query_pairs_mut()
            .append_pair("search_query", &expression);
    }
    url.query_pairs_mut()
        .append_pair("start", "0")
        .append_pair("max_results", &limit.to_string())
        .append_pair(
            "sortBy",
            match (query.sort, query.mode) {
                (Some(PaperSort::Date), _) | (None, PaperMode::Recent) => "submittedDate",
                _ => "relevance",
            },
        )
        .append_pair("sortOrder", "descending");
    Ok(url)
}

fn arxiv_identity_request_url(identity: &PaperIdentity, limit: usize) -> Result<reqwest::Url> {
    let mut url = reqwest::Url::parse(ARXIV_API).expect("static arXiv URL");
    match identity.kind {
        IdentityKind::Arxiv => {
            url.query_pairs_mut()
                .append_pair("id_list", &identity.value);
        }
        IdentityKind::Doi => {
            url.query_pairs_mut().append_pair(
                "search_query",
                &format!("doi:\"{}\"", provider_phrase(&identity.value)),
            );
        }
        _ => {
            return Err(PaperForgeError::InvalidQuery(format!(
                "{} identities are not supported by arxiv",
                identity_kind_slug(identity.kind)
            )));
        }
    }
    url.query_pairs_mut()
        .append_pair("start", "0")
        .append_pair("max_results", &limit.to_string())
        .append_pair("sortBy", "relevance")
        .append_pair("sortOrder", "descending");
    Ok(url)
}

fn arxiv_expression(query: &PaperQuery) -> String {
    let mut required = Vec::new();
    let mut optional = Vec::new();
    let mut excluded = Vec::new();
    for clause in &query.clauses {
        let field = match clause.field {
            TextField::Any => "all",
            TextField::Title => "ti",
            TextField::Abstract => "abs",
            TextField::Author => "au",
            TextField::Topic => "cat",
            TextField::Journal => "jr",
        };
        let value = provider_phrase(&clause.value);
        let part = format!("{field}:\"{value}\"");
        match clause.polarity {
            ClausePolarity::Required => required.push(part),
            ClausePolarity::Optional => optional.push(part),
            ClausePolarity::Excluded => excluded.push(part),
        }
    }
    let mut parts = if required.is_empty() {
        if optional.is_empty() {
            Vec::new()
        } else if optional.len() == 1 {
            optional
        } else {
            vec![format!("({})", optional.join(" OR "))]
        }
    } else {
        vec![required.join(" AND ")]
    };
    if !parts.is_empty() {
        debug_assert_eq!(parts.len(), 1);
    }
    if query.after.is_some() || query.before.is_some() {
        let after = query
            .after
            .as_deref()
            .unwrap_or("0001-01-01")
            .replace('-', "");
        let before = query
            .before
            .as_deref()
            .unwrap_or("9999-12-31")
            .replace('-', "");
        let part = format!("submittedDate:[{after}0000 TO {before}2359]");
        if parts.is_empty() {
            parts.push(part);
        } else {
            parts.push(format!("AND {part}"));
        }
    }
    for part in excluded {
        if parts.is_empty() {
            parts.push("all:*".to_string());
        }
        parts.push(format!("ANDNOT {part}"));
    }
    parts.join(" ")
}

fn europe_pmc_request_url(query: &PaperQuery, limit: usize) -> Result<reqwest::Url> {
    let mut url = reqwest::Url::parse(EUROPE_PMC_API).expect("static Europe PMC URL");
    let mut parts = vec!["OPEN_ACCESS:Y".to_string(), "IN_EPMC:Y".to_string()];
    let has_required = query
        .clauses
        .iter()
        .any(|clause| clause.polarity == ClausePolarity::Required);
    let mut optionals = Vec::new();
    for clause in &query.clauses {
        let field = match clause.field {
            TextField::Any => "",
            TextField::Title => "TITLE:",
            TextField::Abstract => "ABSTRACT:",
            TextField::Author => "AUTH:",
            TextField::Topic => "KW:",
            TextField::Journal => "JOURNAL:",
        };
        let value = provider_phrase(&clause.value);
        let term = format!("{field}\"{value}\"");
        match clause.polarity {
            ClausePolarity::Required => parts.push(term),
            ClausePolarity::Optional if !has_required => optionals.push(term),
            ClausePolarity::Optional => {}
            ClausePolarity::Excluded => parts.push(format!("NOT {term}")),
        }
    }
    if optionals.len() == 1 {
        parts.push(optionals.remove(0));
    } else if !optionals.is_empty() {
        parts.push(format!("({})", optionals.join(" OR ")));
    }
    if !query.identities.is_empty() {
        let identities = query
            .identities
            .iter()
            .map(|identity| {
                let field = match identity.kind {
                    IdentityKind::Pmid => "EXT_ID",
                    IdentityKind::Pmcid => "PMCID",
                    IdentityKind::Doi => "DOI",
                    _ => unreachable!("provider identity was validated"),
                };
                format!("{field}:\"{}\"", provider_phrase(&identity.value))
            })
            .collect::<Vec<_>>()
            .join(" OR ");
        parts.push(format!("({identities})"));
    }
    if let Some(after) = &query.after {
        parts.push(format!("FIRST_PDATE:[{after} TO 9999-12-31]"));
    }
    if let Some(before) = &query.before {
        parts.push(format!("FIRST_PDATE:[0001-01-01 TO {before}]"));
    }
    url.query_pairs_mut()
        .append_pair("query", &parts.join(" AND "))
        .append_pair("format", "json")
        .append_pair("resultType", "core")
        .append_pair("pageSize", &limit.to_string())
        .append_pair(
            "sort",
            if query.sort == Some(PaperSort::Date)
                || (query.sort.is_none() && query.mode == PaperMode::Recent)
            {
                "FIRST_PDATE_D desc"
            } else if query.sort == Some(PaperSort::Citations)
                || (query.sort.is_none() && query.mode == PaperMode::Top)
            {
                "CITED desc"
            } else {
                ""
            },
        );
    Ok(url)
}

fn nasa_request_body(query: &PaperQuery, limit: usize) -> serde_json::Value {
    let required = query
        .clauses
        .iter()
        .filter(|clause| clause.polarity == ClausePolarity::Required)
        .map(|clause| provider_plain_terms(&clause.value))
        .filter(|value| !value.is_empty())
        .collect::<Vec<_>>();
    let optional = query
        .clauses
        .iter()
        .filter(|clause| clause.polarity == ClausePolarity::Optional)
        .map(|clause| provider_plain_terms(&clause.value))
        .filter(|value| !value.is_empty())
        .collect::<Vec<_>>();
    let text = if required.is_empty() {
        optional.join(" OR ")
    } else {
        required.join(" ")
    };
    debug_assert!(
        query
            .identities
            .iter()
            .all(|identity| identity.kind != IdentityKind::Doi),
        "NASA DOI identities are rejected during validation"
    );
    let mut body = serde_json::json!({
        "q": text,
        "distribution": ["PUBLIC"],
        "disseminated": ["DOCUMENT_AND_METADATA"],
        "page": { "from": 0, "size": limit },
        "highlight": false,
        "sort": {
            "field": if query.sort == Some(PaperSort::Date)
                || (query.sort.is_none() && query.mode == PaperMode::Recent)
            { "published" } else { "score" },
            "order": "desc"
        }
    });
    if let Some(object) = body.as_object_mut() {
        if query.after.is_some() || query.before.is_some() {
            object.insert(
                "published".to_string(),
                serde_json::json!({"from": query.after, "to": query.before}),
            );
        }
    }
    body
}

fn provider_phrase(value: &str) -> String {
    let mut escaped = String::new();
    for character in value.chars().filter(|character| !character.is_control()) {
        if matches!(character, '\\' | '"') {
            escaped.push('\\');
        }
        escaped.push(character);
        if escaped.len() >= 1024 {
            break;
        }
    }
    escaped
}

fn provider_plain_terms(value: &str) -> String {
    normalized_text(value)
        .split_whitespace()
        .take(64)
        .collect::<Vec<_>>()
        .join(" ")
}

#[derive(Default)]
struct ArxivEntry {
    id: String,
    title: String,
    summary: String,
    published: String,
    updated: String,
    authors: Vec<String>,
    topics: Vec<String>,
    doi: Option<String>,
    journal: Option<String>,
    license: Option<String>,
    landing: Option<String>,
    pdf: Option<String>,
}

fn parse_arxiv_atom(bytes: &[u8]) -> Result<Vec<PaperSummary>> {
    let mut reader = Reader::from_reader(bytes);
    reader.config_mut().trim_text(true);
    let mut buffer = Vec::new();
    let mut entries = Vec::new();
    let mut entry: Option<ArxivEntry> = None;
    let mut field = String::new();
    let mut in_author = false;
    let mut depth = 0_usize;
    let mut root_seen = false;
    let mut root_closed = false;
    loop {
        match reader.read_event_into(&mut buffer) {
            Ok(Event::Start(start)) => {
                let name = local_xml_name(start.name().as_ref());
                if depth == 0 {
                    if root_seen || !is_atom_feed(&reader, &start)? {
                        return Err(PaperForgeError::Xml(
                            "arXiv response root is not an Atom feed".to_string(),
                        ));
                    }
                    root_seen = true;
                }
                depth += 1;
                if name == "entry" {
                    entry = Some(ArxivEntry::default());
                } else if entry.is_some() {
                    if name == "author" {
                        in_author = true;
                    }
                    if name == "link" {
                        parse_arxiv_link(&reader, &start, entry.as_mut().expect("entry exists"))?;
                    } else if name == "category" {
                        if let Some(term) = xml_attribute(&reader, &start, b"term")? {
                            let term = compact_whitespace(&term);
                            if !term.is_empty() {
                                entry.as_mut().expect("entry exists").topics.push(term);
                            }
                        }
                    } else {
                        field = name;
                    }
                }
            }
            Ok(Event::Empty(empty)) => {
                let name = local_xml_name(empty.name().as_ref());
                if depth == 0 {
                    if root_seen || !is_atom_feed(&reader, &empty)? {
                        return Err(PaperForgeError::Xml(
                            "arXiv response root is not an Atom feed".to_string(),
                        ));
                    }
                    root_seen = true;
                    root_closed = true;
                } else if let Some(current) = entry.as_mut() {
                    if name == "link" {
                        parse_arxiv_link(&reader, &empty, current)?;
                    } else if name == "category" {
                        if let Some(term) = xml_attribute(&reader, &empty, b"term")? {
                            let term = compact_whitespace(&term);
                            if !term.is_empty() {
                                current.topics.push(term);
                            }
                        }
                    }
                }
            }
            Ok(Event::Text(text)) => {
                if let Some(current) = entry.as_mut() {
                    let decoded = reader.decoder().decode(text.as_ref()).map_err(|error| {
                        PaperForgeError::Xml(bounded_diagnostic(&format!(
                            "invalid Atom text: {error}"
                        )))
                    })?;
                    let value = quick_xml::escape::unescape(&decoded)
                        .map_err(|error| {
                            PaperForgeError::Xml(bounded_diagnostic(&error.to_string()))
                        })?
                        .into_owned();
                    assign_arxiv_text(current, &field, in_author, &value);
                }
            }
            Ok(Event::CData(text)) => {
                if let Some(current) = entry.as_mut() {
                    let value = text.decode().map_err(|error| {
                        PaperForgeError::Xml(bounded_diagnostic(&format!(
                            "invalid Atom CDATA: {error}"
                        )))
                    })?;
                    assign_arxiv_text(current, &field, in_author, &value);
                }
            }
            Ok(Event::End(end)) => {
                let name = local_xml_name(end.name().as_ref());
                if name == "author" {
                    in_author = false;
                }
                if name == "entry" {
                    let current = entry.take().expect("entry end follows start");
                    if current.id.contains("/api/errors#") {
                        return Err(PaperForgeError::ProviderApiError {
                            provider: PaperProvider::Arxiv.slug(),
                            message: bounded_diagnostic(&current.title),
                        });
                    }
                    entries.push(arxiv_summary(current)?);
                }
                field.clear();
                depth = depth.checked_sub(1).ok_or_else(|| {
                    PaperForgeError::Xml("arXiv response element depth underflow".to_string())
                })?;
                if depth == 0 {
                    if name != "feed" {
                        return Err(PaperForgeError::Xml(
                            "arXiv response closed an unexpected root".to_string(),
                        ));
                    }
                    root_closed = true;
                }
            }
            Ok(Event::DocType(_)) => {
                return Err(PaperForgeError::Xml(
                    "Atom document types are forbidden".to_string(),
                ));
            }
            Ok(Event::Eof) => {
                if !root_seen || !root_closed || depth != 0 {
                    return Err(PaperForgeError::Xml(
                        "arXiv response is not a complete Atom feed".to_string(),
                    ));
                }
                break;
            }
            Ok(_) => {}
            Err(error) => {
                return Err(PaperForgeError::Xml(bounded_diagnostic(&error.to_string())));
            }
        }
        buffer.clear();
    }
    Ok(entries)
}

fn is_atom_feed(
    reader: &Reader<&[u8]>,
    element: &quick_xml::events::BytesStart<'_>,
) -> Result<bool> {
    Ok(local_xml_name(element.name().as_ref()) == "feed"
        && xml_attributes(reader, element)?
            .iter()
            .any(|(_, value)| value == "http://www.w3.org/2005/Atom"))
}

fn assign_arxiv_text(entry: &mut ArxivEntry, field: &str, in_author: bool, value: &str) {
    let value = compact_whitespace(value);
    if value.is_empty() {
        return;
    }
    match field {
        "id" => entry.id.push_str(&value),
        "title" => append_text(&mut entry.title, &value),
        "summary" => append_text(&mut entry.summary, &value),
        "published" => entry.published.push_str(&value),
        "updated" => entry.updated.push_str(&value),
        "name" if in_author => entry.authors.push(value),
        "doi" => entry.doi = Some(value.to_ascii_lowercase()),
        "journal_ref" => entry.journal = Some(value),
        "license" => entry.license = Some(value),
        _ => {}
    }
}

fn append_text(target: &mut String, value: &str) {
    if !target.is_empty() {
        target.push(' ');
    }
    target.push_str(value);
}

fn parse_arxiv_link(
    reader: &Reader<&[u8]>,
    element: &quick_xml::events::BytesStart<'_>,
    entry: &mut ArxivEntry,
) -> Result<()> {
    let href = xml_attribute(reader, element, b"href")?;
    let rel = xml_attribute(reader, element, b"rel")?.unwrap_or_default();
    let mime = xml_attribute(reader, element, b"type")?.unwrap_or_default();
    let title = xml_attribute(reader, element, b"title")?.unwrap_or_default();
    if let Some(href) = href {
        if rel == "alternate" {
            entry.landing = Some(href.clone());
        }
        if mime.eq_ignore_ascii_case("application/pdf") || title.eq_ignore_ascii_case("pdf") {
            entry.pdf = Some(href);
        }
    }
    Ok(())
}

fn xml_attribute(
    reader: &Reader<&[u8]>,
    element: &quick_xml::events::BytesStart<'_>,
    name: &[u8],
) -> Result<Option<String>> {
    Ok(xml_attributes(reader, element)?
        .into_iter()
        .find_map(|(key, value)| (key.as_bytes() == name).then_some(value)))
}

fn xml_attributes(
    reader: &Reader<impl Read>,
    element: &quick_xml::events::BytesStart<'_>,
) -> Result<Vec<(String, String)>> {
    let mut values = Vec::new();
    for (index, attribute) in element.attributes().with_checks(true).enumerate() {
        if index >= MAX_XML_ATTRIBUTES_PER_ELEMENT {
            return Err(PaperForgeError::Xml(format!(
                "XML element exceeds the {MAX_XML_ATTRIBUTES_PER_ELEMENT} attribute limit"
            )));
        }
        let attribute = attribute
            .map_err(|error| PaperForgeError::Xml(bounded_diagnostic(&error.to_string())))?;
        let key = local_xml_name(attribute.key.as_ref());
        let value = attribute
            .decoded_and_normalized_value(quick_xml::XmlVersion::Implicit1_0, reader.decoder())
            .map_err(|error| PaperForgeError::Xml(bounded_diagnostic(&error.to_string())))?
            .into_owned();
        values.push((key, value));
    }
    Ok(values)
}

fn local_xml_name(name: &[u8]) -> String {
    let local = name.rsplit(|byte| *byte == b':').next().unwrap_or(name);
    String::from_utf8_lossy(local).into_owned()
}

fn arxiv_summary(entry: ArxivEntry) -> Result<PaperSummary> {
    let id_url = reqwest::Url::parse(&entry.id)
        .map_err(|_| PaperForgeError::Xml("arXiv entry has an invalid id URL".to_string()))?;
    if !matches!(id_url.host_str(), Some("arxiv.org" | "export.arxiv.org"))
        || !id_url.username().is_empty()
        || id_url.password().is_some()
    {
        return Err(PaperForgeError::UntrustedUrl(bounded_diagnostic(&entry.id)));
    }
    let native_id = id_url
        .path()
        .trim_start_matches("/abs/")
        .trim_start_matches('/')
        .to_string();
    if !valid_arxiv_id(&native_id) {
        return Err(PaperForgeError::Xml(bounded_diagnostic(&format!(
            "invalid arXiv entry identity '{native_id}'"
        ))));
    }
    let landing = entry
        .landing
        .unwrap_or_else(|| format!("https://arxiv.org/abs/{native_id}"));
    let pdf = entry
        .pdf
        .unwrap_or_else(|| format!("https://arxiv.org/pdf/{native_id}"));
    let landing_url = normalized_arxiv_url(&landing)?;
    let pdf_url = normalized_arxiv_url(&pdf)?;
    trusted_provider_url(PaperProvider::Arxiv, &landing_url, UrlPurpose::Landing)?;
    trusted_provider_url(PaperProvider::Arxiv, &pdf_url, UrlPurpose::Document)?;
    let explicit_license = entry
        .license
        .clone()
        .map(|value| compact_whitespace(&value))
        .filter(|value| !value.is_empty());
    let document_license = explicit_license
        .clone()
        .unwrap_or_else(|| "arXiv.org perpetual, non-exclusive license 1.0".to_string());
    let terms_url = explicit_license
        .clone()
        .or_else(|| Some("https://arxiv.org/licenses/nonexclusive-distrib/1.0/".to_string()));
    let license_evidence = explicit_license.map_or_else(
        || "arxiv:license element absent; non-exclusive distribution terms apply".to_string(),
        |license| format!("explicit arxiv:license={license}"),
    );
    let revision = native_id
        .rfind('v')
        .filter(|index| ascii_digits(&native_id[index + 1..]))
        .map_or_else(
            || entry.updated.clone(),
            |index| native_id[index..].to_string(),
        );
    let attribution = entry.authors.join(", ");
    Ok(PaperSummary {
        provider: PaperProvider::Arxiv,
        id: format!("arxiv:{native_id}"),
        native_id,
        title: compact_whitespace(&entry.title),
        abstract_text: compact_whitespace(&entry.summary),
        authors: entry.authors,
        topics: entry.topics,
        doi: entry.doi,
        journal: entry.journal,
        published: date_prefix(&entry.published),
        updated: date_prefix(&entry.updated),
        citation_count: None,
        source_url: landing_url.to_string(),
        rights: PaperRights {
            metadata_license: "CC0-1.0".to_string(),
            document_license: document_license.clone(),
            search_license: None,
            jats_license: None,
            jats_licenses: Vec::new(),
            jats_permissions: Vec::new(),
            license_conflict: false,
            access_status: "publicly-accessible".to_string(),
            attribution,
            terms_url,
            evidence: vec![license_evidence],
            redistribution_warning: "arXiv distribution does not grant PaperForge or its user redistribution rights; verify the paper license before reuse.".to_string(),
        },
        document: PaperDocument {
            url: pdf_url.to_string(),
            size: None,
            mime: "application/pdf".to_string(),
            format: PaperDocumentFormat::Pdf,
            provider_revision: revision,
        },
        relevance: blank_relevance(),
    })
}

#[derive(Debug, Deserialize)]
struct EuropePmcResponse {
    #[serde(rename = "resultList")]
    result_list: Option<EuropePmcResultList>,
    #[serde(default)]
    error: Option<serde_json::Value>,
}

#[derive(Debug, Deserialize)]
struct EuropePmcResultList {
    #[serde(default)]
    result: Vec<EuropePmcRecord>,
}

#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
struct EuropePmcRecord {
    id: Option<String>,
    pmcid: Option<String>,
    pmid: Option<String>,
    doi: Option<String>,
    title: Option<String>,
    abstract_text: Option<String>,
    author_string: Option<String>,
    journal_info: Option<EuropePmcJournalInfo>,
    first_publication_date: Option<String>,
    first_index_date: Option<String>,
    date_of_revision: Option<String>,
    full_text_received_date: Option<String>,
    cited_by_count: Option<u64>,
    is_open_access: Option<String>,
    #[serde(rename = "inEPMC")]
    in_epmc: Option<String>,
    license: Option<String>,
    #[serde(default)]
    author_list: Option<EuropePmcAuthorList>,
    #[serde(default)]
    keyword_list: Option<EuropePmcKeywordList>,
}

#[derive(Debug, Deserialize)]
struct EuropePmcJournalInfo {
    journal: Option<EuropePmcJournal>,
}

#[derive(Debug, Deserialize)]
struct EuropePmcJournal {
    title: Option<String>,
}

#[derive(Debug, Deserialize)]
struct EuropePmcAuthorList {
    #[serde(default)]
    author: Vec<EuropePmcAuthor>,
}

#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
struct EuropePmcAuthor {
    full_name: Option<String>,
    first_name: Option<String>,
    last_name: Option<String>,
}

#[derive(Debug, Deserialize)]
struct EuropePmcKeywordList {
    #[serde(default)]
    keyword: Vec<String>,
}

#[cfg(test)]
fn parse_europe_pmc_json(bytes: &[u8]) -> Result<Vec<PaperSummary>> {
    Ok(parse_europe_pmc_page(bytes)?.papers)
}

fn parse_europe_pmc_page(bytes: &[u8]) -> Result<ProviderCandidates> {
    let response: EuropePmcResponse = serde_json::from_slice(bytes)?;
    if let Some(error) = response.error {
        return Err(PaperForgeError::ProviderApiError {
            provider: PaperProvider::EuropePmc.slug(),
            message: bounded_diagnostic(&error.to_string()),
        });
    }
    let result_list = response
        .result_list
        .ok_or_else(|| PaperForgeError::ProviderApiError {
            provider: PaperProvider::EuropePmc.slug(),
            message: "response schema omitted resultList".to_string(),
        })?;
    let raw_count = result_list.result.len();
    let papers = result_list
        .result
        .into_iter()
        .filter_map(europe_pmc_summary)
        .collect::<Result<Vec<_>>>()?;
    Ok(ProviderCandidates {
        papers,
        raw_count,
        cap_reached: false,
    })
}

fn europe_pmc_summary(record: EuropePmcRecord) -> Option<Result<PaperSummary>> {
    if !yes(record.is_open_access.as_deref()) || !yes(record.in_epmc.as_deref()) {
        return None;
    }
    let pmcid = record.pmcid?.to_ascii_uppercase();
    if parse_identity("pmcid", &pmcid).is_err() {
        return None;
    }
    let title = compact_whitespace(record.title.as_deref().unwrap_or_default());
    if title.is_empty() {
        return None;
    }
    let authors = record
        .author_list
        .map(|list| {
            list.author
                .into_iter()
                .filter_map(|author| {
                    author
                        .full_name
                        .map(|value| compact_whitespace(&value))
                        .or_else(|| {
                            let joined = format!(
                                "{} {}",
                                author.first_name.unwrap_or_default(),
                                author.last_name.unwrap_or_default()
                            );
                            (!joined.trim().is_empty()).then(|| compact_whitespace(&joined))
                        })
                })
                .collect::<Vec<_>>()
        })
        .filter(|authors| !authors.is_empty())
        .unwrap_or_else(|| {
            record
                .author_string
                .unwrap_or_default()
                .split(',')
                .map(compact_whitespace)
                .filter(|value| !value.is_empty())
                .collect()
        });
    let search_license = record
        .license
        .map(|value| compact_whitespace(&value))
        .filter(|value| !value.is_empty());
    let effective_license = search_license
        .clone()
        .unwrap_or_else(|| "Europe-PMC-open-access-license-unspecified".to_string());
    let revision = newest_provider_revision([
        record.date_of_revision.as_deref(),
        record.full_text_received_date.as_deref(),
        record.first_index_date.as_deref(),
    ])
    .unwrap_or_else(|| format!("unavailable:{pmcid}"));
    let updated = newest_provider_revision([
        record.date_of_revision.as_deref(),
        record.full_text_received_date.as_deref(),
        record.first_index_date.as_deref(),
    ])
    .and_then(|value| date_prefix(&value));
    let document_url =
        format!("https://www.ebi.ac.uk/europepmc/webservices/rest/{pmcid}/fullTextXML");
    let source_id = record.pmid.or(record.id).unwrap_or_else(|| pmcid.clone());
    let source_url = format!(
        "https://www.ebi.ac.uk/europepmc/webservices/rest/search?query=EXT_ID:{source_id}&format=json"
    );
    Some(Ok(PaperSummary {
        provider: PaperProvider::EuropePmc,
        id: format!("europe-pmc:{pmcid}"),
        native_id: pmcid.clone(),
        title,
        abstract_text: compact_whitespace(record.abstract_text.as_deref().unwrap_or_default()),
        authors: authors.clone(),
        topics: record.keyword_list.map_or_else(Vec::new, |list| {
            list.keyword
                .into_iter()
                .map(|value| compact_whitespace(&value))
                .filter(|value| !value.is_empty())
                .collect()
        }),
        doi: record.doi.map(|value| value.to_ascii_lowercase()),
        journal: record
            .journal_info
            .and_then(|info| info.journal)
            .and_then(|journal| journal.title)
            .map(|value| compact_whitespace(&value))
            .filter(|value| !value.is_empty()),
        published: record
            .first_publication_date
            .as_deref()
            .and_then(date_prefix),
        updated,
        citation_count: record.cited_by_count,
        source_url,
        rights: PaperRights {
            metadata_license: "Europe-PMC-metadata-terms".to_string(),
            document_license: effective_license,
            search_license: search_license.clone(),
            jats_license: None,
            jats_licenses: Vec::new(),
            jats_permissions: Vec::new(),
            license_conflict: false,
            access_status: "open-access".to_string(),
            attribution: authors.join(", "),
            terms_url: Some("https://europepmc.org/Copyright".to_string()),
            evidence: vec![search_license.map_or_else(
                || "Europe PMC result license was not supplied".to_string(),
                |license| format!("Europe PMC result license={license}"),
            )],
            redistribution_warning: "Open access does not make every reuse unrestricted; follow the retained article license and JATS permissions.".to_string(),
        },
        document: PaperDocument {
            url: document_url,
            size: None,
            mime: "application/xml".to_string(),
            format: PaperDocumentFormat::JatsXml,
            provider_revision: revision,
        },
        relevance: blank_relevance(),
    }))
}

fn newest_provider_revision<const N: usize>(values: [Option<&str>; N]) -> Option<String> {
    values
        .into_iter()
        .flatten()
        .map(compact_whitespace)
        .filter(|value| !value.is_empty())
        .max()
}

fn yes(value: Option<&str>) -> bool {
    value.is_some_and(|value| matches!(value.to_ascii_uppercase().as_str(), "Y" | "YES" | "TRUE"))
}

fn nasa_values(value: &serde_json::Value) -> Result<Vec<&serde_json::Value>> {
    if let Some(results) = value.get("results").and_then(serde_json::Value::as_array) {
        return Ok(results.iter().collect());
    }
    if let Some(results) = value
        .get("response")
        .and_then(|response| response.get("results"))
        .and_then(serde_json::Value::as_array)
    {
        return Ok(results.iter().collect());
    }
    Err(PaperForgeError::ProviderApiError {
        provider: PaperProvider::NasaNtrs.slug(),
        message: "response schema omitted results".to_string(),
    })
}

fn parse_nasa_candidates(value: &serde_json::Value, limit: usize) -> Result<ProviderCandidates> {
    let values = nasa_values(value)?;
    let raw_count = values.len();
    let papers = values
        .into_iter()
        .map(nasa_summary)
        .collect::<Result<Vec<_>>>()?
        .into_iter()
        .flatten()
        .collect();
    Ok(ProviderCandidates {
        papers,
        raw_count,
        cap_reached: raw_count >= limit,
    })
}

fn nasa_summary(value: &serde_json::Value) -> Result<Option<PaperSummary>> {
    let distribution = json_string(value, &["distribution", "distributionType"]);
    let disseminated = json_string(value, &["disseminated", "dissemination"]);
    let downloads_available = json_bool(value, &["downloadsAvailable"]);
    let cui =
        json_bool(value, &["cui", "isCui"]).or_else(|| json_nested_bool(value, "cui", &["isCui"]));
    let itar = json_nested_string(value, "exportControl", &["itar", "ITAR"])
        .or_else(|| json_string(value, &["itar", "ITAR"]));
    let ear = json_nested_string(value, "exportControl", &["ear", "EAR"])
        .or_else(|| json_string(value, &["ear", "EAR"]));
    let export_control = json_nested_string(
        value,
        "exportControl",
        &["isExportControl", "is_export_control"],
    );
    if !distribution.is_some_and(|item| item.eq_ignore_ascii_case("PUBLIC"))
        || !disseminated.is_some_and(|item| item.eq_ignore_ascii_case("DOCUMENT_AND_METADATA"))
        || itar.is_none_or(|item| !item.eq_ignore_ascii_case("NO"))
        || ear.is_none_or(|item| !item.eq_ignore_ascii_case("NO"))
        || export_control.is_none_or(|item| !item.eq_ignore_ascii_case("NO"))
        || cui != Some(false)
        || downloads_available != Some(true)
    {
        return Ok(None);
    }
    let id = json_scalar_string(value, &["id", "citationId"]);
    let Some(native_id) = id.filter(|id| (8..=20).contains(&id.len()) && ascii_digits(id)) else {
        return Ok(None);
    };
    let downloads = value
        .get("downloads")
        .and_then(serde_json::Value::as_array)
        .or_else(|| {
            value
                .get("downloadLinks")
                .and_then(serde_json::Value::as_array)
        });
    let Some(download) = downloads.and_then(|downloads| {
        downloads.iter().find(|download| {
            json_string(download, &["mime", "mimeType", "mimetype", "contentType"])
                .is_some_and(|mime| mime.eq_ignore_ascii_case("application/pdf"))
                && json_bool(download, &["draft"]) == Some(false)
                && json_string(download, &["type"])
                    .is_some_and(|kind| kind.eq_ignore_ascii_case("STI"))
        })
    }) else {
        return Ok(None);
    };
    let raw_url = json_string(download, &["url", "downloadUrl", "href"])
        .or_else(|| json_nested_string(download, "links", &["original", "download"]));
    let Some(raw_url) = raw_url else {
        return Ok(None);
    };
    let document_url = if raw_url.starts_with('/') {
        format!("https://ntrs.nasa.gov{raw_url}")
    } else {
        raw_url.to_string()
    };
    let parsed = reqwest::Url::parse(&document_url)
        .map_err(|_| PaperForgeError::UntrustedUrl(document_url.clone()))?;
    trusted_provider_url(PaperProvider::NasaNtrs, &parsed, UrlPurpose::Document)?;
    let title = json_string(value, &["title"])
        .map(compact_whitespace)
        .unwrap_or_default();
    if title.is_empty() {
        return Ok(None);
    }
    let authors = nasa_authors(value);
    let copyright = compact_whitespace(
        json_string(
            value,
            &["copyright", "copyrightDetermination", "copyrightStatus"],
        )
        .or_else(|| json_nested_string(value, "copyright", &["determinationType", "licenseType"]))
        .unwrap_or("not-supplied"),
    );
    let public_use = compact_whitespace(
        json_string(
            value,
            &["publicUsePermissions", "publicUseTerms", "useTerms"],
        )
        .or_else(|| json_nested_string(value, "copyright", &["determinationType"]))
        .unwrap_or("not-supplied"),
    );
    let size = json_u64(download, &["size", "fileSize", "bytes"]);
    let revision = compact_whitespace(
        json_string(value, &["modified", "updated", "revision"]).unwrap_or(&native_id),
    );
    Ok(Some(PaperSummary {
        provider: PaperProvider::NasaNtrs,
        id: format!("nasa-ntrs:{native_id}"),
        native_id: native_id.clone(),
        title,
        abstract_text: json_string(value, &["abstract", "abstractText"])
            .map(compact_whitespace)
            .unwrap_or_default(),
        authors: authors.clone(),
        topics: nasa_strings(value, &["subjectCategories", "keywords"]),
        doi: json_string(value, &["doi"])
            .or_else(|| nasa_publication_string(value, &["doi"]))
            .map(|doi| doi.to_ascii_lowercase()),
        journal: json_string(value, &["publication", "journal", "stiType"])
            .or_else(|| nasa_publication_string(value, &["publicationName", "journal"]))
            .map(compact_whitespace)
            .filter(|value| !value.is_empty()),
        published: json_string(value, &["published", "publicationDate", "distributionDate"])
            .and_then(date_prefix),
        updated: json_string(value, &["modified", "updated"]).and_then(date_prefix),
        citation_count: None,
        source_url: format!("https://ntrs.nasa.gov/citations/{native_id}"),
        rights: PaperRights {
            metadata_license: "NASA-NTRS-metadata-terms".to_string(),
            document_license: copyright.clone(),
            search_license: None,
            jats_license: None,
            jats_licenses: Vec::new(),
            jats_permissions: Vec::new(),
            license_conflict: false,
            access_status: "PUBLIC/DOCUMENT_AND_METADATA".to_string(),
            attribution: authors.join(", "),
            terms_url: Some("https://sti.nasa.gov/disclaimers/".to_string()),
            evidence: vec![
                format!("copyright determination={copyright}"),
                format!("public-use terms={public_use}"),
                "isExportControl=NO; ITAR=NO; EAR=NO; CUI=false".to_string(),
            ],
            redistribution_warning: "NASA hosting and PUBLIC distribution do not by themselves establish public-domain or redistribution rights.".to_string(),
        },
        document: PaperDocument {
            url: parsed.to_string(),
            size,
            mime: "application/pdf".to_string(),
            format: PaperDocumentFormat::Pdf,
            provider_revision: revision,
        },
        relevance: blank_relevance(),
    }))
}

fn nasa_authors(value: &serde_json::Value) -> Vec<String> {
    for key in ["authors", "authorAffiliations"] {
        if let Some(values) = value.get(key).and_then(serde_json::Value::as_array) {
            let authors = values
                .iter()
                .filter_map(|author| {
                    author
                        .as_str()
                        .map(compact_whitespace)
                        .or_else(|| {
                            json_string(author, &["name", "fullName"]).map(compact_whitespace)
                        })
                        .or_else(|| {
                            author
                                .get("meta")
                                .and_then(|meta| meta.get("author"))
                                .and_then(|author| json_string(author, &["name", "fullName"]))
                                .map(compact_whitespace)
                        })
                })
                .filter(|author| !author.is_empty())
                .collect::<Vec<_>>();
            if !authors.is_empty() {
                return authors;
            }
        }
    }
    Vec::new()
}

fn nasa_strings(value: &serde_json::Value, keys: &[&str]) -> Vec<String> {
    for key in keys {
        if let Some(items) = value.get(*key).and_then(serde_json::Value::as_array) {
            return items
                .iter()
                .filter_map(|item| {
                    item.as_str()
                        .map(compact_whitespace)
                        .or_else(|| json_string(item, &["name", "value"]).map(compact_whitespace))
                })
                .filter(|item| !item.is_empty())
                .collect();
        }
    }
    Vec::new()
}

fn nasa_publication_string<'a>(value: &'a serde_json::Value, keys: &[&str]) -> Option<&'a str> {
    value
        .get("publications")
        .and_then(serde_json::Value::as_array)
        .and_then(|publications| publications.iter().find_map(|item| json_string(item, keys)))
}

fn json_string<'a>(value: &'a serde_json::Value, keys: &[&str]) -> Option<&'a str> {
    keys.iter()
        .find_map(|key| value.get(*key).and_then(serde_json::Value::as_str))
}

fn json_nested_string<'a>(
    value: &'a serde_json::Value,
    parent: &str,
    keys: &[&str],
) -> Option<&'a str> {
    value
        .get(parent)
        .and_then(|nested| json_string(nested, keys))
}

fn json_bool(value: &serde_json::Value, keys: &[&str]) -> Option<bool> {
    keys.iter()
        .find_map(|key| value.get(*key).and_then(serde_json::Value::as_bool))
}

fn json_nested_bool(value: &serde_json::Value, parent: &str, keys: &[&str]) -> Option<bool> {
    value.get(parent).and_then(|nested| json_bool(nested, keys))
}

fn json_scalar_string(value: &serde_json::Value, keys: &[&str]) -> Option<String> {
    keys.iter().find_map(|key| {
        value.get(*key).and_then(|value| {
            value
                .as_str()
                .map(ToOwned::to_owned)
                .or_else(|| value.as_u64().map(|number| number.to_string()))
        })
    })
}

fn json_u64(value: &serde_json::Value, keys: &[&str]) -> Option<u64> {
    keys.iter()
        .find_map(|key| value.get(*key).and_then(serde_json::Value::as_u64))
}

fn local_filter_rank(papers: &mut Vec<PaperSummary>, query: &PaperQuery) {
    papers.retain_mut(|paper| {
        match_paper(paper, query).is_some_and(|evidence| {
            paper.relevance = evidence;
            true
        })
    });
    papers.sort_by(|left, right| compare_papers(left, right, query));
}

fn finalize_discovery(
    mut papers: Vec<PaperSummary>,
    query: &PaperQuery,
    output_limit: usize,
    candidate_budget: usize,
    raw_candidate_count: usize,
    provider_cap_reached: bool,
) -> DiscoveryResult {
    dedupe_provider(&mut papers);
    let eligible_candidate_count = papers.len();
    local_filter_rank(&mut papers, query);
    papers.truncate(output_limit);
    DiscoveryResult {
        papers,
        candidate_count: raw_candidate_count,
        eligible_candidate_count,
        candidate_cap_reached: provider_cap_reached || raw_candidate_count >= candidate_budget,
        candidate_budget,
    }
}

fn match_paper(paper: &PaperSummary, query: &PaperQuery) -> Option<QueryEvidence> {
    if query.after.as_ref().is_some_and(|after| {
        paper
            .published
            .as_ref()
            .is_none_or(|published| published < after)
    }) || query.before.as_ref().is_some_and(|before| {
        paper
            .published
            .as_ref()
            .is_none_or(|published| published > before)
    }) {
        return None;
    }
    let exact = query
        .identities
        .iter()
        .any(|identity| identity_matches(paper, identity));
    if !query.identities.is_empty() && !exact {
        return None;
    }
    let required = query
        .clauses
        .iter()
        .filter(|clause| clause.polarity == ClausePolarity::Required)
        .collect::<Vec<_>>();
    let optional = query
        .clauses
        .iter()
        .filter(|clause| clause.polarity == ClausePolarity::Optional)
        .collect::<Vec<_>>();
    if query
        .clauses
        .iter()
        .filter(|clause| clause.polarity == ClausePolarity::Excluded)
        .any(|clause| clause_match(paper, clause).is_some())
    {
        return None;
    }
    let required_matches = required
        .iter()
        .filter_map(|clause| clause_match(paper, clause))
        .collect::<Vec<_>>();
    if required_matches.len() != required.len() {
        return None;
    }
    let optional_matches = optional
        .iter()
        .filter_map(|clause| clause_match(paper, clause))
        .collect::<Vec<_>>();
    if required.is_empty() && !optional.is_empty() && optional_matches.is_empty() {
        return None;
    }
    let all_positive = optional_matches.len() == optional.len();
    let title = normalized_text(&paper.title);
    let exact_title = query.clauses.iter().any(|clause| {
        clause.polarity != ClausePolarity::Excluded
            && clause.phrase
            && matches!(clause.field, TextField::Any | TextField::Title)
            && normalized_text(&clause.value) == title
            && clause_match(paper, clause).is_some_and(|matched| matched.title)
    });
    let title_or_phrase = required_matches
        .iter()
        .chain(optional_matches.iter())
        .any(|matched| matched.title || matched.phrase);
    let tier = if exact {
        RelevanceTier::ExactIdentity
    } else if exact_title {
        RelevanceTier::ExactTitlePhrase
    } else if all_positive && title_or_phrase && (!required.is_empty() || !optional.is_empty()) {
        RelevanceTier::AllPositiveTitleOrPhrase
    } else if all_positive && (!required.is_empty() || !optional.is_empty()) {
        RelevanceTier::AllPositive
    } else if !required.is_empty() && !optional_matches.is_empty() {
        RelevanceTier::RequiredPlusOptional
    } else {
        RelevanceTier::FilterOnly
    };
    let mut matched = required_matches
        .iter()
        .chain(optional_matches.iter())
        .map(|item| item.label.clone())
        .collect::<Vec<_>>();
    if exact {
        matched.extend(
            query
                .identities
                .iter()
                .filter(|identity| identity_matches(paper, identity))
                .map(|identity| {
                    format!("{}:{}", identity_kind_slug(identity.kind), identity.value)
                }),
        );
    }
    let score = required_matches
        .iter()
        .chain(optional_matches.iter())
        .map(|matched| matched.score)
        .sum::<i64>()
        + i64::from(exact) * 100;
    Some(QueryEvidence {
        tier,
        score,
        matched,
    })
}

struct ClauseMatch {
    score: i64,
    title: bool,
    phrase: bool,
    label: String,
}

fn clause_match(paper: &PaperSummary, clause: &TextClause) -> Option<ClauseMatch> {
    let needle = normalized_text(&clause.value);
    if needle.is_empty() {
        return None;
    }
    let fields: Vec<(TextField, String)> = match clause.field {
        TextField::Any => vec![
            (TextField::Title, paper.title.clone()),
            (TextField::Author, paper.authors.join(" ")),
            (TextField::Topic, paper.topics.join(" ")),
            (TextField::Abstract, paper.abstract_text.clone()),
            (
                TextField::Journal,
                paper.journal.clone().unwrap_or_default(),
            ),
        ],
        TextField::Title => vec![(TextField::Title, paper.title.clone())],
        TextField::Abstract => vec![(TextField::Abstract, paper.abstract_text.clone())],
        TextField::Author => vec![(TextField::Author, paper.authors.join(" "))],
        TextField::Topic => vec![(TextField::Topic, paper.topics.join(" "))],
        TextField::Journal => vec![(
            TextField::Journal,
            paper.journal.clone().unwrap_or_default(),
        )],
    };
    let mut best: Option<ClauseMatch> = None;
    for (field, value) in fields {
        let value = normalized_text(&value);
        let phrase = contains_normalized_phrase(&value, &needle);
        let matched = if clause.phrase {
            phrase
        } else {
            needle
                .split_whitespace()
                .all(|token| value.split_whitespace().any(|candidate| candidate == token))
        };
        if !matched {
            continue;
        }
        let weight = field_weight(field);
        let score = weight * if phrase { 3 } else { 1 };
        if best.as_ref().is_none_or(|current| current.score < score) {
            best = Some(ClauseMatch {
                score,
                title: field == TextField::Title,
                phrase,
                label: format!("{}:{}", text_field_slug(field), clause.value),
            });
        }
    }
    best
}

fn field_weight(field: TextField) -> i64 {
    match field {
        TextField::Title => 8,
        TextField::Author => 5,
        TextField::Topic => 4,
        TextField::Abstract | TextField::Any => 3,
        TextField::Journal => 2,
    }
}

fn text_field_slug(field: TextField) -> &'static str {
    match field {
        TextField::Any => "any",
        TextField::Title => "title",
        TextField::Abstract => "abstract",
        TextField::Author => "author",
        TextField::Topic => "topic",
        TextField::Journal => "journal",
    }
}

fn identity_matches(paper: &PaperSummary, identity: &PaperIdentity) -> bool {
    match identity.kind {
        IdentityKind::Arxiv => {
            paper.provider == PaperProvider::Arxiv && paper.native_id == identity.value
        }
        IdentityKind::Pmcid => {
            paper.provider == PaperProvider::EuropePmc
                && paper.native_id.eq_ignore_ascii_case(&identity.value)
        }
        IdentityKind::Ntrs => {
            paper.provider == PaperProvider::NasaNtrs && paper.native_id == identity.value
        }
        IdentityKind::Pmid => {
            paper.provider == PaperProvider::EuropePmc
                && europe_pmc_source_ext_id(paper).as_deref() == Some(identity.value.as_str())
        }
        IdentityKind::Doi => paper
            .doi
            .as_ref()
            .is_some_and(|doi| doi.eq_ignore_ascii_case(&identity.value)),
    }
}

fn europe_pmc_source_ext_id(paper: &PaperSummary) -> Option<String> {
    let url = reqwest::Url::parse(&paper.source_url).ok()?;
    let query = url
        .query_pairs()
        .find_map(|(key, value)| (key == "query").then_some(value.into_owned()))?;
    query
        .strip_prefix("EXT_ID:")
        .map(|value| value.trim_matches('"').to_string())
}

fn compare_papers(left: &PaperSummary, right: &PaperSummary, query: &PaperQuery) -> Ordering {
    let strict = query.sort;
    let ordering = match strict {
        Some(PaperSort::Date) => compare_date(right, left)
            .then_with(|| right.relevance.tier.cmp(&left.relevance.tier))
            .then_with(|| right.relevance.score.cmp(&left.relevance.score)),
        Some(PaperSort::Citations) => right
            .citation_count
            .unwrap_or(0)
            .cmp(&left.citation_count.unwrap_or(0))
            .then_with(|| right.relevance.tier.cmp(&left.relevance.tier))
            .then_with(|| right.relevance.score.cmp(&left.relevance.score)),
        Some(PaperSort::Relevance) => compare_relevance(left, right),
        None => match query.mode {
            PaperMode::Standard => compare_relevance(left, right),
            PaperMode::Recent => right
                .relevance
                .tier
                .cmp(&left.relevance.tier)
                .then_with(|| compare_date(right, left))
                .then_with(|| right.relevance.score.cmp(&left.relevance.score)),
            PaperMode::Top => right
                .relevance
                .tier
                .cmp(&left.relevance.tier)
                .then_with(|| {
                    right
                        .citation_count
                        .unwrap_or(0)
                        .cmp(&left.citation_count.unwrap_or(0))
                })
                .then_with(|| right.relevance.score.cmp(&left.relevance.score)),
        },
    };
    ordering.then_with(|| left.id.cmp(&right.id))
}

fn compare_relevance(left: &PaperSummary, right: &PaperSummary) -> Ordering {
    right
        .relevance
        .tier
        .cmp(&left.relevance.tier)
        .then_with(|| right.relevance.score.cmp(&left.relevance.score))
}

fn compare_date(left: &PaperSummary, right: &PaperSummary) -> Ordering {
    left.published
        .as_deref()
        .unwrap_or("")
        .cmp(right.published.as_deref().unwrap_or(""))
}

fn dedupe_provider(papers: &mut Vec<PaperSummary>) {
    let mut seen = HashSet::new();
    papers.retain(|paper| seen.insert((paper.provider, paper.native_id.clone())));
}

fn blank_relevance() -> QueryEvidence {
    QueryEvidence {
        tier: RelevanceTier::FilterOnly,
        score: 0,
        matched: Vec::new(),
    }
}

impl PaperForge {
    fn archive_inner(
        &self,
        papers: &[PaperSummary],
        options: &ArchiveOptions,
        mut progress: Option<&mut dyn FnMut(ArchiveProgress)>,
    ) -> Result<ArchiveSummary> {
        prepare_output(&options.output)?;
        let mut summary = ArchiveSummary {
            selected: papers.len(),
            selected_ids: papers.iter().map(|paper| paper.id.clone()).collect(),
            ..ArchiveSummary::default()
        };
        emit_progress(
            &mut progress,
            progress_snapshot(ArchiveProgressPhase::Starting, papers.len(), &summary),
        );
        for paper in papers {
            match self.archive_one(paper, options, papers.len(), &summary, &mut progress) {
                Ok(ArchiveOutcome::Archived(archived)) => {
                    summary.archived += 1;
                    summary.bytes_written =
                        summary.bytes_written.saturating_add(archived.bytes_written);
                    summary.files_written += archived.files_written;
                    summary.papers.push(archived);
                }
                Ok(ArchiveOutcome::Skipped(skipped)) => {
                    summary.skipped += 1;
                    summary.skipped_papers.push(skipped);
                }
                Err(error) => summary.failed.push(ArchiveFailure {
                    paper_id: paper.id.clone(),
                    output: safe_id(&paper.id)
                        .ok()
                        .map(|identity| options.output.join(identity)),
                    message: error.to_string(),
                }),
            }
            summary.failed_count = summary.failed.len();
            emit_progress(
                &mut progress,
                progress_snapshot(ArchiveProgressPhase::Downloading, papers.len(), &summary),
            );
        }
        emit_progress(
            &mut progress,
            progress_snapshot(ArchiveProgressPhase::Completed, papers.len(), &summary),
        );
        Ok(summary)
    }

    fn archive_one(
        &self,
        selected: &PaperSummary,
        options: &ArchiveOptions,
        total: usize,
        aggregate: &ArchiveSummary,
        progress: &mut Option<&mut dyn FnMut(ArchiveProgress)>,
    ) -> Result<ArchiveOutcome> {
        if selected.provider != self.provider {
            return Err(PaperForgeError::VerificationFailed {
                path: selected.id.clone(),
                message: "paper metadata belongs to another provider".to_string(),
            });
        }
        let mut paper = self.resolve_for_archive(selected)?;
        if paper.id != selected.id {
            return Err(PaperForgeError::VerificationFailed {
                path: selected.id.clone(),
                message: "provider identity changed during archival resolution".to_string(),
            });
        }
        paper.relevance = selected.relevance.clone();
        validate_summary(&paper)?;
        let hard_cap = 1024 * 1024 * 1024_u64;
        let cap = if options.max_download_bytes == 0 {
            hard_cap
        } else {
            options.max_download_bytes.min(hard_cap)
        };
        if paper.document.size.is_some_and(|size| size > cap) {
            return Err(PaperForgeError::ResponseLimitExceeded {
                label: paper.id.clone(),
                size: paper.document.size.unwrap_or_default(),
                limit: cap,
            });
        }
        let destination = options.output.join(safe_id(&paper.id)?);
        let mut recognized = false;
        if destination.exists() {
            let manifest = read_manifest(&destination).map_err(|error| {
                PaperForgeError::UnrecognizedExistingOutput {
                    path: destination.clone(),
                    reason: error.to_string(),
                }
            })?;
            recognize_manifest(&manifest, &paper)?;
            recognized = true;
            if options.skip_existing
                && same_archive_revision(&manifest.paper, &paper)
                && validate_manifest(&destination, &manifest).is_ok()
            {
                return Ok(ArchiveOutcome::Skipped(ArchivedPaper {
                    paper_id: paper.id.clone(),
                    output: destination,
                    bytes_written: manifest.integrity.bytes,
                    files_written: 1,
                    manifest,
                }));
            }
        }
        if destination.exists() && !recognized {
            return Err(PaperForgeError::UnrecognizedExistingOutput {
                path: destination,
                reason: "missing a recognized PaperForge manifest".to_string(),
            });
        }
        let staging = create_staging(&options.output, &paper.id)?;
        let archive_result = (|| -> Result<ArchivedPaper> {
            let relative = format!("paper.{}", paper.document.format.extension());
            let output = staging.join(&relative);
            let downloaded = self.download_document(&paper, &output, cap, |active_bytes| {
                emit_progress(
                    progress,
                    ArchiveProgress {
                        phase: ArchiveProgressPhase::Downloading,
                        total_papers: total,
                        completed_papers: aggregate.archived
                            + aggregate.skipped
                            + aggregate.failed.len(),
                        succeeded_papers: aggregate.archived + aggregate.skipped,
                        failed_papers: aggregate.failed.len(),
                        bytes_written: aggregate.bytes_written.saturating_add(active_bytes),
                        active_paper: Some(paper.id.clone()),
                        active_file: Some(relative.clone()),
                        active_bytes_written: active_bytes,
                        active_total_bytes: paper.document.size,
                    },
                );
            })?;
            if paper.document.format == PaperDocumentFormat::JatsXml {
                enrich_jats_rights(&output, &mut paper.rights)?;
            }
            let manifest = PaperManifest {
                format_version: "1".to_string(),
                paper: paper.clone(),
                integrity: PaperIntegrity {
                    path: relative,
                    bytes: downloaded.bytes,
                    blake3: downloaded.blake3,
                },
                complete: true,
            };
            write_manifest(&staging, &manifest)?;
            validate_manifest(&staging, &manifest)?;
            Ok(ArchivedPaper {
                paper_id: paper.id.clone(),
                output: destination.clone(),
                bytes_written: downloaded.bytes,
                files_written: 1,
                manifest,
            })
        })();
        match archive_result {
            Ok(archived) => {
                if let Err(error) = commit_staging(&staging, &destination) {
                    if staging.exists() {
                        if let Err(cleanup) = fs::remove_dir_all(&staging) {
                            return Err(PaperForgeError::ArchiveTransaction {
                                path: destination,
                                message: format!(
                                    "{error}; also failed to remove staging '{}': {cleanup}",
                                    staging.display()
                                ),
                            });
                        }
                    }
                    return Err(error);
                }
                Ok(ArchiveOutcome::Archived(archived))
            }
            Err(error) => {
                if let Err(cleanup) = fs::remove_dir_all(&staging) {
                    return Err(PaperForgeError::ArchiveTransaction {
                        path: destination,
                        message: format!(
                            "{error}; also failed to remove staging '{}': {cleanup}",
                            staging.display()
                        ),
                    });
                }
                Err(error)
            }
        }
    }

    fn resolve_for_archive(&self, selected: &PaperSummary) -> Result<PaperSummary> {
        let kind = match self.provider {
            PaperProvider::Arxiv => IdentityKind::Arxiv,
            PaperProvider::EuropePmc => IdentityKind::Pmcid,
            PaperProvider::NasaNtrs => IdentityKind::Ntrs,
        };
        if self.provider == PaperProvider::NasaNtrs {
            return self.resolve_nasa_for_archive(selected);
        }
        let query = PaperQuery {
            identities: vec![PaperIdentity {
                kind,
                value: selected.native_id.clone(),
            }],
            limit: Some(1),
            original: format!("{}:{}", identity_kind_slug(kind), selected.native_id),
            ..PaperQuery::default()
        };
        self.discover(&query)?
            .into_iter()
            .next()
            .ok_or_else(|| PaperForgeError::NotFound {
                provider: self.provider.slug(),
                id: selected.native_id.clone(),
            })
    }

    fn resolve_nasa_for_archive(&self, selected: &PaperSummary) -> Result<PaperSummary> {
        self.fetch_nasa_exact(&selected.native_id)?
            .ok_or_else(|| PaperForgeError::NotFound {
                provider: self.provider.slug(),
                id: selected.native_id.clone(),
            })
    }

    fn fetch_nasa_exact(&self, native_id: &str) -> Result<Option<PaperSummary>> {
        let metadata_url = reqwest::Url::parse(&format!(
            "https://ntrs.nasa.gov/api/citations/{}",
            native_id
        ))
        .expect("validated NTRS id forms URL");
        trusted_provider_url(PaperProvider::NasaNtrs, &metadata_url, UrlPurpose::Api)?;
        let metadata = self.api_request(
            || self.client.get(metadata_url.clone()),
            "NASA NTRS metadata",
        )?;
        let mut value: serde_json::Value = serde_json::from_slice(&metadata)?;
        if !value.is_object() {
            return Err(PaperForgeError::ProviderApiError {
                provider: PaperProvider::NasaNtrs.slug(),
                message: "metadata response schema was not an object".to_string(),
            });
        }
        let downloads_url = reqwest::Url::parse(&format!(
            "https://ntrs.nasa.gov/api/citations/{}/downloads",
            native_id
        ))
        .expect("validated NTRS id forms URL");
        trusted_provider_url(PaperProvider::NasaNtrs, &downloads_url, UrlPurpose::Api)?;
        let downloads = self.api_request(
            || self.client.get(downloads_url.clone()),
            "NASA NTRS downloads",
        )?;
        let downloads_value: serde_json::Value = serde_json::from_slice(&downloads)?;
        let list = if downloads_value.is_array() {
            downloads_value
        } else {
            match downloads_value.get("downloads") {
                Some(downloads) if downloads.is_array() => downloads.clone(),
                _ => {
                    return Err(PaperForgeError::ProviderApiError {
                        provider: PaperProvider::NasaNtrs.slug(),
                        message: "downloads response schema omitted its array".to_string(),
                    });
                }
            }
        };
        if let Some(object) = value.as_object_mut() {
            object.insert("downloads".to_string(), list);
        }
        nasa_summary(&value)
    }

    fn download_document<F>(
        &self,
        paper: &PaperSummary,
        output: &Path,
        cap: u64,
        mut progress: F,
    ) -> Result<Downloaded>
    where
        F: FnMut(u64),
    {
        let url = reqwest::Url::parse(&paper.document.url)
            .map_err(|_| PaperForgeError::UntrustedUrl(paper.document.url.clone()))?;
        trusted_provider_url(self.provider, &url, UrlPurpose::Document)?;
        let mut response = None;
        for attempt in 0..MAX_API_ATTEMPTS {
            let arxiv_guard = self.acquire_provider_slot()?;
            let candidate = self.client.get(url.clone()).send()?;
            let status = candidate.status().as_u16();
            let retry_after = retry_after(&candidate);
            if candidate.status().is_success() {
                response = Some((candidate, arxiv_guard));
                break;
            }
            if retryable_status(status) && attempt + 1 < MAX_API_ATTEMPTS {
                drop(arxiv_guard);
                thread::sleep(retry_delay(attempt, retry_after));
                continue;
            }
            return Err(PaperForgeError::ProviderApi {
                provider: self.provider.slug(),
                status,
                body: bounded_error_body(candidate),
            });
        }
        let (mut response, _arxiv_guard) =
            response.expect("download attempt loop returns or succeeds");
        validate_response_mime(&response, paper.document.format)?;
        if let Some(length) = response
            .headers()
            .get(CONTENT_LENGTH)
            .and_then(|value| value.to_str().ok())
            .and_then(|value| value.parse::<u64>().ok())
        {
            if length > cap {
                return Err(PaperForgeError::ResponseLimitExceeded {
                    label: paper.id.clone(),
                    size: length,
                    limit: cap,
                });
            }
        }
        let mut file = File::create(output)?;
        let mut hasher = Blake3::new();
        let mut total = 0_u64;
        let mut next_progress = 0_u64;
        let mut buffer = [0_u8; 64 * 1024];
        loop {
            let read = response.read(&mut buffer)?;
            if read == 0 {
                break;
            }
            total = total.saturating_add(read as u64);
            if total > cap {
                return Err(PaperForgeError::ResponseLimitExceeded {
                    label: paper.id.clone(),
                    size: total,
                    limit: cap,
                });
            }
            file.write_all(&buffer[..read])?;
            hasher.update(&buffer[..read]);
            if total >= next_progress {
                progress(total);
                next_progress = total.saturating_add(PROGRESS_GRANULARITY);
            }
        }
        file.sync_all()?;
        if let Some(expected) = paper.document.size {
            if total != expected {
                return Err(PaperForgeError::VerificationFailed {
                    path: paper.id.clone(),
                    message: format!("expected {expected} bytes but downloaded {total}"),
                });
            }
        }
        verify_document_magic(output, paper.document.format, Some(&paper.native_id))?;
        progress(total);
        Ok(Downloaded {
            bytes: total,
            blake3: hasher.finalize().to_hex().to_string(),
        })
    }
}

struct Downloaded {
    bytes: u64,
    blake3: String,
}

enum ArchiveOutcome {
    Archived(ArchivedPaper),
    Skipped(ArchivedPaper),
}

fn prepare_output(output: &Path) -> Result<()> {
    if output.as_os_str().is_empty() {
        return Err(PaperForgeError::UnsafePath("empty output".to_string()));
    }
    ensure_no_symlink_components(output)?;
    if output.exists() {
        if !output.is_dir() {
            return Err(PaperForgeError::OutputNotDirectory(output.to_path_buf()));
        }
    } else {
        fs::create_dir_all(output)?;
    }
    ensure_not_symlink(output)?;
    refuse_stale_transactions(output)?;
    Ok(())
}

fn refuse_stale_transactions(root: &Path) -> Result<()> {
    for entry in fs::read_dir(root)? {
        let entry = entry?;
        let name = entry.file_name();
        let name = name.to_string_lossy();
        if name.starts_with(".paperforge-stage-") || name.starts_with(".paperforge-backup-") {
            return Err(PaperForgeError::ArchiveTransaction {
                path: entry.path(),
                message: "stale transaction state requires operator review".to_string(),
            });
        }
    }
    Ok(())
}

fn ensure_no_symlink_components(path: &Path) -> Result<()> {
    let mut current = PathBuf::new();
    for component in path.components() {
        match component {
            Component::Prefix(_)
            | Component::RootDir
            | Component::CurDir
            | Component::Normal(_) => {
                current.push(component.as_os_str());
            }
            Component::ParentDir => {
                return Err(PaperForgeError::UnsafePath(path.display().to_string()));
            }
        }
        if current.exists() {
            ensure_not_symlink(&current)?;
        }
    }
    Ok(())
}

fn ensure_not_symlink(path: &Path) -> Result<()> {
    if fs::symlink_metadata(path)?.file_type().is_symlink() {
        return Err(PaperForgeError::UnsafePath(path.display().to_string()));
    }
    Ok(())
}

fn safe_id(value: &str) -> Result<String> {
    if value.is_empty() || value.len() > 512 {
        return Err(PaperForgeError::UnsafePath(value.to_string()));
    }
    let safe = value
        .chars()
        .map(|character| {
            if character.is_ascii_alphanumeric() || matches!(character, '-' | '_' | '.') {
                character
            } else {
                '_'
            }
        })
        .take(160)
        .collect::<String>();
    if safe.is_empty() || matches!(safe.as_str(), "." | "..") {
        return Err(PaperForgeError::UnsafePath(value.to_string()));
    }
    Ok(safe)
}

fn create_staging(root: &Path, id: &str) -> Result<PathBuf> {
    let stem = safe_id(id)?;
    for nonce in 0..100_u32 {
        let path = root.join(format!(
            ".paperforge-stage-{stem}-{}-{nonce}",
            unix_timestamp()?
        ));
        match fs::create_dir(&path) {
            Ok(()) => {
                sync_directory(root)?;
                return Ok(path);
            }
            Err(error) if error.kind() == io::ErrorKind::AlreadyExists => continue,
            Err(error) => return Err(error.into()),
        }
    }
    Err(PaperForgeError::ArchiveTransaction {
        path: root.to_path_buf(),
        message: "could not allocate a staging directory".to_string(),
    })
}

fn write_manifest(directory: &Path, manifest: &PaperManifest) -> Result<()> {
    let bytes = serde_json::to_vec_pretty(manifest)?;
    if bytes.len() as u64 > MAX_MANIFEST_BYTES {
        return Err(PaperForgeError::ResponseLimitExceeded {
            label: "paper manifest".to_string(),
            size: bytes.len() as u64,
            limit: MAX_MANIFEST_BYTES,
        });
    }
    let path = directory.join(MANIFEST_NAME);
    let mut file = File::create(path)?;
    file.write_all(&bytes)?;
    file.write_all(b"\n")?;
    file.sync_all()?;
    sync_directory(directory)?;
    Ok(())
}

fn read_manifest(directory: &Path) -> Result<PaperManifest> {
    ensure_not_symlink(directory)?;
    let path = directory.join(MANIFEST_NAME);
    ensure_not_symlink(&path)?;
    let metadata = fs::metadata(&path)?;
    if metadata.len() > MAX_MANIFEST_BYTES {
        return Err(PaperForgeError::ResponseLimitExceeded {
            label: "paper manifest".to_string(),
            size: metadata.len(),
            limit: MAX_MANIFEST_BYTES,
        });
    }
    Ok(serde_json::from_reader(File::open(path)?)?)
}

/// Read and fully verify a PaperForge manifest, metadata, document, and BLAKE3.
pub fn inspect_archive(directory: impl AsRef<Path>) -> Result<PaperManifest> {
    let directory = directory.as_ref();
    let manifest = read_manifest(directory)?;
    validate_manifest(directory, &manifest)?;
    Ok(manifest)
}

/// Read and verify a PaperForge manifest, document format, identity, size, and BLAKE3.
pub fn verify_archive(directory: impl AsRef<Path>) -> Result<PaperManifest> {
    let directory = directory.as_ref();
    let manifest = read_manifest(directory)?;
    validate_manifest(directory, &manifest)?;
    Ok(manifest)
}

fn recognize_manifest(manifest: &PaperManifest, paper: &PaperSummary) -> Result<()> {
    if manifest.format_version != "1"
        || !manifest.complete
        || manifest.paper.id != paper.id
        || manifest.paper.provider != paper.provider
        || !matches!(manifest.integrity.path.as_str(), "paper.pdf" | "paper.xml")
    {
        return Err(PaperForgeError::UnrecognizedExistingOutput {
            path: PathBuf::from(&paper.id),
            reason: "manifest ownership or identity is invalid".to_string(),
        });
    }
    Ok(())
}

fn same_archive_revision(archived: &PaperSummary, current: &PaperSummary) -> bool {
    !current
        .document
        .provider_revision
        .starts_with("unavailable:")
        && archived.id == current.id
        && archived.document.url == current.document.url
        && archived.document.format == current.document.format
        && archived.document.provider_revision == current.document.provider_revision
        && archived.rights.metadata_license == current.rights.metadata_license
        && archived.rights.search_license == current.rights.search_license
        && archived.rights.access_status == current.rights.access_status
}

fn validate_manifest(directory: &Path, manifest: &PaperManifest) -> Result<()> {
    if !manifest.complete
        || manifest.format_version != "1"
        || !matches!(manifest.integrity.path.as_str(), "paper.pdf" | "paper.xml")
        || manifest.integrity.blake3.len() != 64
        || !manifest
            .integrity
            .blake3
            .bytes()
            .all(|byte| byte.is_ascii_hexdigit())
    {
        return Err(PaperForgeError::VerificationFailed {
            path: directory.display().to_string(),
            message: "manifest structure is invalid".to_string(),
        });
    }
    validate_summary(&manifest.paper)?;
    if manifest.paper.provider == PaperProvider::EuropePmc
        && (manifest.paper.rights.jats_license.as_deref()
            != Some(manifest.paper.rights.document_license.as_str()))
    {
        return Err(PaperForgeError::VerificationFailed {
            path: directory.display().to_string(),
            message: "archived Europe PMC rights lack final JATS resolution".to_string(),
        });
    }
    let path = directory.join(&manifest.integrity.path);
    ensure_not_symlink(&path)?;
    let metadata = fs::metadata(&path)?;
    if !metadata.is_file() || metadata.len() != manifest.integrity.bytes {
        return Err(PaperForgeError::VerificationFailed {
            path: path.display().to_string(),
            message: "archived size does not match the manifest".to_string(),
        });
    }
    let format = if manifest.integrity.path == "paper.pdf" {
        PaperDocumentFormat::Pdf
    } else {
        PaperDocumentFormat::JatsXml
    };
    if format != manifest.paper.document.format {
        return Err(PaperForgeError::VerificationFailed {
            path: path.display().to_string(),
            message: "document format does not match the manifest path".to_string(),
        });
    }
    verify_document_magic(&path, format, Some(&manifest.paper.native_id))?;
    if format == PaperDocumentFormat::JatsXml {
        validate_archived_jats_rights(&path, &manifest.paper.rights)?;
    }
    let actual = blake3_file(&path)?;
    if actual != manifest.integrity.blake3 {
        return Err(PaperForgeError::VerificationFailed {
            path: path.display().to_string(),
            message: "BLAKE3 does not match the manifest".to_string(),
        });
    }
    Ok(())
}

fn commit_staging(staging: &Path, destination: &Path) -> Result<()> {
    let parent = destination
        .parent()
        .ok_or_else(|| PaperForgeError::ArchiveTransaction {
            path: destination.to_path_buf(),
            message: "destination has no parent directory".to_string(),
        })?;
    let destination_name = destination
        .file_name()
        .and_then(|value| value.to_str())
        .ok_or_else(|| PaperForgeError::UnsafePath(destination.display().to_string()))?;
    let backup = parent.join(format!(
        ".paperforge-backup-{destination_name}-{}",
        unix_timestamp()?
    ));
    let had_destination = destination.exists();
    if had_destination {
        if backup.exists() {
            return Err(PaperForgeError::ArchiveTransaction {
                path: destination.to_path_buf(),
                message: format!("backup path '{}' already exists", backup.display()),
            });
        }
        fs::rename(destination, &backup)?;
        if let Err(sync_error) = sync_directory(parent) {
            let rollback = fs::rename(&backup, destination);
            let rollback_sync = sync_directory(parent);
            return Err(PaperForgeError::ArchiveTransaction {
                path: destination.to_path_buf(),
                message: format!(
                    "backup directory sync failed: {sync_error}; rollback={rollback:?}; rollback sync={rollback_sync:?}"
                ),
            });
        }
    }
    if let Err(error) = fs::rename(staging, destination) {
        if had_destination {
            let rollback = fs::rename(&backup, destination);
            let sync = sync_directory(parent);
            return Err(PaperForgeError::ArchiveTransaction {
                path: destination.to_path_buf(),
                message: match (rollback, sync) {
                    (Ok(()), Ok(())) => format!("replacement failed and was rolled back: {error}"),
                    (Ok(()), Err(sync)) => format!(
                        "replacement failed and was rolled back, but directory sync failed: {error}; {sync}"
                    ),
                    (Err(rollback), _) => format!(
                        "replacement failed: {error}; rollback from '{}' failed: {rollback}",
                        backup.display()
                    ),
                },
            });
        }
        return Err(error.into());
    }
    if let Err(sync_error) = sync_directory(parent) {
        let move_new_back = fs::rename(destination, staging);
        let restore_old = if had_destination {
            fs::rename(&backup, destination)
        } else {
            Ok(())
        };
        let rollback_sync = sync_directory(parent);
        return Err(PaperForgeError::ArchiveTransaction {
            path: destination.to_path_buf(),
            message: format!(
                "replacement directory sync failed: {sync_error}; retained-new rollback={move_new_back:?}; old-output restore={restore_old:?}; rollback sync={rollback_sync:?}"
            ),
        });
    }
    if had_destination {
        if let Err(error) = fs::remove_dir_all(&backup) {
            return Err(PaperForgeError::ArchiveTransaction {
                path: destination.to_path_buf(),
                message: format!(
                    "replacement succeeded but backup '{}' could not be removed: {error}",
                    backup.display()
                ),
            });
        }
    }
    if had_destination {
        sync_directory(parent)?;
    }
    Ok(())
}

#[cfg(unix)]
fn sync_directory(path: &Path) -> Result<()> {
    File::open(path)?.sync_all()?;
    Ok(())
}

#[cfg(not(unix))]
fn sync_directory(_path: &Path) -> Result<()> {
    Ok(())
}

#[derive(Clone, Copy)]
enum UrlPurpose {
    Api,
    Landing,
    Document,
}

fn trusted_provider_url(
    provider: PaperProvider,
    url: &reqwest::Url,
    purpose: UrlPurpose,
) -> Result<()> {
    if !secure_url(url) {
        return Err(PaperForgeError::UntrustedUrl(url.to_string()));
    }
    let host = url.host_str().unwrap_or_default();
    let allowed = match (provider, purpose) {
        (PaperProvider::Arxiv, UrlPurpose::Api) => {
            host == "export.arxiv.org" && url.path() == "/api/query"
        }
        (PaperProvider::Arxiv, UrlPurpose::Landing) => {
            matches!(host, "arxiv.org" | "export.arxiv.org") && url.path().starts_with("/abs/")
        }
        (PaperProvider::Arxiv, UrlPurpose::Document) => {
            matches!(host, "arxiv.org" | "export.arxiv.org") && url.path().starts_with("/pdf/")
        }
        (PaperProvider::EuropePmc, UrlPurpose::Api) => {
            host == "www.ebi.ac.uk" && url.path().starts_with("/europepmc/webservices/rest/")
        }
        (PaperProvider::EuropePmc, UrlPurpose::Landing) => {
            host == "www.ebi.ac.uk" && url.path().starts_with("/europepmc/")
        }
        (PaperProvider::EuropePmc, UrlPurpose::Document) => {
            host == "www.ebi.ac.uk"
                && url.path().starts_with("/europepmc/webservices/rest/PMC")
                && url.path().ends_with("/fullTextXML")
                && url.query().is_none()
        }
        (PaperProvider::NasaNtrs, UrlPurpose::Api) => {
            host == "ntrs.nasa.gov" && url.path().starts_with("/api/citations")
        }
        (PaperProvider::NasaNtrs, UrlPurpose::Landing) => {
            host == "ntrs.nasa.gov" && url.path().starts_with("/citations/")
        }
        (PaperProvider::NasaNtrs, UrlPurpose::Document) => {
            host == "ntrs.nasa.gov"
                && url.path().starts_with("/api/citations/")
                && url.path().contains("/downloads/")
                && url.query().is_none()
        }
    };
    if !allowed {
        return Err(PaperForgeError::UntrustedUrl(url.to_string()));
    }
    Ok(())
}

fn secure_url(url: &reqwest::Url) -> bool {
    url.scheme() == "https"
        && url.username().is_empty()
        && url.password().is_none()
        && url.port().is_none_or(|port| port == 443)
        && url.fragment().is_none()
}

fn normalized_arxiv_url(value: &str) -> Result<reqwest::Url> {
    let mut url =
        reqwest::Url::parse(value).map_err(|_| PaperForgeError::UntrustedUrl(value.to_string()))?;
    if url.scheme() == "http"
        && matches!(url.host_str(), Some("arxiv.org" | "export.arxiv.org"))
        && url.username().is_empty()
        && url.password().is_none()
        && url.port().is_none_or(|port| port == 80)
    {
        url.set_port(None)
            .map_err(|()| PaperForgeError::UntrustedUrl(value.to_string()))?;
        url.set_scheme("https")
            .map_err(|()| PaperForgeError::UntrustedUrl(value.to_string()))?;
    }
    Ok(url)
}

fn validate_summary(paper: &PaperSummary) -> Result<()> {
    let expected_id = format!("{}:{}", paper.provider.slug(), paper.native_id);
    let source = reqwest::Url::parse(&paper.source_url)
        .map_err(|_| PaperForgeError::UntrustedUrl(paper.source_url.clone()))?;
    let document = reqwest::Url::parse(&paper.document.url)
        .map_err(|_| PaperForgeError::UntrustedUrl(paper.document.url.clone()))?;
    trusted_provider_url(paper.provider, &source, UrlPurpose::Landing)?;
    trusted_provider_url(paper.provider, &document, UrlPurpose::Document)?;
    let mime_matches = match paper.document.format {
        PaperDocumentFormat::Pdf => paper.document.mime.eq_ignore_ascii_case("application/pdf"),
        PaperDocumentFormat::JatsXml => matches!(
            paper.document.mime.to_ascii_lowercase().as_str(),
            "application/xml" | "text/xml" | "application/jats+xml"
        ),
    };
    let identity_valid = match paper.provider {
        PaperProvider::Arxiv => parse_identity("arxiv", &paper.native_id).is_ok(),
        PaperProvider::EuropePmc => parse_identity("pmcid", &paper.native_id).is_ok(),
        PaperProvider::NasaNtrs => parse_identity("ntrs", &paper.native_id).is_ok(),
    };
    let expected_format = match paper.provider {
        PaperProvider::EuropePmc => PaperDocumentFormat::JatsXml,
        PaperProvider::Arxiv | PaperProvider::NasaNtrs => PaperDocumentFormat::Pdf,
    };
    let url_identity_valid = match paper.provider {
        PaperProvider::Arxiv => {
            source.path().strip_prefix("/abs/") == Some(paper.native_id.as_str())
                && document
                    .path()
                    .strip_prefix("/pdf/")
                    .map(|value| value.strip_suffix(".pdf").unwrap_or(value))
                    == Some(paper.native_id.as_str())
        }
        PaperProvider::EuropePmc => {
            document.path()
                == format!(
                    "/europepmc/webservices/rest/{}/fullTextXML",
                    paper.native_id
                )
        }
        PaperProvider::NasaNtrs => {
            source.path() == format!("/citations/{}", paper.native_id)
                && document
                    .path()
                    .starts_with(&format!("/api/citations/{}/downloads/", paper.native_id))
        }
    };
    if paper.id != expected_id
        || paper.native_id.trim().is_empty()
        || !identity_valid
        || !url_identity_valid
        || paper.title.trim().is_empty()
        || paper.rights.metadata_license.trim().is_empty()
        || paper.rights.document_license.trim().is_empty()
        || paper.rights.access_status.trim().is_empty()
        || paper.rights.redistribution_warning.trim().is_empty()
        || paper.document.provider_revision.trim().is_empty()
        || paper.document.format != expected_format
        || !mime_matches
    {
        return Err(PaperForgeError::VerificationFailed {
            path: paper.id.clone(),
            message: "identity, rights, or document metadata is incomplete".to_string(),
        });
    }
    Ok(())
}

fn validate_response_mime(response: &Response, format: PaperDocumentFormat) -> Result<()> {
    let value = response
        .headers()
        .get(CONTENT_TYPE)
        .and_then(|value| value.to_str().ok())
        .unwrap_or_default()
        .split(';')
        .next()
        .unwrap_or_default()
        .trim()
        .to_ascii_lowercase();
    let valid = match format {
        PaperDocumentFormat::Pdf => value == "application/pdf",
        PaperDocumentFormat::JatsXml => {
            matches!(
                value.as_str(),
                "application/xml" | "text/xml" | "application/jats+xml"
            )
        }
    };
    if !valid {
        return Err(PaperForgeError::VerificationFailed {
            path: "HTTP Content-Type".to_string(),
            message: format!("expected {format:?} MIME but received '{value}'"),
        });
    }
    Ok(())
}

fn verify_document_magic(
    path: &Path,
    format: PaperDocumentFormat,
    expected_identity: Option<&str>,
) -> Result<()> {
    match format {
        PaperDocumentFormat::Pdf => {
            let metadata = fs::metadata(path)?;
            if metadata.len() < MIN_PDF_BYTES {
                return Err(PaperForgeError::VerificationFailed {
                    path: path.display().to_string(),
                    message: format!("PDF is implausibly short at {} bytes", metadata.len()),
                });
            }
            let mut bytes = [0_u8; 5];
            let mut file = File::open(path)?;
            file.read_exact(&mut bytes)
                .map_err(|error| PaperForgeError::VerificationFailed {
                    path: path.display().to_string(),
                    message: format!("PDF header is incomplete: {error}"),
                })?;
            if &bytes != b"%PDF-" {
                return Err(PaperForgeError::VerificationFailed {
                    path: path.display().to_string(),
                    message: "missing %PDF- magic".to_string(),
                });
            }
            let trailer_size = metadata.len().min(PDF_TRAILER_WINDOW);
            file.seek(SeekFrom::End(-(trailer_size as i64)))?;
            let mut trailer = vec![0_u8; trailer_size as usize];
            file.read_exact(&mut trailer)?;
            if !trailer
                .windows(b"%%EOF".len())
                .any(|window| window == b"%%EOF")
            {
                return Err(PaperForgeError::VerificationFailed {
                    path: path.display().to_string(),
                    message: "PDF trailer has no %%EOF marker".to_string(),
                });
            }
        }
        PaperDocumentFormat::JatsXml => {
            let mut reader = Reader::from_reader(BufReader::new(File::open(path)?));
            reader.config_mut().trim_text(true);
            let mut buffer = Vec::new();
            let mut root_seen = false;
            let mut depth = 0_usize;
            let mut in_pmc_article_id = false;
            let mut pmc_ids = Vec::new();
            loop {
                match reader.read_event_into(&mut buffer) {
                    Ok(Event::Start(start)) => {
                        let name = local_xml_name(start.name().as_ref());
                        if !root_seen && name != "article" {
                            return Err(PaperForgeError::VerificationFailed {
                                path: path.display().to_string(),
                                message: "JATS root element is not <article>".to_string(),
                            });
                        }
                        if !root_seen {
                            root_seen = true;
                        }
                        if name == "article-id" {
                            in_pmc_article_id =
                                xml_attributes(&reader, &start)?.iter().any(|(key, value)| {
                                    key == "pub-id-type"
                                        && matches!(
                                            value.to_ascii_lowercase().as_str(),
                                            "pmc" | "pmcid"
                                        )
                                });
                        }
                        depth += 1;
                    }
                    Ok(Event::Empty(start)) => {
                        let name = local_xml_name(start.name().as_ref());
                        if !root_seen && name != "article" {
                            return Err(PaperForgeError::VerificationFailed {
                                path: path.display().to_string(),
                                message: "JATS root element is not <article>".to_string(),
                            });
                        }
                        root_seen = true;
                        let _ = xml_attributes(&reader, &start)?;
                    }
                    Ok(Event::Text(text)) if in_pmc_article_id => {
                        let decoded = reader.decoder().decode(text.as_ref()).map_err(|error| {
                            PaperForgeError::Xml(bounded_diagnostic(&format!(
                                "invalid JATS article-id: {error}"
                            )))
                        })?;
                        let value = quick_xml::escape::unescape(&decoded).map_err(|error| {
                            PaperForgeError::Xml(bounded_diagnostic(&error.to_string()))
                        })?;
                        let value = compact_whitespace(&value);
                        if !value.is_empty() {
                            pmc_ids.push(value);
                        }
                    }
                    Ok(Event::End(end)) => {
                        if local_xml_name(end.name().as_ref()) == "article-id" {
                            in_pmc_article_id = false;
                        }
                        depth = depth.checked_sub(1).ok_or_else(|| {
                            PaperForgeError::VerificationFailed {
                                path: path.display().to_string(),
                                message: "JATS element depth underflow".to_string(),
                            }
                        })?;
                    }
                    Ok(Event::DocType(_)) => {}
                    Ok(Event::Eof) => {
                        break;
                    }
                    Ok(_) => {}
                    Err(error) => {
                        return Err(PaperForgeError::VerificationFailed {
                            path: path.display().to_string(),
                            message: format!("invalid JATS XML: {error}"),
                        });
                    }
                }
                buffer.clear();
            }
            if !root_seen || depth != 0 {
                return Err(PaperForgeError::VerificationFailed {
                    path: path.display().to_string(),
                    message: "JATS XML is incomplete".to_string(),
                });
            }
            if let Some(expected) = expected_identity {
                if !pmc_ids
                    .iter()
                    .any(|value| value.eq_ignore_ascii_case(expected))
                {
                    return Err(PaperForgeError::VerificationFailed {
                        path: path.display().to_string(),
                        message: format!("JATS article-id does not match {expected}"),
                    });
                }
            }
        }
    }
    Ok(())
}

#[derive(Default)]
struct JatsLicenseRecord {
    license_type: Option<String>,
    url: Option<String>,
    permissions: Vec<String>,
}

impl JatsLicenseRecord {
    fn from_element(
        reader: &Reader<impl Read>,
        element: &quick_xml::events::BytesStart<'_>,
    ) -> Result<Self> {
        let mut record = Self::default();
        for (key, value) in xml_attributes(reader, element)? {
            let value = checked_jats_value(&value)?;
            if key == "license-type" && !value.is_empty() {
                record.license_type = Some(value);
            } else if key == "href" && !value.is_empty() {
                record.url = Some(value);
            }
        }
        Ok(record)
    }

    fn effective(&self) -> Option<String> {
        self.url
            .clone()
            .or_else(|| {
                self.license_type
                    .as_ref()
                    .filter(|value| !value.eq_ignore_ascii_case("open-access"))
                    .cloned()
            })
            .or_else(|| self.permissions.first().cloned())
    }

    fn evidence(&self) -> String {
        let mut parts = Vec::new();
        if let Some(value) = &self.license_type {
            parts.push(format!("type={value}"));
        }
        if let Some(value) = &self.url {
            parts.push(format!("url={value}"));
        }
        for value in &self.permissions {
            parts.push(format!("permissions={value}"));
        }
        if parts.is_empty() {
            "JATS license record with no recognized fields".to_string()
        } else {
            format!("JATS license record: {}", parts.join("; "))
        }
    }
}

fn checked_jats_value(value: &str) -> Result<String> {
    let value = compact_whitespace(value);
    if value.len() > MAX_JATS_EVIDENCE_VALUE_BYTES {
        return Err(PaperForgeError::Xml(format!(
            "JATS rights value exceeds the {MAX_JATS_EVIDENCE_VALUE_BYTES} byte limit"
        )));
    }
    Ok(value)
}

fn push_jats_license(
    records: &mut Vec<JatsLicenseRecord>,
    record: JatsLicenseRecord,
) -> Result<()> {
    if records.len() >= MAX_JATS_LICENSE_RECORDS {
        return Err(PaperForgeError::Xml(format!(
            "JATS exceeds the {MAX_JATS_LICENSE_RECORDS} license-record limit"
        )));
    }
    records.push(record);
    Ok(())
}

fn enrich_jats_rights(path: &Path, rights: &mut PaperRights) -> Result<()> {
    let mut reader = Reader::from_reader(BufReader::new(File::open(path)?));
    reader.config_mut().trim_text(true);
    let mut buffer = Vec::new();
    let mut stack = Vec::new();
    let mut copyright_evidence = BTreeSet::new();
    let mut permissions = Vec::new();
    let mut licenses = Vec::new();
    let mut current_license: Option<JatsLicenseRecord> = None;
    let mut license_text = String::new();
    let mut license_ref_text = String::new();
    loop {
        match reader.read_event_into(&mut buffer) {
            Ok(Event::Start(start)) => {
                let name = local_xml_name(start.name().as_ref());
                if name == "license" && stack.iter().any(|item| item == "permissions") {
                    if current_license.is_some() {
                        return Err(PaperForgeError::Xml(
                            "nested JATS license records are unsupported".to_string(),
                        ));
                    }
                    current_license = Some(JatsLicenseRecord::from_element(&reader, &start)?);
                }
                stack.push(name);
            }
            Ok(Event::Empty(start)) => {
                let name = local_xml_name(start.name().as_ref());
                if name == "license" && stack.iter().any(|item| item == "permissions") {
                    push_jats_license(
                        &mut licenses,
                        JatsLicenseRecord::from_element(&reader, &start)?,
                    )?;
                }
            }
            Ok(Event::Text(text)) => {
                if stack.iter().any(|item| item == "permissions") {
                    let decoded = reader.decoder().decode(text.as_ref()).map_err(|error| {
                        PaperForgeError::Xml(bounded_diagnostic(&error.to_string()))
                    })?;
                    let value = match quick_xml::escape::unescape(&decoded) {
                        Ok(value) => value.into_owned(),
                        Err(_) => decoded.to_string(),
                    };
                    if stack.iter().any(|name| name == "license-p") {
                        license_text.push_str(&value);
                        license_text.push(' ');
                        if license_text.len() > MAX_JATS_EVIDENCE_VALUE_BYTES {
                            return Err(PaperForgeError::Xml(format!(
                                "JATS license text exceeds the {MAX_JATS_EVIDENCE_VALUE_BYTES} byte limit"
                            )));
                        }
                    }
                    if stack.last().is_some_and(|name| name == "license_ref") {
                        license_ref_text.push_str(&value);
                        if license_ref_text.len() > MAX_JATS_EVIDENCE_VALUE_BYTES {
                            return Err(PaperForgeError::Xml(format!(
                                "JATS license reference exceeds the {MAX_JATS_EVIDENCE_VALUE_BYTES} byte limit"
                            )));
                        }
                    }
                    if stack.iter().any(|name| {
                        matches!(
                            name.as_str(),
                            "copyright-statement" | "copyright-year" | "copyright-holder"
                        )
                    }) {
                        let value = checked_jats_value(&value)?;
                        if !value.is_empty() {
                            if copyright_evidence.len() >= MAX_JATS_PERMISSIONS
                                && !copyright_evidence.contains(&value)
                            {
                                return Err(PaperForgeError::Xml(format!(
                                    "JATS exceeds the {MAX_JATS_PERMISSIONS} copyright-evidence limit"
                                )));
                            }
                            copyright_evidence.insert(value);
                        }
                    }
                }
            }
            Ok(Event::End(end)) => {
                let name = local_xml_name(end.name().as_ref());
                if name == "license-p" {
                    let value = checked_jats_value(&license_text)?;
                    if !value.is_empty() {
                        if permissions.len() >= MAX_JATS_PERMISSIONS {
                            return Err(PaperForgeError::Xml(format!(
                                "JATS exceeds the {MAX_JATS_PERMISSIONS} permissions limit"
                            )));
                        }
                        permissions.push(value.clone());
                        if let Some(record) = current_license.as_mut() {
                            record.permissions.push(value);
                        }
                    }
                    license_text.clear();
                } else if name == "license_ref" {
                    let value = checked_jats_value(&license_ref_text)?;
                    if !value.is_empty() {
                        if let Some(record) = current_license.as_mut() {
                            if record.url.is_none() {
                                record.url = Some(value);
                            }
                        } else {
                            push_jats_license(
                                &mut licenses,
                                JatsLicenseRecord {
                                    url: Some(value),
                                    ..JatsLicenseRecord::default()
                                },
                            )?;
                        }
                    }
                    license_ref_text.clear();
                } else if name == "license" {
                    if let Some(record) = current_license.take() {
                        push_jats_license(&mut licenses, record)?;
                    }
                }
                stack.pop();
            }
            Ok(Event::DocType(_)) => {}
            Ok(Event::Eof) => break,
            Ok(_) => {}
            Err(error) => {
                return Err(PaperForgeError::Xml(bounded_diagnostic(&error.to_string())));
            }
        }
        buffer.clear();
    }
    if current_license.is_some() {
        return Err(PaperForgeError::Xml(
            "JATS license record is incomplete".to_string(),
        ));
    }
    rights.jats_permissions = permissions.clone();
    rights.jats_licenses = licenses.iter().map(JatsLicenseRecord::evidence).collect();
    rights.evidence.extend(rights.jats_licenses.iter().cloned());
    rights.evidence.extend(
        permissions
            .iter()
            .map(|value| format!("JATS permissions: {value}")),
    );
    rights.evidence.extend(
        copyright_evidence
            .into_iter()
            .map(|value| format!("JATS copyright: {value}")),
    );
    let recognized = licenses
        .iter()
        .filter_map(JatsLicenseRecord::effective)
        .collect::<Vec<_>>();
    let ambiguous = recognized.first().is_some_and(|first| {
        recognized
            .iter()
            .skip(1)
            .any(|value| !licenses_equivalent(first, value))
    });
    let effective = if recognized.is_empty() {
        "Unknown/JATS-unresolved".to_string()
    } else if ambiguous {
        "Unknown/JATS-ambiguous".to_string()
    } else {
        recognized[0].clone()
    };
    rights.license_conflict = ambiguous
        || rights
            .search_license
            .as_deref()
            .is_some_and(|search| !licenses_equivalent(search, &effective));
    rights.terms_url = if !ambiguous && effective.starts_with("http") {
        Some(effective.clone())
    } else {
        None
    };
    rights.document_license = effective.clone();
    rights.jats_license = Some(effective.clone());
    rights
        .evidence
        .push(format!("effective JATS license={effective}"));
    Ok(())
}

fn validate_archived_jats_rights(path: &Path, rights: &PaperRights) -> Result<()> {
    let mut extracted = rights.clone();
    extracted.document_license = extracted
        .search_license
        .clone()
        .unwrap_or_else(|| "Europe-PMC-open-access-license-unspecified".to_string());
    extracted.jats_license = None;
    extracted.jats_licenses.clear();
    extracted.jats_permissions.clear();
    extracted.license_conflict = false;
    extracted.terms_url = Some("https://europepmc.org/Copyright".to_string());
    extracted.evidence.retain(|value| {
        !value.starts_with("JATS ") && !value.starts_with("effective JATS license=")
    });
    enrich_jats_rights(path, &mut extracted)?;
    let retained_jats_evidence = rights
        .evidence
        .iter()
        .filter(|value| value.starts_with("JATS ") || value.starts_with("effective JATS license="))
        .collect::<Vec<_>>();
    let extracted_jats_evidence = extracted
        .evidence
        .iter()
        .filter(|value| value.starts_with("JATS ") || value.starts_with("effective JATS license="))
        .collect::<Vec<_>>();
    if rights.document_license != extracted.document_license
        || rights.jats_license != extracted.jats_license
        || rights.jats_licenses != extracted.jats_licenses
        || rights.jats_permissions != extracted.jats_permissions
        || rights.license_conflict != extracted.license_conflict
        || rights.terms_url != extracted.terms_url
        || retained_jats_evidence != extracted_jats_evidence
    {
        return Err(PaperForgeError::VerificationFailed {
            path: path.display().to_string(),
            message: "manifest JATS rights do not match the archived XML".to_string(),
        });
    }
    Ok(())
}

fn licenses_equivalent(left: &str, right: &str) -> bool {
    match (license_family(left), license_family(right)) {
        (Some(left), Some(right)) => left == right,
        _ => normalized_text(left) == normalized_text(right),
    }
}

fn license_family(value: &str) -> Option<&'static str> {
    let normalized = normalized_text(value);
    if normalized.contains("cc0") || normalized.contains("publicdomain zero") {
        Some("cc0")
    } else if normalized.contains("by nc nd") {
        Some("cc-by-nc-nd")
    } else if normalized.contains("by nc sa") {
        Some("cc-by-nc-sa")
    } else if normalized.contains("by nd") {
        Some("cc-by-nd")
    } else if normalized.contains("by nc") {
        Some("cc-by-nc")
    } else if normalized.contains("by sa") {
        Some("cc-by-sa")
    } else if normalized.contains("creative commons") && normalized.contains("by")
        || normalized.starts_with("cc by")
        || value.to_ascii_lowercase().contains("/licenses/by/")
    {
        Some("cc-by")
    } else {
        None
    }
}

fn blake3_file(path: &Path) -> Result<String> {
    let mut file = File::open(path)?;
    let mut hasher = Blake3::new();
    let mut buffer = [0_u8; 64 * 1024];
    loop {
        let read = file.read(&mut buffer)?;
        if read == 0 {
            break;
        }
        hasher.update(&buffer[..read]);
    }
    Ok(hasher.finalize().to_hex().to_string())
}

fn progress_snapshot(
    phase: ArchiveProgressPhase,
    total: usize,
    summary: &ArchiveSummary,
) -> ArchiveProgress {
    ArchiveProgress {
        phase,
        total_papers: total,
        completed_papers: summary.archived + summary.skipped + summary.failed.len(),
        succeeded_papers: summary.archived + summary.skipped,
        failed_papers: summary.failed.len(),
        bytes_written: summary.bytes_written,
        active_paper: None,
        active_file: None,
        active_bytes_written: 0,
        active_total_bytes: None,
    }
}

fn emit_progress(progress: &mut Option<&mut dyn FnMut(ArchiveProgress)>, update: ArchiveProgress) {
    if let Some(callback) = progress.as_deref_mut() {
        callback(update);
    }
}

fn retry_after(response: &Response) -> Option<Duration> {
    response
        .headers()
        .get(RETRY_AFTER)
        .and_then(|value| value.to_str().ok())
        .and_then(|value| value.parse::<u64>().ok())
        .map(Duration::from_secs)
}

fn retryable_status(status: u16) -> bool {
    status == 429 || (500..=599).contains(&status)
}

fn retry_delay(attempt: usize, retry_after: Option<Duration>) -> Duration {
    retry_after
        .unwrap_or_else(|| Duration::from_secs(1_u64 << attempt.min(2)))
        .min(MAX_RETRY_WAIT)
}

fn read_response_limited(mut response: Response, limit: u64, label: &str) -> Result<Vec<u8>> {
    if let Some(size) = response
        .headers()
        .get(CONTENT_LENGTH)
        .and_then(|value| value.to_str().ok())
        .and_then(|value| value.parse::<u64>().ok())
    {
        if size > limit {
            return Err(PaperForgeError::ResponseLimitExceeded {
                label: label.to_string(),
                size,
                limit,
            });
        }
    }
    let mut bytes = Vec::new();
    response
        .by_ref()
        .take(limit.saturating_add(1))
        .read_to_end(&mut bytes)?;
    if bytes.len() as u64 > limit {
        return Err(PaperForgeError::ResponseLimitExceeded {
            label: label.to_string(),
            size: bytes.len() as u64,
            limit,
        });
    }
    Ok(bytes)
}

fn bounded_error_body(response: Response) -> String {
    match read_response_limited(
        response,
        MAX_ERROR_RESPONSE_BYTES,
        "provider error response",
    ) {
        Ok(bytes) => bounded_diagnostic(&String::from_utf8_lossy(&bytes)),
        Err(error) => bounded_diagnostic(&error.to_string()),
    }
}

fn provider_error_envelope(bytes: &[u8]) -> Option<String> {
    let value: serde_json::Value = serde_json::from_slice(bytes).ok()?;
    for key in ["error", "errors"] {
        if let Some(error) = value.get(key) {
            if !error.is_null() {
                return Some(bounded_diagnostic(&error.to_string()));
            }
        }
    }
    None
}

fn error_chain(error: &dyn std::error::Error) -> String {
    let mut messages = vec![error.to_string()];
    let mut source = error.source();
    while let Some(error) = source {
        let message = error.to_string();
        if !messages.contains(&message) {
            messages.push(message);
        }
        source = error.source();
    }
    bounded_diagnostic(&messages.join(": "))
}

fn unix_timestamp() -> Result<u64> {
    SystemTime::now()
        .duration_since(UNIX_EPOCH)
        .map(|duration| duration.as_secs())
        .map_err(|error| PaperForgeError::Io(io::Error::other(error)))
}

fn normalized_text(value: &str) -> String {
    value
        .nfkc()
        .flat_map(char::to_lowercase)
        .map(|character| {
            if character.is_alphanumeric() {
                character
            } else {
                ' '
            }
        })
        .collect::<String>()
        .split_whitespace()
        .collect::<Vec<_>>()
        .join(" ")
}

fn contains_normalized_phrase(haystack: &str, needle: &str) -> bool {
    format!(" {haystack} ").contains(&format!(" {needle} "))
}

fn compact_whitespace(value: &str) -> String {
    value
        .chars()
        .map(|character| {
            if character.is_control() {
                ' '
            } else {
                character
            }
        })
        .collect::<String>()
        .split_whitespace()
        .collect::<Vec<_>>()
        .join(" ")
}

fn bounded_diagnostic(value: &str) -> String {
    let value = compact_whitespace(value);
    let limit = MAX_ERROR_RESPONSE_BYTES as usize;
    if value.len() <= limit {
        return value;
    }
    let end = value
        .char_indices()
        .map(|(index, _)| index)
        .take_while(|index| *index <= limit)
        .last()
        .unwrap_or(0);
    value[..end].to_string()
}

fn date_prefix(value: &str) -> Option<String> {
    let prefix = value.get(..10)?;
    (parse_date_bound(prefix, false).ok().as_deref() == Some(prefix)).then(|| prefix.to_string())
}

#[cfg(test)]
mod tests {
    use super::*;

    const ARXIV_FIXTURE: &str = r#"<?xml version="1.0" encoding="UTF-8"?>
<feed xmlns="http://www.w3.org/2005/Atom" xmlns:arxiv="http://arxiv.org/schemas/atom">
  <entry>
    <id>http://arxiv.org/abs/2401.12345v2</id>
    <updated>2024-02-03T12:00:00Z</updated>
    <published>2024-01-20T10:00:00Z</published>
    <title>Protein Graph Networks</title>
    <summary>A molecular property model for proteins.</summary>
    <author><name>Jane Smith</name></author>
    <author><name>Alex Doe</name></author>
    <arxiv:doi>10.1234/example.1</arxiv:doi>
    <arxiv:journal_ref>Journal of Graphs</arxiv:journal_ref>
    <arxiv:license>https://creativecommons.org/licenses/by/4.0/</arxiv:license>
    <category term="cs.LG" />
    <link href="http://arxiv.org/abs/2401.12345v2" rel="alternate" type="text/html" />
    <link title="pdf" href="http://arxiv.org/pdf/2401.12345v2" rel="related" type="application/pdf" />
  </entry>
</feed>"#;

    const EUROPE_PMC_FIXTURE: &str = r#"{
      "resultList": {"result": [{
        "id": "12345678",
        "pmid": "12345678",
        "pmcid": "PMC7654321",
        "doi": "10.1000/EP",
        "title": "Protein folding with graphs",
        "abstractText": "Molecular property prediction.",
        "authorString": "Smith J, Doe A",
        "journalInfo": {"journal": {"title": "Open Biology"}},
        "firstPublicationDate": "2023-05-06",
        "firstIndexDate": "2023-05-10",
        "fullTextReceivedDate": "2023-05-11",
        "dateOfRevision": "2024-06-12",
        "citedByCount": 42,
        "isOpenAccess": "Y",
        "inEPMC": "Y",
        "license": "CC BY 4.0",
        "authorList": {"author": [{"fullName": "Jane Smith"}]},
        "keywordList": {"keyword": ["protein", "graphs"]}
      }]}
    }"#;

    fn nasa_fixture(itar: &str, host: &str) -> serde_json::Value {
        serde_json::json!({
            "id": "20230001234",
            "title": "Public aerodynamic report",
            "abstract": "Graph methods for flight.",
            "authors": [{"name": "NASA Researcher"}],
            "subjectCategories": ["Aerodynamics"],
            "published": "2023-08-01",
            "modified": "2023-09-01",
            "distribution": "PUBLIC",
            "disseminated": "DOCUMENT_AND_METADATA",
            "exportControl": {"isExportControl": "NO", "itar": itar, "ear": "NO"},
            "cui": false,
            "downloadsAvailable": true,
            "copyrightDetermination": "Copyright - Other",
            "publicUsePermissions": "Review source terms",
            "downloads": [{
                "mimeType": "application/pdf",
                "draft": false,
                "type": "STI",
                "url": format!("https://{host}/api/citations/20230001234/downloads/report.pdf"),
                "size": 1234
            }]
        })
    }

    fn sample_paper(id: &str, title: &str, date: &str, citations: Option<u64>) -> PaperSummary {
        PaperSummary {
            provider: PaperProvider::NasaNtrs,
            id: format!("nasa-ntrs:{id}"),
            native_id: id.to_string(),
            title: title.to_string(),
            abstract_text: "Molecular property graphs in an abstract".to_string(),
            authors: vec!["Jane Smith".to_string()],
            topics: vec!["Machine Learning".to_string()],
            doi: Some(format!("10.1000/{id}")),
            journal: Some("Journal of Tests".to_string()),
            published: Some(date.to_string()),
            updated: Some(date.to_string()),
            citation_count: citations,
            source_url: format!("https://ntrs.nasa.gov/citations/{id}"),
            rights: PaperRights {
                metadata_license: "metadata-terms".to_string(),
                document_license: "copyright status supplied".to_string(),
                search_license: None,
                jats_license: None,
                jats_licenses: Vec::new(),
                jats_permissions: Vec::new(),
                license_conflict: false,
                access_status: "PUBLIC/DOCUMENT_AND_METADATA".to_string(),
                attribution: "Jane Smith".to_string(),
                terms_url: Some("https://sti.nasa.gov/disclaimers/".to_string()),
                evidence: vec!["ITAR=NO; EAR=NO; CUI=false".to_string()],
                redistribution_warning: "Do not infer redistribution rights.".to_string(),
            },
            document: PaperDocument {
                url: format!("https://ntrs.nasa.gov/api/citations/{id}/downloads/report.pdf"),
                size: None,
                mime: "application/pdf".to_string(),
                format: PaperDocumentFormat::Pdf,
                provider_revision: date.to_string(),
            },
            relevance: blank_relevance(),
        }
    }

    fn test_pdf(label: &str) -> Vec<u8> {
        let mut bytes = format!("%PDF-1.7\n{label}\n").into_bytes();
        bytes.resize(MIN_PDF_BYTES as usize + 32, b' ');
        bytes.extend_from_slice(b"\n%%EOF\n");
        bytes
    }

    fn test_europe_rights(search_license: &str) -> PaperRights {
        PaperRights {
            metadata_license: "metadata".to_string(),
            document_license: search_license.to_string(),
            search_license: Some(search_license.to_string()),
            jats_license: None,
            jats_licenses: Vec::new(),
            jats_permissions: Vec::new(),
            license_conflict: false,
            access_status: "open-access".to_string(),
            attribution: String::new(),
            terms_url: Some("https://europepmc.org/Copyright".to_string()),
            evidence: vec![format!("result license={search_license}")],
            redistribution_warning: "review".to_string(),
        }
    }

    #[test]
    fn parses_rich_query() {
        let query = parse_paper_query(
            r#"graph networks +"molecular property" -survey title:protein author:"Jane Smith" topic:cs.LG after:2022 before:2026 @recent @limit:25"#,
        )
        .unwrap();
        assert_eq!(query.clauses.len(), 7);
        assert_eq!(query.after.as_deref(), Some("2022-01-01"));
        assert_eq!(query.before.as_deref(), Some("2026-12-31"));
        assert_eq!(query.mode, PaperMode::Recent);
        assert_eq!(query.limit, Some(25));
        assert!(query.clauses[2].phrase);
        assert_eq!(query.clauses[2].polarity, ClausePolarity::Required);
    }

    #[test]
    fn lexer_rejects_unterminated_quote() {
        assert!(parse_paper_query("title:\"unfinished").is_err());
    }

    #[test]
    fn parser_rejects_unknown_fields_and_directives() {
        assert!(parse_paper_query("publisher:test").is_err());
        assert!(parse_paper_query("protein @unknown").is_err());
        assert!(parse_paper_query("protein @limit:101").is_err());
    }

    #[test]
    fn parses_exact_urls_and_preserves_arxiv_version() {
        let arxiv = parse_paper_query("https://arxiv.org/pdf/2401.12345v2.pdf").unwrap();
        assert_eq!(arxiv.identities[0].value, "2401.12345v2");
        let pmc = parse_paper_query("https://europepmc.org/articles/PMC7654321").unwrap();
        assert_eq!(pmc.identities[0].kind, IdentityKind::Pmcid);
        let ntrs = parse_paper_query("https://ntrs.nasa.gov/citations/20230001234").unwrap();
        assert_eq!(ntrs.identities[0].kind, IdentityKind::Ntrs);
        let doi = parse_paper_query("https://doi.org/10.1000/ABC.1").unwrap();
        assert_eq!(doi.identities[0].value, "10.1000/abc.1");
    }

    #[test]
    fn exact_urls_reject_unsupported_hosts_and_queries() {
        assert!(parse_paper_query("https://example.org/abs/2401.12345").is_err());
        assert!(parse_paper_query("http://arxiv.org/abs/2401.12345").is_err());
        assert!(parse_paper_query("https://arxiv.org/abs/2401.12345?download=1").is_err());
    }

    #[test]
    fn validates_identity_shapes() {
        assert!(parse_paper_query("arxiv:0704.0001v1").is_ok());
        assert!(parse_paper_query("pmcid:PMC123456").is_ok());
        assert!(parse_paper_query("doi:11.1000/nope").is_err());
        assert!(parse_paper_query("pmid:not-digits").is_err());
    }

    #[test]
    fn validates_calendar_bounds() {
        let query = parse_paper_query("after:2024-02 before:2024-02").unwrap();
        assert_eq!(query.after.as_deref(), Some("2024-02-01"));
        assert_eq!(query.before.as_deref(), Some("2024-02-29"));
        assert!(parse_paper_query("after:2023-02-29").is_err());
        assert!(parse_paper_query("after:2025 before:2024").is_err());
    }

    #[test]
    fn rejects_conflicting_modes_and_unbounded_negation() {
        assert!(parse_paper_query("protein @recent @top").is_err());
        assert!(parse_paper_query("-survey").is_err());
        assert!(parse_paper_query("-survey after:2020").is_ok());
    }

    #[test]
    fn provider_serializers_fail_closed_for_unsupported_arxiv_grammar() {
        let query = parse_paper_query(r#"title:"protein\" OR all:*""#).unwrap();
        assert!(validate_query_for_provider(&query, PaperProvider::Arxiv).is_err());
        let europe = europe_pmc_request_url(&query, 5).unwrap();
        let serialized = europe
            .query_pairs()
            .find(|(key, _)| key == "query")
            .unwrap()
            .1;
        assert!(serialized.contains("\\\""));
        let nasa = nasa_request_body(&query, 5).to_string();
        assert!(!nasa.contains("all:*"));
    }

    #[test]
    fn provider_serializers_preserve_optional_semantics_and_sort_direction() {
        let optional = parse_paper_query("graph networks").unwrap();
        assert_eq!(
            arxiv_expression(&optional),
            "(all:\"graph\" OR all:\"networks\")"
        );
        let europe = europe_pmc_request_url(&optional, 100).unwrap();
        let expression = europe
            .query_pairs()
            .find(|(key, _)| key == "query")
            .unwrap()
            .1;
        assert!(expression.contains("(\"graph\" OR \"networks\")"));

        let required = parse_paper_query("+protein graph networks").unwrap();
        assert_eq!(arxiv_expression(&required), "all:\"protein\"");
        let nasa = nasa_request_body(&required, 100);
        assert_eq!(nasa["q"], "protein");

        let recent = arxiv_request_url(&parse_paper_query("graph @recent").unwrap(), 100).unwrap();
        let pairs = recent
            .query_pairs()
            .collect::<std::collections::HashMap<_, _>>();
        assert_eq!(
            pairs.get("sortBy").map(|value| value.as_ref()),
            Some("submittedDate")
        );
        assert_eq!(
            pairs.get("sortOrder").map(|value| value.as_ref()),
            Some("descending")
        );
    }

    #[test]
    fn exact_identity_requests_are_separate_and_nasa_doi_is_rejected() {
        let arxiv = parse_paper_query("arxiv:2401.12345v2 doi:10.1000/example").unwrap();
        let first = arxiv_identity_request_url(&arxiv.identities[0], 5).unwrap();
        assert!(first.query_pairs().any(|(key, _)| key == "id_list"));
        assert!(!first.query_pairs().any(|(key, _)| key == "search_query"));
        let second = arxiv_identity_request_url(&arxiv.identities[1], 5).unwrap();
        assert!(second.query_pairs().any(|(key, _)| key == "search_query"));
        assert!(!second.query_pairs().any(|(key, _)| key == "id_list"));

        let nasa_doi = parse_paper_query("doi:10.1000/example").unwrap();
        assert!(validate_query_for_provider(&nasa_doi, PaperProvider::NasaNtrs).is_err());
    }

    #[test]
    fn provider_capabilities_reject_absent_citations_and_order_europe_top() {
        for provider in [PaperProvider::Arxiv, PaperProvider::NasaNtrs] {
            assert!(
                validate_query_for_provider(&parse_paper_query("graph @top").unwrap(), provider)
                    .is_err()
            );
            assert!(
                validate_query_for_provider(
                    &parse_paper_query("graph @sort:citations").unwrap(),
                    provider
                )
                .is_err()
            );
        }
        let top = europe_pmc_request_url(&parse_paper_query("graph @top").unwrap(), 100).unwrap();
        assert!(
            top.query_pairs()
                .any(|(key, value)| key == "sort" && value == "CITED desc")
        );
    }

    #[test]
    fn publicly_constructed_queries_receive_full_validation() {
        let malformed = PaperQuery {
            clauses: vec![TextClause {
                field: TextField::Any,
                polarity: ClausePolarity::Optional,
                value: String::new(),
                phrase: false,
            }],
            original: "manually-built".to_string(),
            ..PaperQuery::default()
        };
        assert!(validate_query_for_provider(&malformed, PaperProvider::EuropePmc).is_err());
        let non_phrase_whitespace = PaperQuery {
            clauses: vec![TextClause {
                field: TextField::Any,
                polarity: ClausePolarity::Optional,
                value: "graph networks".to_string(),
                phrase: false,
            }],
            original: "manually-built".to_string(),
            ..PaperQuery::default()
        };
        assert!(validate_query_for_provider(&non_phrase_whitespace, PaperProvider::Arxiv).is_err());
        assert!(parse_paper_query(r#""graph networks""#).is_ok());
        let overlong = "x".repeat(MAX_QUERY_BYTES + 1);
        assert!(parse_paper_query(&overlong).is_err());
    }

    #[test]
    fn provider_directive_must_match_client() {
        let query = parse_paper_query("protein @provider:arxiv").unwrap();
        assert!(validate_query_for_provider(&query, PaperProvider::Arxiv).is_ok());
        assert!(validate_query_for_provider(&query, PaperProvider::EuropePmc).is_err());
    }

    #[test]
    fn ranking_uses_tiers_and_whole_tokens() {
        let query = parse_paper_query(r#"+graph "molecular property""#).unwrap();
        let mut exact = sample_paper(
            "20230001234",
            "Molecular Property Graph",
            "2022-01-01",
            None,
        );
        let mut partial = sample_paper("20230001235", "Unrelated", "2024-01-01", None);
        partial.abstract_text = "graph molecular material property".to_string();
        let exact_evidence = match_paper(&exact, &query).unwrap();
        let partial_evidence = match_paper(&partial, &query).unwrap();
        assert_eq!(exact_evidence.tier, RelevanceTier::AllPositiveTitleOrPhrase);
        assert!(partial_evidence.tier < exact_evidence.tier);
        assert!(
            !partial_evidence
                .matched
                .iter()
                .any(|item| item.contains("molecular property"))
        );
        exact.title = "Cartography".to_string();
        let token_query = parse_paper_query("graph").unwrap();
        exact.abstract_text.clear();
        assert!(match_paper(&exact, &token_query).is_none());
    }

    #[test]
    fn discovery_evidence_is_candidate_bounded_and_identity_union_is_local() {
        let query = parse_paper_query("graph @limit:1").unwrap();
        let candidates = vec![
            sample_paper("20230001234", "Graph alpha", "2022-01-01", None),
            sample_paper("20230001235", "Graph beta", "2023-01-01", None),
            sample_paper("20230001236", "Graph gamma", "2024-01-01", None),
        ];
        let result = finalize_discovery(candidates, &query, 1, 100, 3, false);
        assert_eq!(result.papers.len(), 1);
        assert_eq!(result.candidate_count, 3);
        assert_eq!(result.eligible_candidate_count, 3);
        assert!(!result.candidate_cap_reached);
        assert_eq!(result.candidate_budget, 100);

        let identities = parse_paper_query("ntrs:20230001234 ntrs:20230001235").unwrap();
        assert!(
            match_paper(
                &sample_paper("20230001234", "A", "2023-01-01", None),
                &identities
            )
            .is_some()
        );
        assert!(
            match_paper(
                &sample_paper("20230001235", "B", "2023-01-01", None),
                &identities
            )
            .is_some()
        );
    }

    #[test]
    fn exact_title_tier_requires_matching_title_field_evidence() {
        let query = parse_paper_query(r#"+graph author:"Exact title""#).unwrap();
        let mut paper = sample_paper("20230001234", "Exact title", "2023-01-01", None);
        paper.authors = vec!["Different Author".to_string()];
        paper.abstract_text = "graph evidence".to_string();
        let evidence = match_paper(&paper, &query).unwrap();
        assert_ne!(evidence.tier, RelevanceTier::ExactTitlePhrase);
    }

    #[test]
    fn strict_and_mode_sorts_are_deterministic() {
        let mut older_cited = sample_paper("20230001234", "Graph alpha", "2020-01-01", Some(50));
        let mut newer = sample_paper("20230001235", "Graph beta", "2024-01-01", Some(2));
        let date_query = parse_paper_query("graph @sort:date").unwrap();
        local_filter_rank(&mut vec![older_cited.clone(), newer.clone()], &date_query);
        let mut papers = vec![older_cited.clone(), newer.clone()];
        local_filter_rank(&mut papers, &date_query);
        assert_eq!(papers[0].native_id, "20230001235");
        let cited_query = parse_paper_query("graph @sort:citations").unwrap();
        local_filter_rank(&mut papers, &cited_query);
        assert_eq!(papers[0].native_id, "20230001234");
        older_cited.relevance = blank_relevance();
        newer.relevance = blank_relevance();
    }

    #[test]
    fn parses_arxiv_atom_fixture_without_fabricated_citations() {
        let papers = parse_arxiv_atom(ARXIV_FIXTURE.as_bytes()).unwrap();
        assert_eq!(papers.len(), 1);
        let paper = &papers[0];
        assert_eq!(paper.native_id, "2401.12345v2");
        assert_eq!(paper.document.provider_revision, "v2");
        assert_eq!(paper.citation_count, None);
        assert_eq!(paper.document.url, "https://arxiv.org/pdf/2401.12345v2");
        assert_eq!(paper.rights.metadata_license, "CC0-1.0");
        assert!(paper.rights.document_license.contains("creativecommons"));
    }

    #[test]
    fn absent_arxiv_license_is_not_fabricated_as_explicit_evidence() {
        let fixture = ARXIV_FIXTURE.replace(
            "    <arxiv:license>https://creativecommons.org/licenses/by/4.0/</arxiv:license>\n",
            "",
        );
        let paper = parse_arxiv_atom(fixture.as_bytes()).unwrap().remove(0);
        assert_eq!(
            paper.rights.document_license,
            "arXiv.org perpetual, non-exclusive license 1.0"
        );
        assert_eq!(
            paper.rights.terms_url.as_deref(),
            Some("https://arxiv.org/licenses/nonexclusive-distrib/1.0/")
        );
        assert!(paper.rights.evidence[0].contains("element absent"));
        assert!(!paper.rights.evidence[0].contains("explicit"));
    }

    #[test]
    fn atom_parser_rejects_doctype() {
        let xml = b"<!DOCTYPE feed [<!ENTITY x SYSTEM 'file:///etc/passwd'>]><feed/>";
        assert!(parse_arxiv_atom(xml).is_err());
        assert!(parse_arxiv_atom(b"<html/>").is_err());
        assert!(parse_arxiv_atom(b"<feed/>").is_err());
    }

    #[test]
    fn parses_europe_pmc_fixture_as_jats_with_real_citations() {
        let papers = parse_europe_pmc_json(EUROPE_PMC_FIXTURE.as_bytes()).unwrap();
        assert_eq!(papers.len(), 1);
        assert_eq!(papers[0].native_id, "PMC7654321");
        assert_eq!(papers[0].citation_count, Some(42));
        assert_eq!(papers[0].document.format, PaperDocumentFormat::JatsXml);
        assert_eq!(papers[0].rights.document_license, "CC BY 4.0");
        assert_eq!(papers[0].journal.as_deref(), Some("Open Biology"));
        assert_eq!(papers[0].document.provider_revision, "2024-06-12");
    }

    #[test]
    fn nasa_security_gate_requires_every_condition_and_first_party_pdf() {
        assert!(
            nasa_summary(&nasa_fixture("NO", "ntrs.nasa.gov"))
                .unwrap()
                .is_some()
        );
        assert!(
            nasa_summary(&nasa_fixture("YES", "ntrs.nasa.gov"))
                .unwrap()
                .is_none()
        );
        assert!(nasa_summary(&nasa_fixture("NO", "evil.example")).is_err());
        let mut export_controlled = nasa_fixture("NO", "ntrs.nasa.gov");
        export_controlled["exportControl"]["isExportControl"] = serde_json::json!("YES");
        assert!(nasa_summary(&export_controlled).unwrap().is_none());
        let mut missing_export_control = nasa_fixture("NO", "ntrs.nasa.gov");
        missing_export_control["exportControl"]
            .as_object_mut()
            .unwrap()
            .remove("isExportControl");
        assert!(nasa_summary(&missing_export_control).unwrap().is_none());
        let mut draft = nasa_fixture("NO", "ntrs.nasa.gov");
        draft["downloads"][0]["draft"] = serde_json::json!(true);
        assert!(nasa_summary(&draft).unwrap().is_none());
        let mut attachment = nasa_fixture("NO", "ntrs.nasa.gov");
        attachment["downloads"][0]["type"] = serde_json::json!("ATTACHMENT");
        assert!(nasa_summary(&attachment).unwrap().is_none());
    }

    #[test]
    fn nasa_candidate_evidence_counts_raw_rejected_records_and_page_fullness() {
        let eligible = nasa_fixture("NO", "ntrs.nasa.gov");
        let mut rejected = nasa_fixture("NO", "ntrs.nasa.gov");
        rejected["exportControl"]["isExportControl"] = serde_json::json!("YES");
        let mut results = vec![eligible];
        results.extend(std::iter::repeat_n(rejected, 99));
        let candidates =
            parse_nasa_candidates(&serde_json::json!({"results": results}), 100).unwrap();
        assert_eq!(candidates.raw_count, 100);
        assert_eq!(candidates.papers.len(), 1);
        assert!(candidates.cap_reached);
        let query = parse_paper_query("graph").unwrap();
        let result = finalize_discovery(
            candidates.papers,
            &query,
            20,
            100,
            candidates.raw_count,
            candidates.cap_reached,
        );
        assert_eq!(result.candidate_count, 100);
        assert_eq!(result.eligible_candidate_count, 1);
        assert!(result.candidate_cap_reached);
    }

    #[test]
    fn pmid_identity_matching_is_exact() {
        let mut paper = parse_europe_pmc_json(EUROPE_PMC_FIXTURE.as_bytes())
            .unwrap()
            .remove(0);
        paper.source_url =
            "https://www.ebi.ac.uk/europepmc/webservices/rest/search?query=EXT_ID:1234&format=json"
                .to_string();
        let short = PaperIdentity {
            kind: IdentityKind::Pmid,
            value: "123".to_string(),
        };
        let exact = PaperIdentity {
            kind: IdentityKind::Pmid,
            value: "1234".to_string(),
        };
        assert!(!identity_matches(&paper, &short));
        assert!(identity_matches(&paper, &exact));
    }

    #[test]
    fn provider_schema_drift_is_an_error_not_an_empty_result() {
        assert!(parse_europe_pmc_json(br#"{"version":"future"}"#).is_err());
        assert!(nasa_values(&serde_json::json!({"stats": {"total": 0}})).is_err());
        assert!(parse_arxiv_atom(b"<html/>").is_err());
    }

    #[test]
    fn rights_keep_metadata_and_document_licenses_separate() {
        let arxiv = parse_arxiv_atom(ARXIV_FIXTURE.as_bytes())
            .unwrap()
            .remove(0);
        let ep = parse_europe_pmc_json(EUROPE_PMC_FIXTURE.as_bytes())
            .unwrap()
            .remove(0);
        assert_ne!(arxiv.rights.metadata_license, arxiv.rights.document_license);
        assert_ne!(ep.rights.metadata_license, ep.rights.document_license);
        assert!(
            arxiv
                .rights
                .redistribution_warning
                .contains("does not grant")
        );
    }

    #[test]
    fn trusted_url_policy_is_path_and_host_specific() {
        let valid = reqwest::Url::parse(
            "https://www.ebi.ac.uk/europepmc/webservices/rest/PMC123/fullTextXML",
        )
        .unwrap();
        assert!(
            trusted_provider_url(PaperProvider::EuropePmc, &valid, UrlPurpose::Document).is_ok()
        );
        let invalid = reqwest::Url::parse("https://www.ebi.ac.uk/redirect?to=evil").unwrap();
        assert!(
            trusted_provider_url(PaperProvider::EuropePmc, &invalid, UrlPurpose::Document).is_err()
        );
    }

    #[test]
    fn verifies_pdf_and_jats_magic_without_loading_jats_dtd() {
        let temp = tempfile::tempdir().unwrap();
        let pdf = temp.path().join("paper.pdf");
        fs::write(&pdf, test_pdf("body")).unwrap();
        assert!(verify_document_magic(&pdf, PaperDocumentFormat::Pdf, None).is_ok());
        fs::write(&pdf, b"not a pdf").unwrap();
        assert!(verify_document_magic(&pdf, PaperDocumentFormat::Pdf, None).is_err());
        let xml = temp.path().join("paper.xml");
        fs::write(&xml, b"<?xml version='1.0'?><article><front><article-meta><article-id pub-id-type='pmc'>PMC123</article-id></article-meta></front></article>").unwrap();
        assert!(verify_document_magic(&xml, PaperDocumentFormat::JatsXml, Some("PMC123")).is_ok());
        assert!(verify_document_magic(&xml, PaperDocumentFormat::JatsXml, Some("PMC999")).is_err());
        fs::write(&xml, b"<!DOCTYPE article SYSTEM 'unfetched.dtd'><article/>").unwrap();
        assert!(verify_document_magic(&xml, PaperDocumentFormat::JatsXml, None).is_ok());
        fs::write(&xml, b"<!DOCTYPE article><html/>").unwrap();
        assert!(verify_document_magic(&xml, PaperDocumentFormat::JatsXml, None).is_err());
    }

    #[test]
    fn jats_permissions_override_conflicting_search_license_and_capture_nested_text() {
        let temp = tempfile::tempdir().unwrap();
        let xml = temp.path().join("paper.xml");
        fs::write(
            &xml,
            br#"<article><front><article-meta><permissions><license license-type="open-access" xlink:href="https://creativecommons.org/licenses/by-nc/4.0/" xmlns:xlink="http://www.w3.org/1999/xlink"><license-p><bold>Reuse</bold> for noncommercial purposes with attribution.</license-p></license></permissions></article-meta></front></article>"#,
        )
        .unwrap();
        let mut rights = PaperRights {
            metadata_license: "metadata".to_string(),
            document_license: "CC BY 4.0 from result".to_string(),
            search_license: Some("CC BY 4.0 from result".to_string()),
            jats_license: None,
            jats_licenses: Vec::new(),
            jats_permissions: Vec::new(),
            license_conflict: false,
            access_status: "open-access".to_string(),
            attribution: String::new(),
            terms_url: None,
            evidence: vec!["result license".to_string()],
            redistribution_warning: "review".to_string(),
        };
        enrich_jats_rights(&xml, &mut rights).unwrap();
        assert_eq!(
            rights.document_license,
            "https://creativecommons.org/licenses/by-nc/4.0/"
        );
        assert!(rights.license_conflict);
        assert_eq!(rights.jats_license, Some(rights.document_license.clone()));
        assert_eq!(rights.jats_licenses.len(), 1);
        assert!(
            rights
                .evidence
                .iter()
                .any(|item| item.contains("type=open-access"))
        );
        assert!(
            rights
                .evidence
                .iter()
                .any(|item| item.contains("Reuse for noncommercial purposes"))
        );
    }

    #[test]
    fn jats_without_license_never_retains_search_license_as_effective() {
        let temp = tempfile::tempdir().unwrap();
        let xml = temp.path().join("paper.xml");
        fs::write(
            &xml,
            br#"<article><front><article-meta><permissions><copyright-statement>Copyright holder</copyright-statement></permissions></article-meta></front></article>"#,
        )
        .unwrap();
        let mut rights = test_europe_rights("CC BY 4.0");
        enrich_jats_rights(&xml, &mut rights).unwrap();
        assert_eq!(rights.search_license.as_deref(), Some("CC BY 4.0"));
        assert_eq!(rights.document_license, "Unknown/JATS-unresolved");
        assert_eq!(
            rights.jats_license.as_deref(),
            Some("Unknown/JATS-unresolved")
        );
        assert!(rights.jats_licenses.is_empty());
        assert!(rights.license_conflict);
    }

    #[test]
    fn multiple_different_jats_licenses_are_preserved_and_resolve_unknown() {
        let temp = tempfile::tempdir().unwrap();
        let xml = temp.path().join("paper.xml");
        fs::write(
            &xml,
            br#"<article xmlns:xlink="http://www.w3.org/1999/xlink"><front><article-meta><permissions><license xlink:href="https://creativecommons.org/licenses/by/4.0/"><license-p>Attribution required.</license-p></license><license xlink:href="https://creativecommons.org/licenses/by-nc-nd/4.0/"><license-p>Noncommercial and no derivatives.</license-p></license></permissions></article-meta></front></article>"#,
        )
        .unwrap();
        let mut rights = test_europe_rights("CC BY 4.0");
        enrich_jats_rights(&xml, &mut rights).unwrap();
        assert_eq!(rights.document_license, "Unknown/JATS-ambiguous");
        assert_eq!(rights.jats_licenses.len(), 2);
        assert_eq!(rights.jats_permissions.len(), 2);
        assert!(rights.license_conflict);
        assert!(rights.jats_licenses[0].contains("licenses/by/4.0"));
        assert!(rights.jats_licenses[1].contains("licenses/by-nc-nd/4.0"));
    }

    #[test]
    fn cc_by_nd_is_distinct_and_conflicts_with_cc_by() {
        assert_eq!(license_family("CC BY-ND 4.0"), Some("cc-by-nd"));
        assert!(!licenses_equivalent("CC BY 4.0", "CC BY-ND 4.0"));
        let temp = tempfile::tempdir().unwrap();
        let xml = temp.path().join("paper.xml");
        fs::write(
            &xml,
            br#"<article xmlns:xlink="http://www.w3.org/1999/xlink"><front><article-meta><permissions><license xlink:href="https://creativecommons.org/licenses/by-nd/4.0/"><license-p>No derivatives.</license-p></license></permissions></article-meta></front></article>"#,
        )
        .unwrap();
        let mut rights = test_europe_rights("CC BY 4.0");
        enrich_jats_rights(&xml, &mut rights).unwrap();
        assert_eq!(
            rights.document_license,
            "https://creativecommons.org/licenses/by-nd/4.0/"
        );
        assert!(rights.license_conflict);
    }

    #[test]
    fn xml_attribute_count_is_bounded() {
        let temp = tempfile::tempdir().unwrap();
        let xml = temp.path().join("paper.xml");
        let attributes = (0..=MAX_XML_ATTRIBUTES_PER_ELEMENT)
            .map(|index| format!(" a{index}=\"x\""))
            .collect::<String>();
        fs::write(
            &xml,
            format!(
                "<article><front><permissions><license{attributes}><license-p>terms</license-p></license></permissions></front></article>"
            ),
        )
        .unwrap();
        let mut rights = sample_paper("20230001234", "Graph", "2023-01-01", None).rights;
        let started = Instant::now();
        assert!(enrich_jats_rights(&xml, &mut rights).is_err());
        assert!(started.elapsed() < Duration::from_secs(2));
    }

    #[test]
    fn manifest_corruption_is_recognized_and_atomic_repair_validates() {
        let temp = tempfile::tempdir().unwrap();
        let destination = temp.path().join("nasa-ntrs_20230001234");
        fs::create_dir(&destination).unwrap();
        let file = destination.join("paper.pdf");
        fs::write(&file, test_pdf("original")).unwrap();
        let paper = sample_paper("20230001234", "Graph report", "2023-01-01", None);
        let manifest = PaperManifest {
            format_version: "1".to_string(),
            paper: paper.clone(),
            integrity: PaperIntegrity {
                path: "paper.pdf".to_string(),
                bytes: fs::metadata(&file).unwrap().len(),
                blake3: blake3_file(&file).unwrap(),
            },
            complete: true,
        };
        write_manifest(&destination, &manifest).unwrap();
        assert!(validate_manifest(&destination, &manifest).is_ok());
        let mut tampered = manifest.clone();
        tampered.paper.id = "nasa-ntrs:20230009999".to_string();
        write_manifest(&destination, &tampered).unwrap();
        assert!(inspect_archive(&destination).is_err());
        assert!(verify_archive(&destination).is_err());
        tampered.paper = paper.clone();
        tampered.paper.source_url = "https://ntrs.nasa.gov/citations/20230009999".to_string();
        write_manifest(&destination, &tampered).unwrap();
        assert!(inspect_archive(&destination).is_err());
        write_manifest(&destination, &manifest).unwrap();
        fs::write(&file, b"corrupt").unwrap();
        assert!(validate_manifest(&destination, &manifest).is_err());

        let staging = create_staging(temp.path(), &paper.id).unwrap();
        let repaired_file = staging.join("paper.pdf");
        fs::write(&repaired_file, test_pdf("repaired")).unwrap();
        let repaired = PaperManifest {
            integrity: PaperIntegrity {
                path: "paper.pdf".to_string(),
                bytes: fs::metadata(&repaired_file).unwrap().len(),
                blake3: blake3_file(&repaired_file).unwrap(),
            },
            ..manifest
        };
        write_manifest(&staging, &repaired).unwrap();
        commit_staging(&staging, &destination).unwrap();
        let retained = read_manifest(&destination).unwrap();
        assert!(validate_manifest(&destination, &retained).is_ok());
        assert_eq!(inspect_archive(&destination).unwrap(), retained);
        assert_eq!(verify_archive(&destination).unwrap(), retained);
    }

    #[test]
    fn manifest_jats_rights_must_match_valid_archived_xml() {
        let temp = tempfile::tempdir().unwrap();
        let directory = temp.path().join("europe-pmc_PMC7654321");
        fs::create_dir(&directory).unwrap();
        let file = directory.join("paper.xml");
        fs::write(
            &file,
            br#"<article xmlns:xlink="http://www.w3.org/1999/xlink"><front><article-meta><article-id pub-id-type="pmc">PMC7654321</article-id><permissions><license xlink:href="https://creativecommons.org/licenses/by-nd/4.0/"><license-p>No derivatives.</license-p></license></permissions></article-meta></front></article>"#,
        )
        .unwrap();
        let mut paper = parse_europe_pmc_json(EUROPE_PMC_FIXTURE.as_bytes())
            .unwrap()
            .remove(0);
        enrich_jats_rights(&file, &mut paper.rights).unwrap();
        let manifest = PaperManifest {
            format_version: "1".to_string(),
            paper,
            integrity: PaperIntegrity {
                path: "paper.xml".to_string(),
                bytes: fs::metadata(&file).unwrap().len(),
                blake3: blake3_file(&file).unwrap(),
            },
            complete: true,
        };
        write_manifest(&directory, &manifest).unwrap();
        assert!(verify_archive(&directory).is_ok());
        let mut tampered = manifest;
        tampered.paper.rights.document_license = "CC BY 4.0".to_string();
        tampered.paper.rights.jats_license = Some("CC BY 4.0".to_string());
        write_manifest(&directory, &tampered).unwrap();
        assert!(inspect_archive(&directory).is_err());
        assert!(verify_archive(&directory).is_err());
    }

    #[test]
    fn unavailable_revision_forces_revalidation_and_stale_transactions_are_refused() {
        let mut archived = sample_paper("20230001234", "Graph", "2023-01-01", None);
        let mut current = archived.clone();
        current.document.provider_revision = "unavailable:PMC123".to_string();
        assert!(!same_archive_revision(&archived, &current));
        archived.rights.search_license = Some("cc by".to_string());
        current.document.provider_revision = archived.document.provider_revision.clone();
        assert!(!same_archive_revision(&archived, &current));

        let temp = tempfile::tempdir().unwrap();
        fs::create_dir(temp.path().join(".paperforge-stage-stale")).unwrap();
        assert!(prepare_output(temp.path()).is_err());
    }

    #[test]
    fn provider_display_text_and_diagnostics_strip_controls_and_are_bounded() {
        assert_eq!(
            compact_whitespace("safe\u{1b}[31m title\nnext"),
            "safe [31m title next"
        );
        let diagnostic = bounded_diagnostic(&format!("{}\u{1b}", "x".repeat(20_000)));
        assert!(!diagnostic.contains('\u{1b}'));
        assert!(diagnostic.len() <= MAX_ERROR_RESPONSE_BYTES as usize);
    }

    #[test]
    fn defaults_limits_and_progress_are_bounded_and_consistent() {
        let options = ArchiveOptions::default();
        assert_eq!(options.output, PathBuf::from("papers"));
        assert_eq!(options.max_download_bytes, 256 * 1024 * 1024);
        let client =
            PaperForge::with_user_agent(PaperProvider::Arxiv, "test/1 contact@example.org")
                .unwrap()
                .with_discovery_limit(0);
        assert_eq!(client.discovery_limit, 1);
        let summary = ArchiveSummary {
            selected: 3,
            archived: 1,
            skipped: 1,
            failed: vec![ArchiveFailure {
                paper_id: "x".to_string(),
                output: None,
                message: "failure".to_string(),
            }],
            ..ArchiveSummary::default()
        };
        let progress = progress_snapshot(ArchiveProgressPhase::Completed, 3, &summary);
        assert_eq!(progress.completed_papers, 3);
        assert_eq!(progress.succeeded_papers, 2);
        assert_eq!(progress.failed_papers, 1);
    }

    #[test]
    fn arxiv_gate_is_process_global_and_exclusive_across_clients() {
        let first =
            PaperForge::with_user_agent(PaperProvider::Arxiv, "first/1 contact@example.org")
                .unwrap();
        let second =
            PaperForge::with_user_agent(PaperProvider::Arxiv, "second/1 contact@example.org")
                .unwrap();
        let guard = first.acquire_provider_slot().unwrap();
        assert!(
            ARXIV_REQUEST_GATE.get().unwrap().try_lock().is_err(),
            "the request gate must remain exclusive while a response is consumed"
        );
        drop(guard);
        ARXIV_REQUEST_GATE.get().unwrap().lock().unwrap().last_start =
            Some(Instant::now() - ARXIV_INTERVAL);
        assert!(second.acquire_provider_slot().unwrap().is_some());
    }

    #[test]
    #[ignore = "bounded live arXiv PDF archive and verified skip"]
    fn live_arxiv_exact_pdf_archive_and_skip() -> Result<()> {
        let temp = tempfile::tempdir()?;
        let forge = PaperForge::with_user_agent(
            PaperProvider::Arxiv,
            "paperforge-live-test/0.1 (contact@example.org)",
        )?
        .with_discovery_limit(1);
        let papers = forge.discover(&parse_paper_query("arxiv:2106.09685v2")?)?;
        let options = ArchiveOptions {
            output: temp.path().join("papers"),
            skip_existing: false,
            max_download_bytes: 64 * 1024 * 1024,
        };
        let first = forge.archive(&papers, &options)?;
        assert!(first.is_success() && first.archived == 1);
        assert_eq!(first.papers[0].manifest.paper.id, papers[0].id);
        verify_archive(&first.papers[0].output)?;
        let second = forge.archive(
            &papers,
            &ArchiveOptions {
                skip_existing: true,
                ..options
            },
        )?;
        assert_eq!(second.skipped, 1);
        Ok(())
    }

    #[test]
    #[ignore = "bounded live Europe PMC JATS archive"]
    fn live_europe_pmc_jats_archive() -> Result<()> {
        let temp = tempfile::tempdir()?;
        let forge = PaperForge::with_user_agent(
            PaperProvider::EuropePmc,
            "paperforge-live-test/0.1 (contact@example.org)",
        )?
        .with_discovery_limit(1);
        let papers = forge.discover(&parse_paper_query("pmcid:PMC7537588 @limit:1")?)?;
        let result = forge.archive(
            &papers,
            &ArchiveOptions {
                output: temp.path().join("papers"),
                max_download_bytes: 32 * 1024 * 1024,
                ..ArchiveOptions::default()
            },
        )?;
        assert!(result.is_success() && result.archived == 1);
        let rights = &result.papers[0].manifest.paper.rights;
        assert!(rights.jats_license.is_some() || !rights.jats_permissions.is_empty());
        verify_archive(&result.papers[0].output)?;
        Ok(())
    }

    #[test]
    #[ignore = "bounded live NASA NTRS query and eligible PDF archive"]
    fn live_nasa_query_pdf_archive() -> Result<()> {
        let temp = tempfile::tempdir()?;
        let forge = PaperForge::with_user_agent(
            PaperProvider::NasaNtrs,
            "paperforge-live-test/0.1 (contact@example.org)",
        )?
        .with_discovery_limit(1);
        let papers = forge.discover(&parse_paper_query("ntrs:20230001234 @limit:1")?)?;
        let result = forge.archive(
            &papers,
            &ArchiveOptions {
                output: temp.path().join("papers"),
                max_download_bytes: 64 * 1024 * 1024,
                ..ArchiveOptions::default()
            },
        )?;
        assert!(result.is_success() && result.archived == 1);
        assert!(
            result.papers[0]
                .manifest
                .paper
                .rights
                .evidence
                .iter()
                .any(|item| item.contains("isExportControl=NO"))
        );
        verify_archive(&result.papers[0].output)?;
        Ok(())
    }
}