use std::collections::BTreeMap;
use camino::{Utf8Path, Utf8PathBuf};
use chrono::Utc;
use serde_json::Value;
use crate::dry_run::{build_fetch_plan, try_build_fetch_plan, FetchPlan};
use crate::http::HttpError;
use crate::provenance::{Capability, LogEvent, LogResult, RowInput};
use crate::source::{FetchContext, FetchError, FetchResult, Source};
use crate::sources::arxiv::ArxivSource;
use crate::sources::crossref::CrossrefSource;
use crate::sources::unpaywall::UnpaywallSource;
use crate::store::{DoigetExtension, Metadata, Store};
use crate::DenialContext;
use crate::{ArxivId, CapabilityProfile, Doi, Ref, Safekey, MAX_BATCH_REFS, SCHEMA_VERSION};
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
#[non_exhaustive]
pub struct MetadataOnlyOutcome {
pub source: String,
pub resolver_profile: String,
pub license: Option<String>,
pub oa_url: Option<String>,
#[serde(default)]
pub oa_status: Option<String>,
pub metadata: Value,
}
pub async fn metadata_only(
ref_: &Ref,
profile: &CapabilityProfile,
ctx: &FetchContext,
) -> Result<MetadataOnlyOutcome, FetchError> {
let cache_root = if resolver_base_overridden() {
None
} else {
ctx.cache_root.as_deref()
};
if let Some(root) = cache_root {
if let Some(cached) = crate::resolver_cache::read(root, ref_) {
return Ok(cached);
}
}
let outcome = match ref_ {
Ref::Doi(doi) => metadata_only_doi(doi, ref_, profile, ctx).await?,
Ref::Arxiv(id) => {
let arxiv = arxiv_source_from_env();
let metadata = arxiv.fetch_metadata_only(id, ctx).await?;
MetadataOnlyOutcome {
source: arxiv.name().to_string(),
resolver_profile: arxiv.name().to_string(),
license: Some("arxiv-default".to_string()),
oa_url: None,
oa_status: Some("green".to_string()),
metadata,
}
}
};
if let Some(root) = cache_root {
crate::resolver_cache::write(root, ref_, &outcome);
}
Ok(outcome)
}
fn resolver_base_overridden() -> bool {
[
"DOIGET_CROSSREF_BASE",
"DOIGET_UNPAYWALL_BASE",
"DOIGET_ARXIV_BASE",
]
.iter()
.any(|k| std::env::var_os(k).is_some())
}
pub async fn resolve_only(
ref_: &Ref,
profile: &CapabilityProfile,
ctx: &FetchContext,
) -> Result<MetadataOnlyOutcome, FetchError> {
metadata_only(ref_, profile, ctx).await
}
pub async fn metadata_only_to_store(
ref_: &Ref,
profile: &CapabilityProfile,
ctx: &FetchContext,
store: &dyn Store,
) -> Result<MetadataOnlyOutcome, FetchError> {
let outcome = metadata_only(ref_, profile, ctx).await?;
let safekey = ref_.safekey();
let metadata = build_metadata_only_metadata(ref_, &outcome);
write_metadata_and_pdf(store, &safekey, &metadata, None, ctx)?;
Ok(outcome)
}
fn build_metadata_only_metadata(ref_: &Ref, outcome: &MetadataOnlyOutcome) -> Metadata {
let (doi, arxiv_id) = match ref_ {
Ref::Doi(d) => (Some(d.clone()), None),
Ref::Arxiv(a) => (None, Some(a.clone())),
};
let ref_id = ref_.as_input_str().to_string();
let title = match extract_metadata_title(&outcome.metadata) {
Some(t) => t,
None => {
tracing::warn!(
ref_id = %ref_id,
source = %outcome.source,
"metadata-only: no usable title in resolver payload; \
persisting the ref id as the title placeholder"
);
ref_id
}
};
Metadata {
schema_version: SCHEMA_VERSION.to_string(),
title,
authors: extract_metadata_authors(&outcome.metadata),
year: None,
doi,
arxiv_id,
arxiv_categories: Vec::new(),
abstract_: None,
venue: None,
volume: None,
issue: None,
pages: None,
publisher: None,
issn: None,
isbn: None,
type_: None,
keywords: Vec::new(),
url: outcome.oa_url.clone(),
pdf_path: None,
doiget: Some(DoigetExtension {
fetched_at: Utc::now(),
source: outcome.source.clone(),
license: outcome
.license
.clone()
.unwrap_or_else(|| "unknown".to_string()),
oa_status: outcome.oa_status.clone(),
size_bytes: 0,
mcp_call_id: None,
tags: Vec::new(),
collections: Vec::new(),
annotation: None,
}),
other: BTreeMap::new(),
}
}
#[must_use]
pub fn cite_metadata(ref_: &Ref, outcome: &MetadataOnlyOutcome) -> Metadata {
let mut m = build_metadata_only_metadata(ref_, outcome);
if outcome.source == "crossref" {
let f = extract_crossref_fields(&outcome.metadata);
if let Some(title) = f.title {
m.title = title;
}
if !f.authors.is_empty() {
m.authors = f.authors;
}
m.year = f.year;
m.venue = f.venue;
m.volume = f.volume;
m.issue = f.issue;
m.pages = f.pages;
m.type_ = f.type_;
m.publisher = outcome
.metadata
.get("publisher")
.and_then(Value::as_str)
.map(str::to_string);
m.issn = outcome
.metadata
.get("ISSN")
.and_then(Value::as_array)
.and_then(|a| a.first())
.and_then(Value::as_str)
.map(str::to_string);
} else if outcome.source == "arxiv" {
m.year = outcome
.metadata
.get("published")
.and_then(Value::as_str)
.and_then(parse_rfc3339_year);
m.arxiv_categories = extract_arxiv_categories(&outcome.metadata);
}
m
}
fn parse_rfc3339_year(s: &str) -> Option<i32> {
chrono::DateTime::parse_from_rfc3339(s)
.ok()
.map(|dt| chrono::Datelike::year(&dt))
}
fn extract_arxiv_categories(atom: &Value) -> Vec<String> {
atom.get("categories")
.and_then(Value::as_array)
.map(|a| {
a.iter()
.filter_map(Value::as_str)
.map(str::to_string)
.collect()
})
.unwrap_or_default()
}
fn normalize_page_range(page: &str) -> String {
if page.contains("--") || !page.contains('-') {
return page.to_string();
}
page.replace('-', "--")
}
fn extract_metadata_title(meta: &Value) -> Option<String> {
let t = meta.get("title")?;
let s = match t.as_str() {
Some(s) => s.trim().to_string(),
None => t
.as_array()?
.iter()
.filter_map(Value::as_str)
.map(str::trim)
.find(|s| !s.is_empty())?
.to_string(),
};
if s.is_empty() {
None
} else {
Some(s)
}
}
fn extract_metadata_authors(meta: &Value) -> Vec<String> {
if let Some(arr) = meta.get("authors").and_then(Value::as_array) {
let v: Vec<String> = arr
.iter()
.filter_map(|a| a.as_str().map(str::to_string))
.collect();
if !v.is_empty() {
return v;
}
}
for key in ["author", "z_authors"] {
if let Some(arr) = meta.get(key).and_then(Value::as_array) {
let v: Vec<String> = arr
.iter()
.filter_map(|a| {
let given = a.get("given").and_then(Value::as_str).unwrap_or("");
let family = a.get("family").and_then(Value::as_str).unwrap_or("");
let name = format!("{given} {family}");
let name = name.trim();
if name.is_empty() {
a.get("name").and_then(Value::as_str).map(str::to_string)
} else {
Some(name.to_string())
}
})
.collect();
if !v.is_empty() {
return v;
}
}
}
Vec::new()
}
const FALLBACK_CONTACT_EMAIL: &str = "doiget@localhost";
fn env_nonempty(key: &str) -> Option<String> {
std::env::var(key).ok().filter(|s| !s.trim().is_empty())
}
fn resolve_contact_email() -> String {
contact_email_or_placeholder()
}
#[must_use]
pub fn configured_contact_email() -> Option<String> {
env_nonempty("DOIGET_CONTACT_EMAIL")
.or_else(|| crate::user_extension::load_or_default().contact_email)
}
#[must_use]
pub fn contact_email_or_placeholder() -> String {
configured_contact_email().unwrap_or_else(|| FALLBACK_CONTACT_EMAIL.to_string())
}
struct ContactAddresses {
crossref: String,
unpaywall: String,
}
fn resolve_contact_emails() -> ContactAddresses {
let env_contact = env_nonempty("DOIGET_CONTACT_EMAIL");
let env_unpaywall = env_nonempty("DOIGET_UNPAYWALL_EMAIL");
if let (Some(crossref), Some(unpaywall)) = (&env_contact, &env_unpaywall) {
return ContactAddresses {
crossref: crossref.clone(),
unpaywall: unpaywall.clone(),
};
}
let file = crate::user_extension::load_or_default();
let crossref = env_contact
.or(file.contact_email)
.unwrap_or_else(|| FALLBACK_CONTACT_EMAIL.to_string());
let unpaywall = env_unpaywall
.or(file.unpaywall_email)
.unwrap_or_else(|| crossref.clone());
ContactAddresses {
crossref,
unpaywall,
}
}
fn arxiv_source_from_env() -> ArxivSource {
if let Ok(s) = std::env::var("DOIGET_ARXIV_BASE") {
match url::Url::parse(&s) {
Ok(url) => return ArxivSource::with_base(url),
Err(e) => tracing::warn!(
value = %s,
error = %e,
"DOIGET_ARXIV_BASE is not a valid URL; using the default arXiv base"
),
}
}
ArxivSource::new()
}
fn crossref_source_from_env(contact: &str) -> CrossrefSource {
if let Ok(s) = std::env::var("DOIGET_CROSSREF_BASE") {
match url::Url::parse(&s) {
Ok(url) => return CrossrefSource::with_base(url, contact.to_string()),
Err(e) => tracing::warn!(
value = %s,
error = %e,
"DOIGET_CROSSREF_BASE is not a valid URL; using the default Crossref base"
),
}
}
CrossrefSource::new(contact.to_string())
}
fn unpaywall_source_from_env(contact: &str) -> UnpaywallSource {
if let Ok(s) = std::env::var("DOIGET_UNPAYWALL_BASE") {
match url::Url::parse(&s) {
Ok(url) => return UnpaywallSource::with_base(url, contact.to_string()),
Err(e) => tracing::warn!(
value = %s,
error = %e,
"DOIGET_UNPAYWALL_BASE is not a valid URL; using the default Unpaywall base"
),
}
}
UnpaywallSource::new(contact.to_string())
}
async fn metadata_only_doi(
_doi: &Doi,
ref_: &Ref,
profile: &CapabilityProfile,
ctx: &FetchContext,
) -> Result<MetadataOnlyOutcome, FetchError> {
let contact = resolve_contact_email();
let crossref = crossref_source_from_env(&contact);
match crossref.fetch(ref_, profile, ctx).await {
Ok(res) => {
let metadata = res.metadata_json.unwrap_or(Value::Null);
let oa_url = extract_crossref_publisher_url(&metadata);
Ok(MetadataOnlyOutcome {
source: crossref.name().to_string(),
resolver_profile: crossref.name().to_string(),
license: None,
oa_url,
oa_status: None,
metadata,
})
}
Err(crossref_err) => {
let unpaywall = unpaywall_source_from_env(&contact);
match unpaywall.fetch(ref_, profile, ctx).await {
Ok(res) => {
let metadata = res.metadata_json.unwrap_or(Value::Null);
let oa_url = extract_unpaywall_oa_url(&metadata);
let oa_status = extract_unpaywall_oa_status(&metadata);
let license = if res.license == "unknown" {
None
} else {
Some(res.license)
};
Ok(MetadataOnlyOutcome {
source: unpaywall.name().to_string(),
resolver_profile: unpaywall.name().to_string(),
license,
oa_url,
oa_status,
metadata,
})
}
Err(_unpaywall_err) => {
Err(crossref_err)
}
}
}
}
}
fn extract_crossref_publisher_url(msg: &Value) -> Option<String> {
let arr = msg.get("link")?.as_array()?;
let eligible = || {
arr.iter().filter(|e| {
e.get("intended-application")
.and_then(Value::as_str)
.is_some_and(|a| a.eq_ignore_ascii_case("unspecified"))
})
};
let url_of = |e: &Value| {
e.get("URL")
.and_then(Value::as_str)
.filter(|s| !s.is_empty())
.map(str::to_string)
};
eligible()
.find(|e| {
e.get("content-type")
.and_then(Value::as_str)
.is_some_and(|c| c.eq_ignore_ascii_case("application/pdf"))
})
.and_then(url_of)
.or_else(|| eligible().find_map(url_of))
}
fn extract_unpaywall_oa_url(meta: &Value) -> Option<String> {
let loc = meta.get("best_oa_location")?;
loc.get("url_for_pdf")
.and_then(Value::as_str)
.or_else(|| loc.get("url").and_then(Value::as_str))
.map(|s| s.to_string())
}
fn extract_unpaywall_oa_status(meta: &Value) -> Option<String> {
meta.get("oa_status")
.and_then(Value::as_str)
.filter(|s| !s.is_empty())
.map(str::to_string)
}
#[derive(Debug, Clone)]
#[non_exhaustive]
pub enum PdfLegStatus {
Fetched,
NoOaUrl,
Blocked {
code: crate::ErrorCode,
message: String,
denial: Option<crate::DenialContext>,
suggested_arxiv_id: Option<String>,
},
PreprintFallback {
arxiv_id: String,
original_block: String,
},
TdmFetched {
source: String,
original_block: String,
},
}
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct FetchPaperOutcome {
pub source: String,
pub resolver_profile: String,
pub license: String,
pub oa_status: Option<String>,
pub path: Utf8PathBuf,
pub size_bytes: u64,
pub schema_version: String,
pub pdf_leg: PdfLegStatus,
pub safekey: String,
pub canonical_digest: String,
pub title: String,
pub authors: Vec<String>,
pub year: Option<i32>,
pub attempts: Vec<SourceAttempt>,
}
impl FetchPaperOutcome {
#[doc(hidden)]
pub fn for_test_synthetic(
safekey: impl Into<String>,
source: impl Into<String>,
pdf_leg: PdfLegStatus,
) -> Self {
let safekey: String = safekey.into();
let source: String = source.into();
Self {
source: source.clone(),
resolver_profile: source.clone(),
license: "unknown".to_string(),
oa_status: None,
path: Utf8PathBuf::from(format!("/tmp/{safekey}.pdf")),
size_bytes: 0,
schema_version: SCHEMA_VERSION.to_string(),
pdf_leg,
safekey: safekey.clone(),
canonical_digest: "00".repeat(32),
title: String::new(),
authors: Vec::new(),
year: None,
attempts: Vec::new(),
}
}
#[doc(hidden)]
pub fn for_test_synthetic_with_attempts(
safekey: impl Into<String>,
source: impl Into<String>,
pdf_leg: PdfLegStatus,
attempts: Vec<SourceAttempt>,
) -> Self {
Self {
attempts,
..Self::for_test_synthetic(safekey, source, pdf_leg)
}
}
}
pub async fn fetch_paper(
ref_: &Ref,
profile: &CapabilityProfile,
ctx: &FetchContext,
store: &dyn Store,
store_root: &Utf8Path,
) -> Result<FetchPaperOutcome, FetchError> {
let safekey = ref_.safekey();
match ref_ {
Ref::Arxiv(id) => {
fetch_paper_arxiv(id, ref_, profile, ctx, store, store_root, &safekey).await
}
Ref::Doi(doi) => {
fetch_paper_doi(doi, ref_, profile, ctx, store, store_root, &safekey).await
}
}
}
pub fn fetch_paper_plan(ref_: &Ref, store_root: &Utf8Path) -> FetchPlan {
build_fetch_plan(ref_, store_root)
}
pub fn try_fetch_paper_plan(ref_: &Ref, store_root: &Utf8Path) -> Result<FetchPlan, FetchError> {
try_build_fetch_plan(ref_, store_root)
}
async fn fetch_paper_arxiv(
id: &ArxivId,
ref_: &Ref,
profile: &CapabilityProfile,
ctx: &FetchContext,
store: &dyn Store,
store_root: &Utf8Path,
safekey: &Safekey,
) -> Result<FetchPaperOutcome, FetchError> {
let source = arxiv_source_from_env();
if !source.can_serve(profile, ref_) {
return Err(FetchError::NotEligible {
source_key: source.name().to_string(),
});
}
let FetchResult {
license,
pdf_bytes,
final_url,
metadata_json,
..
} = source.fetch(ref_, profile, ctx).await?;
let pdf = pdf_bytes.ok_or_else(|| FetchError::SourceSchema {
hint: "arxiv source returned no PDF bytes".to_string(),
})?;
let size_bytes = pdf.len() as u64;
let (title, authors, year, arxiv_categories) = match &metadata_json {
Some(atom) => (
extract_metadata_title(atom).unwrap_or_else(|| format!("arxiv:{}", id.as_str())),
extract_metadata_authors(atom),
atom.get("published")
.and_then(Value::as_str)
.and_then(parse_rfc3339_year),
extract_arxiv_categories(atom),
),
None => (
format!("arxiv:{}", id.as_str()),
Vec::new(),
None,
Vec::new(),
),
};
let metadata = Metadata {
schema_version: SCHEMA_VERSION.to_string(),
title,
authors,
year,
doi: None,
arxiv_id: Some(id.clone()),
arxiv_categories,
abstract_: None,
venue: None,
volume: None,
issue: None,
pages: None,
publisher: None,
issn: None,
isbn: None,
type_: None,
keywords: Vec::new(),
url: final_url.as_ref().map(|u| u.to_string()),
pdf_path: Some(format!("{}.pdf", safekey.as_str())),
doiget: Some(DoigetExtension {
fetched_at: Utc::now(),
source: "arxiv".to_string(),
license: license.clone(),
oa_status: Some("green".to_string()),
size_bytes,
mcp_call_id: None,
tags: Vec::new(),
collections: Vec::new(),
annotation: None,
}),
other: BTreeMap::new(),
};
let tmp = stage_pdf_to_tempfile(&pdf)?;
let pdf_src = Utf8Path::from_path(tmp.path())
.ok_or_else(|| FetchError::SourceSchema {
hint: "staging tempfile path is not UTF-8".to_string(),
})?
.to_path_buf();
write_metadata_and_pdf(store, safekey, &metadata, Some(&pdf_src), ctx)?;
drop(tmp);
let path = store_root.join(format!("{}.pdf", safekey.as_str()));
let canonical_digest =
crate::CanonicalRef::new(crate::SourceType::Arxiv, id.as_str(), "arxiv", None).digest_hex();
Ok(FetchPaperOutcome {
source: "arxiv".to_string(),
resolver_profile: "arxiv".to_string(),
license,
oa_status: Some("green".to_string()),
path,
size_bytes,
schema_version: SCHEMA_VERSION.to_string(),
pdf_leg: PdfLegStatus::Fetched,
safekey: safekey.as_str().to_string(),
canonical_digest,
title: metadata.title.clone(),
authors: metadata.authors.clone(),
year: metadata.year,
attempts: Vec::new(),
})
}
async fn fetch_paper_doi(
doi: &Doi,
ref_: &Ref,
profile: &CapabilityProfile,
ctx: &FetchContext,
store: &dyn Store,
store_root: &Utf8Path,
safekey: &Safekey,
) -> Result<FetchPaperOutcome, FetchError> {
let addresses = resolve_contact_emails();
let contact = addresses.crossref;
let unpaywall_contact = addresses.unpaywall;
let crossref = crossref_source_from_env(&contact);
let (cross, crossref_err) = match crossref.fetch(ref_, profile, ctx).await {
Ok(r) => (Some(r), None),
Err(e) => {
tracing::warn!(
error = %e,
"crossref fetch failed; continuing with unpaywall-only metadata + OA leg"
);
(None, Some(e))
}
};
let crossref_meta = cross
.as_ref()
.and_then(|c| c.metadata_json.clone())
.unwrap_or(Value::Null);
#[allow(unused_mut)]
let mut extracted = extract_crossref_fields(&crossref_meta);
#[allow(unused_mut)]
let mut attempts: Vec<SourceAttempt> = Vec::new();
#[cfg(any(
feature = "tdm-elsevier",
feature = "tdm-aps",
feature = "tdm-springer",
feature = "tdm-ieee"
))]
let tdm_meta = resolve_tdm_chain(ref_, profile, ctx, cross.is_some(), &mut attempts).await;
#[cfg(not(any(
feature = "tdm-elsevier",
feature = "tdm-aps",
feature = "tdm-springer",
feature = "tdm-ieee"
)))]
let tdm_meta: Option<Value> = None;
#[cfg(feature = "metadata")]
let optional_resolved = resolve_optional_chain(
ref_,
profile,
ctx,
cross.is_some() || tdm_meta.is_some(),
&mut extracted,
&mut attempts,
)
.await;
#[cfg(feature = "metadata")]
let optional_meta = optional_resolved
.as_ref()
.map(|(_, m)| m.clone())
.or(tdm_meta);
#[cfg(not(feature = "metadata"))]
let optional_meta: Option<Value> = tdm_meta;
let _ = &optional_meta;
let unpaywall = unpaywall_source_from_env(&unpaywall_contact);
let upw_result = unpaywall.fetch(ref_, profile, ctx).await;
let (mut license, source_label, oa_chain, oa_status) = match upw_result {
Ok(r) => {
let chain = extract_oa_url_chain(r.metadata_json.as_ref());
let oa_status = r
.metadata_json
.as_ref()
.and_then(extract_unpaywall_oa_status);
let label = if r.license != "unknown" {
"unpaywall".to_string()
} else {
"crossref".to_string()
};
(r.license, label, chain, oa_status)
}
Err(e) => {
tracing::warn!(
error = %e,
doi = %doi.as_str(),
"unpaywall fetch failed; OA chain will be empty (downstream PdfLegStatus::NoOaUrl \
is conservative — Unpaywall was unreachable, not authoritatively oa-free)"
);
(
"unknown".to_string(),
"crossref".to_string(),
Vec::new(),
None,
)
}
};
let (pdf_leg, pdf_bytes) = if oa_chain.is_empty() {
(PdfLegStatus::NoOaUrl, None)
} else {
let mut succeeded: Option<Vec<u8>> = None;
let mut last_err: Option<HttpError> = None;
let total = oa_chain.len();
for (idx, candidate) in oa_chain.iter().enumerate() {
let attempt = idx + 1;
tracing::debug!(
attempt,
total,
url = %candidate,
"trying OA PDF candidate (ADR-0029 chain)"
);
match try_fetch_oa_pdf(doi, candidate, ctx).await {
Ok((bytes, _final_url)) => {
if attempt > 1 {
tracing::info!(
attempt,
total,
url = %candidate,
"OA PDF chain succeeded on fallback candidate (ADR-0029)"
);
}
succeeded = Some(bytes);
break;
}
Err(e) => {
tracing::warn!(
attempt,
total,
url = %candidate,
error = %e,
"OA PDF candidate failed; advancing to next (ADR-0029 chain)"
);
last_err = Some(e);
}
}
}
match (succeeded, last_err) {
(Some(bytes), _) => (PdfLegStatus::Fetched, Some(bytes)),
(None, Some(e)) => {
let fe = FetchError::Http(e);
let denial: Option<crate::DenialContext> = (&fe).into();
let message = fe.to_string();
let code: crate::ErrorCode = fe.into();
let suggested_arxiv_id = oa_chain.iter().find_map(extract_arxiv_id_from_url);
(
PdfLegStatus::Blocked {
code,
message,
denial,
suggested_arxiv_id,
},
None,
)
}
(None, None) => {
tracing::error!(
total = oa_chain.len(),
"OA PDF chain walker exhausted without recording success or error \
(defensive fallback — should be unreachable)"
);
(
PdfLegStatus::Blocked {
code: crate::ErrorCode::InternalError,
message:
"OA PDF chain walker exhausted without recording success or error \
(orchestrator bug — please report)"
.to_string(),
denial: None,
suggested_arxiv_id: None,
},
None,
)
}
}
};
let (pdf_leg, pdf_bytes, arxiv_id_for_metadata, fallback_license) =
try_arxiv_preprint_fallback(doi, pdf_leg, pdf_bytes, profile, ctx).await;
#[cfg(feature = "metadata")]
let (pdf_leg, pdf_bytes) = try_optional_source_oa_fallback(
doi,
pdf_leg,
pdf_bytes,
profile,
ctx,
&mut attempts,
optional_resolved.as_ref().map(|(n, m)| (*n, m)),
)
.await;
#[cfg(any(
feature = "tdm-elsevier",
feature = "tdm-aps",
feature = "tdm-springer",
feature = "tdm-ieee"
))]
let (pdf_leg, pdf_bytes) =
try_tdm_content_fallback(doi, pdf_leg, pdf_bytes, profile, ctx, &mut attempts).await;
if let Some(fl) = fallback_license {
license = fl;
}
#[cfg(any(
feature = "tdm-elsevier",
feature = "tdm-aps",
feature = "tdm-springer",
feature = "tdm-ieee"
))]
if matches!(pdf_leg, PdfLegStatus::TdmFetched { .. }) {
license = "unknown".to_string();
}
if let Some(e) = crossref_err {
if pdf_bytes.is_none() {
if !attempts.is_empty() {
let trace = render_attempts(&attempts);
let lead = if nothing_was_consulted(&attempts) {
"no optional source was consulted for this DOI"
} else {
"the optional sources were consulted and did not resolve it"
};
return Err(FetchError::NotFound {
hint: format!(
"{e}
= note: {lead}:
{trace}"
),
});
}
return Err(e);
}
}
let (final_source_label, size_bytes, pdf_path_relative, pdf_staged) = match &pdf_bytes {
Some(bytes) => {
let staged = stage_pdf_to_tempfile(bytes)?;
let label = match &pdf_leg {
PdfLegStatus::PreprintFallback { .. } => "arxiv".to_string(),
PdfLegStatus::TdmFetched { source, .. } => source.clone(),
_ => "oa-publisher".to_string(),
};
(
label,
bytes.len() as u64,
Some(format!("{}.pdf", safekey.as_str())),
Some(staged),
)
}
None => (source_label, 0u64, None, None),
};
let metadata = Metadata {
schema_version: SCHEMA_VERSION.to_string(),
title: extracted.title.unwrap_or_else(|| doi.as_str().to_string()),
authors: extracted.authors,
year: extracted.year,
doi: Some(doi.clone()),
arxiv_id: arxiv_id_for_metadata,
arxiv_categories: Vec::new(),
abstract_: None,
venue: extracted.venue,
volume: extracted.volume,
issue: extracted.issue,
pages: extracted.pages,
publisher: None,
issn: None,
isbn: None,
type_: extracted.type_,
keywords: Vec::new(),
url: cross
.as_ref()
.and_then(|c| c.final_url.as_ref())
.map(|u| u.to_string()),
pdf_path: pdf_path_relative,
doiget: Some(DoigetExtension {
fetched_at: Utc::now(),
source: final_source_label.clone(),
license: license.clone(),
oa_status: oa_status.clone(),
size_bytes,
mcp_call_id: None,
tags: Vec::new(),
collections: Vec::new(),
annotation: None,
}),
other: BTreeMap::new(),
};
let pdf_src_path = pdf_staged
.as_ref()
.and_then(|tmp| Utf8Path::from_path(tmp.path()).map(|p| p.to_path_buf()));
write_metadata_and_pdf(store, safekey, &metadata, pdf_src_path.as_deref(), ctx)?;
drop(pdf_staged);
let path = if pdf_bytes.is_some() {
store_root.join(format!("{}.pdf", safekey.as_str()))
} else {
store_root
.join(".metadata")
.join(format!("{}.toml", safekey.as_str()))
};
let canonical_digest = crate::CanonicalRef::new(
crate::SourceType::Doi,
doi.as_str(),
&final_source_label,
None,
)
.digest_hex();
Ok(FetchPaperOutcome {
source: final_source_label.clone(),
resolver_profile: final_source_label,
license,
oa_status,
path,
size_bytes,
schema_version: SCHEMA_VERSION.to_string(),
pdf_leg,
safekey: safekey.as_str().to_string(),
canonical_digest,
title: metadata.title.clone(),
authors: metadata.authors.clone(),
year: metadata.year,
attempts,
})
}
#[cfg(feature = "metadata")]
fn optional_source_oa_url<'a>(source: &str, meta: &'a Value) -> Option<&'a str> {
match source {
"openalex" => crate::sources::openalex::open_access_pdf_url(meta),
"core" => crate::sources::core_oa::open_access_pdf_url(meta),
"hal" => crate::sources::hal::open_access_pdf_url(meta),
"europe-pmc" => crate::sources::europepmc::open_access_pdf_url(meta),
_ => None,
}
}
#[cfg(feature = "metadata")]
async fn try_optional_source_oa_fallback(
doi: &Doi,
pdf_leg: PdfLegStatus,
pdf_bytes: Option<Vec<u8>>,
profile: &CapabilityProfile,
ctx: &FetchContext,
attempts: &mut Vec<SourceAttempt>,
already_resolved: Option<(&'static str, &Value)>,
) -> (PdfLegStatus, Option<Vec<u8>>) {
if pdf_bytes.is_some() || !matches!(pdf_leg, PdfLegStatus::Blocked { .. }) {
return (pdf_leg, pdf_bytes);
}
let (name, meta) = match already_resolved {
Some((n, m)) => (n, m.clone()),
None => {
let ref_ = Ref::Doi(doi.clone());
let mut discard = CrossrefFields::default();
let mut fresh: Vec<SourceAttempt> = Vec::new();
let r =
resolve_optional_chain(&ref_, profile, ctx, false, &mut discard, &mut fresh).await;
if !fresh.is_empty() {
attempts.retain(|a| !fresh.iter().any(|f| f.source == a.source));
attempts.extend(fresh);
}
match r {
Some((n, m)) => (n, m),
None => return (pdf_leg, pdf_bytes),
}
}
};
let Some(raw) = optional_source_oa_url(name, &meta) else {
tracing::debug!(
source = name,
doi = %doi.as_str(),
"optional source resolved but reported no document URL"
);
return (pdf_leg, pdf_bytes);
};
let Ok(url) = url::Url::parse(raw) else {
tracing::warn!(
source = name,
url = raw,
"optional source reported an unparsable document URL; keeping Blocked"
);
return (pdf_leg, pdf_bytes);
};
tracing::info!(
source = name,
doi = %doi.as_str(),
url = %url,
"OA chain exhausted; trying a copy reported by an optional source (#445)"
);
match try_fetch_oa_pdf(doi, &url, ctx).await {
Ok((bytes, _final_url)) => (PdfLegStatus::Fetched, Some(bytes)),
Err(e) => {
tracing::warn!(
source = name,
error = %e,
"optional-source copy also failed; keeping the original block"
);
(pdf_leg, pdf_bytes)
}
}
}
#[cfg(any(
feature = "tdm-elsevier",
feature = "tdm-aps",
feature = "tdm-springer",
feature = "tdm-ieee"
))]
#[allow(clippy::vec_init_then_push)]
async fn try_tdm_content_fallback(
doi: &Doi,
pdf_leg: PdfLegStatus,
pdf_bytes: Option<Vec<u8>>,
profile: &CapabilityProfile,
ctx: &FetchContext,
attempts: &mut Vec<SourceAttempt>,
) -> (PdfLegStatus, Option<Vec<u8>>) {
struct ContentEntry<'a> {
name: &'static str,
enable_hint: &'static [&'static str],
prefixes: &'static [&'static str],
publisher: &'static str,
src: &'a dyn crate::source::Source,
}
if pdf_bytes.is_some() {
return (pdf_leg, pdf_bytes);
}
let PdfLegStatus::Blocked {
message: ref blocked_message,
..
} = pdf_leg
else {
return (pdf_leg, pdf_bytes);
};
let original_block = blocked_message.clone();
let ref_ = Ref::Doi(doi.clone());
#[cfg(feature = "tdm-aps")]
let aps = optional_base("DOIGET_APS_BASE").map_or_else(
crate::sources::tdm_aps::TdmApsSource::new,
crate::sources::tdm_aps::TdmApsSource::with_base,
);
#[cfg(feature = "tdm-elsevier")]
let elsevier = optional_base("DOIGET_ELSEVIER_BASE").map_or_else(
crate::sources::tdm_elsevier::TdmElsevierSource::new,
crate::sources::tdm_elsevier::TdmElsevierSource::with_base,
);
#[cfg(feature = "tdm-springer")]
let springer = optional_base("DOIGET_SPRINGER_BASE").map_or_else(
crate::sources::tdm_springer::TdmSpringerSource::new,
crate::sources::tdm_springer::TdmSpringerSource::with_base,
);
#[cfg(feature = "tdm-ieee")]
let ieee = optional_base("DOIGET_IEEE_BASE").map_or_else(
crate::sources::tdm_ieee::TdmIeeeSource::new,
crate::sources::tdm_ieee::TdmIeeeSource::with_base,
);
#[allow(unused_mut)]
let mut chain: Vec<ContentEntry<'_>> = Vec::new();
#[cfg(feature = "tdm-aps")]
chain.push(ContentEntry {
name: "tdm-aps",
enable_hint: &["DOIGET_KEY_APS", "DOIGET_AGREE_TDM_APS"],
prefixes: crate::sources::tdm_aps::PUBLISHER_PREFIXES,
publisher: "American Physical Society (APS)",
src: &aps,
});
#[cfg(feature = "tdm-elsevier")]
chain.push(ContentEntry {
name: "tdm-elsevier",
enable_hint: &["DOIGET_KEY_ELSEVIER", "DOIGET_AGREE_TDM_ELSEVIER"],
prefixes: crate::sources::tdm_elsevier::PUBLISHER_PREFIXES,
publisher: "Elsevier BV",
src: &elsevier,
});
#[cfg(feature = "tdm-springer")]
chain.push(ContentEntry {
name: "tdm-springer",
enable_hint: &["DOIGET_KEY_SPRINGER", "DOIGET_AGREE_TDM_SPRINGER"],
prefixes: crate::sources::tdm_springer::PUBLISHER_PREFIXES,
publisher: "Springer Nature",
src: &springer,
});
#[cfg(feature = "tdm-ieee")]
chain.push(ContentEntry {
name: "tdm-ieee",
enable_hint: &["DOIGET_KEY_IEEE", "DOIGET_AGREE_TDM_IEEE"],
prefixes: crate::sources::tdm_ieee::PUBLISHER_PREFIXES,
publisher: "IEEE",
src: &ieee,
});
fn record(attempts: &mut Vec<SourceAttempt>, name: &'static str, outcome: AttemptOutcome) {
attempts.retain(|a| a.source != name);
attempts.push(SourceAttempt::new(name, outcome));
}
for e in chain {
debug_assert_eq!(e.name, e.src.name(), "chain name must match Source::name");
if !e.prefixes.contains(&doi.prefix()) {
record(
attempts,
e.name,
AttemptOutcome::WrongPublisher {
detail: format!("DOI prefix {} is not {}", doi.prefix(), e.publisher),
},
);
continue;
}
if !e.src.can_serve(profile, &ref_) {
record(
attempts,
e.name,
AttemptOutcome::Disabled { env: e.enable_hint },
);
continue;
}
match e.src.fetch_content(&ref_, profile, ctx).await {
Ok(None) => {}
Ok(Some(bytes)) => {
tracing::info!(
source = e.name,
doi = %doi.as_str(),
size = bytes.len(),
"OA routes exhausted; the publisher served its own copy under the user's TDM agreement (#458)"
);
record(attempts, e.name, AttemptOutcome::Resolved);
return (
PdfLegStatus::TdmFetched {
source: e.name.to_string(),
original_block,
},
Some(bytes.to_vec()),
);
}
Err(err) => {
tracing::warn!(
source = e.name,
error = %err,
"TDM content leg failed; keeping the original block"
);
record(attempts, e.name, classify_attempt(&err));
}
}
}
(pdf_leg, pdf_bytes)
}
async fn try_arxiv_preprint_fallback(
doi: &Doi,
pdf_leg: PdfLegStatus,
oa_pdf_bytes: Option<Vec<u8>>,
profile: &CapabilityProfile,
ctx: &FetchContext,
) -> (
PdfLegStatus,
Option<Vec<u8>>,
Option<ArxivId>,
Option<String>,
) {
let (arxiv_id_str, original_block) = match &pdf_leg {
PdfLegStatus::Blocked {
suggested_arxiv_id: Some(s),
message,
..
} => (s.clone(), message.clone()),
_ => return (pdf_leg, oa_pdf_bytes, None, None),
};
let arxiv_id = match ArxivId::parse(&arxiv_id_str) {
Ok(id) => id,
Err(e) => {
tracing::warn!(
error = %e,
arxiv_id = %arxiv_id_str,
doi = %doi.as_str(),
"preprint fallback: could not parse suggested_arxiv_id; keeping Blocked"
);
return (pdf_leg, oa_pdf_bytes, None, None);
}
};
tracing::info!(
doi = %doi.as_str(),
arxiv_id = %arxiv_id.as_str(),
"OA PDF blocked; attempting arXiv preprint fallback (issue #325)"
);
let arxiv_ref = Ref::Arxiv(arxiv_id.clone());
let arxiv_source = arxiv_source_from_env();
match arxiv_source.fetch(&arxiv_ref, profile, ctx).await {
Ok(result) => match result.pdf_bytes {
Some(bytes) => {
tracing::info!(
doi = %doi.as_str(),
arxiv_id = %arxiv_id.as_str(),
size = bytes.len(),
"arXiv preprint fallback succeeded; storing under DOI safekey (issue #325)"
);
let license = result.license;
(
PdfLegStatus::PreprintFallback {
arxiv_id: arxiv_id.as_str().to_string(),
original_block,
},
Some(bytes.to_vec()),
Some(arxiv_id),
Some(license),
)
}
None => {
tracing::warn!(
doi = %doi.as_str(),
arxiv_id = %arxiv_id.as_str(),
"preprint fallback: arXiv source returned no PDF bytes; keeping Blocked"
);
(pdf_leg, oa_pdf_bytes, None, None)
}
},
Err(e) => {
tracing::warn!(
error = %e,
doi = %doi.as_str(),
arxiv_id = %arxiv_id.as_str(),
"preprint fallback: arXiv fetch also failed; keeping Blocked"
);
(pdf_leg, oa_pdf_bytes, None, None)
}
}
}
fn stage_pdf_to_tempfile(bytes: &[u8]) -> Result<tempfile::NamedTempFile, FetchError> {
let tmp = tempfile::NamedTempFile::new().map_err(|e| FetchError::SourceSchema {
hint: format!("creating PDF staging tempfile: {e}"),
})?;
std::fs::write(tmp.path(), bytes).map_err(|e| FetchError::SourceSchema {
hint: format!("staging PDF bytes: {e}"),
})?;
Ok(tmp)
}
fn write_metadata_and_pdf(
store: &dyn Store,
safekey: &Safekey,
metadata: &Metadata,
pdf_src: Option<&Utf8Path>,
ctx: &FetchContext,
) -> Result<(), FetchError> {
let store_path_relative = if pdf_src.is_some() {
format!("{}.pdf", safekey.as_str())
} else {
format!(".metadata/{}.toml", safekey.as_str())
};
let size_bytes = metadata.doiget.as_ref().map(|d| d.size_bytes).unwrap_or(0);
let license = metadata.doiget.as_ref().map(|d| d.license.as_str());
let source_name = metadata.doiget.as_ref().map(|d| d.source.as_str());
let canonical_digest: Option<String> = match (metadata.doi.as_ref(), metadata.arxiv_id.as_ref())
{
(Some(d), _) => source_name.map(|s| {
crate::CanonicalRef::new(crate::SourceType::Doi, d.as_str(), s, None).digest_hex()
}),
(None, Some(a)) => source_name.map(|s| {
crate::CanonicalRef::new(crate::SourceType::Arxiv, a.as_str(), s, None).digest_hex()
}),
(None, None) => None,
};
match store.write(safekey, metadata, pdf_src) {
Ok(()) => {
ctx.log.append(RowInput {
event: LogEvent::StoreWrite,
result: LogResult::Ok,
capability: Capability::Oa,
ref_: metadata
.doi
.as_ref()
.map(|d| d.as_str())
.or_else(|| metadata.arxiv_id.as_ref().map(|a| a.as_str())),
source: source_name,
error_code: None,
size_bytes: Some(size_bytes),
license,
store_path: Some(&store_path_relative),
canonical_digest: canonical_digest.as_deref(),
})?;
Ok(())
}
Err(e) => {
if let Err(log_err) = ctx.log.append(RowInput {
event: LogEvent::StoreWrite,
result: LogResult::Err,
capability: Capability::Oa,
ref_: metadata
.doi
.as_ref()
.map(|d| d.as_str())
.or_else(|| metadata.arxiv_id.as_ref().map(|a| a.as_str())),
source: source_name,
error_code: Some("STORE_ERROR"),
size_bytes: None,
license: None,
store_path: Some(&store_path_relative),
canonical_digest: canonical_digest.as_deref(),
}) {
tracing::error!(
store_err = %e,
log_err = %log_err,
"BOTH store.write AND provenance log append failed; \
audit trail is broken for this attempt"
);
}
Err(FetchError::SourceSchema {
hint: format!("store write failed: {e}"),
})
}
}
}
async fn try_fetch_oa_pdf(
doi: &Doi,
url: &url::Url,
ctx: &FetchContext,
) -> Result<(Vec<u8>, url::Url), HttpError> {
const SOURCE: &str = "oa-publisher";
let _permit = ctx.rate_limiter.acquire(SOURCE).await;
let canonical =
crate::CanonicalRef::new(crate::SourceType::Doi, doi.as_str(), SOURCE, None).digest_hex();
if let Some(allowlist) = ctx.http.source_allowlist(SOURCE) {
let host = url
.host_str()
.map(|h| h.to_ascii_lowercase())
.unwrap_or_default();
if !allowlist.matches(&host) {
let e = HttpError::RedirectDenied {
source_key: SOURCE.to_string(),
host: host.clone(),
expected_hosts: allowlist.redirect_hosts.clone(),
};
tracing::info!(
oa_url = %url,
denied_host = %host,
"OA URL host outside oa-publisher allowlist (pre-fetch check, \
docs/REDIRECT_ALLOWLIST.md §1 / issue #145)"
);
let _ = ctx.log.append(RowInput {
event: LogEvent::Fetch,
result: LogResult::Err,
capability: Capability::Oa,
ref_: Some(doi.as_str()),
source: Some(SOURCE),
error_code: Some(crate::ErrorCode::NetworkError.as_wire()),
size_bytes: None,
license: None,
store_path: None,
canonical_digest: Some(&canonical),
});
return Err(e);
}
}
match ctx.http.fetch_pdf(SOURCE, url.clone()).await {
Ok((body, final_url)) => {
let size_bytes = body.len() as u64;
if let Err(e) = ctx.log.append(RowInput {
event: LogEvent::Fetch,
result: LogResult::Ok,
capability: Capability::Oa,
ref_: Some(doi.as_str()),
source: Some(SOURCE),
error_code: None,
size_bytes: Some(size_bytes),
license: None,
store_path: None,
canonical_digest: Some(&canonical),
}) {
tracing::warn!(error = %e, "appending oa-publisher Fetch ok row failed");
}
Ok((body.to_vec(), final_url))
}
Err(e) => {
match &e {
HttpError::RedirectDenied { host, .. } => {
tracing::info!(
oa_url = %url,
denied_host = %host,
"OA URL host outside oa-publisher allowlist"
);
}
HttpError::NotAPdf { .. } => {
tracing::info!(
oa_url = %url,
"OA URL did not return a PDF magic byte"
);
}
other => {
tracing::warn!(
oa_url = %url,
error = %other,
"OA PDF fetch failed"
);
}
}
let _ = ctx.log.append(RowInput {
event: LogEvent::Fetch,
result: LogResult::Err,
capability: Capability::Oa,
ref_: Some(doi.as_str()),
source: Some(SOURCE),
error_code: Some(crate::ErrorCode::NetworkError.as_wire()),
size_bytes: None,
license: None,
store_path: None,
canonical_digest: Some(&canonical),
});
Err(e)
}
}
}
#[derive(Default)]
pub(crate) struct CrossrefFields {
pub(crate) title: Option<String>,
pub(crate) authors: Vec<String>,
pub(crate) year: Option<i32>,
pub(crate) venue: Option<String>,
pub(crate) volume: Option<String>,
pub(crate) issue: Option<String>,
pub(crate) pages: Option<String>,
pub(crate) type_: Option<String>,
}
#[cfg(feature = "metadata")]
pub(crate) fn extract_datacite_fields(attributes: &Value) -> CrossrefFields {
let title = attributes
.get("titles")
.and_then(|v| v.as_array())
.and_then(|arr| arr.first())
.and_then(|t| t.get("title"))
.and_then(|v| v.as_str())
.map(str::to_string);
let authors = attributes
.get("creators")
.and_then(|v| v.as_array())
.map(|arr| {
arr.iter()
.filter_map(|c| {
c.get("name")
.and_then(|v| v.as_str())
.map(str::to_string)
.or_else(|| {
let given = c.get("givenName").and_then(|v| v.as_str());
let family = c.get("familyName").and_then(|v| v.as_str());
match (given, family) {
(Some(g), Some(f)) => Some(format!("{g} {f}")),
(None, Some(f)) => Some(f.to_string()),
_ => None,
}
})
})
.collect()
})
.unwrap_or_default();
let year = attributes
.get("publicationYear")
.and_then(serde_json::Value::as_i64)
.and_then(|y| i32::try_from(y).ok());
let venue = attributes.get("publisher").and_then(|v| {
v.as_str()
.map(str::to_string)
.or_else(|| v.get("name").and_then(|n| n.as_str()).map(str::to_string))
});
let type_ = crate::sources::datacite::resource_type_general(attributes).map(str::to_string);
CrossrefFields {
title,
authors,
year,
venue,
volume: None,
issue: None,
pages: None,
type_,
}
}
pub(crate) fn extract_crossref_fields(msg: &Value) -> CrossrefFields {
let title = msg
.get("title")
.and_then(|v| v.as_array())
.and_then(|arr| arr.first())
.and_then(|v| v.as_str())
.map(|s| s.to_string());
let authors = msg
.get("author")
.and_then(|v| v.as_array())
.map(|arr| {
arr.iter()
.filter_map(|a| {
let family = a.get("family").and_then(|v| v.as_str());
let given = a.get("given").and_then(|v| v.as_str());
match (family, given) {
(Some(f), Some(g)) => Some(format!("{f}, {g}")),
(Some(f), None) => Some(f.to_string()),
(None, Some(g)) => Some(g.to_string()),
_ => None,
}
})
.collect()
})
.unwrap_or_default();
let year = msg
.get("issued")
.and_then(|v| v.get("date-parts"))
.and_then(|v| v.as_array())
.and_then(|arr| arr.first())
.and_then(|v| v.as_array())
.and_then(|arr| arr.first())
.and_then(|v| v.as_i64())
.and_then(|n| i32::try_from(n).ok());
let venue = msg
.get("container-title")
.and_then(|v| v.as_array())
.and_then(|arr| arr.first())
.and_then(|v| v.as_str())
.map(|s| s.to_string());
let type_ = msg
.get("type")
.and_then(|v| v.as_str())
.map(|s| s.to_string());
let volume = msg
.get("volume")
.and_then(|v| v.as_str())
.map(|s| s.to_string());
let issue = msg
.get("issue")
.and_then(|v| v.as_str())
.map(|s| s.to_string());
let pages = msg
.get("page")
.and_then(|v| v.as_str())
.map(normalize_page_range);
CrossrefFields {
title,
authors,
year,
venue,
volume,
issue,
pages,
type_,
}
}
fn extract_oa_url_chain(meta: Option<&Value>) -> Vec<url::Url> {
let meta = match meta {
Some(m) => m,
None => return Vec::new(),
};
let mut out: Vec<url::Url> = Vec::new();
let mut seen: std::collections::HashSet<String> = std::collections::HashSet::new();
let mut push_unique = |u: url::Url| {
let key = u.as_str().to_string();
if seen.insert(key) {
out.push(u);
}
};
if let Some(best) = meta.get("best_oa_location") {
if let Some(u) = pull_oa_url_from_location(best) {
push_unique(u);
}
}
if let Some(arr) = meta.get("oa_locations").and_then(|v| v.as_array()) {
for loc in arr {
if let Some(u) = pull_oa_url_from_location(loc) {
push_unique(u);
}
}
}
out
}
fn pull_oa_url_from_location(loc: &Value) -> Option<url::Url> {
let candidate = loc
.get("url_for_pdf")
.and_then(|v| v.as_str())
.or_else(|| loc.get("url").and_then(|v| v.as_str()))?;
url::Url::parse(candidate).ok()
}
fn extract_arxiv_id_from_url(url: &url::Url) -> Option<String> {
let host = url.host_str()?;
let is_arxiv = matches!(
host,
"arxiv.org" | "www.arxiv.org" | "export.arxiv.org" | "e-print.arxiv.org"
);
if !is_arxiv {
return None;
}
let path = url.path();
let raw = if path.starts_with("/pdf/") {
let s = path.strip_prefix("/pdf/")?;
s.strip_suffix(".pdf").unwrap_or(s)
} else if path.starts_with("/abs/") {
path.strip_prefix("/abs/")?
} else {
return None;
};
Some(strip_arxiv_version(raw).to_string())
}
fn strip_arxiv_version(id: &str) -> &str {
if let Some(v_pos) = id.rfind('v') {
let before_v = id[..v_pos].chars().next_back();
let suffix = &id[v_pos + 1..];
if before_v.is_some_and(|c| c.is_ascii_digit())
&& !suffix.is_empty()
&& suffix.bytes().all(|b| b.is_ascii_digit())
{
return &id[..v_pos];
}
}
id
}
#[derive(Debug)]
pub struct BatchResultEntry {
pub ref_: Ref,
pub outcome: Result<FetchPaperOutcome, FetchError>,
}
#[derive(Debug)]
#[non_exhaustive]
pub struct BatchOutcome {
pub results: Vec<BatchResultEntry>,
}
pub async fn batch_fetch(
refs: &[Ref],
profile: &CapabilityProfile,
ctx: &FetchContext,
store: &dyn Store,
store_root: &Utf8Path,
) -> Result<BatchOutcome, FetchError> {
if refs.len() > MAX_BATCH_REFS {
return Err(FetchError::TooManyRefs {
got: refs.len(),
max: MAX_BATCH_REFS,
});
}
let mut results = Vec::with_capacity(refs.len());
for ref_ in refs {
let outcome = fetch_paper(ref_, profile, ctx, store, store_root).await;
results.push(BatchResultEntry {
ref_: ref_.clone(),
outcome,
});
}
Ok(BatchOutcome { results })
}
pub fn batch_fetch_plans(
refs: &[Ref],
store_root: &Utf8Path,
) -> Result<Vec<(Ref, FetchPlan)>, FetchError> {
if refs.len() > MAX_BATCH_REFS {
return Err(FetchError::TooManyRefs {
got: refs.len(),
max: MAX_BATCH_REFS,
});
}
refs.iter()
.map(|r| try_build_fetch_plan(r, store_root).map(|p| (r.clone(), p)))
.collect()
}
#[cfg(test)]
#[allow(clippy::expect_used, clippy::unwrap_used, clippy::panic)]
mod tests {
use super::*;
struct LadderEnv {
vars: Vec<(&'static str, Option<std::ffi::OsString>)>,
}
impl LadderEnv {
fn scoped(dir: &str) -> Self {
let mut vars = Vec::new();
for v in ["DOIGET_CONTACT_EMAIL", "DOIGET_UNPAYWALL_EMAIL"] {
vars.push((v, std::env::var_os(v)));
std::env::remove_var(v);
}
for v in ["XDG_CONFIG_HOME", "APPDATA", "HOME", "USERPROFILE"] {
vars.push((v, std::env::var_os(v)));
std::env::set_var(v, dir);
}
Self { vars }
}
fn set(&mut self, var: &'static str, value: &str) {
self.vars.push((var, std::env::var_os(var)));
std::env::set_var(var, value);
}
}
impl Drop for LadderEnv {
fn drop(&mut self) {
for (var, prior) in self.vars.iter().rev() {
match prior {
Some(v) => std::env::set_var(var, v),
None => std::env::remove_var(var),
}
}
}
}
fn write_config(td: &tempfile::TempDir, body: &str) -> String {
let dir = camino::Utf8PathBuf::from_path_buf(td.path().to_path_buf())
.expect("temp path is UTF-8");
std::fs::create_dir_all(dir.join("doiget").as_std_path()).expect("mkdir");
std::fs::write(dir.join("doiget").join("config.toml").as_std_path(), body)
.expect("write config.toml");
dir.to_string()
}
#[test]
#[serial_test::serial]
fn resolve_contact_email_reads_the_config_file_rung() {
let td = tempfile::TempDir::new().expect("tempdir");
let dir = write_config(
&td,
"[network]\ncontact_email = \"file@institution.edu\"\nunpaywall_email = \"up@institution.edu\"\n",
);
let _env = LadderEnv::scoped(&dir);
let ContactAddresses {
crossref: contact,
unpaywall,
} = resolve_contact_emails();
assert_eq!(contact, "file@institution.edu");
assert_eq!(unpaywall, "up@institution.edu");
}
#[test]
#[serial_test::serial]
fn the_env_rung_outranks_the_file_and_unpaywall_falls_back_to_contact() {
let td = tempfile::TempDir::new().expect("tempdir");
let dir = write_config(&td, "[network]\ncontact_email = \"file@institution.edu\"\n");
let mut env = LadderEnv::scoped(&dir);
env.set("DOIGET_CONTACT_EMAIL", "env@institution.edu");
let ContactAddresses {
crossref: contact,
unpaywall,
} = resolve_contact_emails();
assert_eq!(contact, "env@institution.edu");
assert_eq!(
unpaywall, "env@institution.edu",
"no unpaywall rung is set, so it must inherit the resolved contact"
);
}
#[test]
#[serial_test::serial]
fn an_absent_config_still_yields_the_documented_fallback() {
let td = tempfile::TempDir::new().expect("tempdir");
let dir = camino::Utf8PathBuf::from_path_buf(td.path().to_path_buf())
.expect("temp path is UTF-8")
.to_string();
let _env = LadderEnv::scoped(&dir);
let ContactAddresses {
crossref: contact,
unpaywall,
} = resolve_contact_emails();
assert_eq!(contact, FALLBACK_CONTACT_EMAIL);
assert_eq!(unpaywall, FALLBACK_CONTACT_EMAIL);
}
#[test]
#[serial_test::serial]
fn a_blank_value_on_either_rung_is_treated_as_unset() {
let td = tempfile::TempDir::new().expect("tempdir");
let dir = write_config(&td, "[network]\ncontact_email = \" \"\n");
let mut env = LadderEnv::scoped(&dir);
env.set("DOIGET_CONTACT_EMAIL", "");
assert_eq!(resolve_contact_email(), FALLBACK_CONTACT_EMAIL);
}
fn crossref_outcome() -> MetadataOnlyOutcome {
MetadataOnlyOutcome {
source: "crossref".to_string(),
resolver_profile: "crossref".to_string(),
license: None,
oa_url: None,
oa_status: None,
metadata: serde_json::json!({
"title": ["Rigorous results on valence-bond ground states"],
"author": [
{ "family": "Affleck", "given": "Ian" },
{ "family": "Lieb", "given": "Elliott H." },
],
"issued": { "date-parts": [[1988, 6, 1]] },
"container-title": ["Physical Review Letters"],
"publisher": "American Physical Society",
"ISSN": ["0031-9007", "1079-7114"],
"volume": "59",
"issue": "7",
"page": "799-802",
"type": "journal-article",
}),
}
}
#[test]
fn cite_metadata_enriches_from_crossref_envelope() {
let ref_ = Ref::parse("10.1103/PhysRevLett.59.799").unwrap();
let m = cite_metadata(&ref_, &crossref_outcome());
assert_eq!(m.title, "Rigorous results on valence-bond ground states");
assert_eq!(m.authors, vec!["Affleck, Ian", "Lieb, Elliott H."]);
assert_eq!(m.year, Some(1988));
assert_eq!(m.venue.as_deref(), Some("Physical Review Letters"));
assert_eq!(m.publisher.as_deref(), Some("American Physical Society"));
assert_eq!(m.issn.as_deref(), Some("0031-9007"));
assert_eq!(m.volume.as_deref(), Some("59"));
assert_eq!(m.issue.as_deref(), Some("7"));
assert_eq!(m.pages.as_deref(), Some("799--802"));
assert_eq!(m.type_.as_deref(), Some("journal-article"));
}
#[test]
fn cite_metadata_non_crossref_keeps_minimal_baseline() {
let ref_ = Ref::parse("arxiv:2401.12345").unwrap();
let outcome = MetadataOnlyOutcome {
source: "arxiv".to_string(),
resolver_profile: "arxiv".to_string(),
license: Some("arxiv-default".to_string()),
oa_url: None,
oa_status: Some("green".to_string()),
metadata: serde_json::json!({ "title": "An arXiv Preprint" }),
};
let m = cite_metadata(&ref_, &outcome);
assert_eq!(m.title, "An arXiv Preprint");
assert_eq!(m.year, None);
assert_eq!(m.venue, None);
assert_eq!(m.publisher, None);
assert_eq!(m.issn, None);
assert!(m.arxiv_id.is_some());
}
#[test]
fn cite_metadata_arxiv_overlay_fills_year_and_categories() {
let ref_ = Ref::parse("arxiv:2401.12345").unwrap();
let outcome = MetadataOnlyOutcome {
source: "arxiv".to_string(),
resolver_profile: "arxiv".to_string(),
license: Some("arxiv-default".to_string()),
oa_url: None,
oa_status: Some("green".to_string()),
metadata: serde_json::json!({
"title": "An arXiv Preprint",
"published": "2024-03-15T00:00:00Z",
"categories": ["cond-mat.str-el", "cond-mat.dis-nn"],
}),
};
let m = cite_metadata(&ref_, &outcome);
assert_eq!(m.year, Some(2024));
assert_eq!(
m.arxiv_categories,
vec!["cond-mat.str-el".to_string(), "cond-mat.dis-nn".to_string()]
);
let bad = MetadataOnlyOutcome {
metadata: serde_json::json!({ "title": "x", "published": "not-a-date" }),
..outcome
};
assert_eq!(cite_metadata(&ref_, &bad).year, None);
}
#[test]
fn test_extract_arxiv_id_from_url() {
let urls = [
("https://arxiv.org/pdf/1901.12345.pdf", Some("1901.12345")),
("https://arxiv.org/abs/1901.12345", Some("1901.12345")),
("https://arxiv.org/pdf/1901.12345v2.pdf", Some("1901.12345")),
("https://arxiv.org/abs/1901.12345v3", Some("1901.12345")),
(
"https://www.arxiv.org/pdf/cond-mat/9501001.pdf",
Some("cond-mat/9501001"),
),
(
"https://export.arxiv.org/abs/cond-mat/9501001",
Some("cond-mat/9501001"),
),
(
"https://arxiv.org/pdf/cond-mat/9501001v1.pdf",
Some("cond-mat/9501001"),
),
(
"https://e-print.arxiv.org/pdf/2401.12345.pdf",
Some("2401.12345"),
),
("https://example.org/pdf/1901.12345.pdf", None),
];
for (url_str, expected) in urls {
let url = url::Url::parse(url_str).unwrap();
assert_eq!(
extract_arxiv_id_from_url(&url),
expected.map(String::from),
"url: {url_str}"
);
}
}
#[test]
fn test_strip_arxiv_version() {
assert_eq!(strip_arxiv_version("2401.12345v2"), "2401.12345");
assert_eq!(strip_arxiv_version("2401.12345v10"), "2401.12345");
assert_eq!(strip_arxiv_version("2401.12345"), "2401.12345");
assert_eq!(
strip_arxiv_version("cond-mat/9501001v3"),
"cond-mat/9501001"
);
assert_eq!(strip_arxiv_version("quant-phv5"), "quant-phv5");
}
#[test]
fn extract_crossref_publisher_url_takes_a_general_entry() {
let msg = serde_json::json!({
"link": [
{"URL": "https://example.org/free.pdf",
"intended-application": "unspecified"},
{"URL": "https://example.org/alt.pdf",
"intended-application": "unspecified"}
]
});
assert_eq!(
extract_crossref_publisher_url(&msg),
Some("https://example.org/free.pdf".to_string())
);
}
#[test]
fn extract_crossref_publisher_url_returns_none_when_absent() {
assert!(extract_crossref_publisher_url(&serde_json::json!({})).is_none());
}
#[test]
fn extract_crossref_publisher_url_skips_empty_url_strings() {
let msg = serde_json::json!({
"link": [
{"URL": "", "intended-application": "unspecified"},
{"URL": "https://example.org/real.pdf",
"intended-application": "unspecified"}
]
});
assert_eq!(
extract_crossref_publisher_url(&msg),
Some("https://example.org/real.pdf".to_string())
);
}
#[test]
fn extract_crossref_publisher_url_refuses_programme_scoped_links() {
for scoped in ["text-mining", "similarity-checking", "syndication"] {
let msg = serde_json::json!({
"link": [{"URL": "https://example.org/scoped.pdf",
"intended-application": scoped}]
});
assert_eq!(
extract_crossref_publisher_url(&msg),
None,
"{scoped} is scoped to a programme doiget is not in on this path"
);
}
}
#[test]
fn extract_crossref_publisher_url_refuses_an_unlabelled_entry() {
let msg = serde_json::json!({"link": [{"URL": "https://example.org/x.pdf"}]});
assert_eq!(extract_crossref_publisher_url(&msg), None);
}
#[test]
fn extract_crossref_publisher_url_prefers_a_pdf_content_type() {
let msg = serde_json::json!({
"link": [
{"URL": "https://example.org/landing",
"content-type": "text/html",
"intended-application": "unspecified"},
{"URL": "https://example.org/real.pdf",
"content-type": "application/pdf",
"intended-application": "unspecified"}
]
});
assert_eq!(
extract_crossref_publisher_url(&msg),
Some("https://example.org/real.pdf".to_string())
);
}
#[test]
fn extract_unpaywall_oa_url_prefers_url_for_pdf() {
let meta = serde_json::json!({
"best_oa_location": {
"url_for_pdf": "https://example.org/pdf",
"url": "https://example.org/landing"
}
});
assert_eq!(
extract_unpaywall_oa_url(&meta),
Some("https://example.org/pdf".to_string())
);
}
#[test]
fn extract_unpaywall_oa_url_falls_back_to_url() {
let meta = serde_json::json!({
"best_oa_location": {
"url": "https://example.org/landing"
}
});
assert_eq!(
extract_unpaywall_oa_url(&meta),
Some("https://example.org/landing".to_string())
);
}
#[test]
fn extract_unpaywall_oa_url_returns_none_when_absent() {
let meta = serde_json::json!({});
assert!(extract_unpaywall_oa_url(&meta).is_none());
}
#[test]
fn extract_unpaywall_oa_status_present_absent_and_empty() {
assert_eq!(
extract_unpaywall_oa_status(&serde_json::json!({"oa_status": "gold"})).as_deref(),
Some("gold")
);
assert!(extract_unpaywall_oa_status(&serde_json::json!({})).is_none());
assert!(extract_unpaywall_oa_status(&serde_json::json!({"oa_status": ""})).is_none());
}
#[test]
fn extract_crossref_fields_parses_minimal_shape() {
let msg = serde_json::json!({
"title": ["Example Title"],
"author": [{ "family": "Smith", "given": "Alice" }],
"issued": { "date-parts": [[2024, 1, 15]] },
"container-title": ["Phys. Rev. X"],
"type": "journal-article"
});
let f = extract_crossref_fields(&msg);
assert_eq!(f.title.as_deref(), Some("Example Title"));
assert_eq!(f.authors, vec!["Smith, Alice".to_string()]);
assert_eq!(f.year, Some(2024));
assert_eq!(f.venue.as_deref(), Some("Phys. Rev. X"));
assert_eq!(f.type_.as_deref(), Some("journal-article"));
}
#[test]
fn extract_crossref_fields_tolerates_missing() {
let f = extract_crossref_fields(&serde_json::json!({}));
assert!(f.title.is_none());
assert!(f.authors.is_empty());
assert!(f.year.is_none());
assert!(f.venue.is_none());
assert!(f.type_.is_none());
}
#[test]
fn extract_oa_url_chain_prefers_best_url_for_pdf() {
let meta = serde_json::json!({
"best_oa_location": {
"url_for_pdf": "https://example.org/pdf",
"url": "https://example.org/landing"
}
});
let chain = extract_oa_url_chain(Some(&meta));
assert_eq!(chain.len(), 1);
assert_eq!(chain[0].as_str(), "https://example.org/pdf");
}
#[test]
fn extract_oa_url_chain_falls_back_to_url_when_url_for_pdf_absent() {
let meta = serde_json::json!({
"best_oa_location": {
"url": "https://example.org/landing"
}
});
let chain = extract_oa_url_chain(Some(&meta));
assert_eq!(chain.len(), 1);
assert_eq!(chain[0].as_str(), "https://example.org/landing");
}
#[test]
fn extract_oa_url_chain_is_empty_when_no_locations() {
let meta = serde_json::json!({});
assert!(extract_oa_url_chain(Some(&meta)).is_empty());
assert!(extract_oa_url_chain(None).is_empty());
}
#[test]
fn extract_oa_url_chain_appends_oa_locations_after_best() {
let meta = serde_json::json!({
"best_oa_location": {
"url_for_pdf": "https://publisher.example.org/pdf"
},
"oa_locations": [
{"url_for_pdf": "https://publisher.example.org/pdf"},
{"url_for_pdf": "https://arxiv.org/pdf/2401.12345"},
{"url": "https://repo.example.edu/handle/123"}
]
});
let chain = extract_oa_url_chain(Some(&meta));
let strs: Vec<&str> = chain.iter().map(|u| u.as_str()).collect();
assert_eq!(
strs,
vec![
"https://publisher.example.org/pdf",
"https://arxiv.org/pdf/2401.12345",
"https://repo.example.edu/handle/123",
],
"chain ordering MUST be best_oa_location first, oa_locations[] verbatim after"
);
}
#[test]
fn extract_oa_url_chain_dedupes_repeated_urls() {
let meta = serde_json::json!({
"best_oa_location": {
"url_for_pdf": "https://example.org/pdf"
},
"oa_locations": [
{"url_for_pdf": "https://example.org/pdf"},
{"url_for_pdf": "https://example.org/pdf"},
{"url_for_pdf": "https://arxiv.org/pdf/2401.12345"}
]
});
let chain = extract_oa_url_chain(Some(&meta));
assert_eq!(chain.len(), 2);
assert_eq!(chain[0].as_str(), "https://example.org/pdf");
assert_eq!(chain[1].as_str(), "https://arxiv.org/pdf/2401.12345");
}
#[test]
fn extract_oa_url_chain_skips_unparsable_urls() {
let meta = serde_json::json!({
"best_oa_location": {
"url_for_pdf": "https://good.example.org/pdf"
},
"oa_locations": [
{"url_for_pdf": "not a url"},
{"url_for_pdf": "https://arxiv.org/pdf/2401.12345"}
]
});
let chain = extract_oa_url_chain(Some(&meta));
assert_eq!(chain.len(), 2);
assert_eq!(chain[0].as_str(), "https://good.example.org/pdf");
assert_eq!(chain[1].as_str(), "https://arxiv.org/pdf/2401.12345");
}
#[test]
fn fetch_paper_plan_matches_build_fetch_plan() {
use crate::{ArxivId, Doi};
let r = Ref::Doi(Doi("10.1234/example".to_string()));
let root = Utf8PathBuf::from("/tmp/doiget-test");
let plan_a = fetch_paper_plan(&r, &root);
let plan_b = build_fetch_plan(&r, &root);
assert_eq!(plan_a.metadata_sources, plan_b.metadata_sources);
assert_eq!(plan_a.target_pdf_path, plan_b.target_pdf_path);
assert_eq!(plan_a.target_metadata_path, plan_b.target_metadata_path);
let r2 = Ref::Arxiv(ArxivId("2401.12345".to_string()));
let plan_c = fetch_paper_plan(&r2, &root);
let plan_d = build_fetch_plan(&r2, &root);
assert_eq!(plan_c.pdf_sources[0].key, plan_d.pdf_sources[0].key);
}
#[test]
fn batch_fetch_plans_returns_plan_per_ref_in_order() {
use crate::{ArxivId, Doi};
let refs = vec![
Ref::Doi(Doi("10.1234/alpha".to_string())),
Ref::Arxiv(ArxivId("2401.12345".to_string())),
];
let root = Utf8PathBuf::from("/tmp/doiget-batch-test");
let plans = batch_fetch_plans(&refs, &root).expect("under cap returns Ok");
assert_eq!(plans.len(), 2);
assert!(matches!(plans[0].0, Ref::Doi(_)));
assert!(matches!(plans[1].0, Ref::Arxiv(_)));
assert_eq!(plans[0].1.metadata_sources, vec!["crossref", "unpaywall"]);
assert_eq!(plans[1].1.pdf_sources[0].key, "arxiv");
}
#[test]
fn batch_fetch_plans_too_many_refs_returns_err() {
use crate::Doi;
let n = MAX_BATCH_REFS + 1;
let refs: Vec<Ref> = (0..n)
.map(|i| Ref::Doi(Doi(format!("10.1234/n{}", i))))
.collect();
let root = Utf8PathBuf::from("/tmp/doiget-toomany");
let err = batch_fetch_plans(&refs, &root).expect_err("over cap returns Err");
match err {
FetchError::TooManyRefs { got, max } => {
assert_eq!(got, n);
assert_eq!(max, MAX_BATCH_REFS);
}
other => panic!("expected TooManyRefs, got: {other:?}"),
}
}
#[tokio::test]
async fn batch_fetch_too_many_refs_returns_err_before_any_fetch() {
use crate::http::{tier_1_allowlist, HttpClient};
use crate::provenance::ProvenanceLog;
use crate::rate_limiter::RateLimiter;
use crate::store::FsStore;
use crate::{Doi, RateLimits};
use std::sync::Arc;
let td = tempfile::TempDir::new().expect("tempdir");
let log_path = Utf8Path::from_path(td.path())
.expect("utf-8")
.join("log.jsonl");
let store_root = Utf8Path::from_path(td.path())
.expect("utf-8")
.join("papers");
let ctx = FetchContext {
http: Arc::new(HttpClient::new(tier_1_allowlist()).expect("http client")),
rate_limiter: Arc::new(RateLimiter::new(RateLimits::HARD_CODED)),
log: Arc::new(
ProvenanceLog::open(log_path, "01J0000000000000000000TEST".into())
.expect("provenance log"),
),
session_id: "01J0000000000000000000TEST".into(),
cache_root: None,
};
let profile = CapabilityProfile::for_tests();
let store = FsStore::new(store_root.clone()).expect("fs store");
let n = MAX_BATCH_REFS + 1;
let refs: Vec<Ref> = (0..n)
.map(|i| Ref::Doi(Doi(format!("10.1234/n{}", i))))
.collect();
let err = batch_fetch(&refs, &profile, &ctx, &store, &store_root)
.await
.expect_err("over cap returns Err");
match err {
FetchError::TooManyRefs { got, max } => {
assert_eq!(got, n);
assert_eq!(max, MAX_BATCH_REFS);
}
other => panic!("expected TooManyRefs, got: {other:?}"),
}
}
#[tokio::test]
async fn try_fetch_oa_pdf_non_pdf_body_is_err_not_silent_none() {
use crate::http::HttpClient;
use crate::provenance::ProvenanceLog;
use crate::rate_limiter::RateLimiter;
use crate::{Doi, RateLimits};
use std::sync::Arc;
use wiremock::matchers::method;
use wiremock::{Mock, MockServer, ResponseTemplate};
let server = MockServer::start().await;
Mock::given(method("GET"))
.respond_with(
ResponseTemplate::new(200).set_body_bytes(b"<html>not a pdf</html>".to_vec()),
)
.mount(&server)
.await;
let host = server
.uri()
.parse::<url::Url>()
.expect("uri")
.host_str()
.expect("host")
.to_string();
let td = tempfile::TempDir::new().expect("tempdir");
let log_path = Utf8Path::from_path(td.path())
.expect("utf-8")
.join("log.jsonl");
let ctx = FetchContext {
http: Arc::new(HttpClient::new_for_tests_allow_http("oa-publisher", &host)),
rate_limiter: Arc::new(RateLimiter::new(RateLimits::HARD_CODED)),
log: Arc::new(
ProvenanceLog::open(log_path, "01J0000000000000000000TEST".into())
.expect("provenance log"),
),
session_id: "01J0000000000000000000TEST".into(),
cache_root: None,
};
let doi = Doi("10.1234/example".to_string());
let url: url::Url = format!("{}/oa.pdf", server.uri()).parse().expect("url");
let res = try_fetch_oa_pdf(&doi, &url, &ctx).await;
match res {
Err(HttpError::NotAPdf { .. }) => {}
other => panic!("expected Err(NotAPdf), got: {other:?}"),
}
}
#[tokio::test]
async fn try_fetch_oa_pdf_off_allowlist_host_no_redirect_is_redirect_denied_145() {
use crate::http::HttpClient;
use crate::provenance::ProvenanceLog;
use crate::rate_limiter::RateLimiter;
use crate::{DenialContext, DenialReason, Doi, RateLimits};
use std::sync::Arc;
use wiremock::matchers::method;
use wiremock::{Mock, MockServer, ResponseTemplate};
let server = MockServer::start().await;
Mock::given(method("GET"))
.respond_with(ResponseTemplate::new(200).set_body_bytes(b"%PDF-1.7 real pdf".to_vec()))
.mount(&server)
.await;
let td = tempfile::TempDir::new().expect("tempdir");
let log_path = Utf8Path::from_path(td.path())
.expect("utf-8")
.join("log.jsonl");
let ctx = FetchContext {
http: Arc::new(HttpClient::new_for_tests_allow_http(
"oa-publisher",
"allowed-publisher.example.com",
)),
rate_limiter: Arc::new(RateLimiter::new(RateLimits::HARD_CODED)),
log: Arc::new(
ProvenanceLog::open(log_path.clone(), "01J0000000000000000000TEST".into())
.expect("provenance log"),
),
session_id: "01J0000000000000000000TEST".into(),
cache_root: None,
};
let doi = Doi("10.1234/example".to_string());
let off_host_url: url::Url = format!("{}/oa.pdf", server.uri()).parse().expect("url");
let res = try_fetch_oa_pdf(&doi, &off_host_url, &ctx).await;
let err = match res {
Err(e @ HttpError::RedirectDenied { .. }) => e,
other => {
panic!("expected Err(RedirectDenied) from the pre-fetch check, got: {other:?}")
}
};
match &err {
HttpError::RedirectDenied {
source_key,
host,
expected_hosts,
} => {
assert_eq!(source_key, "oa-publisher");
assert_eq!(
host,
off_host_url
.host_str()
.expect("wiremock host")
.to_ascii_lowercase()
.as_str()
);
assert_eq!(
expected_hosts,
&vec!["allowed-publisher.example.com".to_string()]
);
}
_ => unreachable!(),
}
assert!(
server
.received_requests()
.await
.unwrap_or_default()
.is_empty(),
"the off-allowlist OA URL must NOT be fetched: the pre-check \
(REDIRECT_ALLOWLIST.md §1) rejects it before any request is \
issued; wiremock recorded request(s)",
);
let dc: Option<DenialContext> = (&err).into();
let dc = dc.expect("pre-fetch RedirectDenied -> Some(DenialContext)");
assert_eq!(dc.reason, DenialReason::RedirectNotInAllowlist);
assert_eq!(dc.source.as_deref(), Some("oa-publisher"));
assert_eq!(
dc.attempted,
Some(off_host_url.host_str().expect("host").to_ascii_lowercase()),
"attempted host must be the rejected OA URL host, lowercased — \
identical to what the redirect closure records",
);
assert_eq!(
dc.expected,
Some(vec!["allowed-publisher.example.com".to_string()]),
);
let log_txt = std::fs::read_to_string(&log_path).expect("read provenance log");
let fetch_err_row = log_txt
.lines()
.filter_map(|l| serde_json::from_str::<serde_json::Value>(l).ok())
.find(|v| {
v.get("event").and_then(|e| e.as_str()) == Some("fetch")
&& v.get("result").and_then(|r| r.as_str()) == Some("err")
})
.expect("a Fetch/err provenance row was written");
assert_eq!(
fetch_err_row.get("source").and_then(|s| s.as_str()),
Some("oa-publisher"),
);
assert_eq!(
fetch_err_row.get("error_code").and_then(|c| c.as_str()),
Some("NETWORK_ERROR"),
);
assert_eq!(
fetch_err_row.get("ref").and_then(|r| r.as_str()),
Some("10.1234/example"),
);
}
#[tokio::test]
async fn try_fetch_oa_pdf_on_allowlist_host_still_fetches_pdf_no_regression_145() {
use crate::http::HttpClient;
use crate::provenance::ProvenanceLog;
use crate::rate_limiter::RateLimiter;
use crate::{Doi, RateLimits};
use std::sync::Arc;
use wiremock::matchers::method;
use wiremock::{Mock, MockServer, ResponseTemplate};
let server = MockServer::start().await;
let body = b"%PDF-1.7\nhello pdf".to_vec();
Mock::given(method("GET"))
.respond_with(ResponseTemplate::new(200).set_body_bytes(body.clone()))
.mount(&server)
.await;
let host = server
.uri()
.parse::<url::Url>()
.expect("uri")
.host_str()
.expect("host")
.to_string();
let td = tempfile::TempDir::new().expect("tempdir");
let log_path = Utf8Path::from_path(td.path())
.expect("utf-8")
.join("log.jsonl");
let ctx = FetchContext {
http: Arc::new(HttpClient::new_for_tests_allow_http("oa-publisher", &host)),
rate_limiter: Arc::new(RateLimiter::new(RateLimits::HARD_CODED)),
log: Arc::new(
ProvenanceLog::open(log_path, "01J0000000000000000000TEST".into())
.expect("provenance log"),
),
session_id: "01J0000000000000000000TEST".into(),
cache_root: None,
};
let doi = Doi("10.1234/example".to_string());
let url: url::Url = format!("{}/oa.pdf", server.uri()).parse().expect("url");
let (bytes, _final_url) = try_fetch_oa_pdf(&doi, &url, &ctx)
.await
.expect("on-allowlist OA URL still fetches the PDF");
assert_eq!(bytes, body, "PDF bytes must be returned unchanged");
}
#[test]
fn pre_fetch_denial_produces_byte_identical_denial_context_as_redirect_denied_145() {
use crate::{DenialContext, DenialReason};
let pre_fetch = HttpError::RedirectDenied {
source_key: "oa-publisher".to_string(),
host: "attacker.test".to_string(),
expected_hosts: vec!["*.springer.com".to_string(), "*.plos.org".to_string()],
};
let redirect_closure = HttpError::RedirectDenied {
source_key: "oa-publisher".to_string(),
host: "attacker.test".to_string(),
expected_hosts: vec!["*.springer.com".to_string(), "*.plos.org".to_string()],
};
let dc_pre: Option<DenialContext> = (&pre_fetch).into();
let dc_red: Option<DenialContext> = (&redirect_closure).into();
let dc_pre = dc_pre.expect("pre-fetch -> Some");
let dc_red = dc_red.expect("redirect -> Some");
assert_eq!(dc_pre, dc_red);
assert_eq!(dc_pre.reason, DenialReason::RedirectNotInAllowlist);
assert_eq!(dc_pre.source.as_deref(), Some("oa-publisher"));
assert_eq!(dc_pre.attempted.as_deref(), Some("attacker.test"));
assert_eq!(
dc_pre.expected,
Some(vec!["*.springer.com".to_string(), "*.plos.org".to_string()]),
);
assert_eq!(dc_pre.hop_index, None);
assert_eq!(dc_pre.cap, None);
assert_eq!(dc_pre.actual, None);
}
async fn md139_harness() -> (
wiremock::MockServer,
FetchContext,
crate::store::FsStore,
Utf8PathBuf,
tempfile::TempDir,
) {
use crate::http::HttpClient;
use crate::provenance::ProvenanceLog;
use crate::rate_limiter::RateLimiter;
use crate::store::FsStore;
use crate::RateLimits;
use std::sync::Arc;
use wiremock::matchers::method;
use wiremock::{Mock, MockServer, ResponseTemplate};
let server = MockServer::start().await;
Mock::given(method("GET"))
.respond_with(ResponseTemplate::new(200).set_body_string(
r#"{"status":"ok","message":{"title":["Example Paper"],"author":[{"given":"Ada","family":"Lovelace"}]}}"#,
))
.mount(&server)
.await;
std::env::set_var("DOIGET_CROSSREF_BASE", server.uri());
let host = server
.uri()
.parse::<url::Url>()
.expect("uri")
.host_str()
.expect("host")
.to_string();
let td = tempfile::TempDir::new().expect("tempdir");
let base = Utf8Path::from_path(td.path()).expect("utf-8");
let log_path = base.join("log.jsonl");
let store_root = base.join("papers");
let ctx = FetchContext {
http: Arc::new(HttpClient::new_for_tests_allow_http_multi(&[
("crossref", &host),
("unpaywall", &host),
])),
rate_limiter: Arc::new(RateLimiter::new(RateLimits::HARD_CODED)),
log: Arc::new(
ProvenanceLog::open(log_path, "01J0000000000000000000TEST".into())
.expect("provenance log"),
),
session_id: "01J0000000000000000000TEST".into(),
cache_root: None,
};
let store = FsStore::new(store_root.clone()).expect("fs store");
(server, ctx, store, store_root, td)
}
fn metadata_dir_tomls(store_root: &Utf8Path) -> Vec<Utf8PathBuf> {
let md = store_root.join(".metadata");
match std::fs::read_dir(md.as_std_path()) {
Ok(rd) => rd
.filter_map(|e| e.ok())
.filter_map(|e| Utf8PathBuf::from_path_buf(e.path()).ok())
.filter(|p| p.extension() == Some("toml"))
.collect(),
Err(_) => Vec::new(),
}
}
#[tokio::test]
#[serial_test::serial]
async fn metadata_only_to_store_writes_metadata_toml_139() {
let (_server, ctx, store, store_root, _td) = md139_harness().await;
let profile = CapabilityProfile::from_env().expect("clean env");
let ref_ = Ref::Doi(Doi("10.1234/example".to_string()));
let outcome = metadata_only_to_store(&ref_, &profile, &ctx, &store)
.await
.expect("metadata_only_to_store ok");
assert_eq!(outcome.source, "crossref");
let tomls = metadata_dir_tomls(&store_root);
assert_eq!(
tomls.len(),
1,
"exactly one .metadata/*.toml must be written (MCP_TOOLS.md §11 SIDE EFFECT, #139); got {tomls:?}"
);
let body = std::fs::read_to_string(&tomls[0]).expect("read metadata toml");
let meta: crate::store::Metadata = toml::from_str(&body).expect("parse metadata toml");
assert_eq!(meta.title, "Example Paper");
assert_eq!(
meta.doi.as_ref().map(|d| d.as_str()),
Some("10.1234/example")
);
let ext = meta.doiget.expect("[doiget] table present");
assert_eq!(ext.source, "crossref");
assert_eq!(ext.size_bytes, 0, "metadata-only entry has no PDF");
std::env::remove_var("DOIGET_CROSSREF_BASE");
}
#[tokio::test]
#[serial_test::serial]
async fn resolve_only_and_pure_metadata_only_write_nothing_139() {
let (_server, ctx, _store, store_root, _td) = md139_harness().await;
let profile = CapabilityProfile::from_env().expect("clean env");
let ref_ = Ref::Doi(Doi("10.1234/example".to_string()));
let r = resolve_only(&ref_, &profile, &ctx)
.await
.expect("resolve_only ok");
assert_eq!(r.source, "crossref");
assert!(
metadata_dir_tomls(&store_root).is_empty(),
"resolve_only MUST NOT write a metadata TOML (docs/MCP_TOOLS.md §1; #139)"
);
let m = metadata_only(&ref_, &profile, &ctx)
.await
.expect("metadata_only ok");
assert_eq!(m.source, "crossref");
assert!(
metadata_dir_tomls(&store_root).is_empty(),
"pure metadata_only MUST NOT write to the store (#139)"
);
std::env::remove_var("DOIGET_CROSSREF_BASE");
}
#[tokio::test]
#[serial_test::serial]
async fn metadata_only_to_store_arxiv_writes_metadata_toml_139() {
use crate::http::HttpClient;
use crate::provenance::ProvenanceLog;
use crate::rate_limiter::RateLimiter;
use crate::store::FsStore;
use crate::RateLimits;
use std::sync::Arc;
use wiremock::matchers::method;
use wiremock::{Mock, MockServer, ResponseTemplate};
let atom = r#"<?xml version="1.0" encoding="UTF-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
<entry>
<id>http://arxiv.org/abs/2401.12345v1</id>
<published>2024-01-15T00:00:00Z</published>
<title>Example arXiv Paper Title</title>
<summary>Example abstract.</summary>
<author><name>Jane Doe</name></author>
<category term="cs.LG" scheme="http://arxiv.org/schemas/atom"/>
</entry>
</feed>"#;
let server = MockServer::start().await;
Mock::given(method("GET"))
.respond_with(ResponseTemplate::new(200).set_body_string(atom))
.mount(&server)
.await;
std::env::set_var("DOIGET_ARXIV_BASE", server.uri());
let host = server
.uri()
.parse::<url::Url>()
.expect("uri")
.host_str()
.expect("host")
.to_string();
let td = tempfile::TempDir::new().expect("tempdir");
let base = Utf8Path::from_path(td.path()).expect("utf-8");
let store_root = base.join("papers");
let ctx = FetchContext {
http: Arc::new(HttpClient::new_for_tests_allow_http("arxiv", &host)),
rate_limiter: Arc::new(RateLimiter::new(RateLimits::HARD_CODED)),
log: Arc::new(
ProvenanceLog::open(base.join("log.jsonl"), "01J0000000000000000000TEST".into())
.expect("provenance log"),
),
session_id: "01J0000000000000000000TEST".into(),
cache_root: None,
};
let store = FsStore::new(store_root.clone()).expect("fs store");
let profile = CapabilityProfile::from_env().expect("clean env");
let ref_ = Ref::Arxiv(crate::ArxivId::parse("2401.12345").expect("arxiv id"));
let outcome = metadata_only_to_store(&ref_, &profile, &ctx, &store)
.await
.expect("metadata_only_to_store (arxiv) ok");
assert_eq!(outcome.source, "arxiv");
let tomls = metadata_dir_tomls(&store_root);
assert_eq!(
tomls.len(),
1,
"arXiv metadata-only must write one TOML; got {tomls:?}"
);
let meta: crate::store::Metadata =
toml::from_str(&std::fs::read_to_string(&tomls[0]).expect("read")).expect("parse");
assert_eq!(meta.title, "Example arXiv Paper Title");
assert_eq!(
meta.arxiv_id.as_ref().map(|a| a.as_str()),
Some("2401.12345")
);
assert!(meta.doi.is_none(), "arXiv entry has no DOI");
let ext = meta.doiget.expect("[doiget] table");
assert_eq!(ext.source, "arxiv");
assert_eq!(ext.license, "arxiv-default");
std::env::remove_var("DOIGET_ARXIV_BASE");
}
#[test]
fn extract_metadata_title_handles_string_array_missing_blank() {
use serde_json::json;
assert_eq!(
extract_metadata_title(&json!({"title": "Hello"})),
Some("Hello".to_string())
);
assert_eq!(
extract_metadata_title(&json!({"title": ["Real Title"]})),
Some("Real Title".to_string())
);
assert_eq!(extract_metadata_title(&json!({"x": 1})), None);
assert_eq!(extract_metadata_title(&json!({"title": " "})), None);
assert_eq!(extract_metadata_title(&json!({"title": []})), None);
assert_eq!(
extract_metadata_title(&json!({"title": [" ", "Real Title"]})),
Some("Real Title".to_string())
);
assert_eq!(extract_metadata_title(&json!({"title": [" ", ""]})), None);
}
#[test]
fn extract_metadata_authors_handles_each_resolver_shape() {
use serde_json::json;
assert_eq!(
extract_metadata_authors(&json!({"authors": ["Jane Doe", "John Roe"]})),
vec!["Jane Doe".to_string(), "John Roe".to_string()]
);
assert_eq!(
extract_metadata_authors(&json!({"author": [{"given": "Ada", "family": "Lovelace"}]})),
vec!["Ada Lovelace".to_string()]
);
assert_eq!(
extract_metadata_authors(&json!({"author": [{"family": "Onsager"}]})),
vec!["Onsager".to_string()]
);
assert_eq!(
extract_metadata_authors(&json!({"author": [{"name": "K. Wilson"}]})),
vec!["K. Wilson".to_string()]
);
assert_eq!(
extract_metadata_authors(&json!({"z_authors": [{"given": "L", "family": "Kadanoff"}]})),
vec!["L Kadanoff".to_string()]
);
assert!(extract_metadata_authors(&json!({"x": 1})).is_empty());
assert!(extract_metadata_authors(&json!({"authors": []})).is_empty());
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum AttemptOutcome {
Disabled {
env: &'static [&'static str],
},
NotApplicable,
WrongPublisher {
detail: String,
},
NotNeeded,
NoRecord,
NotOpenAccess {
detail: String,
},
Denied {
denial: DenialContext,
},
Failed {
detail: String,
},
Resolved,
}
impl AttemptOutcome {
#[must_use]
pub fn was_consulted(&self) -> bool {
matches!(
self,
Self::NoRecord
| Self::NotOpenAccess { .. }
| Self::Denied { .. }
| Self::Failed { .. }
| Self::Resolved
)
}
#[must_use]
pub fn wire(&self) -> &'static str {
match self {
Self::Disabled { .. } => "not_consulted_disabled",
Self::NotApplicable => "not_consulted_not_applicable",
Self::WrongPublisher { .. } => "not_consulted_wrong_publisher",
Self::NotNeeded => "not_consulted_not_needed",
Self::NoRecord => "consulted_no_record",
Self::NotOpenAccess { .. } => "consulted_not_open_access",
Self::Denied { .. } => "consulted_denied",
Self::Failed { .. } => "consulted_failed",
Self::Resolved => "consulted_resolved",
}
}
#[must_use]
pub fn detail(&self) -> Option<&str> {
match self {
Self::Disabled { .. } | Self::Denied { .. } => None,
Self::WrongPublisher { detail }
| Self::NotOpenAccess { detail }
| Self::Failed { detail } => Some(detail),
_ => None,
}
}
#[must_use]
pub fn required_env(&self) -> Option<&'static [&'static str]> {
match self {
Self::Disabled { env } => Some(env),
_ => None,
}
}
#[must_use]
pub fn denial(&self) -> Option<&DenialContext> {
match self {
Self::Denied { denial } => Some(denial),
_ => None,
}
}
#[must_use]
pub fn render(&self) -> String {
match self {
Self::Disabled { env } => {
format!("not consulted (set {} to enable)", env.join(" + "))
}
Self::NotApplicable => "not consulted (cannot serve this ref kind)".to_string(),
Self::WrongPublisher { detail } => format!("not consulted ({detail})"),
Self::NotNeeded => "not consulted (an earlier source answered)".to_string(),
Self::NoRecord => "consulted: no record".to_string(),
Self::NotOpenAccess { detail } => {
format!("consulted: found, not open access ({detail})")
}
Self::Denied { denial } => match &denial.attempted {
Some(a) => format!("consulted: refused ({:?}, {a})", denial.reason),
None => format!("consulted: refused ({:?})", denial.reason),
},
Self::Failed { detail } => format!("consulted: failed ({detail})"),
Self::Resolved => "consulted: resolved".to_string(),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub struct SourceAttempt {
pub source: &'static str,
pub outcome: AttemptOutcome,
}
impl SourceAttempt {
#[must_use]
pub fn new(source: &'static str, outcome: AttemptOutcome) -> Self {
Self { source, outcome }
}
}
#[must_use]
pub fn attempts_to_value(attempts: &[SourceAttempt]) -> serde_json::Value {
serde_json::Value::Array(
attempts
.iter()
.map(|a| {
let mut o = serde_json::Map::new();
o.insert("source".into(), serde_json::json!(a.source));
o.insert("outcome".into(), serde_json::json!(a.outcome.wire()));
if let Some(d) = a.outcome.detail() {
o.insert("detail".into(), serde_json::json!(d));
}
if let Some(env) = a.outcome.required_env() {
o.insert("detail".into(), serde_json::json!(env.join(" + ")));
o.insert("required_env".into(), serde_json::json!(env));
}
if let Some(dc) = a.outcome.denial() {
o.insert("detail".into(), serde_json::json!(a.outcome.render()));
o.insert("denial_context".into(), serde_json::json!(dc));
let rem = crate::remediation::for_denial(dc);
if !rem.is_empty() {
o.insert("remediation".into(), serde_json::json!(rem));
}
}
o.insert(
"consulted".into(),
serde_json::json!(a.outcome.was_consulted()),
);
serde_json::Value::Object(o)
})
.collect(),
)
}
#[must_use]
pub fn render_attempts(attempts: &[SourceAttempt]) -> String {
attempts
.iter()
.map(|a| format!(" {:<12} {}", a.source, a.outcome.render()))
.collect::<Vec<_>>()
.join("\n")
}
#[must_use]
pub fn nothing_was_consulted(attempts: &[SourceAttempt]) -> bool {
!attempts.is_empty() && attempts.iter().all(|a| !a.outcome.was_consulted())
}
#[cfg(any(
feature = "metadata",
feature = "tdm-elsevier",
feature = "tdm-aps",
feature = "tdm-springer",
feature = "tdm-ieee"
))]
fn classify_attempt(e: &FetchError) -> AttemptOutcome {
match e {
FetchError::NotFound { .. } => AttemptOutcome::NoRecord,
FetchError::SourceSchema { hint } if is_access_refusal(hint) => {
AttemptOutcome::NotOpenAccess {
detail: hint.clone(),
}
}
other => match Option::<DenialContext>::from(other) {
Some(denial) if denial.reason != crate::DenialReason::CapabilityNotGranted => {
AttemptOutcome::Denied { denial }
}
_ => AttemptOutcome::Failed {
detail: other.to_string(),
},
},
}
}
#[cfg(all(
test,
any(
feature = "metadata",
feature = "tdm-elsevier",
feature = "tdm-aps",
feature = "tdm-springer",
feature = "tdm-ieee"
)
))]
#[allow(clippy::expect_used, clippy::unwrap_used, clippy::panic)]
mod attempt_denial_tests {
use super::*;
use crate::http::HttpError;
use crate::DenialReason;
fn redirect_denial() -> FetchError {
FetchError::Http(HttpError::RedirectDenied {
source_key: "hal".to_string(),
host: "cdn.example.org".to_string(),
expected_hosts: vec!["hal.science".to_string()],
})
}
#[test]
fn a_policy_refusal_keeps_its_denial_context() {
let outcome = classify_attempt(&redirect_denial());
let denial = outcome
.denial()
.expect("a redirect denial must survive classification");
assert_eq!(denial.reason, DenialReason::RedirectNotInAllowlist);
assert_eq!(denial.attempted.as_deref(), Some("cdn.example.org"));
assert_eq!(outcome.wire(), "consulted_denied");
assert!(
outcome.was_consulted(),
"a refusal means a request went out"
);
}
#[test]
fn a_denied_row_carries_a_remediation_on_the_wire() {
let attempts = vec![SourceAttempt::new(
"hal",
classify_attempt(&redirect_denial()),
)];
let v = attempts_to_value(&attempts);
let row = &v[0];
assert_eq!(row["outcome"], serde_json::json!("consulted_denied"));
assert_eq!(
row["denial_context"]["reason"],
serde_json::json!("redirect_not_in_allowlist"),
"row: {row}"
);
let rem = row["remediation"]
.as_array()
.unwrap_or_else(|| panic!("a redirect denial has a config channel; row: {row}"));
assert!(!rem.is_empty());
assert!(
row["detail"].is_string(),
"`detail` must stay populated for a #459-era consumer; row: {row}"
);
}
#[test]
fn an_ungranted_capability_is_not_reported_as_consulted_and_denied() {
let outcome = classify_attempt(&FetchError::NotEligible {
source_key: "tdm-aps".into(),
});
assert!(
outcome.denial().is_none(),
"got {outcome:?}: this never reached the network"
);
assert_eq!(outcome.wire(), "consulted_failed");
}
#[test]
fn the_accessors_narrow_rather_than_generalise() {
let failed = AttemptOutcome::Failed {
detail: "connection reset".to_string(),
};
assert!(failed.denial().is_none());
assert!(failed.required_env().is_none());
assert!(failed.detail().is_some());
let disabled = AttemptOutcome::Disabled {
env: &["DOIGET_ENABLE_HAL"],
};
assert!(disabled.denial().is_none());
assert!(
disabled.detail().is_none(),
"`Disabled` carries structure; the joined string is built at the wire"
);
assert!(!disabled.was_consulted());
assert_eq!(
disabled.render(),
"not consulted (set DOIGET_ENABLE_HAL to enable)"
);
}
#[test]
fn a_denial_without_an_attempted_host_still_renders() {
let outcome = AttemptOutcome::Denied {
denial: DenialContext {
reason: DenialReason::SizeCapExceeded,
source: Some("core".to_string()),
attempted: None,
expected: None,
hop_index: None,
cap: None,
actual: None,
},
};
assert_eq!(outcome.render(), "consulted: refused (SizeCapExceeded)");
let v = attempts_to_value(&[SourceAttempt::new("core", outcome)]);
assert!(v[0].get("remediation").is_none(), "row: {}", v[0]);
assert!(v[0].get("denial_context").is_some(), "row: {}", v[0]);
}
#[test]
fn a_disabled_row_lists_its_variables_instead_of_joining_them() {
let attempts = vec![SourceAttempt::new(
"tdm-aps",
AttemptOutcome::Disabled {
env: &["DOIGET_KEY_APS", "DOIGET_AGREE_TDM_APS"],
},
)];
let v = attempts_to_value(&attempts);
let row = &v[0];
assert_eq!(
row["required_env"],
serde_json::json!(["DOIGET_KEY_APS", "DOIGET_AGREE_TDM_APS"]),
"row: {row}"
);
assert_eq!(
row["detail"],
serde_json::json!("DOIGET_KEY_APS + DOIGET_AGREE_TDM_APS"),
"row: {row}"
);
}
}
#[cfg(any(
feature = "metadata",
feature = "tdm-elsevier",
feature = "tdm-aps",
feature = "tdm-springer",
feature = "tdm-ieee"
))]
fn is_access_refusal(hint: &str) -> bool {
hint.contains("not open access")
|| hint.contains("openAccess")
|| hint.contains("no retrievable PDF")
}
#[cfg(feature = "metadata")]
async fn resolve_optional_chain(
ref_: &Ref,
profile: &CapabilityProfile,
ctx: &FetchContext,
crossref_answered: bool,
extracted: &mut CrossrefFields,
attempts: &mut Vec<SourceAttempt>,
) -> Option<(&'static str, Value)> {
let datacite = optional_base("DOIGET_DATACITE_BASE").map_or_else(
crate::sources::datacite::DataCiteSource::new,
crate::sources::datacite::DataCiteSource::with_base,
);
let epmc = optional_base("DOIGET_EUROPE_PMC_BASE").map_or_else(
crate::sources::europepmc::EuropePmcSource::new,
crate::sources::europepmc::EuropePmcSource::with_base,
);
let openaire = optional_base("DOIGET_OPENAIRE_BASE").map_or_else(
crate::sources::openaire::OpenAireSource::new,
crate::sources::openaire::OpenAireSource::with_base,
);
let hal = optional_base("DOIGET_HAL_BASE").map_or_else(
crate::sources::hal::HalSource::new,
crate::sources::hal::HalSource::with_base,
);
let core = optional_base("DOIGET_CORE_BASE").map_or_else(
crate::sources::core_oa::CoreSource::new,
crate::sources::core_oa::CoreSource::with_base,
);
let openalex_contact = resolve_contact_email();
let openalex = match optional_base("DOIGET_OPENALEX_BASE") {
Some(base) => crate::sources::openalex::OpenalexSource::with_base(base, openalex_contact),
None => crate::sources::openalex::OpenalexSource::new(openalex_contact),
};
let chain: Vec<(
&'static str,
&'static [&'static str],
&dyn crate::source::Source,
)> = vec![
("datacite", &["DOIGET_ENABLE_DATACITE"], &datacite),
("europe-pmc", &["DOIGET_ENABLE_EUROPE_PMC"], &epmc),
("openaire", &["DOIGET_ENABLE_OPENAIRE"], &openaire),
("hal", &["DOIGET_ENABLE_HAL"], &hal),
("core", &["DOIGET_ENABLE_CORE"], &core),
("openalex", &["DOIGET_ENABLE_OPENALEX"], &openalex),
];
let mut resolved: Option<(&'static str, Value)> = None;
for (name, env, src) in chain {
debug_assert_eq!(name, src.name(), "chain name must match Source::name");
if crossref_answered || resolved.is_some() {
attempts.push(SourceAttempt::new(name, AttemptOutcome::NotNeeded));
continue;
}
if !src.can_serve(profile, ref_) {
let outcome = if matches!(ref_, Ref::Doi(_)) {
AttemptOutcome::Disabled { env }
} else {
AttemptOutcome::NotApplicable
};
attempts.push(SourceAttempt::new(name, outcome));
continue;
}
match src.fetch(ref_, profile, ctx).await {
Ok(r) => {
if let Some(meta) = r.metadata_json.as_ref() {
*extracted = extract_optional_fields(name, meta);
}
attempts.push(SourceAttempt::new(name, AttemptOutcome::Resolved));
resolved = r.metadata_json.map(|m| (name, m));
}
Err(e) => {
tracing::debug!(source = name, error = %e, "optional source did not resolve");
attempts.push(SourceAttempt::new(name, classify_attempt(&e)));
}
}
}
resolved
}
#[cfg(any(
feature = "tdm-elsevier",
feature = "tdm-aps",
feature = "tdm-springer",
feature = "tdm-ieee"
))]
#[allow(clippy::vec_init_then_push)]
async fn resolve_tdm_chain(
ref_: &Ref,
profile: &CapabilityProfile,
ctx: &FetchContext,
crossref_answered: bool,
attempts: &mut Vec<SourceAttempt>,
) -> Option<Value> {
struct Entry<'a> {
name: &'static str,
enable_hint: &'static [&'static str],
prefixes: &'static [&'static str],
publisher: &'static str,
src: &'a dyn crate::source::Source,
}
#[cfg(feature = "tdm-aps")]
let aps = optional_base("DOIGET_APS_BASE").map_or_else(
crate::sources::tdm_aps::TdmApsSource::new,
crate::sources::tdm_aps::TdmApsSource::with_base,
);
#[cfg(feature = "tdm-elsevier")]
let elsevier = optional_base("DOIGET_ELSEVIER_BASE").map_or_else(
crate::sources::tdm_elsevier::TdmElsevierSource::new,
crate::sources::tdm_elsevier::TdmElsevierSource::with_base,
);
#[cfg(feature = "tdm-springer")]
let springer = optional_base("DOIGET_SPRINGER_BASE").map_or_else(
crate::sources::tdm_springer::TdmSpringerSource::new,
crate::sources::tdm_springer::TdmSpringerSource::with_base,
);
#[cfg(feature = "tdm-ieee")]
let ieee = optional_base("DOIGET_IEEE_BASE").map_or_else(
crate::sources::tdm_ieee::TdmIeeeSource::new,
crate::sources::tdm_ieee::TdmIeeeSource::with_base,
);
#[allow(unused_mut)]
let mut chain: Vec<Entry<'_>> = Vec::new();
#[cfg(feature = "tdm-aps")]
chain.push(Entry {
name: "tdm-aps",
enable_hint: &["DOIGET_KEY_APS", "DOIGET_AGREE_TDM_APS"],
prefixes: crate::sources::tdm_aps::PUBLISHER_PREFIXES,
publisher: "American Physical Society (APS)",
src: &aps,
});
#[cfg(feature = "tdm-elsevier")]
chain.push(Entry {
name: "tdm-elsevier",
enable_hint: &["DOIGET_KEY_ELSEVIER", "DOIGET_AGREE_TDM_ELSEVIER"],
prefixes: crate::sources::tdm_elsevier::PUBLISHER_PREFIXES,
publisher: "Elsevier BV",
src: &elsevier,
});
#[cfg(feature = "tdm-springer")]
chain.push(Entry {
name: "tdm-springer",
enable_hint: &["DOIGET_KEY_SPRINGER", "DOIGET_AGREE_TDM_SPRINGER"],
prefixes: crate::sources::tdm_springer::PUBLISHER_PREFIXES,
publisher: "Springer Nature",
src: &springer,
});
#[cfg(feature = "tdm-ieee")]
chain.push(Entry {
name: "tdm-ieee",
enable_hint: &["DOIGET_KEY_IEEE", "DOIGET_AGREE_TDM_IEEE"],
prefixes: crate::sources::tdm_ieee::PUBLISHER_PREFIXES,
publisher: "IEEE",
src: &ieee,
});
let mut resolved: Option<Value> = None;
for e in chain {
debug_assert_eq!(e.name, e.src.name(), "chain name must match Source::name");
if crossref_answered || resolved.is_some() {
attempts.push(SourceAttempt::new(e.name, AttemptOutcome::NotNeeded));
continue;
}
let Ref::Doi(doi) = ref_ else {
attempts.push(SourceAttempt::new(e.name, AttemptOutcome::NotApplicable));
continue;
};
if !e.prefixes.contains(&doi.prefix()) {
attempts.push(SourceAttempt::new(
e.name,
AttemptOutcome::WrongPublisher {
detail: format!("DOI prefix {} is not {}", doi.prefix(), e.publisher),
},
));
continue;
}
if !e.src.can_serve(profile, ref_) {
attempts.push(SourceAttempt::new(
e.name,
AttemptOutcome::Disabled { env: e.enable_hint },
));
continue;
}
match e.src.fetch(ref_, profile, ctx).await {
Ok(r) => {
attempts.push(SourceAttempt::new(e.name, AttemptOutcome::Resolved));
resolved = r.metadata_json;
}
Err(err) => {
tracing::debug!(source = e.name, error = %err, "TDM source did not resolve");
attempts.push(SourceAttempt::new(e.name, classify_attempt(&err)));
}
}
}
resolved
}
#[cfg(any(
feature = "metadata",
feature = "tdm-elsevier",
feature = "tdm-aps",
feature = "tdm-springer",
feature = "tdm-ieee"
))]
fn optional_base(env: &str) -> Option<url::Url> {
let raw = std::env::var(env).ok()?;
match url::Url::parse(&raw) {
Ok(u) => Some(u),
Err(e) => {
tracing::warn!(value = %raw, error = %e, env, "base override is not a valid URL; using the default");
None
}
}
}
#[cfg(feature = "metadata")]
fn extract_optional_fields(source: &str, meta: &Value) -> CrossrefFields {
match source {
"datacite" => extract_datacite_fields(meta),
_ => CrossrefFields::default(),
}
}
#[cfg(all(test, feature = "metadata"))]
#[allow(clippy::expect_used, clippy::unwrap_used, clippy::panic)]
mod chain_tests {
use super::*;
use std::sync::Arc;
use camino::Utf8PathBuf;
use tempfile::TempDir;
use wiremock::matchers::method;
use wiremock::{Mock, MockServer, ResponseTemplate};
use crate::http::HttpClient;
use crate::provenance::ProvenanceLog;
use crate::rate_limiter::RateLimiter;
use crate::{CapabilityProfile, Doi, MetadataAccess, RateLimits, Ref};
struct BaseGuard(Vec<(&'static str, Option<String>)>);
impl BaseGuard {
fn to(uri: &str) -> Self {
const VARS: &[&str] = &[
"DOIGET_DATACITE_BASE",
"DOIGET_EUROPE_PMC_BASE",
"DOIGET_OPENAIRE_BASE",
"DOIGET_HAL_BASE",
"DOIGET_CORE_BASE",
"DOIGET_OPENALEX_BASE",
];
Self(
VARS.iter()
.map(|v| {
let old = std::env::var(v).ok();
std::env::set_var(v, uri);
(*v, old)
})
.collect(),
)
}
}
impl Drop for BaseGuard {
fn drop(&mut self) {
for (v, old) in &self.0 {
match old {
Some(o) => std::env::set_var(v, o),
None => std::env::remove_var(v),
}
}
}
}
fn ctx_for(host: &str) -> (TempDir, FetchContext) {
let td = TempDir::new().expect("tempdir");
let dir = Utf8PathBuf::try_from(td.path().to_path_buf()).expect("utf-8");
let http = Arc::new(HttpClient::new_for_tests_allow_http_multi(&[
("datacite", host),
("europe-pmc", host),
("openaire", host),
("hal", host),
("core", host),
("openalex", host),
]));
let session_id = "01J0000000000000000000TEST".to_string();
let log = Arc::new(
ProvenanceLog::open(dir.join("t.jsonl"), session_id.clone()).expect("log opens"),
);
(
td,
FetchContext {
http,
rate_limiter: Arc::new(RateLimiter::new(RateLimits::HARD_CODED)),
log,
session_id,
cache_root: None,
},
)
}
fn all_off() -> CapabilityProfile {
let mut p = CapabilityProfile::for_tests();
p.metadata = MetadataAccess {
openalex: false,
semantic_scholar: false,
doaj: false,
datacite: false,
hal: false,
openaire: false,
core: false,
europe_pmc: false,
};
p
}
fn all_on() -> CapabilityProfile {
let mut p = all_off();
p.metadata.datacite = true;
p.metadata.hal = true;
p.metadata.openaire = true;
p.metadata.core = true;
p.metadata.europe_pmc = true;
p.metadata.openalex = true;
p
}
fn outcome<'a>(attempts: &'a [SourceAttempt], name: &str) -> &'a AttemptOutcome {
&attempts
.iter()
.find(|a| a.source == name)
.unwrap_or_else(|| panic!("no attempt recorded for {name}; got {attempts:?}"))
.outcome
}
#[test]
fn every_oa_url_bearing_source_has_a_dispatch_arm() {
let cases: &[(&str, serde_json::Value, &str)] = &[
(
"core",
serde_json::json!({ "downloadUrl": "https://core.example/1.pdf" }),
"https://core.example/1.pdf",
),
(
"hal",
serde_json::json!({
"openAccess_bool": true,
"fileMain_s": "https://hal.example/2.pdf"
}),
"https://hal.example/2.pdf",
),
(
"europe-pmc",
serde_json::json!({
"fullTextUrlList": {
"fullTextUrl": [{
"documentStyle": "pdf",
"availabilityCode": "OA",
"url": "https://epmc.example/3.pdf"
}]
}
}),
"https://epmc.example/3.pdf",
),
(
"openalex",
serde_json::json!({
"locations": [
{ "is_oa": true, "pdf_url": "https://repo.example.ac.uk/4.pdf" }
]
}),
"https://repo.example.ac.uk/4.pdf",
),
];
let mut broken: Vec<String> = Vec::new();
for (source, meta, expected) in cases {
let got = optional_source_oa_url(source, meta);
if got != Some(*expected) {
broken.push(format!("{source}: expected {expected:?}, got {got:?}"));
}
}
assert!(
broken.is_empty(),
"these sources are in the optional chain but their document URL never reaches the \
content leg -- a missing or wrong `optional_source_oa_url` arm:\n {}",
broken.join("\n ")
);
assert_eq!(
optional_source_oa_url("datacite", &serde_json::json!({ "downloadUrl": "x" })),
None,
"datacite reports no document URL; it must not fall through to another arm"
);
}
#[tokio::test]
#[serial_test::serial]
async fn every_optional_source_is_actually_reached_by_the_production_chain() {
let server = MockServer::start().await;
let _bases = BaseGuard::to(&server.uri());
Mock::given(method("GET"))
.respond_with(ResponseTemplate::new(200).set_body_string(
r#"{"results":[],"response":{"docs":[]},"resultList":{"result":[]}}"#,
))
.mount(&server)
.await;
let (_td, ctx) = ctx_for(&server.address().to_string());
let ref_ = Ref::Doi(Doi::parse("10.1234/example").expect("doi"));
let mut fields = CrossrefFields::default();
let mut attempts = Vec::new();
resolve_optional_chain(&ref_, &all_on(), &ctx, false, &mut fields, &mut attempts).await;
let names: Vec<&str> = attempts.iter().map(|a| a.source).collect();
assert_eq!(
names,
vec![
"datacite",
"europe-pmc",
"openaire",
"hal",
"core",
"openalex"
],
"the trace must list every source, in chain order"
);
for a in &attempts {
assert!(
a.outcome.was_consulted(),
"{} was NOT reached by the production chain: {:?}",
a.source,
a.outcome
);
}
assert_eq!(
server.received_requests().await.expect("recorded").len(),
6,
"each enabled source must issue exactly one request"
);
}
#[tokio::test]
#[serial_test::serial]
async fn flags_off_means_never_consulted_and_says_which_var_to_set() {
let server = MockServer::start().await;
let _bases = BaseGuard::to(&server.uri());
let (_td, ctx) = ctx_for(&server.address().to_string());
let ref_ = Ref::Doi(Doi::parse("10.1234/example").expect("doi"));
let mut fields = CrossrefFields::default();
let mut attempts = Vec::new();
resolve_optional_chain(&ref_, &all_off(), &ctx, false, &mut fields, &mut attempts).await;
assert!(
server
.received_requests()
.await
.expect("recorded")
.is_empty(),
"a disabled chain must make NO request"
);
assert!(
nothing_was_consulted(&attempts),
"the trace must report that nothing was reached"
);
assert_eq!(
outcome(&attempts, "hal"),
&AttemptOutcome::Disabled {
env: &["DOIGET_ENABLE_HAL"]
},
"a disabled source must name the variable that enables it"
);
let rendered = render_attempts(&attempts);
assert!(
rendered.contains("not consulted (set DOIGET_ENABLE_HAL to enable)"),
"rendered trace must be actionable; got:\n{rendered}"
);
}
#[tokio::test]
#[serial_test::serial]
async fn never_consulted_and_consulted_but_empty_render_differently() {
let server = MockServer::start().await;
let _bases = BaseGuard::to(&server.uri());
Mock::given(method("GET"))
.respond_with(ResponseTemplate::new(200).set_body_string(r#"{"results":[]}"#))
.mount(&server)
.await;
let (_td, ctx) = ctx_for(&server.address().to_string());
let ref_ = Ref::Doi(Doi::parse("10.1234/example").expect("doi"));
let mut f1 = CrossrefFields::default();
let mut consulted = Vec::new();
let mut on = all_off();
on.metadata.datacite = true;
resolve_optional_chain(&ref_, &on, &ctx, false, &mut f1, &mut consulted).await;
let mut f2 = CrossrefFields::default();
let mut skipped = Vec::new();
resolve_optional_chain(&ref_, &all_off(), &ctx, false, &mut f2, &mut skipped).await;
let a = outcome(&consulted, "datacite");
let b = outcome(&skipped, "datacite");
assert!(a.was_consulted(), "(a) must be consulted, got {a:?}");
assert!(!b.was_consulted(), "(b) must NOT be consulted, got {b:?}");
assert_ne!(
a.render(),
b.render(),
"the two states MUST NOT render identically"
);
assert!(a.render().starts_with("consulted:"), "{}", a.render());
assert!(b.render().starts_with("not consulted"), "{}", b.render());
}
#[tokio::test]
#[serial_test::serial]
async fn a_crossref_hit_skips_the_chain_without_pretending_it_was_disabled() {
let server = MockServer::start().await;
let _bases = BaseGuard::to(&server.uri());
let (_td, ctx) = ctx_for(&server.address().to_string());
let ref_ = Ref::Doi(Doi::parse("10.1234/example").expect("doi"));
let mut fields = CrossrefFields::default();
let mut attempts = Vec::new();
resolve_optional_chain(&ref_, &all_on(), &ctx, true, &mut fields, &mut attempts).await;
assert!(
server
.received_requests()
.await
.expect("recorded")
.is_empty(),
"a Crossref hit must cost no extra requests"
);
for a in &attempts {
assert_eq!(
a.outcome,
AttemptOutcome::NotNeeded,
"{} must be NotNeeded, not Disabled — the flags ARE on",
a.source
);
}
}
#[tokio::test]
#[serial_test::serial]
async fn an_access_refusal_is_recorded_distinctly_from_a_miss() {
let server = MockServer::start().await;
let _bases = BaseGuard::to(&server.uri());
Mock::given(method("GET"))
.respond_with(ResponseTemplate::new(200).set_body_string(
r#"{"resultList":{"result":[{"doi":"10.1234/x","isOpenAccess":"N","inEPMC":"Y"}]}}"#,
))
.mount(&server)
.await;
let (_td, ctx) = ctx_for(&server.address().to_string());
let ref_ = Ref::Doi(Doi::parse("10.1234/x").expect("doi"));
let mut fields = CrossrefFields::default();
let mut attempts = Vec::new();
let mut on = all_off();
on.metadata.europe_pmc = true;
resolve_optional_chain(&ref_, &on, &ctx, false, &mut fields, &mut attempts).await;
let o = outcome(&attempts, "europe-pmc");
assert!(
matches!(o, AttemptOutcome::NotOpenAccess { .. }),
"a closed record must be NotOpenAccess, not NoRecord/Failed; got {o:?}"
);
assert!(o.was_consulted(), "it WAS reached");
assert!(
o.render().contains("not open access"),
"the reason must survive into the message; got {}",
o.render()
);
}
#[tokio::test]
#[serial_test::serial]
async fn a_closed_subset_record_with_a_free_pdf_resolves_and_carries_its_url() {
let server = MockServer::start().await;
let _bases = BaseGuard::to(&server.uri());
Mock::given(method("GET"))
.respond_with(ResponseTemplate::new(200).set_body_string(
r#"{"resultList":{"result":[{"doi":"10.1098/rspa.2014.0585",
"isOpenAccess":"N","inEPMC":"Y","fullTextUrlList":{"fullTextUrl":[
{"availability":"Free","availabilityCode":"F","documentStyle":"pdf",
"site":"Europe_PMC",
"url":"https://europepmc.org/articles/PMC4277194?pdf=render"}]}}]}}"#,
))
.mount(&server)
.await;
let (_td, ctx) = ctx_for(&server.address().to_string());
let ref_ = Ref::Doi(Doi::parse("10.1098/rspa.2014.0585").expect("doi"));
let mut fields = CrossrefFields::default();
let mut attempts = Vec::new();
let mut on = all_off();
on.metadata.europe_pmc = true;
let resolved =
resolve_optional_chain(&ref_, &on, &ctx, false, &mut fields, &mut attempts).await;
let (source, record) = resolved.expect("a Free PDF entry must resolve, not refuse");
assert_eq!(source, "europe-pmc");
let row = outcome(&attempts, "europe-pmc");
assert!(
matches!(row, AttemptOutcome::Resolved),
"the row must read as resolved, not as an access refusal; got {row:?}"
);
assert_eq!(
optional_source_oa_url("europe-pmc", &record),
Some("https://europepmc.org/articles/PMC4277194?pdf=render"),
"the URL the oa-publisher leg fetches must survive the chain"
);
}
#[tokio::test]
#[serial_test::serial]
async fn an_arxiv_ref_is_not_applicable_rather_than_disabled() {
let server = MockServer::start().await;
let _bases = BaseGuard::to(&server.uri());
let (_td, ctx) = ctx_for(&server.address().to_string());
let ref_ = Ref::Arxiv(crate::ArxivId::parse("2401.12345").expect("arxiv"));
let mut fields = CrossrefFields::default();
let mut attempts = Vec::new();
resolve_optional_chain(&ref_, &all_on(), &ctx, false, &mut fields, &mut attempts).await;
for a in &attempts {
assert_eq!(
a.outcome,
AttemptOutcome::NotApplicable,
"{} must be NotApplicable for an arXiv ref",
a.source
);
assert!(
!a.outcome.render().contains("set DOIGET_"),
"must not suggest a variable that would not help: {}",
a.outcome.render()
);
}
assert!(server
.received_requests()
.await
.expect("recorded")
.is_empty());
}
}
#[cfg(all(
test,
any(
feature = "tdm-aps",
feature = "tdm-elsevier",
feature = "tdm-springer",
feature = "tdm-ieee"
)
))]
#[allow(clippy::expect_used, clippy::unwrap_used, clippy::panic)]
mod tdm_singleton_reach_tests {
use super::*;
use std::sync::Arc;
use camino::Utf8PathBuf;
use tempfile::TempDir;
use crate::http::HttpClient;
use crate::provenance::ProvenanceLog;
use crate::rate_limiter::RateLimiter;
use crate::{CapabilityProfile, Doi, RateLimits, Ref};
fn ctx() -> (TempDir, FetchContext) {
let td = TempDir::new().expect("tempdir");
let dir = Utf8PathBuf::try_from(td.path().to_path_buf()).expect("utf-8");
let session_id = "01J0000000000000000000SNG".to_string();
let log = Arc::new(
ProvenanceLog::open(dir.join("t.jsonl"), session_id.clone()).expect("log opens"),
);
let c = FetchContext {
http: Arc::new(HttpClient::new_for_tests_allow_http(
"tdm-probe",
"127.0.0.1:1",
)),
rate_limiter: Arc::new(RateLimiter::new(RateLimits::HARD_CODED)),
log,
session_id,
cache_root: None,
};
(td, c)
}
#[tokio::test]
#[serial_test::serial]
#[allow(clippy::vec_init_then_push)]
async fn every_compiled_publisher_is_in_the_chain() {
let mut cases: Vec<(&str, &str)> = Vec::new();
#[cfg(feature = "tdm-aps")]
cases.push(("tdm-aps", "10.1103/PhysRevX.10.011001"));
#[cfg(feature = "tdm-elsevier")]
cases.push(("tdm-elsevier", "10.1016/j.example.2024.001"));
#[cfg(feature = "tdm-springer")]
cases.push(("tdm-springer", "10.1007/s00220-024-05001-x"));
#[cfg(feature = "tdm-ieee")]
cases.push(("tdm-ieee", "10.1109/TSP.2018.2812747"));
assert!(!cases.is_empty(), "the guard must have checked something");
let (_td, c) = ctx();
let profile = CapabilityProfile::for_tests();
for (name, doi) in cases {
let ref_ = Ref::Doi(Doi::parse(doi).expect("doi"));
let mut attempts = Vec::new();
resolve_tdm_chain(&ref_, &profile, &c, false, &mut attempts).await;
assert!(
attempts.iter().any(|a| a.source == name),
"`{name}` is compiled but absent from the chain for its own DOI {doi}; \
the production path cannot reach it. attempts: {attempts:?}"
);
}
}
}
#[cfg(all(
test,
feature = "tdm-aps",
feature = "tdm-elsevier",
feature = "tdm-springer",
feature = "tdm-ieee"
))]
#[allow(clippy::expect_used, clippy::unwrap_used, clippy::panic)]
mod tdm_chain_tests {
use super::*;
use std::sync::Arc;
use camino::Utf8PathBuf;
use tempfile::TempDir;
use wiremock::matchers::{header, method, path, path_regex};
use wiremock::{Mock, MockServer, ResponseTemplate};
use crate::http::HttpClient;
use crate::provenance::ProvenanceLog;
use crate::rate_limiter::RateLimiter;
use crate::{CapabilityProfile, Doi, RateLimits, Ref, TdmGrant};
struct BaseGuard(Vec<(&'static str, Option<String>)>);
impl BaseGuard {
fn to(uri: &str) -> Self {
const VARS: &[&str] = &[
"DOIGET_APS_BASE",
"DOIGET_ELSEVIER_BASE",
"DOIGET_SPRINGER_BASE",
"DOIGET_IEEE_BASE",
];
Self(
VARS.iter()
.map(|v| {
let old = std::env::var(v).ok();
std::env::set_var(v, uri);
(*v, old)
})
.collect(),
)
}
}
impl Drop for BaseGuard {
fn drop(&mut self) {
for (v, old) in &self.0 {
match old {
Some(o) => std::env::set_var(v, o),
None => std::env::remove_var(v),
}
}
}
}
struct OaBaseGuard(Vec<(&'static str, Option<String>)>);
impl OaBaseGuard {
fn to(uri: &str) -> Self {
const VARS: &[&str] = &[
"DOIGET_CROSSREF_BASE",
"DOIGET_UNPAYWALL_BASE",
"DOIGET_ARXIV_BASE",
];
Self(
VARS.iter()
.map(|v| {
let old = std::env::var(v).ok();
std::env::set_var(v, uri);
(*v, old)
})
.collect(),
)
}
}
impl Drop for OaBaseGuard {
fn drop(&mut self) {
for (v, old) in &self.0 {
match old {
Some(o) => std::env::set_var(v, o),
None => std::env::remove_var(v),
}
}
}
}
fn ctx_for(host: &str) -> (TempDir, FetchContext) {
let td = TempDir::new().expect("tempdir");
let dir = Utf8PathBuf::try_from(td.path().to_path_buf()).expect("utf-8");
let http = Arc::new(HttpClient::new_for_tests_allow_http_multi(&[
("tdm-aps", host),
("tdm-elsevier", host),
("tdm-springer", host),
("tdm-ieee", host),
]));
let session_id = "01J0000000000000000000TDM".to_string();
let log = Arc::new(
ProvenanceLog::open(dir.join("t.jsonl"), session_id.clone()).expect("log opens"),
);
(
td,
FetchContext {
http,
rate_limiter: Arc::new(RateLimiter::new(RateLimits::HARD_CODED)),
log,
session_id,
cache_root: None,
},
)
}
fn grant(agree_var: &str) -> TdmGrant {
TdmGrant {
api_key: secrecy::SecretString::from("test-key".to_string()),
agree_env_var: agree_var.to_string(),
..Default::default()
}
}
fn all_gates_open() -> CapabilityProfile {
let mut p = CapabilityProfile::for_tests();
p.tdm_aps = Some(grant("DOIGET_AGREE_TDM_APS"));
p.tdm_elsevier = Some(grant("DOIGET_AGREE_TDM_ELSEVIER"));
p.tdm_springer = Some(grant("DOIGET_AGREE_TDM_SPRINGER"));
p.tdm_ieee = Some(grant("DOIGET_AGREE_TDM_IEEE"));
p
}
fn all_gates_closed() -> CapabilityProfile {
let mut p = CapabilityProfile::for_tests();
p.tdm_aps = None;
p.tdm_elsevier = None;
p.tdm_springer = None;
p.tdm_ieee = None;
p
}
fn outcome<'a>(attempts: &'a [SourceAttempt], name: &str) -> &'a AttemptOutcome {
&attempts
.iter()
.find(|a| a.source == name)
.unwrap_or_else(|| panic!("no attempt recorded for {name}; got {attempts:?}"))
.outcome
}
#[tokio::test]
#[serial_test::serial]
async fn every_tdm_source_is_reached_for_its_own_publishers_doi() {
for (doi, expected) in [
("10.1103/PhysRevX.10.011001", "tdm-aps"),
("10.1016/j.example.2024.001", "tdm-elsevier"),
("10.1007/s00220-024-05001-x", "tdm-springer"),
("10.1109/TSP.2018.2812747", "tdm-ieee"),
("10.23919/example.2024.001", "tdm-ieee"),
] {
let server = MockServer::start().await;
Mock::given(method("GET"))
.respond_with(ResponseTemplate::new(200).set_body_string("{}"))
.mount(&server)
.await;
let _bases = BaseGuard::to(&server.uri());
let (_td, ctx) = ctx_for(&server.address().to_string());
let ref_ = Ref::Doi(Doi::parse(doi).expect("doi"));
let mut attempts = Vec::new();
resolve_tdm_chain(&ref_, &all_gates_open(), &ctx, false, &mut attempts).await;
let o = outcome(&attempts, expected);
assert!(
o.was_consulted(),
"{expected} was NOT reached for {doi}: {o:?}"
);
assert_eq!(
server.received_requests().await.expect("recorded").len(),
1,
"{expected} must issue exactly one request for {doi}"
);
}
}
#[tokio::test]
#[serial_test::serial]
async fn a_foreign_doi_is_wrong_publisher_not_disabled() {
let server = MockServer::start().await;
let _bases = BaseGuard::to(&server.uri());
let (_td, ctx) = ctx_for(&server.address().to_string());
let ref_ = Ref::Doi(Doi::parse("10.1090/s0025-5718-04-01692-8").expect("doi"));
let mut attempts = Vec::new();
resolve_tdm_chain(&ref_, &all_gates_open(), &ctx, false, &mut attempts).await;
for name in ["tdm-aps", "tdm-elsevier", "tdm-springer", "tdm-ieee"] {
let o = outcome(&attempts, name);
assert!(
matches!(o, AttemptOutcome::WrongPublisher { .. }),
"{name} must be WrongPublisher for an AMS DOI, got {o:?}"
);
assert!(
!o.render().contains("DOIGET_KEY"),
"must not suggest a credential that would not help: {}",
o.render()
);
assert!(
o.render().contains("10.1090"),
"the message must name the prefix that did not match: {}",
o.render()
);
}
assert!(
server
.received_requests()
.await
.expect("recorded")
.is_empty(),
"a foreign DOI must cost the publisher nothing"
);
}
#[tokio::test]
#[serial_test::serial]
async fn closed_gates_name_the_key_and_the_agreement() {
let server = MockServer::start().await;
let _bases = BaseGuard::to(&server.uri());
let (_td, ctx) = ctx_for(&server.address().to_string());
let ref_ = Ref::Doi(Doi::parse("10.1103/PhysRevX.10.011001").expect("doi"));
let mut attempts = Vec::new();
resolve_tdm_chain(&ref_, &all_gates_closed(), &ctx, false, &mut attempts).await;
let o = outcome(&attempts, "tdm-aps");
assert_eq!(
o,
&AttemptOutcome::Disabled {
env: &["DOIGET_KEY_APS", "DOIGET_AGREE_TDM_APS"]
},
"a closed Tier-3 gate must name the key AND the agreement"
);
assert!(!o.was_consulted());
assert!(
server
.received_requests()
.await
.expect("recorded")
.is_empty(),
"closed gates must make NO request"
);
}
#[tokio::test]
#[serial_test::serial]
async fn a_crossref_hit_skips_the_tdm_chain() {
let server = MockServer::start().await;
let _bases = BaseGuard::to(&server.uri());
let (_td, ctx) = ctx_for(&server.address().to_string());
let ref_ = Ref::Doi(Doi::parse("10.1103/PhysRevX.10.011001").expect("doi"));
let mut attempts = Vec::new();
resolve_tdm_chain(&ref_, &all_gates_open(), &ctx, true, &mut attempts).await;
for name in ["tdm-aps", "tdm-elsevier", "tdm-springer", "tdm-ieee"] {
assert_eq!(
outcome(&attempts, name),
&AttemptOutcome::NotNeeded,
"{name} must be NotNeeded -- the gates ARE open"
);
}
assert!(server
.received_requests()
.await
.expect("recorded")
.is_empty());
}
#[tokio::test]
#[serial_test::serial]
async fn fetch_paper_actually_reaches_the_tdm_chain() {
let server = MockServer::start().await;
Mock::given(method("GET"))
.respond_with(ResponseTemplate::new(404))
.mount(&server)
.await;
let _bases = BaseGuard::to(&server.uri());
let _oa = OaBaseGuard::to(&server.uri());
let (_td, ctx) = ctx_for(&server.address().to_string());
let store_td = TempDir::new().expect("tempdir");
let root = Utf8PathBuf::try_from(store_td.path().to_path_buf()).expect("utf-8");
let store = crate::store::FsStore::new(root.clone()).expect("store");
let ref_ = Ref::Doi(Doi::parse("10.1103/PhysRevX.10.011001").expect("doi"));
let err = fetch_paper(&ref_, &all_gates_open(), &ctx, &store, &root)
.await
.expect_err("everything 404s, so the fetch must fail");
let paths: Vec<String> = server
.received_requests()
.await
.expect("recorded")
.iter()
.map(|r| r.url.path().to_string())
.collect();
const APS_DOCUMENTED_PREFIX: &str = "/v2/journals/articles/";
assert!(
paths.iter().any(|p| p.contains(APS_DOCUMENTED_PREFIX)),
"fetch_paper never reached tdm-aps at its documented endpoint; paths were {paths:?}"
);
let hint = err.to_string();
assert!(
hint.contains("tdm-aps") && hint.contains("consulted:"),
"the trace must record tdm-aps as consulted; got:
{hint}"
);
}
fn ctx_for_content(host: &str) -> (TempDir, FetchContext) {
let td = TempDir::new().expect("tempdir");
let dir = Utf8PathBuf::try_from(td.path().to_path_buf()).expect("utf-8");
let http = Arc::new(HttpClient::new_for_tests_allow_http_multi(&[
("crossref", host),
("unpaywall", host),
("oa-publisher", host),
("tdm-aps", host),
("tdm-elsevier", host),
("tdm-springer", host),
("tdm-ieee", host),
]));
let session_id = "01J0000000000000000000CNT".to_string();
let log = Arc::new(
ProvenanceLog::open(dir.join("t.jsonl"), session_id.clone()).expect("log opens"),
);
(
td,
FetchContext {
http,
rate_limiter: Arc::new(RateLimiter::new(RateLimits::HARD_CODED)),
log,
session_id,
cache_root: None,
},
)
}
fn crossref_body() -> serde_json::Value {
serde_json::json!({
"status": "ok",
"message": {
"title": ["A paper APS published"],
"author": [{ "family": "Doe", "given": "Jane" }],
"issued": { "date-parts": [[2026, 1, 1]] },
"container-title": ["Physical Review X"],
"type": "journal-article"
}
})
}
fn unpaywall_body(oa_url: &str) -> serde_json::Value {
serde_json::json!({
"doi": "10.1103/PhysRevX.10.011001",
"is_oa": true,
"title": "A paper APS published",
"best_oa_location": {
"url": oa_url,
"url_for_pdf": oa_url,
"license": "cc-by"
}
})
}
const PDF_BYTES: &[u8] = b"%PDF-1.7\nthe publisher's own copy\n%%EOF\n";
async fn mount_oa_blocked(server: &MockServer, aps: ResponseTemplate) {
let oa_url = format!("{}/oa/file.pdf", server.uri());
Mock::given(method("GET"))
.and(path_regex("^/works/"))
.respond_with(ResponseTemplate::new(200).set_body_json(crossref_body()))
.mount(server)
.await;
Mock::given(method("GET"))
.and(path_regex("^/v2/journals/articles/"))
.and(header("accept", "application/pdf"))
.respond_with(aps)
.mount(server)
.await;
Mock::given(method("GET"))
.and(path("/oa/file.pdf"))
.respond_with(ResponseTemplate::new(403))
.mount(server)
.await;
Mock::given(method("GET"))
.respond_with(ResponseTemplate::new(200).set_body_json(unpaywall_body(&oa_url)))
.mount(server)
.await;
}
fn aps_pdf_requests(reqs: &[wiremock::Request]) -> usize {
reqs.iter()
.filter(|r| {
r.url.path().starts_with("/v2/journals/articles/")
&& r.headers
.get("accept")
.and_then(|v| v.to_str().ok())
.is_some_and(|v| v.contains("application/pdf"))
})
.count()
}
#[tokio::test]
#[serial_test::serial]
async fn tdm_content_leg_serves_the_pdf_when_the_oa_route_is_blocked() {
let server = MockServer::start().await;
mount_oa_blocked(
&server,
ResponseTemplate::new(200).set_body_bytes(PDF_BYTES),
)
.await;
let _bases = BaseGuard::to(&server.uri());
let _oa = OaBaseGuard::to(&server.uri());
let (_td, ctx) = ctx_for_content(&server.address().to_string());
let store_td = TempDir::new().expect("tempdir");
let root = Utf8PathBuf::try_from(store_td.path().to_path_buf()).expect("utf-8");
let store = crate::store::FsStore::new(root.clone()).expect("store");
let ref_ = Ref::Doi(Doi::parse("10.1103/PhysRevX.10.011001").expect("doi"));
let outcome = fetch_paper(&ref_, &all_gates_open(), &ctx, &store, &root)
.await
.expect("the TDM content leg should have supplied the PDF");
let reqs = server.received_requests().await.expect("recorded");
assert_eq!(
aps_pdf_requests(&reqs),
1,
"expected exactly one Accept: application/pdf request to the APS article endpoint; \
paths were {:?}",
reqs.iter().map(|r| r.url.path()).collect::<Vec<_>>()
);
match &outcome.pdf_leg {
PdfLegStatus::TdmFetched {
source,
original_block,
} => {
assert_eq!(source, "tdm-aps");
assert!(
!original_block.is_empty(),
"the OA refusal must be carried forward, not discarded"
);
}
other => panic!("expected TdmFetched, got {other:?}"),
}
assert_eq!(outcome.source, "tdm-aps");
assert_eq!(outcome.size_bytes, PDF_BYTES.len() as u64);
assert_eq!(
outcome.license, "unknown",
"a TDM-retrieved copy must not inherit the OA location's licence"
);
}
#[tokio::test]
#[serial_test::serial]
async fn tdm_content_leg_is_not_consulted_for_another_publishers_doi() {
let server = MockServer::start().await;
mount_oa_blocked(
&server,
ResponseTemplate::new(200).set_body_bytes(PDF_BYTES),
)
.await;
let _bases = BaseGuard::to(&server.uri());
let _oa = OaBaseGuard::to(&server.uri());
let (_td, ctx) = ctx_for_content(&server.address().to_string());
let store_td = TempDir::new().expect("tempdir");
let root = Utf8PathBuf::try_from(store_td.path().to_path_buf()).expect("utf-8");
let store = crate::store::FsStore::new(root.clone()).expect("store");
let ref_ = Ref::Doi(Doi::parse("10.1016/j.physrep.2020.01.001").expect("doi"));
let _ = fetch_paper(&ref_, &all_gates_open(), &ctx, &store, &root).await;
let reqs = server.received_requests().await.expect("recorded");
assert_eq!(
aps_pdf_requests(&reqs),
0,
"an Elsevier DOI reached the APS content endpoint; paths were {:?}",
reqs.iter().map(|r| r.url.path()).collect::<Vec<_>>()
);
}
#[tokio::test]
#[serial_test::serial]
async fn content_leg_reports_a_foreign_doi_as_wrong_publisher_not_disabled() {
let server = MockServer::start().await;
let _bases = BaseGuard::to(&server.uri());
let (_td, ctx) = ctx_for_content(&server.address().to_string());
let doi = Doi::parse("10.1016/j.physrep.2020.01.001").expect("doi");
let blocked = PdfLegStatus::Blocked {
code: crate::ErrorCode::NetworkError,
message: "the open route refused us".to_string(),
denial: None,
suggested_arxiv_id: None,
};
let mut attempts: Vec<SourceAttempt> = Vec::new();
let (leg, bytes) =
try_tdm_content_fallback(&doi, blocked, None, &all_gates_open(), &ctx, &mut attempts)
.await;
assert!(bytes.is_none(), "no publisher owns this DOI here");
assert!(matches!(leg, PdfLegStatus::Blocked { .. }));
assert!(
matches!(outcome(&attempts, "tdm-aps"), AttemptOutcome::WrongPublisher { .. }),
"an Elsevier DOI must read as WrongPublisher for tdm-aps, not Disabled; got {attempts:?}"
);
}
#[tokio::test]
#[serial_test::serial]
async fn tdm_content_leg_rejects_a_non_pdf_body_and_keeps_the_original_block() {
let server = MockServer::start().await;
mount_oa_blocked(
&server,
ResponseTemplate::new(200).set_body_string("<html>Access denied</html>"),
)
.await;
let _bases = BaseGuard::to(&server.uri());
let _oa = OaBaseGuard::to(&server.uri());
let (_td, ctx) = ctx_for_content(&server.address().to_string());
let store_td = TempDir::new().expect("tempdir");
let root = Utf8PathBuf::try_from(store_td.path().to_path_buf()).expect("utf-8");
let store = crate::store::FsStore::new(root.clone()).expect("store");
let ref_ = Ref::Doi(Doi::parse("10.1103/PhysRevX.10.011001").expect("doi"));
let outcome = fetch_paper(&ref_, &all_gates_open(), &ctx, &store, &root)
.await
.expect("a metadata-only outcome is still an outcome");
let reqs = server.received_requests().await.expect("recorded");
assert_eq!(aps_pdf_requests(&reqs), 1);
match &outcome.pdf_leg {
PdfLegStatus::Blocked { message, .. } => {
assert!(
!message.is_empty(),
"the ORIGINAL OA refusal must survive, not the TDM failure"
);
}
other => panic!("expected the original Blocked leg to survive, got {other:?}"),
}
assert_eq!(outcome.size_bytes, 0, "nothing should have been stored");
}
}
#[cfg(all(test, feature = "metadata"))]
#[allow(clippy::expect_used, clippy::unwrap_used, clippy::panic)]
mod oa_fallthrough_tests {
use super::*;
use std::sync::Arc;
use camino::Utf8PathBuf;
use tempfile::TempDir;
use wiremock::matchers::{path, path_regex};
use wiremock::{Mock, MockServer, ResponseTemplate};
use crate::http::HttpClient;
use crate::provenance::ProvenanceLog;
use crate::rate_limiter::RateLimiter;
use crate::{CapabilityProfile, Doi, RateLimits, Ref};
struct EnvSet(Vec<(&'static str, Option<String>)>);
impl EnvSet {
fn new(pairs: &[(&'static str, String)]) -> Self {
Self(
pairs
.iter()
.map(|(k, v)| {
let old = std::env::var(k).ok();
std::env::set_var(k, v);
(*k, old)
})
.collect(),
)
}
}
impl Drop for EnvSet {
fn drop(&mut self) {
for (k, old) in &self.0 {
match old {
Some(v) => std::env::set_var(k, v),
None => std::env::remove_var(k),
}
}
}
}
async fn server_with_a_rate_limited_publisher_and_a_repository_copy() -> MockServer {
let server = MockServer::start().await;
let base = server.uri();
Mock::given(path_regex("^/works/"))
.respond_with(ResponseTemplate::new(200).set_body_string(
"{\"status\":\"ok\",\"message\":{\"title\":[\"Computing multiple roots\"]}}",
))
.mount(&server)
.await;
Mock::given(path_regex("^/10\\.1090"))
.respond_with(ResponseTemplate::new(200).set_body_string(format!(
"{{\"doi\":\"10.1090/example\",\"is_oa\":true,\"oa_status\":\"bronze\",\"best_oa_location\":\
{{\"url_for_pdf\":\"{base}/blocked.pdf\",\"license\":\"cc-by\"}}}}"
)))
.mount(&server)
.await;
Mock::given(path("/blocked.pdf"))
.respond_with(ResponseTemplate::new(429))
.mount(&server)
.await;
Mock::given(path("/v3/search/works"))
.respond_with(ResponseTemplate::new(200).set_body_string(format!(
"{{\"totalHits\":1,\"results\":[{{\"id\":1,\
\"title\":\"Computing multiple roots\",\"doi\":\"10.1090/example\",\
\"downloadUrl\":\"{base}/repo.pdf\"}}]}}"
)))
.mount(&server)
.await;
Mock::given(path("/repo.pdf"))
.respond_with(
ResponseTemplate::new(200).set_body_bytes(b"%PDF-1.4\nrepository copy\n".to_vec()),
)
.mount(&server)
.await;
server
}
fn ctx_for(host: &str) -> (TempDir, FetchContext) {
let td = TempDir::new().expect("tempdir");
let dir = Utf8PathBuf::try_from(td.path().to_path_buf()).expect("utf-8");
let host_only = host.split(':').next().unwrap_or(host);
let http = Arc::new(HttpClient::new_for_tests_allow_http_multi(&[
("crossref", host),
("unpaywall", host),
("oa-publisher", host_only),
("core", host),
]));
let session_id = "01J000000000000000000FALL".to_string();
let log = Arc::new(
ProvenanceLog::open(dir.join("t.jsonl"), session_id.clone()).expect("log opens"),
);
(
td,
FetchContext {
http,
rate_limiter: Arc::new(RateLimiter::new(RateLimits::HARD_CODED)),
log,
session_id,
cache_root: None,
},
)
}
async fn run_fetch(
server: &MockServer,
core_enabled: bool,
) -> (FetchPaperOutcome, Vec<String>) {
let base = server.uri();
let mut env = vec![
("DOIGET_CROSSREF_BASE", base.clone()),
("DOIGET_UNPAYWALL_BASE", base.clone()),
("DOIGET_ARXIV_BASE", base.clone()),
("DOIGET_CORE_BASE", base.clone()),
("DOIGET_CONTACT_EMAIL", "test@example.org".to_string()),
];
if core_enabled {
env.push(("DOIGET_ENABLE_CORE", "1".to_string()));
} else {
std::env::remove_var("DOIGET_ENABLE_CORE");
}
let _env = EnvSet::new(&env);
let profile = CapabilityProfile::from_env().expect("profile");
let (_td, ctx) = ctx_for(&server.address().to_string());
let store_td = TempDir::new().expect("tempdir");
let root = Utf8PathBuf::try_from(store_td.path().to_path_buf()).expect("utf-8");
let store = crate::store::FsStore::new(root.clone()).expect("store");
let ref_ = Ref::Doi(Doi::parse("10.1090/example").expect("doi"));
let outcome = fetch_paper(&ref_, &profile, &ctx, &store, &root)
.await
.expect("crossref answered, so the fetch resolves either way");
let paths = server
.received_requests()
.await
.expect("recorded")
.iter()
.map(|r| r.url.path().to_string())
.collect();
(outcome, paths)
}
#[tokio::test]
#[serial_test::serial]
async fn a_rate_limited_publisher_falls_through_to_an_enabled_source() {
let server = server_with_a_rate_limited_publisher_and_a_repository_copy().await;
let (outcome, paths) = run_fetch(&server, true).await;
assert!(
paths.iter().any(|p| p == "/v3/search/works"),
"CORE was never consulted; paths were {paths:?}"
);
assert!(
paths.iter().any(|p| p == "/repo.pdf"),
"the copy CORE reported was never fetched; paths were {paths:?}; leg={:?}",
outcome.pdf_leg
);
assert!(
matches!(outcome.pdf_leg, PdfLegStatus::Fetched),
"the run should have recovered; got {:?}",
outcome.pdf_leg
);
}
#[tokio::test]
#[serial_test::serial]
async fn with_no_source_enabled_the_run_is_unchanged() {
let server = server_with_a_rate_limited_publisher_and_a_repository_copy().await;
let (outcome, paths) = run_fetch(&server, false).await;
assert!(
!paths.iter().any(|p| p == "/v3/search/works"),
"a disabled source must cost nothing; paths were {paths:?}"
);
assert!(
matches!(outcome.pdf_leg, PdfLegStatus::Blocked { .. }),
"without a fallback source this must still be Blocked; got {:?}",
outcome.pdf_leg
);
}
#[cfg(feature = "tdm-ieee")]
#[tokio::test]
#[serial_test::serial]
async fn the_fallback_preserves_the_tier_3_rows_it_used_to_delete() {
let server = server_with_a_rate_limited_publisher_and_a_repository_copy().await;
let (outcome, paths) = run_fetch(&server, true).await;
assert!(
paths.iter().any(|p| p == "/v3/search/works"),
"the fallback did not run, so this test cannot see the bug; paths were {paths:?}"
);
let sources: Vec<&str> = outcome.attempts.iter().map(|a| a.source).collect();
assert!(
sources.contains(&"tdm-ieee"),
"the Tier-3 row was dropped from the trace by the fallback; trace held {sources:?}"
);
assert!(
sources.contains(&"core"),
"the refreshed Tier-2 rows are missing; trace held {sources:?}"
);
}
}