use std::sync::Arc;
use anyhow::{anyhow, Context, Result};
use camino::{Utf8Path, Utf8PathBuf};
use super::output::print_err;
#[cfg(feature = "citation")]
use doiget_core::http::tier_2_allowlist;
use doiget_core::http::{
discovery_allowlist, fulltext_allowlist, oa_publisher_allowlist, tier_1_allowlist,
tier_3_allowlists, HttpClient,
};
use doiget_core::orchestrator::{
fetch_paper as core_fetch_paper, FetchPaperOutcome, PdfLegStatus, SourceAttempt,
};
use doiget_core::provenance::{Capability, LogEvent, LogResult, ProvenanceLog, RowInput};
use doiget_core::rate_limiter::RateLimiter;
use doiget_core::source::{FetchContext, FetchError};
use doiget_core::store::FsStore;
use doiget_core::{CapabilityProfile, DenialContext, DenialReason, ErrorCode, RateLimits, Ref};
pub(crate) fn new_session_id() -> String {
ulid::Ulid::generate().to_string()
}
pub use doiget_core::dry_run::{
build_dry_run_envelope, build_fetch_plan, FetchPlan, PdfSourcePlan, RateLimitBudget,
};
#[allow(clippy::print_stdout)]
pub fn emit_dry_run_plan_to_stdout(ref_: &Ref, plan: &FetchPlan) -> Result<()> {
let envelope = build_dry_run_envelope(ref_, plan);
let s = serde_json::to_string(&envelope).context("serializing dry-run envelope to JSON")?;
println!("{s}");
Ok(())
}
pub(crate) fn resolve_log_path() -> Result<Utf8PathBuf> {
if let Some(s) = read_env_utf8("DOIGET_LOG_PATH")? {
return Ok(Utf8PathBuf::from(s));
}
let cfg = config_dir_utf8()?;
Ok(cfg.join("doiget").join("access.jsonl"))
}
fn read_env_utf8(key: &str) -> Result<Option<String>> {
match std::env::var(key) {
Ok(s) => Ok(Some(s)),
Err(std::env::VarError::NotPresent) => Ok(None),
Err(std::env::VarError::NotUnicode(_)) => Err(anyhow!("{key} is not valid UTF-8")),
}
}
fn home_dir_utf8() -> Result<Utf8PathBuf> {
if let Some(s) = read_env_utf8("HOME")? {
return Ok(Utf8PathBuf::from(s));
}
if let Some(s) = read_env_utf8("USERPROFILE")? {
return Ok(Utf8PathBuf::from(s));
}
Err(anyhow!("neither HOME nor USERPROFILE is set"))
}
pub(crate) fn config_dir_utf8() -> Result<Utf8PathBuf> {
if let Some(s) = read_env_utf8("XDG_CONFIG_HOME")? {
return Ok(Utf8PathBuf::from(s));
}
if let Some(s) = read_env_utf8("APPDATA")? {
return Ok(Utf8PathBuf::from(s));
}
let home = home_dir_utf8()?;
Ok(home.join(".config"))
}
pub(crate) fn cache_dir_utf8() -> Result<Utf8PathBuf> {
if let Some(s) = read_env_utf8("DOIGET_CACHE_ROOT")? {
return Ok(Utf8PathBuf::from(s));
}
if let Some(s) = read_env_utf8("XDG_CACHE_HOME")? {
return Ok(Utf8PathBuf::from(s).join("doiget"));
}
if let Some(s) = read_env_utf8("LOCALAPPDATA")? {
return Ok(Utf8PathBuf::from(s).join("doiget").join("cache"));
}
let home = home_dir_utf8()?;
Ok(home.join(".cache").join("doiget"))
}
pub(crate) fn build_resolve_context() -> Result<FetchContext> {
let session_id = new_session_id();
let log_path = resolve_log_path()?;
let http = Arc::new(build_http_client(None)?);
let rate_limiter = Arc::new(RateLimiter::new(RateLimits::HARD_CODED));
let log = Arc::new(
ProvenanceLog::open(log_path, session_id.clone())
.context("failed to open provenance log")?,
);
let cache_root = cache_dir_utf8().ok();
Ok(FetchContext {
http,
rate_limiter,
log,
session_id,
cache_root,
})
}
pub(crate) fn build_http_client(user_agent: Option<&str>) -> Result<HttpClient> {
let arxiv = std::env::var("DOIGET_ARXIV_BASE").ok();
let crossref = std::env::var("DOIGET_CROSSREF_BASE").ok();
let unpaywall = std::env::var("DOIGET_UNPAYWALL_BASE").ok();
let oa_publisher = std::env::var("DOIGET_OA_PUBLISHER_BASE").ok();
let openalex_base = std::env::var("DOIGET_OPENALEX_BASE").ok();
let ar5iv_base = std::env::var("DOIGET_AR5IV_BASE").ok();
if arxiv.is_none()
&& crossref.is_none()
&& unpaywall.is_none()
&& oa_publisher.is_none()
&& openalex_base.is_none()
&& ar5iv_base.is_none()
{
let mut allowlists = tier_1_allowlist();
allowlists.extend(oa_publisher_allowlist());
allowlists.extend(discovery_allowlist());
allowlists.extend(fulltext_allowlist());
#[cfg(feature = "citation")]
allowlists.extend(tier_2_allowlist());
allowlists.extend(tier_3_allowlists());
match config_dir_utf8() {
Ok(cfg_dir) => {
let path = cfg_dir.join("doiget").join("config.toml");
match doiget_core::user_extension::load(&path) {
Ok(cfg) => {
let mut hosts = cfg.additional_hosts;
if cfg.trust_academic_repos {
hosts.extend(doiget_core::user_extension::academic_repo_hosts());
}
if cfg.trust_oa_registries {
hosts.extend(doiget_core::user_extension::oa_registry_hosts());
}
if !hosts.is_empty() {
tracing::info!(
count = hosts.len(),
trust_academic_repos = cfg.trust_academic_repos,
trust_oa_registries = cfg.trust_oa_registries,
path = %path,
"merging user-extension allowlist hosts (ADR-0028 D2)"
);
doiget_core::user_extension::merge_into_allowlists(
&mut allowlists,
&hosts,
);
}
}
Err(e) => {
tracing::warn!(
error = %e,
path = %path,
"failed to load user-extension allowlist; \
falling back to curated set only"
);
}
}
}
Err(e) => {
tracing::debug!(
error = %e,
"config dir unresolvable; \
user-extension allowlist disabled (curated set only)"
);
}
}
return match user_agent {
Some(ua) => HttpClient::new_with_user_agent(allowlists, ua),
None => HttpClient::new(allowlists),
}
.context("building HTTP client");
}
let mut owned: Vec<(String, String)> = Vec::new();
for (source, base) in [
("arxiv", arxiv.as_deref()),
("crossref", crossref.as_deref()),
("unpaywall", unpaywall.as_deref()),
("oa-publisher", oa_publisher.as_deref()),
("openalex", openalex_base.as_deref()),
("ar5iv", ar5iv_base.as_deref()),
] {
if let Some(b) = base {
let url = url::Url::parse(b)
.with_context(|| format!("DOIGET_*_BASE for {source} is not a URL: {b}"))?;
let host = url
.host_str()
.ok_or_else(|| anyhow!("base URL has no host: {b}"))?;
owned.push((source.to_string(), host.to_string()));
}
}
let entries: Vec<(&str, &str)> = owned
.iter()
.map(|(s, h)| (s.as_str(), h.as_str()))
.collect();
Ok(HttpClient::new_for_tests_allow_http_multi(&entries))
}
#[allow(dead_code)]
pub(crate) struct OrchestratorConfig {
pub(crate) store_root: Utf8PathBuf,
pub(crate) log_path: Utf8PathBuf,
pub(crate) contact_email: String,
pub(crate) unpaywall_email: String,
}
impl OrchestratorConfig {
fn from_env() -> Result<Self> {
let store_root = super::resolve_store_root()?;
let log_path = resolve_log_path()?;
let contact_email =
std::env::var("DOIGET_CONTACT_EMAIL").unwrap_or_else(|_| "doiget@localhost".into());
let unpaywall_email =
std::env::var("DOIGET_UNPAYWALL_EMAIL").unwrap_or_else(|_| contact_email.clone());
Ok(Self {
store_root,
log_path,
contact_email,
unpaywall_email,
})
}
}
pub(crate) struct FetchHarness {
pub(crate) http: Arc<HttpClient>,
pub(crate) rate_limiter: Arc<RateLimiter>,
pub(crate) log: Arc<ProvenanceLog>,
pub(crate) store: FsStore,
pub(crate) profile: CapabilityProfile,
pub(crate) session_id: String,
#[allow(dead_code)]
pub(crate) cfg: OrchestratorConfig,
}
impl FetchHarness {
pub(crate) fn from_env() -> Result<Self> {
Self::from_env_with_ua(None)
}
pub(crate) fn from_env_with_ua(user_agent: Option<&str>) -> Result<Self> {
let cfg = OrchestratorConfig::from_env()?;
if let Some(parent) = cfg.log_path.parent() {
if !parent.as_str().is_empty() {
std::fs::create_dir_all(parent.as_std_path())
.with_context(|| format!("creating log dir {parent}"))?;
}
}
let session_id = new_session_id();
let log = Arc::new(
ProvenanceLog::open(cfg.log_path.clone(), session_id.clone())
.context("opening provenance log")?,
);
let http = Arc::new(build_http_client(user_agent)?);
let rate_limiter = Arc::new(RateLimiter::new(RateLimits::HARD_CODED));
let store = FsStore::new(cfg.store_root.clone()).context("opening store")?;
let profile = CapabilityProfile::from_env().context("resolving capability profile")?;
Ok(Self {
http,
rate_limiter,
log,
store,
profile,
session_id,
cfg,
})
}
pub(crate) fn fetch_context(&self) -> FetchContext {
FetchContext {
http: self.http.clone(),
rate_limiter: self.rate_limiter.clone(),
log: self.log.clone(),
session_id: self.session_id.clone(),
cache_root: None,
}
}
pub(crate) fn log_session_start(&self, ref_input: Option<&str>) -> Result<()> {
self.log
.append(RowInput {
event: LogEvent::SessionStart,
result: LogResult::Ok,
capability: Capability::Oa,
ref_: ref_input,
source: None,
error_code: None,
size_bytes: None,
license: None,
store_path: None,
canonical_digest: None,
})
.context("appending SessionStart row")?;
Ok(())
}
pub(crate) fn log_session_end(&self, ok: bool, ref_input: Option<&str>) {
let result = if ok { LogResult::Ok } else { LogResult::Err };
let _ = self.log.append(RowInput {
event: LogEvent::SessionEnd,
result,
capability: Capability::Oa,
ref_: ref_input,
source: None,
error_code: None,
size_bytes: None,
license: None,
store_path: None,
canonical_digest: None,
});
}
pub(crate) async fn fetch_one(&self, ref_: &Ref) -> Result<FetchPaperOutcome, FetchError> {
let ctx = self.fetch_context();
core_fetch_paper(ref_, &self.profile, &ctx, &self.store, self.store.root()).await
}
}
pub(crate) fn outcome_is_clean_success(outcome: &FetchPaperOutcome) -> bool {
!matches!(outcome.pdf_leg, PdfLegStatus::Blocked { .. })
}
fn emit_success_line(ref_: &Ref, outcome: &FetchPaperOutcome) {
let label = match ref_ {
Ref::Arxiv(id) => format!("arxiv:{}", id.as_str()),
Ref::Doi(doi) => format!("doi:{}", doi.as_str()),
};
match &outcome.pdf_leg {
PdfLegStatus::Fetched => {
print_success(format_args!(
"fetched {} ({} bytes) -> {}",
label, outcome.size_bytes, outcome.path
));
}
PdfLegStatus::NoOaUrl => {
print_success(format_args!(
"fetched {} (metadata-only: no OA PDF available) -> {}",
label, outcome.path
));
}
PdfLegStatus::PreprintFallback { arxiv_id, .. } => {
print_success(format_args!(
"fetched {} ({} bytes) via arXiv preprint arxiv:{} -> {}",
label, outcome.size_bytes, arxiv_id, outcome.path
));
}
PdfLegStatus::Blocked {
code,
message,
denial,
suggested_arxiv_id,
} => {
let effective = effective_blocked_code(*code, denial.as_ref());
render_blocked_error(
ref_,
outcome,
effective,
message,
denial.as_ref(),
suggested_arxiv_id.as_deref(),
);
}
_ => {
if outcome.size_bytes == 0 {
print_success(format_args!(
"fetched {} (metadata-only) -> {}",
label, outcome.path
));
} else {
print_success(format_args!(
"fetched {} ({} bytes) -> {}",
label, outcome.size_bytes, outcome.path
));
}
}
}
if !matches!(outcome.pdf_leg, PdfLegStatus::Blocked { .. }) {
emit_identity_line(outcome);
}
}
fn emit_identity_line(outcome: &FetchPaperOutcome) {
let by = match outcome.authors.as_slice() {
[] => String::new(),
[a] => format!(" by {a}"),
[a, ..] => format!(" by {a} et al."),
};
let year = match outcome.year {
Some(y) => format!(" ({y})"),
None => String::new(),
};
let oa = outcome.oa_status.as_deref().unwrap_or("?");
print_success(format_args!(
" \"{}\"{}{} [{}/{}]",
outcome.title, by, year, outcome.source, oa
));
}
pub async fn run_with_options(
input: String,
dry_run: bool,
link: Option<Utf8PathBuf>,
_mode: super::output::OutputMode,
) -> Result<()> {
let ref_ = match Ref::parse(&input) {
Ok(r) => r,
Err(e) => {
print_err(format_args!(
"error[{}]: invalid ref: {e}",
ErrorCode::InvalidRef.as_wire()
));
return Err(anyhow::Error::new(CliExit(cli_exit_code(
ErrorCode::InvalidRef,
))));
}
};
if dry_run {
let store_root = super::resolve_store_root()?;
let plan = build_fetch_plan(&ref_, &store_root);
emit_dry_run_plan_to_stdout(&ref_, &plan)?;
return Ok(());
}
let harness = FetchHarness::from_env()?;
harness.log_session_start(Some(ref_.as_input_str()))?;
let result = harness.fetch_one(&ref_).await;
let session_ok = match &result {
Ok(o) => outcome_is_clean_success(o),
Err(_) => false,
};
harness.log_session_end(session_ok, Some(ref_.as_input_str()));
match result {
Ok(outcome) => {
if let PdfLegStatus::Blocked {
code,
message,
denial,
suggested_arxiv_id,
} = &outcome.pdf_leg
{
let effective = effective_blocked_code(*code, denial.as_ref());
render_blocked_error(
&ref_,
&outcome,
effective,
message,
denial.as_ref(),
suggested_arxiv_id.as_deref(),
);
return Err(anyhow::Error::new(CliExit(cli_exit_code(effective))));
}
emit_success_line(&ref_, &outcome);
if let Some(dir) = link.as_deref() {
emit_link_result(&ref_, &outcome, dir);
}
Ok(())
}
Err(e) => {
render_fetch_error(&e);
let code: ErrorCode = (&e).into();
Err(anyhow::Error::new(CliExit(cli_exit_code(code))))
}
}
}
fn emit_link_result(ref_: &Ref, outcome: &FetchPaperOutcome, dir: &Utf8Path) {
let label = match ref_ {
Ref::Arxiv(id) => format!("arxiv:{}", id.as_str()),
Ref::Doi(doi) => format!("doi:{}", doi.as_str()),
};
if !matches!(
outcome.pdf_leg,
PdfLegStatus::Fetched | PdfLegStatus::PreprintFallback { .. }
) {
print_success(format_args!(
"note: --link skipped for {label} (no PDF — metadata-only fetch)"
));
return;
}
let name = fetch_link_filename(
&outcome.title,
&outcome.authors,
outcome.year,
&outcome.safekey,
);
match link_artifact(dir, &outcome.path, &name) {
Ok((path, kind)) => print_success(format_args!("linked {label} -> {path} ({kind})")),
Err(e) => print_err(format_args!("warning: --link failed for {label}: {e}")),
}
}
fn fetch_link_filename(
title: &str,
authors: &[String],
year: Option<i32>,
safekey: &str,
) -> String {
let surname = authors
.first()
.map(|a| slugify(a.split_whitespace().last().unwrap_or(a)))
.unwrap_or_default();
let year = year.map(|y| y.to_string()).unwrap_or_default();
let title_slug: String = slugify(title)
.split('-')
.take(6)
.collect::<Vec<_>>()
.join("-");
let mut stem = format!("{surname}{year}");
if !stem.is_empty() && !title_slug.is_empty() {
stem.push('-');
}
stem.push_str(&title_slug);
let stem: String = stem.chars().take(80).collect();
let stem = stem.trim_matches('-');
if stem.is_empty() {
format!("{safekey}.pdf")
} else {
format!("{stem}.pdf")
}
}
fn slugify(s: &str) -> String {
s.chars()
.map(|c| {
if c.is_ascii_alphanumeric() {
c.to_ascii_lowercase()
} else {
'-'
}
})
.collect::<String>()
.split('-')
.filter(|p| !p.is_empty())
.collect::<Vec<_>>()
.join("-")
}
fn link_artifact(
dir: &Utf8Path,
src: &Utf8Path,
name: &str,
) -> Result<(Utf8PathBuf, &'static str)> {
std::fs::create_dir_all(dir.as_std_path())
.with_context(|| format!("creating link dir {dir}"))?;
let dst = dir.join(name);
if let Ok(meta) = std::fs::symlink_metadata(dst.as_std_path()) {
if meta.file_type().is_symlink() {
std::fs::remove_file(dst.as_std_path())
.with_context(|| format!("replacing existing symlink {dst}"))?;
} else {
anyhow::bail!(
"refusing to overwrite existing file {dst} (not a doiget symlink) — \
remove it or choose another --link dir"
);
}
}
match make_symlink(src, &dst) {
Ok(()) => Ok((dst, "symlink")),
Err(_) => {
std::fs::copy(src.as_std_path(), dst.as_std_path())
.with_context(|| format!("copying {src} -> {dst}"))?;
Ok((dst, "copy"))
}
}
}
#[cfg(unix)]
fn make_symlink(src: &Utf8Path, dst: &Utf8Path) -> std::io::Result<()> {
std::os::unix::fs::symlink(src.as_std_path(), dst.as_std_path())
}
#[cfg(windows)]
fn make_symlink(src: &Utf8Path, dst: &Utf8Path) -> std::io::Result<()> {
std::os::windows::fs::symlink_file(src.as_std_path(), dst.as_std_path())
}
#[cfg(not(any(unix, windows)))]
fn make_symlink(_src: &Utf8Path, _dst: &Utf8Path) -> std::io::Result<()> {
Err(std::io::Error::new(
std::io::ErrorKind::Unsupported,
"symlinks unsupported on this platform",
))
}
#[allow(clippy::print_stderr)]
fn print_success(args: std::fmt::Arguments<'_>) {
eprintln!("{args}");
}
#[derive(Debug)]
pub struct CliExit(pub i32);
impl std::fmt::Display for CliExit {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "exiting with status {}", self.0)
}
}
impl std::error::Error for CliExit {}
pub(crate) fn effective_blocked_code(code: ErrorCode, denial: Option<&DenialContext>) -> ErrorCode {
match denial.map(|d| d.reason) {
Some(
DenialReason::RedirectNotInAllowlist
| DenialReason::InsecureScheme
| DenialReason::HostInBlockList,
) => ErrorCode::CapabilityDenied,
_ => code,
}
}
fn denial_reason_wire(reason: DenialReason) -> &'static str {
match reason {
DenialReason::RedirectNotInAllowlist => "redirect_not_in_allowlist",
DenialReason::InsecureScheme => "insecure_scheme",
DenialReason::HostInBlockList => "host_in_block_list",
_ => "policy_denied",
}
}
pub(crate) fn cli_exit_code(code: ErrorCode) -> i32 {
match code {
ErrorCode::CapabilityDenied => 3,
ErrorCode::StoreError | ErrorCode::LogError => 4,
ErrorCode::FetchTimeout => 124,
ErrorCode::Ambiguous => 2,
_ => 1,
}
}
fn denial_note_lines(dc: &DenialContext, config_path: Option<&camino::Utf8Path>) -> Vec<String> {
let attempted = dc.attempted.as_deref().unwrap_or("(unknown)");
let mut out = vec![match &dc.expected {
Some(exp) if !exp.is_empty() => {
format!(
" = note: attempted {attempted}; allowed: {}",
exp.join(", ")
)
}
_ => format!(" = note: attempted {attempted}"),
}];
if dc.reason != DenialReason::RedirectNotInAllowlist {
return out;
}
let where_ = config_path.map_or_else(
|| "your doiget config.toml".to_string(),
|p| p.as_str().to_string(),
);
out.push(format!(
" = help: that host is not on the allowlist yet; widen it in {where_}"
));
out.push(
" [network] trust_academic_repos = true # 15 curated academic suffixes"
.to_string(),
);
out.push(
" [network] trust_oa_registries = true # DOAJ, SciELO, Zenodo, OSF, HAL"
.to_string(),
);
if dc.attempted.is_some() {
for (pattern, why) in doiget_core::remediation::widening_suggestions(attempted) {
out.push(format!(
" [[network.additional_hosts]] host = \"{pattern}\" # {why}"
));
}
}
out.push(" see docs/CONFIG.md §3.1 for both".to_string());
out
}
fn print_denial_notes(dc: &DenialContext) {
for line in denial_note_lines(dc, super::user_config_path().as_deref()) {
print_err(format_args!("{line}"));
}
}
pub(crate) fn render_fetch_error(e: &FetchError) {
let code: ErrorCode = e.into();
print_err(format_args!("error[{}]: {}", code.as_wire(), e));
if let Some(dc) = Option::<DenialContext>::from(e) {
print_denial_notes(&dc);
}
}
fn render_blocked_error(
ref_: &Ref,
outcome: &FetchPaperOutcome,
code: ErrorCode,
message: &str,
denial: Option<&DenialContext>,
suggested_arxiv_id: Option<&str>,
) {
let label = match ref_ {
Ref::Arxiv(id) => format!("arxiv:{}", id.as_str()),
Ref::Doi(doi) => format!("doi:{}", doi.as_str()),
};
match denial.map(|d| d.reason) {
Some(
reason @ (DenialReason::RedirectNotInAllowlist
| DenialReason::InsecureScheme
| DenialReason::HostInBlockList),
) => {
print_err(format_args!(
"error[{}]: {label}: an OA PDF was found but its host is blocked by \
supply-chain policy ({}): {message}",
code.as_wire(),
denial_reason_wire(reason)
));
}
_ => {
print_err(format_args!(
"error[{}]: {label}: an OA PDF was found but could not be retrieved: {message}",
code.as_wire()
));
}
}
if let Some(dc) = denial {
print_denial_notes(dc);
}
print_err(format_args!(
" = note: metadata-only record written to {}",
outcome.path
));
if let Some(arxiv_id) = suggested_arxiv_id {
print_err(format_args!(
" = suggest: Try fetching the arXiv version: doiget fetch arxiv:{}",
arxiv_id
));
}
for line in blocked_trace_lines(&outcome.attempts, message) {
print_err(format_args!("{line}"));
}
}
fn blocked_trace_lines(attempts: &[SourceAttempt], message: &str) -> Vec<String> {
let mut out = Vec::new();
if message.contains("429") {
out.push(
" = suggest: HTTP 429 is a rate limit, not a policy block — it is transient. Retry \
later, and set DOIGET_CONTACT_EMAIL for the polite pool."
.to_string(),
);
}
if attempts.is_empty() {
return out;
}
let lead = if doiget_core::orchestrator::nothing_was_consulted(attempts) {
"no other source was consulted for this DOI"
} else {
"the other sources were consulted and offered no alternative copy"
};
out.push(format!(" = note: {lead}:"));
out.extend(
doiget_core::orchestrator::render_attempts(attempts)
.lines()
.map(|l| format!(" {l}")),
);
out
}
#[cfg(test)]
#[allow(clippy::expect_used, clippy::unwrap_used, clippy::panic)]
mod tests {
use super::*;
use serial_test::serial;
struct EnvGuard {
key: &'static str,
prev: Option<String>,
}
impl EnvGuard {
fn save(key: &'static str) -> Self {
Self {
key,
prev: std::env::var(key).ok(),
}
}
}
impl Drop for EnvGuard {
fn drop(&mut self) {
match &self.prev {
Some(v) => std::env::set_var(self.key, v),
None => std::env::remove_var(self.key),
}
}
}
#[test]
#[serial]
#[cfg(any(
feature = "tdm-aps",
feature = "tdm-elsevier",
feature = "tdm-springer",
feature = "tdm-ieee"
))]
#[allow(clippy::vec_init_then_push)]
fn the_production_client_registers_every_tier_3_source_key() {
let _g: Vec<EnvGuard> = [
"DOIGET_ARXIV_BASE",
"DOIGET_CROSSREF_BASE",
"DOIGET_UNPAYWALL_BASE",
"DOIGET_OA_PUBLISHER_BASE",
"DOIGET_OPENALEX_BASE",
"DOIGET_AR5IV_BASE",
]
.iter()
.map(|k| {
let g = EnvGuard::save(k);
std::env::remove_var(k);
g
})
.collect();
let client = build_http_client(None).expect("production client builds");
let mut keys: Vec<&str> = Vec::new();
#[cfg(feature = "tdm-aps")]
keys.push("tdm-aps");
#[cfg(feature = "tdm-elsevier")]
keys.push("tdm-elsevier");
#[cfg(feature = "tdm-springer")]
keys.push("tdm-springer");
#[cfg(feature = "tdm-ieee")]
keys.push("tdm-ieee");
assert!(!keys.is_empty(), "the guard must have checked something");
for key in keys {
assert!(
client.source_allowlist(key).is_some(),
"the production client has no allowlist for `{key}`; the orchestrator \
reaches this source and the fetch would die at UnknownSource (#454)"
);
}
}
#[test]
fn new_session_id_is_26_chars() {
let id = new_session_id();
assert_eq!(id.len(), 26, "session id must be 26 chars: {:?}", id);
assert!(
id.chars().all(|c| c.is_ascii_alphanumeric()),
"ulid must be ASCII alphanumeric: {:?}",
id
);
}
#[test]
#[serial]
fn build_http_client_merges_user_extension_into_oa_publisher_allowlist() {
use std::io::Write;
let td = tempfile::TempDir::new().expect("tempdir");
let cfg_dir = td.path().join("doiget");
std::fs::create_dir_all(&cfg_dir).expect("mkdir doiget/");
let cfg_path = cfg_dir.join("config.toml");
let mut f = std::fs::File::create(&cfg_path).expect("create config.toml");
f.write_all(
br#"
[[network.additional_hosts]]
host = "ruj.uj.edu.pl"
note = "Jagiellonian"
[[network.additional_hosts]]
host = "*.uj.edu.pl"
"#,
)
.expect("write config.toml");
drop(f);
let _g0 = EnvGuard::save("XDG_CONFIG_HOME");
let _g1 = EnvGuard::save("APPDATA");
let _g2 = EnvGuard::save("HOME");
let _g3 = EnvGuard::save("USERPROFILE");
let _g4 = EnvGuard::save("DOIGET_ARXIV_BASE");
let _g5 = EnvGuard::save("DOIGET_CROSSREF_BASE");
let _g6 = EnvGuard::save("DOIGET_UNPAYWALL_BASE");
let _g7 = EnvGuard::save("DOIGET_OA_PUBLISHER_BASE");
let _g8 = EnvGuard::save("DOIGET_OPENALEX_BASE");
std::env::set_var("XDG_CONFIG_HOME", td.path());
std::env::set_var("APPDATA", td.path());
std::env::set_var("HOME", td.path());
std::env::set_var("USERPROFILE", td.path());
std::env::remove_var("DOIGET_ARXIV_BASE");
std::env::remove_var("DOIGET_CROSSREF_BASE");
std::env::remove_var("DOIGET_UNPAYWALL_BASE");
std::env::remove_var("DOIGET_OA_PUBLISHER_BASE");
std::env::remove_var("DOIGET_OPENALEX_BASE");
let client = build_http_client(None).expect("HttpClient builds");
let oa = client
.source_allowlist("oa-publisher")
.expect("oa-publisher source registered");
assert!(
oa.redirect_hosts.iter().any(|p| p == "*.aps.org"),
"curated *.aps.org MUST still be present after merge; got {:?}",
oa.redirect_hosts
);
assert!(
oa.matches("ruj.uj.edu.pl"),
"literal `ruj.uj.edu.pl` from user config MUST match"
);
assert!(
oa.matches("alpha.uj.edu.pl"),
"wildcard `*.uj.edu.pl` from user config MUST match alpha.uj.edu.pl"
);
assert!(
!oa.matches("ruj.uj.edu.ru"),
"host outside the suffix MUST NOT match"
);
}
#[test]
#[serial]
fn build_http_client_merges_oa_registries_when_flag_is_set() {
use std::io::Write;
let td = tempfile::TempDir::new().expect("tempdir");
let cfg_dir = td.path().join("doiget");
std::fs::create_dir_all(&cfg_dir).expect("mkdir doiget/");
let mut f = std::fs::File::create(cfg_dir.join("config.toml")).expect("create config");
f.write_all(b"[network]\ntrust_oa_registries = true\n")
.expect("write config.toml");
drop(f);
struct EnvGuard {
key: &'static str,
prev: Option<String>,
}
impl EnvGuard {
fn save(key: &'static str) -> Self {
Self {
key,
prev: std::env::var(key).ok(),
}
}
}
impl Drop for EnvGuard {
fn drop(&mut self) {
match &self.prev {
Some(v) => std::env::set_var(self.key, v),
None => std::env::remove_var(self.key),
}
}
}
let _g: Vec<EnvGuard> = [
"XDG_CONFIG_HOME",
"APPDATA",
"HOME",
"USERPROFILE",
"DOIGET_ARXIV_BASE",
"DOIGET_CROSSREF_BASE",
"DOIGET_UNPAYWALL_BASE",
"DOIGET_OA_PUBLISHER_BASE",
"DOIGET_OPENALEX_BASE",
]
.iter()
.map(|k| EnvGuard::save(k))
.collect();
for k in ["XDG_CONFIG_HOME", "APPDATA", "HOME", "USERPROFILE"] {
std::env::set_var(k, td.path());
}
for k in [
"DOIGET_ARXIV_BASE",
"DOIGET_CROSSREF_BASE",
"DOIGET_UNPAYWALL_BASE",
"DOIGET_OA_PUBLISHER_BASE",
"DOIGET_OPENALEX_BASE",
] {
std::env::remove_var(k);
}
let client = build_http_client(None).expect("HttpClient builds");
let oa = client
.source_allowlist("oa-publisher")
.expect("oa-publisher source registered");
assert!(
oa.matches("zenodo.org"),
"the zenodo apex must match with the flag set; got {:?}",
oa.redirect_hosts
);
assert!(oa.matches("data.zenodo.org"), "wildcard covers subdomains");
assert!(oa.matches("hal.science"), "hal apex must match");
assert!(
oa.redirect_hosts.iter().any(|p| p == "*.aps.org"),
"the curated allowlist MUST survive the merge"
);
assert!(
!oa.matches("strathprints.strath.ac.uk"),
"trust_oa_registries MUST NOT imply trust_academic_repos"
);
assert!(
!oa.matches("evil.example.com"),
"unrelated host still denied"
);
}
#[test]
#[serial]
fn build_http_client_registers_openalex_for_discovery() {
struct EnvGuard {
key: &'static str,
prev: Option<String>,
}
impl EnvGuard {
fn save(key: &'static str) -> Self {
Self {
key,
prev: std::env::var(key).ok(),
}
}
}
impl Drop for EnvGuard {
fn drop(&mut self) {
match &self.prev {
Some(v) => std::env::set_var(self.key, v),
None => std::env::remove_var(self.key),
}
}
}
let td = tempfile::TempDir::new().expect("tempdir");
let _g0 = EnvGuard::save("XDG_CONFIG_HOME");
let _g1 = EnvGuard::save("APPDATA");
let _g2 = EnvGuard::save("HOME");
let _g3 = EnvGuard::save("USERPROFILE");
let _g4 = EnvGuard::save("DOIGET_ARXIV_BASE");
let _g5 = EnvGuard::save("DOIGET_CROSSREF_BASE");
let _g6 = EnvGuard::save("DOIGET_UNPAYWALL_BASE");
let _g7 = EnvGuard::save("DOIGET_OA_PUBLISHER_BASE");
let _g8 = EnvGuard::save("DOIGET_OPENALEX_BASE");
std::env::set_var("XDG_CONFIG_HOME", td.path());
std::env::set_var("APPDATA", td.path());
std::env::set_var("HOME", td.path());
std::env::set_var("USERPROFILE", td.path());
std::env::remove_var("DOIGET_ARXIV_BASE");
std::env::remove_var("DOIGET_CROSSREF_BASE");
std::env::remove_var("DOIGET_UNPAYWALL_BASE");
std::env::remove_var("DOIGET_OA_PUBLISHER_BASE");
std::env::remove_var("DOIGET_OPENALEX_BASE");
let client = build_http_client(None).expect("HttpClient builds");
let oa = client
.source_allowlist("openalex")
.expect("openalex source registered for discovery (ADR-0031 D2)");
assert!(
oa.matches("api.openalex.org"),
"api.openalex.org MUST be on the discovery allowlist; got {:?}",
oa.redirect_hosts
);
}
#[test]
fn fetch_paper_outcome_is_reachable_from_cli() {
let _ = std::any::type_name::<doiget_core::orchestrator::FetchPaperOutcome>();
}
#[test]
fn ambiguous_maps_to_exit_code_2() {
assert_eq!(cli_exit_code(ErrorCode::Ambiguous), 2);
}
fn denial(reason: DenialReason) -> DenialContext {
DenialContext {
reason,
source: None,
attempted: None,
expected: None,
hop_index: None,
cap: None,
actual: None,
}
}
#[test]
fn policy_denials_reclassify_network_error_to_capability_denied() {
for r in [
DenialReason::RedirectNotInAllowlist,
DenialReason::InsecureScheme,
DenialReason::HostInBlockList,
] {
let d = denial(r);
assert_eq!(
effective_blocked_code(ErrorCode::NetworkError, Some(&d)),
ErrorCode::CapabilityDenied,
"policy reason {r:?} must promote NetworkError -> CapabilityDenied"
);
assert_eq!(
cli_exit_code(effective_blocked_code(ErrorCode::NetworkError, Some(&d))),
3,
"policy reason {r:?} must map to exit 3 (docs/ERRORS.md §4/§6.1)"
);
}
}
#[test]
fn absent_denial_context_keeps_network_error() {
assert_eq!(
effective_blocked_code(ErrorCode::NetworkError, None),
ErrorCode::NetworkError
);
assert_eq!(
cli_exit_code(effective_blocked_code(ErrorCode::NetworkError, None)),
1
);
}
#[test]
fn non_policy_denials_keep_core_code() {
for r in [
DenialReason::SizeCapExceeded,
DenialReason::ContentTypeMismatch,
] {
let d = denial(r);
assert_eq!(
effective_blocked_code(ErrorCode::NetworkError, Some(&d)),
ErrorCode::NetworkError,
"non-policy reason {r:?} must NOT be reclassified"
);
}
}
#[test]
fn denial_reason_wire_matches_serde_snake_case() {
for r in [
DenialReason::RedirectNotInAllowlist,
DenialReason::InsecureScheme,
DenialReason::HostInBlockList,
] {
let serde_form = serde_json::to_string(&r).expect("serialize DenialReason");
let serde_token = serde_form.trim_matches('"');
assert_eq!(
denial_reason_wire(r),
serde_token,
"CLI wire token for {r:?} must equal the serde snake_case form"
);
}
}
#[test]
#[serial]
fn denial_help_names_the_file_the_reader_loads() {
struct EnvGuard(&'static str, Option<String>);
impl Drop for EnvGuard {
fn drop(&mut self) {
match &self.1 {
Some(v) => std::env::set_var(self.0, v),
None => std::env::remove_var(self.0),
}
}
}
let td = tempfile::TempDir::new().expect("tempdir");
let _g: Vec<EnvGuard> = ["XDG_CONFIG_HOME", "APPDATA", "HOME", "USERPROFILE"]
.iter()
.map(|k| EnvGuard(k, std::env::var(k).ok()))
.collect();
std::env::set_var("XDG_CONFIG_HOME", td.path());
let reader = super::config_dir_utf8()
.expect("reader resolves")
.join("doiget")
.join("config.toml");
let helped = crate::commands::user_config_path().expect("help path resolves");
assert_eq!(
helped, reader,
"the denial help must name the config.toml the reader loads"
);
let mut dc = denial(DenialReason::RedirectNotInAllowlist);
dc.attempted = Some("strathprints.strath.ac.uk".to_string());
let joined = denial_note_lines(&dc, Some(helped.as_path())).join("\n");
assert!(
joined.contains(reader.as_str()),
"rendered help must carry that path; got:\n{joined}"
);
}
#[test]
fn redirect_denial_names_both_allowlist_keys_and_the_config_file() {
let mut dc = denial(DenialReason::RedirectNotInAllowlist);
dc.attempted = Some("strathprints.strath.ac.uk".to_string());
dc.expected = Some(vec!["*.springer.com".to_string()]);
let cfg = camino::Utf8PathBuf::from("/home/alice/.config/doiget/config.toml");
let lines = denial_note_lines(&dc, Some(cfg.as_path()));
let joined = lines.join("\n");
assert!(
joined.contains("attempted strathprints.strath.ac.uk; allowed: *.springer.com"),
"the pre-existing note must survive; got:\n{joined}"
);
assert!(
joined.contains("trust_academic_repos = true"),
"the curated-set knob must be named; got:\n{joined}"
);
assert!(
joined.contains("[[network.additional_hosts]] host = \"strathprints.strath.ac.uk\""),
"the per-host escape hatch must echo the attempted host; got:\n{joined}"
);
assert!(
joined.contains("/home/alice/.config/doiget/config.toml"),
"the file the user must edit must be named; got:\n{joined}"
);
assert!(
joined.contains("docs/CONFIG.md §3.1"),
"the schema section must be named; got:\n{joined}"
);
}
#[test]
fn non_allowlist_denials_get_no_allowlist_help() {
for reason in [DenialReason::InsecureScheme, DenialReason::HostInBlockList] {
let mut dc = denial(reason);
dc.attempted = Some("evil.example.com".to_string());
let lines = denial_note_lines(&dc, None);
assert_eq!(
lines.len(),
1,
"{reason:?} must emit the note only, got: {lines:?}"
);
assert!(
!lines[0].contains("trust_academic_repos"),
"{reason:?} is not fixed by widening the allowlist: {lines:?}"
);
}
}
#[test]
fn redirect_denial_help_degrades_without_config_dir_or_host() {
let lines = denial_note_lines(&denial(DenialReason::RedirectNotInAllowlist), None);
let joined = lines.join("\n");
assert!(joined.contains("your doiget config.toml"), "{joined}");
assert!(joined.contains("trust_academic_repos = true"), "{joined}");
assert!(
!joined.contains("additional_hosts]] host ="),
"no attempted host means no copy-pasteable host line; got:\n{joined}"
);
}
#[test]
fn slugify_lowercases_and_collapses_non_alnum() {
assert_eq!(
slugify("Attention Is All You Need"),
"attention-is-all-you-need"
);
assert_eq!(slugify("Foo/Bar: Baz!!"), "foo-bar-baz");
assert_eq!(slugify(" spaced "), "spaced");
assert_eq!(slugify("!!!"), ""); }
#[test]
fn fetch_link_filename_builds_readable_name() {
let name = fetch_link_filename(
"Attention Is All You Need",
&["Ashish Vaswani".to_string()],
Some(2017),
"arxiv_1706.03762",
);
assert_eq!(name, "vaswani2017-attention-is-all-you-need.pdf");
}
#[test]
fn fetch_link_filename_falls_back_to_safekey() {
assert_eq!(
fetch_link_filename("", &[], None, "doi_10.1234_x"),
"doi_10.1234_x.pdf"
);
assert_eq!(
fetch_link_filename("…—", &[], None, "doi_10.1234_y"),
"doi_10.1234_y.pdf"
);
}
#[test]
fn link_artifact_creates_readable_artifact() {
let td = tempfile::TempDir::new().expect("tempdir");
let dir = camino::Utf8Path::from_path(td.path()).expect("utf8");
let src = dir.join("src.pdf");
std::fs::write(src.as_std_path(), b"%PDF-DATA").expect("write src");
let (dst, _kind) = link_artifact(dir, &src, "out.pdf").expect("link");
assert!(dst.exists(), "linked artifact must exist: {dst}");
assert_eq!(
std::fs::read(dst.as_std_path()).expect("read dst"),
b"%PDF-DATA",
"linked artifact (symlink or copy) must resolve to the source bytes"
);
}
#[test]
fn link_artifact_refuses_to_clobber_unrelated_file() {
let td = tempfile::TempDir::new().expect("tempdir");
let dir = camino::Utf8Path::from_path(td.path()).expect("utf8");
let src = dir.join("src.pdf");
std::fs::write(src.as_std_path(), b"%PDF-DATA").expect("write src");
let taken = dir.join("taken.pdf");
std::fs::write(taken.as_std_path(), b"USER-DATA").expect("write taken");
let err = link_artifact(dir, &src, "taken.pdf").expect_err("must refuse");
assert!(
err.to_string().contains("refusing to overwrite"),
"error must explain the refusal: {err}"
);
assert_eq!(
std::fs::read(taken.as_std_path()).expect("read taken"),
b"USER-DATA",
"the user's file must be left untouched"
);
}
#[test]
fn a_refused_hop_also_offers_the_registrable_domain() {
let mut dc = denial(DenialReason::RedirectNotInAllowlist);
dc.attempted = Some("pubs.ams.org".to_string());
let joined = denial_note_lines(&dc, None).join("\n");
assert!(joined.contains(r#"host = "pubs.ams.org""#), "{joined}");
assert!(
joined.contains(r#"host = "*.ams.org""#),
"the whole-publisher wildcard is what ends the loop in one step:\n{joined}"
);
assert!(
joined.contains(r#"host = "ams.org""#),
"a single-suffix wildcard does not match the apex, so offer it too:\n{joined}"
);
}
#[test]
fn every_suggestion_is_a_pattern_the_validator_accepts() {
for host in [
"pubs.ams.org",
"www.ams.org",
"ams.org",
"strathprints.strath.ac.uk",
"repository.ruj.uj.edu.pl",
"link.springer.com",
] {
for (pattern, _) in doiget_core::remediation::widening_suggestions(host) {
doiget_core::user_extension::validate_pattern(&pattern).unwrap_or_else(|e| {
panic!("suggested `{pattern}` for `{host}`, which the validator rejects: {e:?}")
});
}
}
}
#[test]
fn a_public_suffix_is_never_offered() {
for (host, forbidden) in [
("foo.co.uk", "co.uk"),
("foo.ac.jp", "ac.jp"),
("foo.com.au", "com.au"),
("example.org", "org"),
] {
let joined: String = doiget_core::remediation::widening_suggestions(host)
.into_iter()
.map(|(p, _)| p)
.collect::<Vec<_>>()
.join(" ");
assert!(
!joined
.split(' ')
.any(|p| p == forbidden || p == format!("*.{forbidden}")),
"offered the public suffix `{forbidden}` for `{host}`: {joined}"
);
}
}
#[test]
fn a_four_label_academic_host_still_gets_its_institution_wildcard() {
let got: Vec<String> =
doiget_core::remediation::widening_suggestions("strathprints.strath.ac.uk")
.into_iter()
.map(|(p, _)| p)
.collect();
assert!(
got.iter().any(|p| p == "*.strath.ac.uk"),
"expected the institution wildcard; got {got:?}"
);
}
#[test]
fn an_apex_host_offers_its_subdomains() {
let got: Vec<String> = doiget_core::remediation::widening_suggestions("ams.org")
.into_iter()
.map(|(p, _)| p)
.collect();
assert_eq!(got, vec!["ams.org".to_string(), "*.ams.org".to_string()]);
}
#[test]
fn a_rate_limited_block_says_the_limit_is_transient() {
let joined = blocked_trace_lines(&[], "network error: HTTP 429 from https://ams.org/x.pdf")
.join("\n");
assert!(joined.contains("429"), "{joined}");
assert!(joined.contains("transient"), "{joined}");
assert!(
joined.contains("Retry later"),
"say what to DO, not just what happened:\n{joined}"
);
for line in blocked_trace_lines(&[], "network error: HTTP 429 from https://ams.org/x.pdf") {
assert!(
!line.trim_start().contains(" "),
"a lost line continuation left source indentation in the message:\n{line}"
);
}
}
#[test]
fn a_policy_block_is_not_described_as_transient() {
let joined =
blocked_trace_lines(&[], "redirect target x.example not in allowlist").join("\n");
assert!(
!joined.contains("transient"),
"an allowlist denial is permanent until reconfigured:\n{joined}"
);
}
#[test]
fn a_blocked_leg_reports_which_other_sources_were_consulted() {
use doiget_core::orchestrator::{AttemptOutcome, SourceAttempt};
let attempts = vec![
SourceAttempt::new("core", AttemptOutcome::NoRecord),
SourceAttempt::new(
"hal",
AttemptOutcome::Disabled {
env: "DOIGET_ENABLE_HAL",
},
),
];
let joined = blocked_trace_lines(&attempts, "HTTP 429").join("\n");
assert!(
joined.contains("the other sources were consulted"),
"at least one WAS consulted:\n{joined}"
);
assert!(
joined.contains("core") && joined.contains("no record"),
"{joined}"
);
assert!(
joined.contains("DOIGET_ENABLE_HAL"),
"a source that was never asked must still name its switch:\n{joined}"
);
}
#[test]
fn a_blocked_leg_with_nothing_consulted_says_so() {
use doiget_core::orchestrator::{AttemptOutcome, SourceAttempt};
let attempts = vec![
SourceAttempt::new(
"core",
AttemptOutcome::Disabled {
env: "DOIGET_ENABLE_CORE",
},
),
SourceAttempt::new(
"hal",
AttemptOutcome::Disabled {
env: "DOIGET_ENABLE_HAL",
},
),
];
let joined = blocked_trace_lines(&attempts, "HTTP 429").join("\n");
assert!(
joined.contains("no other source was consulted"),
"must not imply the paper is unavailable elsewhere:\n{joined}"
);
}
#[test]
fn no_attempts_means_no_trace_block() {
let lines = blocked_trace_lines(&[], "not-a-pdf body");
assert!(lines.is_empty(), "{lines:?}");
}
}