use std::time::Duration;
use serde::de::DeserializeOwned;
use serde::{Deserialize, Deserializer, Serialize};
use url::Url;
use crate::tei::{Author, Biblio};
pub const DEFAULT_OPENALEX_URL: &str = "https://api.openalex.org";
const SEARCH_LIMIT: u32 = 10;
const MIN_TITLE_LEN: usize = 8;
const CONTAINMENT_MIN_TITLE_LEN: usize = 20;
const MAX_ERROR_MESSAGE_LEN: usize = 500;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum MatchKind {
Doi,
Title,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Completion {
pub biblio: Biblio,
pub openalex_id: String,
pub matched_by: MatchKind,
}
#[derive(Debug, thiserror::Error)]
#[non_exhaustive]
pub enum Error {
#[error("invalid OpenAlex base URL {base_url:?}: {source}")]
InvalidBaseUrl {
base_url: String,
#[source]
source: url::ParseError,
},
#[error("unsupported URL scheme {scheme:?} in {base_url:?}")]
UnsupportedScheme {
base_url: String,
scheme: String,
},
#[error("OpenAlex HTTP transport error: {0}")]
Http(#[from] reqwest::Error),
#[error("OpenAlex API at {url} returned HTTP {status}: {message}")]
HttpStatus {
status: u16,
url: String,
message: String,
},
#[error("could not parse OpenAlex response from {url}: {source}")]
Parse {
url: String,
#[source]
source: serde_json::Error,
},
}
#[derive(Debug, Clone)]
pub struct Completer {
base_url: Url,
http: reqwest::Client,
}
impl Completer {
pub fn new() -> Self {
Self {
base_url: Url::parse(DEFAULT_OPENALEX_URL).expect("built-in OpenAlex URL is valid"),
http: reqwest::Client::new(),
}
}
pub fn with_base_url(base_url: impl AsRef<str>) -> Result<Self, Error> {
Ok(Self {
base_url: parse_base_url(base_url.as_ref())?,
http: reqwest::Client::new(),
})
}
pub fn with_timeout(timeout: Duration) -> Result<Self, Error> {
Self::builder().timeout(timeout).build()
}
pub fn builder() -> CompleterBuilder {
CompleterBuilder::default()
}
pub fn base_url(&self) -> &str {
self.base_url.as_str()
}
pub async fn complete(&self, biblio: &Biblio) -> Result<Option<Completion>, Error> {
let Some((work, matched_by)) = self.find_work(biblio).await? else {
return Ok(None);
};
Ok(Some(Completion {
biblio: merge(biblio, &work),
openalex_id: work.id.clone(),
matched_by,
}))
}
async fn find_work(&self, biblio: &Biblio) -> Result<Option<(Work, MatchKind)>, Error> {
if let Some(doi) = biblio.doi.as_deref().and_then(normalize_doi) {
if let Some(work) = self.lookup_by_doi(&doi).await? {
return Ok(Some((work, MatchKind::Doi)));
}
}
let Some(title) = biblio.title.as_deref() else {
return Ok(None);
};
let term = sanitize_search_term(title);
if term.chars().count() < MIN_TITLE_LEN {
return Ok(None);
}
let candidates = self.search_by_title(&term).await?;
Ok(select_match(biblio, candidates).map(|work| (work, MatchKind::Title)))
}
async fn lookup_by_doi(&self, doi: &str) -> Result<Option<Work>, Error> {
let mut url = self.works_url();
url.path_segments_mut()
.expect("http(s) URLs can be a base")
.push(&format!("doi:{doi}"));
let response = self.send(&url).await?;
match response.status() {
reqwest::StatusCode::OK => {}
reqwest::StatusCode::NOT_FOUND => return Ok(None),
_ => return Err(self.status_error(&url, response).await),
}
let body = response.text().await?;
Ok(Some(parse_json(&url, &body)?))
}
async fn search_by_title(&self, title: &str) -> Result<Vec<Work>, Error> {
let mut url = self.works_url();
url.query_pairs_mut()
.append_pair("filter", &format!("title.search:{title}"))
.append_pair("per-page", &SEARCH_LIMIT.to_string())
.append_pair("sort", "cited_by_count:desc");
let response = self.send(&url).await?;
if !response.status().is_success() {
return Err(self.status_error(&url, response).await);
}
let body = response.text().await?;
let response: WorkResponse = parse_json(&url, &body)?;
Ok(response.results)
}
fn works_url(&self) -> Url {
let mut url = self.base_url.clone();
url.path_segments_mut()
.expect("http(s) URLs can be a base")
.pop_if_empty()
.push("works");
url
}
async fn send(&self, url: &Url) -> Result<reqwest::Response, Error> {
let request = self.http.get(url.clone()).header(
reqwest::header::USER_AGENT,
concat!("grobid-rs/", env!("CARGO_PKG_VERSION")),
);
Ok(request.send().await?)
}
async fn status_error(&self, url: &Url, response: reqwest::Response) -> Error {
let status = response.status().as_u16();
let message = response.text().await.unwrap_or_default();
Error::HttpStatus {
status,
url: url.to_string(),
message: truncate_message(message),
}
}
}
impl Default for Completer {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug, Clone)]
pub struct CompleterBuilder {
base_url: Url,
http: Option<reqwest::Client>,
timeout: Option<Duration>,
connect_timeout: Option<Duration>,
}
impl Default for CompleterBuilder {
fn default() -> Self {
Self {
base_url: Url::parse(DEFAULT_OPENALEX_URL).expect("built-in OpenAlex URL is valid"),
http: None,
timeout: None,
connect_timeout: None,
}
}
}
impl CompleterBuilder {
pub fn base_url(mut self, base_url: impl AsRef<str>) -> Result<Self, Error> {
self.base_url = parse_base_url(base_url.as_ref())?;
Ok(self)
}
pub fn http_client(mut self, client: reqwest::Client) -> Self {
self.http = Some(client);
self
}
pub fn timeout(mut self, timeout: Duration) -> Self {
self.timeout = Some(timeout);
self
}
pub fn connect_timeout(mut self, timeout: Duration) -> Self {
self.connect_timeout = Some(timeout);
self
}
pub fn build(self) -> Result<Completer, Error> {
let http = match self.http {
Some(client) => client,
None => {
let mut builder = reqwest::Client::builder();
if let Some(timeout) = self.timeout {
builder = builder.timeout(timeout);
}
if let Some(connect_timeout) = self.connect_timeout {
builder = builder.connect_timeout(connect_timeout);
}
builder.build()?
}
};
Ok(Completer {
base_url: self.base_url,
http,
})
}
}
fn parse_base_url(base_url: &str) -> Result<Url, Error> {
let parsed = Url::parse(base_url).map_err(|source| Error::InvalidBaseUrl {
base_url: base_url.to_string(),
source,
})?;
match parsed.scheme() {
"http" | "https" => Ok(parsed),
scheme => Err(Error::UnsupportedScheme {
base_url: base_url.to_string(),
scheme: scheme.to_string(),
}),
}
}
#[derive(Debug, Default, Deserialize)]
#[serde(default)]
struct Work {
#[serde(deserialize_with = "null_default")]
id: String,
title: Option<String>,
display_name: Option<String>,
doi: Option<String>,
#[serde(deserialize_with = "null_default")]
publication_year: u32,
#[serde(deserialize_with = "null_default")]
publication_date: String,
#[serde(deserialize_with = "null_default")]
authorships: Vec<Authorship>,
#[serde(deserialize_with = "null_default")]
biblio: WorkBiblio,
#[serde(deserialize_with = "null_default")]
ids: WorkIds,
primary_location: Option<Location>,
#[serde(deserialize_with = "null_default")]
locations: Vec<Location>,
#[serde(deserialize_with = "null_default")]
cited_by_count: u32,
#[serde(rename = "type", deserialize_with = "null_default")]
work_type: String,
}
#[derive(Debug, Default, Deserialize)]
#[serde(default)]
struct WorkBiblio {
volume: Option<String>,
issue: Option<String>,
first_page: Option<String>,
last_page: Option<String>,
}
#[derive(Debug, Default, Deserialize)]
#[serde(default)]
struct WorkIds {
pmid: Option<String>,
}
#[derive(Debug, Default, Deserialize)]
#[serde(default)]
struct Authorship {
#[serde(deserialize_with = "null_default")]
author: AuthorName,
}
#[derive(Debug, Default, Deserialize)]
#[serde(default)]
struct AuthorName {
#[serde(deserialize_with = "null_default")]
display_name: String,
orcid: Option<String>,
}
#[derive(Debug, Default, Deserialize)]
#[serde(default)]
struct Location {
landing_page_url: Option<String>,
source: Option<Source>,
}
#[derive(Debug, Default, Deserialize)]
#[serde(default)]
struct Source {
#[serde(deserialize_with = "null_default")]
display_name: String,
host_organization_name: Option<String>,
#[serde(deserialize_with = "null_default")]
issn: Vec<String>,
issn_l: Option<String>,
#[serde(rename = "type")]
source_type: Option<String>,
}
#[derive(Debug, Default, Deserialize)]
#[serde(default)]
struct WorkResponse {
#[serde(deserialize_with = "null_default")]
results: Vec<Work>,
}
fn null_default<'de, D, T>(deserializer: D) -> Result<T, D::Error>
where
T: Default + Deserialize<'de>,
D: Deserializer<'de>,
{
Ok(Option::<T>::deserialize(deserializer)?.unwrap_or_default())
}
fn parse_json<T: DeserializeOwned>(url: &Url, body: &str) -> Result<T, Error> {
serde_json::from_str(body).map_err(|source| Error::Parse {
url: url.to_string(),
source,
})
}
fn truncate_message(mut message: String) -> String {
if message.len() > MAX_ERROR_MESSAGE_LEN {
let mut end = MAX_ERROR_MESSAGE_LEN;
while !message.is_char_boundary(end) {
end -= 1;
}
message.truncate(end);
message.push('…');
}
message.trim().to_string()
}
fn select_match(biblio: &Biblio, candidates: Vec<Work>) -> Option<Work> {
let query = biblio.title.as_deref()?;
let parsed_title = normalize_title(query);
let parsed_year = parsed_year(biblio);
candidates
.into_iter()
.filter_map(|work| {
let title = normalize_title(work_title(&work));
if title.is_empty() || !titles_match(&parsed_title, &title) {
return None;
}
if !year_compatible(biblio, &work) || !authors_compatible(biblio, &work) {
return None;
}
let exact_title = u8::from(title == parsed_title);
let exact_year =
u8::from(work.publication_year != 0 && parsed_year == Some(work.publication_year));
Some((exact_title, exact_year, work.cited_by_count, work))
})
.max_by_key(|(exact_title, exact_year, cited_by_count, _)| {
(*exact_title, *exact_year, *cited_by_count)
})
.map(|(_, _, _, work)| work)
}
fn work_title(work: &Work) -> &str {
work.title
.as_deref()
.or(work.display_name.as_deref())
.unwrap_or_default()
}
fn titles_match(parsed: &str, candidate: &str) -> bool {
let parsed = normalize_title(parsed);
let candidate = normalize_title(candidate);
if parsed.is_empty() || candidate.is_empty() {
return false;
}
if parsed == candidate {
return true;
}
let (longer, shorter) = if parsed.len() >= candidate.len() {
(parsed.as_str(), candidate.as_str())
} else {
(candidate.as_str(), parsed.as_str())
};
shorter.chars().count() >= CONTAINMENT_MIN_TITLE_LEN && longer.contains(shorter)
}
fn sanitize_search_term(raw: &str) -> String {
let replaced: String = raw
.chars()
.map(|c| match c {
',' | '|' | '!' | '"' | '*' | '?' => ' ',
other => other,
})
.collect();
replaced.split_whitespace().collect::<Vec<_>>().join(" ")
}
fn normalize_title(raw: &str) -> String {
let mut normalized = String::with_capacity(raw.len());
let mut pending_space = false;
for c in raw.chars().flat_map(char::to_lowercase) {
if c.is_alphanumeric() {
if pending_space && !normalized.is_empty() {
normalized.push(' ');
}
pending_space = false;
normalized.push(c);
} else {
pending_space = true;
}
}
normalized
}
fn parsed_year(biblio: &Biblio) -> Option<u32> {
let digits: String = biblio
.date
.as_deref()?
.chars()
.take_while(|c| c.is_ascii_digit())
.take(4)
.collect();
digits.parse().ok()
}
fn year_compatible(biblio: &Biblio, work: &Work) -> bool {
let Some(parsed) = parsed_year(biblio) else {
return true;
};
work.publication_year == 0 || parsed.abs_diff(work.publication_year) <= 1
}
fn authors_compatible(biblio: &Biblio, work: &Work) -> bool {
let Some(candidate) = work
.authorships
.first()
.map(|authorship| authorship.author.display_name.as_str())
else {
return true;
};
let Some(parsed) = biblio
.authors
.first()
.and_then(|author| author.surname.as_deref().or(author.full_name.as_deref()))
else {
return true;
};
match (name_token(parsed), name_token(candidate)) {
(Some(parsed), Some(candidate)) => parsed == candidate,
_ => true,
}
}
fn name_token(name: &str) -> Option<String> {
normalize_title(name)
.split_whitespace()
.last()
.map(str::to_string)
}
fn normalize_doi(raw: &str) -> Option<String> {
let trimmed = raw.trim();
let lower = trimmed.to_ascii_lowercase();
let stripped = [
"https://doi.org/",
"http://doi.org/",
"https://dx.doi.org/",
"http://dx.doi.org/",
"doi:",
]
.iter()
.find_map(|prefix| lower.starts_with(prefix).then(|| &trimmed[prefix.len()..]))
.unwrap_or(trimmed)
.trim();
(!stripped.is_empty()).then(|| stripped.to_string())
}
fn merge(biblio: &Biblio, work: &Work) -> Biblio {
let mut merged = biblio.clone();
fill(&mut merged.title, nonempty(Some(work_title(work))));
let date = work_date(work);
fill(&mut merged.date, date.as_deref());
if merged.authors.is_empty() {
merged.authors = work_authors(work);
}
fill_source_fields(&mut merged, work, work_source(work));
let volume = work.biblio.volume.as_deref();
fill(&mut merged.volume, volume);
let issue = work.biblio.issue.as_deref();
fill(&mut merged.issue, issue);
let first_page = work.biblio.first_page.as_deref();
fill(&mut merged.first_page, first_page);
let last_page = work.biblio.last_page.as_deref();
fill(&mut merged.last_page, last_page);
let pages = work_pages(work);
fill(&mut merged.pages, pages.as_deref());
let doi = work.doi.as_deref().and_then(normalize_doi);
fill(&mut merged.doi, doi.as_deref());
fill(&mut merged.pmid, work.ids.pmid.as_deref());
fill(&mut merged.url, work_url(work));
merged
}
fn work_source(work: &Work) -> Option<&Source> {
work.primary_location
.as_ref()
.and_then(|location| location.source.as_ref())
.or_else(|| {
work.locations
.iter()
.find_map(|location| location.source.as_ref())
})
}
fn fill_source_fields(merged: &mut Biblio, work: &Work, source: Option<&Source>) {
let Some(source) = source else {
return;
};
match source.source_type.as_deref() {
Some("journal") => {
fill(
&mut merged.journal,
nonempty(Some(source.display_name.as_str())),
);
fill(&mut merged.issn, source_issn(source));
}
Some("repository" | "metadata" | "other") => {}
_ => match work.work_type.as_str() {
"article" | "review" | "editorial" | "letter" => {
fill(
&mut merged.journal,
nonempty(Some(source.display_name.as_str())),
);
fill(&mut merged.issn, source_issn(source));
}
"book-chapter" | "proceedings-article" => {
fill(
&mut merged.book_title,
nonempty(Some(source.display_name.as_str())),
);
}
_ => {}
},
}
match work.work_type.as_str() {
"article"
| "review"
| "editorial"
| "letter"
| "book"
| "book-chapter"
| "proceedings-article" => {
fill(
&mut merged.publisher,
nonempty(source.host_organization_name.as_deref()),
);
}
"report" | "dissertation" => {
fill(
&mut merged.institution,
nonempty(source.host_organization_name.as_deref()),
);
}
_ => {}
}
}
fn source_issn(source: &Source) -> Option<&str> {
source
.issn
.iter()
.map(String::as_str)
.find(|issn| !issn.trim().is_empty())
.or_else(|| {
source
.issn_l
.as_deref()
.filter(|issn| !issn.trim().is_empty())
})
}
fn work_date(work: &Work) -> Option<String> {
if let Some(date) = nonempty(Some(work.publication_date.as_str())) {
return Some(date.to_string());
}
(work.publication_year > 0).then(|| work.publication_year.to_string())
}
fn work_pages(work: &Work) -> Option<String> {
let first = nonempty(work.biblio.first_page.as_deref());
let last = nonempty(work.biblio.last_page.as_deref());
match (first, last) {
(Some(first), Some(last)) if first != last => Some(format!("{first}-{last}")),
(Some(page), _) | (_, Some(page)) => Some(page.to_string()),
(None, None) => None,
}
}
fn work_url(work: &Work) -> Option<&str> {
work.primary_location
.as_ref()
.and_then(|location| nonempty(location.landing_page_url.as_deref()))
.or_else(|| {
work.locations
.iter()
.find_map(|location| nonempty(location.landing_page_url.as_deref()))
})
.or(Some(work.id.as_str()))
}
fn work_authors(work: &Work) -> Vec<Author> {
work.authorships
.iter()
.filter_map(|authorship| {
let name = authorship.author.display_name.trim();
if name.is_empty() {
return None;
}
let (given_name, surname) = split_name(name);
Some(Author {
full_name: Some(name.to_string()),
given_name,
surname,
orcid: nonempty(authorship.author.orcid.as_deref()).map(str::to_string),
..Author::default()
})
})
.collect()
}
fn split_name(name: &str) -> (Option<String>, Option<String>) {
match name.rsplit_once(char::is_whitespace) {
Some((given, surname)) => (
nonempty(Some(given)).map(str::to_string),
nonempty(Some(surname)).map(str::to_string),
),
None => (None, Some(name.to_string())),
}
}
fn fill(target: &mut Option<String>, value: Option<&str>) {
if nonempty(target.as_deref()).is_some() {
return;
}
if let Some(value) = nonempty(value) {
*target = Some(value.to_string());
}
}
fn nonempty(value: Option<&str>) -> Option<&str> {
value.map(str::trim).filter(|value| !value.is_empty())
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::{json, Value};
const BASE_WORK: &str = r#"
{
"id": "https://openalex.org/W1",
"title": "Example",
"display_name": null,
"doi": null,
"publication_year": 2020,
"publication_date": "2020-01-01",
"authorships": [],
"biblio": {},
"ids": {
"openalex": "https://openalex.org/W1"
},
"primary_location": null,
"locations": [],
"cited_by_count": 0,
"type": "article",
"abstract_inverted_index": null,
"grants": []
}
"#;
fn work_fixture(overrides: Value) -> Work {
let mut base: Value = serde_json::from_str(BASE_WORK).expect("base fixture is valid JSON");
{
let base = base.as_object_mut().expect("fixture is an object");
let overrides = overrides.as_object().expect("overrides are an object");
for (key, value) in overrides {
base.insert(key.clone(), value.clone());
}
}
serde_json::from_value(base).expect("fixture deserializes into a work")
}
fn source(display_name: &str, source_type: &str, host: &str) -> Value {
json!({
"display_name": display_name,
"host_organization_name": host,
"issn": ["1234-5678"],
"type": source_type
})
}
fn authorship(display_name: &str) -> Value {
json!({ "author": { "display_name": display_name } })
}
fn model_biblio() -> Biblio {
Biblio {
title: Some("The Barc model for continuous variables".to_string()),
date: Some("2000".to_string()),
authors: vec![Author {
surname: Some("Kahle".to_string()),
..Author::default()
}],
..Biblio::default()
}
}
fn model_candidate(overrides: Value) -> Work {
let mut base = json!({
"title": "The Barc model for continuous variables",
"publication_year": 2000,
"authorships": [authorship("Brewster Kahle")]
});
{
let base = base.as_object_mut().expect("candidate is an object");
for (key, value) in overrides.as_object().expect("overrides are an object") {
base.insert(key.clone(), value.clone());
}
}
work_fixture(base)
}
#[test]
fn test_default_completer() {
let completer = Completer::new();
assert_eq!(completer.base_url(), "https://api.openalex.org/");
assert_eq!(
completer.works_url().as_str(),
"https://api.openalex.org/works"
);
}
#[test]
fn test_with_base_url_validation() {
assert!(matches!(
Completer::with_base_url("not a url"),
Err(Error::InvalidBaseUrl { .. })
));
assert!(matches!(
Completer::with_base_url("ftp://example.org"),
Err(Error::UnsupportedScheme { .. })
));
let completer =
Completer::with_base_url("http://127.0.0.1:8080/mirror").expect("valid base url");
assert_eq!(
completer.works_url().as_str(),
"http://127.0.0.1:8080/mirror/works"
);
}
#[test]
fn test_normalize_doi() {
assert_eq!(normalize_doi(" 10.1/x ").as_deref(), Some("10.1/x"));
assert_eq!(
normalize_doi("https://doi.org/10.1/X").as_deref(),
Some("10.1/X")
);
assert_eq!(
normalize_doi("HTTPS://DOI.ORG/10.1/X").as_deref(),
Some("10.1/X")
);
assert_eq!(
normalize_doi("http://dx.doi.org/10.1/x").as_deref(),
Some("10.1/x")
);
assert_eq!(normalize_doi("doi:10.1/x").as_deref(), Some("10.1/x"));
assert_eq!(normalize_doi("10.1/x"), Some("10.1/x".to_string()));
assert_eq!(normalize_doi(" "), None);
}
#[test]
fn test_normalize_title() {
assert_eq!(normalize_title(" Hello, World! "), "hello world");
assert_eq!(
normalize_title("C++ & Rust: a comparison"),
"c rust a comparison"
);
assert_eq!(normalize_title("ÄÖÜ"), "äöü");
assert_eq!(normalize_title("!!!"), "");
assert_eq!(normalize_title(""), "");
}
#[test]
fn test_sanitize_search_term() {
assert_eq!(
sanitize_search_term("Sexual selection, sensory systems and sensory exploitation"),
"Sexual selection sensory systems and sensory exploitation"
);
assert_eq!(
sanitize_search_term("Newborns' preferential tracking ... decline*"),
"Newborns' preferential tracking ... decline"
);
assert_eq!(
sanitize_search_term("alpha|beta!gamma\"delta*epsilon?zeta"),
"alpha beta gamma delta epsilon zeta"
);
assert_eq!(sanitize_search_term(" a , b "), "a b");
assert_eq!(
sanitize_search_term("p < 0.05: 95% of C++"),
"p < 0.05: 95% of C++"
);
assert_eq!(sanitize_search_term("!!!"), "");
}
#[test]
fn test_titles_match_equal_variants() {
for (parsed, candidate) in [
("The Barc model", "the barc model"),
("Deep Learning!", "deep learning"),
("A study\t of\n things", "a study of things"),
("Zürich", "zürich"),
("C++: a language", "c a language"),
] {
assert!(
titles_match(parsed, candidate),
"{parsed:?} should match {candidate:?}"
);
}
}
#[test]
fn test_titles_match_containment() {
assert!(titles_match(
"The Barc model for continuous variables: a tutorial",
"The Barc model for continuous variables"
));
assert!(titles_match(
"The Barc model for continuous variables",
"The Barc model for continuous variables: a tutorial"
));
}
#[test]
fn test_titles_match_rejects_different_titles() {
for (parsed, candidate) in [
("The Barc model", "A different paper"),
("A study of things", "A study of other things"),
("The Barc model", ""),
("", "The Barc model"),
("Deep learning", "Deep learning for everything"),
] {
assert!(
!titles_match(parsed, candidate),
"{parsed:?} should not match {candidate:?}"
);
}
}
#[test]
fn test_year_compatible() {
let biblio = |date: &str| Biblio {
date: Some(date.to_string()),
..Biblio::default()
};
let work = |year: u32| work_fixture(json!({ "publication_year": year }));
assert!(year_compatible(&biblio("2019"), &work(2019)));
assert!(year_compatible(&biblio("2019-03-01"), &work(2019)));
assert!(year_compatible(&biblio("2019"), &work(2018)));
assert!(year_compatible(&biblio("2019"), &work(2020)));
assert!(!year_compatible(&biblio("2019"), &work(2017)));
assert!(!year_compatible(&biblio("2019"), &work(2021)));
assert!(year_compatible(&Biblio::default(), &work(1999)));
assert!(year_compatible(&biblio("2019"), &work(0)));
assert!(year_compatible(&biblio("in press"), &work(1999)));
}
#[test]
fn test_authors_compatible() {
let parsed = |surname: &str| Biblio {
authors: vec![Author {
surname: Some(surname.to_string()),
..Author::default()
}],
..Biblio::default()
};
let work = |name: &str| work_fixture(json!({ "authorships": [authorship(name)] }));
assert!(authors_compatible(
&parsed("Kahle"),
&work("Brewster Kahle")
));
assert!(authors_compatible(
&parsed("van der Berg"),
&work("Cornelis van der Berg")
));
assert!(authors_compatible(&parsed("Kahle"), &work("B. Kahle")));
assert!(!authors_compatible(
&parsed("Kahle"),
&work("Brewster Priem")
));
assert!(authors_compatible(&Biblio::default(), &work("Anyone Else")));
assert!(authors_compatible(&parsed("Kahle"), &work("")));
assert!(authors_compatible(
&parsed("Kahle"),
&work_fixture(json!({ "authorships": [] }))
));
let full_name = Biblio {
authors: vec![Author {
full_name: Some("Brewster Kahle".to_string()),
..Author::default()
}],
..Biblio::default()
};
assert!(authors_compatible(&full_name, &work("Brewster Kahle")));
}
#[test]
fn test_split_name() {
assert_eq!(
split_name("Brewster Kahle"),
(Some("Brewster".to_string()), Some("Kahle".to_string()))
);
assert_eq!(
split_name("Aristotle"),
(None, Some("Aristotle".to_string()))
);
assert_eq!(
split_name("Cornelis van der Berg"),
(
Some("Cornelis van der".to_string()),
Some("Berg".to_string())
)
);
assert_eq!(
split_name("Brewster Kahle"),
(Some("Brewster".to_string()), Some("Kahle".to_string()))
);
}
#[test]
fn test_work_authors() {
let work = work_fixture(json!({
"authorships": [
authorship("Brewster Kahle"),
{
"author": {
"display_name": "Cornelis van der Berg",
"orcid": "https://orcid.org/0000-0001-2345-6789"
}
},
authorship("Aristotle"),
{ "author": { "display_name": " " } },
{ "author": null }
]
}));
let authors = work_authors(&work);
assert_eq!(authors.len(), 3);
assert_eq!(authors[0].given_name.as_deref(), Some("Brewster"));
assert_eq!(authors[0].surname.as_deref(), Some("Kahle"));
assert_eq!(authors[0].full_name.as_deref(), Some("Brewster Kahle"));
assert_eq!(authors[1].given_name.as_deref(), Some("Cornelis van der"));
assert_eq!(authors[1].surname.as_deref(), Some("Berg"));
assert_eq!(
authors[1].orcid.as_deref(),
Some("https://orcid.org/0000-0001-2345-6789")
);
assert_eq!(authors[2].given_name, None);
assert_eq!(authors[2].surname.as_deref(), Some("Aristotle"));
}
#[test]
fn test_select_match_filters_incompatible_candidates() {
let biblio = model_biblio();
let wrong_title = model_candidate(json!({ "title": "A different paper entirely" }));
assert!(select_match(&biblio, vec![wrong_title]).is_none());
let wrong_year = model_candidate(json!({ "publication_year": 1980 }));
assert!(select_match(&biblio, vec![wrong_year]).is_none());
let wrong_author = model_candidate(json!({ "authorships": [authorship("Someone Else")] }));
assert!(select_match(&biblio, vec![wrong_author]).is_none());
let untitled = model_candidate(json!({ "title": null, "display_name": null }));
assert!(select_match(&biblio, vec![untitled]).is_none());
assert!(select_match(&biblio, vec![]).is_none());
assert!(select_match(&Biblio::default(), vec![model_candidate(json!({}))]).is_none());
let wrong = model_candidate(json!({ "title": "A different paper entirely" }));
let right = model_candidate(json!({}));
let selected = select_match(&biblio, vec![wrong, right]).expect("second candidate matches");
assert_eq!(
selected.title.as_deref(),
Some("The Barc model for continuous variables")
);
}
#[test]
fn test_select_match_prefers_exact_title_over_containment() {
let containment = model_candidate(json!({
"title": "The Barc model for continuous variables and their applications",
"cited_by_count": 10_000
}));
let exact = model_candidate(json!({ "cited_by_count": 1 }));
let selected = select_match(&model_biblio(), vec![containment, exact]).expect("match");
assert_eq!(
selected.title.as_deref(),
Some("The Barc model for continuous variables")
);
}
#[test]
fn test_select_match_prefers_exact_year_then_citations() {
let off_by_one = model_candidate(json!({
"publication_year": 2001,
"cited_by_count": 9999
}));
let exact = model_candidate(json!({
"publication_year": 2000,
"cited_by_count": 1
}));
let selected =
select_match(&model_biblio(), vec![off_by_one, exact]).expect("exact year wins");
assert_eq!(selected.publication_year, 2000);
let rarely_cited = model_candidate(json!({ "cited_by_count": 1 }));
let canonical = model_candidate(json!({ "cited_by_count": 100 }));
let selected = select_match(&model_biblio(), vec![rarely_cited, canonical]).expect("match");
assert_eq!(selected.cited_by_count, 100);
}
#[test]
fn test_work_pages() {
let work = work_fixture(json!({
"biblio": { "first_page": "17", "last_page": "27" }
}));
assert_eq!(work_pages(&work).as_deref(), Some("17-27"));
let work = work_fixture(json!({
"biblio": { "first_page": "17", "last_page": "17" }
}));
assert_eq!(work_pages(&work).as_deref(), Some("17"));
let work = work_fixture(json!({ "biblio": { "first_page": "17" } }));
assert_eq!(work_pages(&work).as_deref(), Some("17"));
let work = work_fixture(json!({ "biblio": { "last_page": "27" } }));
assert_eq!(work_pages(&work).as_deref(), Some("27"));
let work = work_fixture(json!({ "biblio": {} }));
assert_eq!(work_pages(&work), None);
}
#[test]
fn test_work_date() {
let work = work_fixture(json!({
"publication_date": "2019-05-01",
"publication_year": 2019
}));
assert_eq!(work_date(&work).as_deref(), Some("2019-05-01"));
let work = work_fixture(json!({
"publication_date": "",
"publication_year": 2019
}));
assert_eq!(work_date(&work).as_deref(), Some("2019"));
let work = work_fixture(json!({
"publication_date": null,
"publication_year": 0
}));
assert_eq!(work_date(&work), None);
}
#[test]
fn test_work_url_fallbacks() {
let work = work_fixture(json!({
"primary_location": { "landing_page_url": "https://primary.example" },
"locations": [{ "landing_page_url": "https://other.example" }]
}));
assert_eq!(work_url(&work), Some("https://primary.example"));
let work = work_fixture(json!({
"primary_location": { "landing_page_url": null },
"locations": [{ "landing_page_url": "https://other.example" }]
}));
assert_eq!(work_url(&work), Some("https://other.example"));
let work = work_fixture(json!({ "primary_location": null, "locations": [] }));
assert_eq!(work_url(&work), Some("https://openalex.org/W1"));
}
#[test]
fn test_source_issn() {
let work = work_fixture(json!({
"primary_location": { "source": source("J", "journal", "P") }
}));
assert_eq!(source_issn(work_source(&work).unwrap()), Some("1234-5678"));
let work = work_fixture(json!({
"primary_location": { "source": {
"display_name": "J",
"issn": [],
"issn_l": "9999-0000"
}}
}));
assert_eq!(source_issn(work_source(&work).unwrap()), Some("9999-0000"));
let work = work_fixture(json!({
"primary_location": { "source": { "display_name": "J", "issn": ["", " "] } }
}));
assert_eq!(source_issn(work_source(&work).unwrap()), None);
}
#[test]
fn test_merge_fills_missing_fields() {
let work = work_fixture(json!({
"title": "The Barc model for continuous variables",
"doi": "https://doi.org/10.1002/example",
"publication_date": "2000-03-01",
"publication_year": 2000,
"ids": { "pmid": "12345678" },
"authorships": [{
"author": {
"display_name": "Brewster Kahle",
"orcid": "https://orcid.org/0000-0001-2345-6789"
}
}],
"biblio": {
"volume": "18",
"issue": "1",
"first_page": "17",
"last_page": "27"
},
"primary_location": {
"landing_page_url": "https://example.org/article",
"source": source("Cell Biochemistry and Function", "journal", "Wiley")
}
}));
let biblio = Biblio {
title: Some("The Barc model for continuous variables".to_string()),
..Biblio::default()
};
let merged = merge(&biblio, &work);
assert_eq!(merged.date.as_deref(), Some("2000-03-01"));
assert_eq!(
merged.journal.as_deref(),
Some("Cell Biochemistry and Function")
);
assert_eq!(merged.publisher.as_deref(), Some("Wiley"));
assert_eq!(merged.issn.as_deref(), Some("1234-5678"));
assert_eq!(merged.volume.as_deref(), Some("18"));
assert_eq!(merged.issue.as_deref(), Some("1"));
assert_eq!(merged.first_page.as_deref(), Some("17"));
assert_eq!(merged.last_page.as_deref(), Some("27"));
assert_eq!(merged.pages.as_deref(), Some("17-27"));
assert_eq!(merged.doi.as_deref(), Some("10.1002/example"));
assert_eq!(merged.pmid.as_deref(), Some("12345678"));
assert_eq!(merged.url.as_deref(), Some("https://example.org/article"));
assert_eq!(merged.authors.len(), 1);
assert_eq!(
merged.authors[0].orcid.as_deref(),
Some("https://orcid.org/0000-0001-2345-6789")
);
}
#[test]
fn test_merge_keeps_existing_fields() {
let work = work_fixture(json!({
"title": "OpenAlex title",
"doi": "https://doi.org/10.1002/example",
"publication_year": 2000,
"authorships": [authorship("Brewster Kahle")],
"biblio": { "volume": "18" },
"primary_location": {
"source": source("OpenAlex Journal", "journal", "Wiley")
}
}));
let biblio = Biblio {
title: Some("Parsed title".to_string()),
journal: Some("Parsed Journal".to_string()),
volume: Some("1".to_string()),
authors: vec![Author {
surname: Some("Parsed".to_string()),
..Author::default()
}],
..Biblio::default()
};
let merged = merge(&biblio, &work);
assert_eq!(merged.title.as_deref(), Some("Parsed title"));
assert_eq!(merged.journal.as_deref(), Some("Parsed Journal"));
assert_eq!(merged.volume.as_deref(), Some("1"));
assert_eq!(merged.authors[0].surname.as_deref(), Some("Parsed"));
assert_eq!(merged.doi.as_deref(), Some("10.1002/example"));
}
#[test]
fn test_merge_never_overwrites_other_identifiers() {
let work = work_fixture(json!({ "title": "Example", "doi": "10.1/x" }));
let biblio = Biblio {
pmcid: Some("PMC1".to_string()),
arxiv_id: Some("1706.03762".to_string()),
pii: Some("S0000-0000(00)00000-0".to_string()),
ark: Some("ark:/12345/x".to_string()),
istex_id: Some("istex-1".to_string()),
..Biblio::default()
};
let merged = merge(&biblio, &work);
assert_eq!(merged.pmcid.as_deref(), Some("PMC1"));
assert_eq!(merged.arxiv_id.as_deref(), Some("1706.03762"));
assert_eq!(merged.pii.as_deref(), Some("S0000-0000(00)00000-0"));
assert_eq!(merged.ark.as_deref(), Some("ark:/12345/x"));
assert_eq!(merged.istex_id.as_deref(), Some("istex-1"));
}
#[test]
fn test_merge_container_types() {
let chapter = work_fixture(json!({
"type": "book-chapter",
"primary_location": {
"source": source("Handbook of Examples", "book series", "Springer")
}
}));
let merged = merge(&Biblio::default(), &chapter);
assert_eq!(merged.book_title.as_deref(), Some("Handbook of Examples"));
assert_eq!(merged.journal, None);
assert_eq!(merged.publisher.as_deref(), Some("Springer"));
let preprint = work_fixture(json!({
"type": "preprint",
"primary_location": {
"source": source("arXiv", "repository", "Cornell University")
}
}));
let merged = merge(&Biblio::default(), &preprint);
assert_eq!(merged.journal, None);
assert_eq!(merged.book_title, None);
assert_eq!(merged.publisher, None);
assert_eq!(merged.institution, None);
let report = work_fixture(json!({
"type": "report",
"primary_location": {
"source": source("Reports", "repository", "Some Institute")
}
}));
let merged = merge(&Biblio::default(), &report);
assert_eq!(merged.institution.as_deref(), Some("Some Institute"));
}
#[test]
fn test_work_parsing_tolerates_schema_drift() {
let work: Work = serde_json::from_str(
r#"{
"id": "https://openalex.org/W1",
"title": "Example",
"type": "article",
"awards": [],
"is_xpac": false,
"cited_by_api_url": null
}"#,
)
.expect("tolerant parse");
assert_eq!(work.title.as_deref(), Some("Example"));
assert_eq!(work.work_type, "article");
assert_eq!(work.publication_year, 0);
assert!(work.authorships.is_empty());
}
#[test]
fn test_work_parsing_tolerates_nulls_and_missing_fields() {
let work: Work = serde_json::from_str(
r#"{
"id": "https://openalex.org/W1",
"title": null,
"display_name": null,
"doi": null,
"publication_year": null,
"publication_date": null,
"authorships": null,
"biblio": null,
"ids": null,
"primary_location": null,
"locations": null,
"cited_by_count": null,
"type": null
}"#,
)
.expect("tolerant parse");
assert_eq!(work.id, "https://openalex.org/W1");
assert_eq!(work.doi, None);
assert_eq!(work.publication_year, 0);
assert_eq!(work.publication_date, "");
assert!(work.authorships.is_empty());
assert!(work.locations.is_empty());
let work: Work = serde_json::from_str(
r#"{
"id": "W",
"authorships": [
{ "author": null },
{ "author": { "display_name": null, "orcid": null } }
],
"primary_location": { "source": null },
"locations": [
{ "landing_page_url": null, "source": { "display_name": null, "issn": null } }
]
}"#,
)
.expect("tolerant parse");
assert!(work.authorships[0].author.display_name.is_empty());
assert_eq!(work.authorships[1].author.orcid, None);
assert!(work.locations[0]
.source
.as_ref()
.expect("source")
.issn
.is_empty());
}
#[test]
fn test_work_response_parsing_defaults() {
let response: WorkResponse = serde_json::from_str("{}").expect("tolerant parse");
assert!(response.results.is_empty());
let response: WorkResponse =
serde_json::from_str(r#"{ "results": null }"#).expect("tolerant parse");
assert!(response.results.is_empty());
}
#[tokio::test]
async fn test_complete_without_doi_or_title_skips_lookup() {
let result = Completer::new().complete(&Biblio::default()).await;
assert!(matches!(result, Ok(None)));
}
#[test]
fn test_truncate_message() {
assert_eq!(truncate_message(" short ".to_string()), "short");
let long = "€".repeat(200);
let truncated = truncate_message(long);
assert!(truncated.ends_with('…'));
assert!(truncated.len() <= MAX_ERROR_MESSAGE_LEN + '…'.len_utf8());
}
#[test]
#[ignore = "known weakness: titles are not folded to ASCII, so diacritics differ"]
fn known_weakness_diacritics_are_not_folded() {
assert!(titles_match("Müller model", "Muller model"));
}
#[test]
#[ignore = "known weakness: apostrophes split words (Alzheimer's vs Alzheimers)"]
fn known_weakness_possessive_apostrophes() {
assert!(titles_match("Alzheimer's disease", "Alzheimers disease"));
}
#[test]
#[ignore = "known weakness: compounds with and without hyphens are not equal"]
fn known_weakness_hyphen_compounds() {
assert!(titles_match(
"Co-operation and competition",
"Cooperation and competition"
));
}
#[test]
#[ignore = "known weakness: containment needs 20 characters, so short subtitles are rejected"]
fn known_weakness_short_subtitle() {
assert!(titles_match("Deep learning: a review", "Deep learning"));
}
#[test]
#[ignore = "known weakness: the ±1 year tolerance can bind a re-edition"]
fn known_weakness_year_tolerance_binds_neighbouring_edition() {
let biblio = Biblio {
date: Some("2019".to_string()),
..Biblio::default()
};
let work = work_fixture(json!({ "publication_year": 2020 }));
assert!(!year_compatible(&biblio, &work));
}
#[test]
#[ignore = "known weakness: surname-first display names are mis-split"]
fn known_weakness_surname_first_names() {
let work = work_fixture(json!({ "authorships": [authorship("Xi Jinping")] }));
let authors = work_authors(&work);
assert_eq!(authors[0].surname.as_deref(), Some("Xi"));
}
#[test]
#[ignore = "known weakness: compound surnames lose all but the last word"]
fn known_weakness_compound_surnames() {
let work = work_fixture(json!({
"authorships": [authorship("Gabriel García Márquez")]
}));
let authors = work_authors(&work);
assert_eq!(authors[0].surname.as_deref(), Some("García Márquez"));
}
#[test]
#[ignore = "known weakness: 'Surname, Given' display names are not reordered"]
fn known_weakness_comma_names() {
let work = work_fixture(json!({ "authorships": [authorship("Kahle, Brewster")] }));
let authors = work_authors(&work);
assert_eq!(authors[0].surname.as_deref(), Some("Kahle"));
}
#[test]
#[ignore = "known weakness: title-only matches cannot be verified"]
fn known_weakness_unverifiable_title_only_match() {
let biblio = Biblio {
title: Some("Editorial".to_string()),
..Biblio::default()
};
let wrong = work_fixture(json!({
"title": "Editorial",
"publication_year": 2021,
"cited_by_count": 1
}));
assert!(select_match(&biblio, vec![wrong]).is_none());
}
#[test]
fn completer_builder_sets_base_url() {
let completer = Completer::builder()
.base_url("http://localhost:9999")
.expect("valid URL")
.timeout(Duration::from_secs(3))
.connect_timeout(Duration::from_secs(1))
.build()
.expect("client");
assert_eq!(completer.base_url(), "http://localhost:9999/");
}
#[test]
fn completer_builder_validates_base_url() {
let error = Completer::builder()
.base_url("not a url")
.expect_err("invalid URL must fail");
assert!(matches!(error, Error::InvalidBaseUrl { .. }), "{error:?}");
let error = Completer::builder()
.base_url("ftp://example.org")
.expect_err("unsupported scheme must fail");
assert!(
matches!(error, Error::UnsupportedScheme { .. }),
"{error:?}"
);
}
#[tokio::test]
async fn completer_timeout_applies() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
.await
.expect("bind");
let addr = listener.local_addr().expect("local addr");
tokio::spawn(async move {
if let Ok((stream, _)) = listener.accept().await {
let _stream = stream;
std::future::pending::<()>().await;
}
});
let completer = Completer::builder()
.base_url(format!("http://{addr}"))
.expect("valid URL")
.timeout(Duration::from_millis(200))
.build()
.expect("client");
let biblio = Biblio {
doi: Some("10.1234/example".to_string()),
..Biblio::default()
};
let started = std::time::Instant::now();
let error = completer
.complete(&biblio)
.await
.expect_err("the request must time out");
assert!(matches!(error, Error::Http(_)), "{error:?}");
assert!(
started.elapsed() < Duration::from_secs(5),
"timeout took {:?}",
started.elapsed()
);
}
}