1use std::collections::HashSet;
7
8use async_trait::async_trait;
9use serde::Deserialize;
10use url::Url;
11
12use crate::provenance::{Capability, LogEvent, LogResult, RowInput};
13use crate::source::{FetchContext, FetchError, FetchResult, Source};
14use crate::{CapabilityProfile, Ref};
15
16const DEFAULT_BASE: &str = "https://api.crossref.org";
19
20const MIN_CITATION_SCORE: f64 = 0.5;
23
24#[derive(Clone, Debug)]
28pub struct CrossrefSource {
29 base: Url,
34 #[allow(dead_code)]
39 contact_email: String,
40}
41
42impl CrossrefSource {
43 #[must_use]
48 pub fn new(contact_email: String) -> Self {
49 Self {
50 #[allow(clippy::expect_used)]
54 base: Url::parse(DEFAULT_BASE).expect("hard-coded base URL is valid"),
55 contact_email,
56 }
57 }
58
59 pub fn with_base(base: Url, contact_email: String) -> Self {
66 Self {
67 base,
68 contact_email,
69 }
70 }
71
72 fn request_url(&self, doi: &crate::Doi) -> Result<Url, FetchError> {
77 let path = format!("/works/{}", doi.as_str());
82 self.base.join(&path).map_err(|e| FetchError::SourceSchema {
83 hint: format!("crossref URL construction failed: {e}"),
84 })
85 }
86
87 pub async fn resolve_citation(
89 &self,
90 query: &str,
91 rows: u8,
92 ctx: &FetchContext,
93 ) -> Result<Vec<crate::ResolvedCandidate>, FetchError> {
94 let _permit = ctx.rate_limiter.acquire(self.name()).await;
96
97 let mut url = self
100 .base
101 .join("/works")
102 .map_err(|e| FetchError::SourceSchema {
103 hint: format!("crossref resolve_citation URL construction failed: {e}"),
104 })?;
105 url.query_pairs_mut()
106 .append_pair("query.bibliographic", query)
107 .append_pair("rows", &rows.to_string())
108 .append_pair("mailto", &self.contact_email);
109
110 let (body, _final_url) = ctx.http.fetch_bytes(self.name(), url).await?;
112
113 let envelope: serde_json::Value =
115 serde_json::from_slice(&body).map_err(|e| FetchError::SourceSchema {
116 hint: format!("crossref returned non-JSON for search: {e}"),
117 })?;
118
119 let items = envelope
120 .get("message")
121 .and_then(|m| m.get("items"))
122 .and_then(|i| i.as_array())
123 .ok_or_else(|| FetchError::SourceSchema {
124 hint: "crossref response missing message.items".to_string(),
125 })?;
126
127 let query_tokens = {
129 let mut t: Vec<String> = query
130 .split(|c: char| !c.is_alphanumeric())
131 .map(|s| s.to_lowercase())
132 .filter(|s| !s.is_empty())
133 .collect();
134 t.sort();
135 t.dedup();
136 t
137 };
138
139 if query_tokens.is_empty() {
140 return Ok(Vec::new());
141 }
142
143 let mut candidates = Vec::new();
144
145 for item in items {
146 let doi = match item.get("DOI").and_then(|v| v.as_str()) {
147 Some(d) => d.to_string(),
148 None => continue,
149 };
150
151 let fields = crate::orchestrator::extract_crossref_fields(item);
152
153 let mut candidate_text = String::new();
155 if let Some(t) = &fields.title {
156 candidate_text.push_str(&t.to_lowercase());
157 candidate_text.push(' ');
158 }
159 for author in &fields.authors {
163 candidate_text.push_str(&author.to_lowercase());
164 candidate_text.push(' ');
165 }
166 if let Some(v) = &fields.venue {
167 candidate_text.push_str(&v.to_lowercase());
168 candidate_text.push(' ');
169 }
170 if let Some(y) = fields.year {
171 candidate_text.push_str(&y.to_string());
172 candidate_text.push(' ');
173 }
174
175 let candidate_tokens: HashSet<String> = candidate_text
177 .split(|c: char| !c.is_alphanumeric())
178 .map(|s| s.to_lowercase())
179 .filter(|s| !s.is_empty())
180 .collect();
181
182 let matched: Vec<String> = query_tokens
189 .iter()
190 .filter(|q| candidate_tokens.contains(*q))
191 .cloned()
192 .collect();
193
194 let score = matched.len() as f64 / query_tokens.len() as f64;
195
196 if score >= MIN_CITATION_SCORE {
197 let first_author = fields.authors.first().cloned().unwrap_or_default();
198 candidates.push(crate::ResolvedCandidate {
199 doi,
200 title: fields.title.unwrap_or_default(),
201 author: first_author,
202 year: fields.year,
203 score,
204 confidence: crate::Confidence::from_score(score),
205 matched,
206 source: "crossref".to_string(),
207 });
208 }
209 }
210
211 candidates.sort_by(|a, b| {
213 b.score
214 .partial_cmp(&a.score)
215 .unwrap_or(std::cmp::Ordering::Equal)
216 });
217
218 Ok(candidates)
219 }
220}
221
222#[async_trait]
223impl Source for CrossrefSource {
224 fn name(&self) -> &str {
225 "crossref"
226 }
227
228 fn can_serve(&self, _profile: &CapabilityProfile, ref_: &Ref) -> bool {
229 matches!(ref_, Ref::Doi(_))
230 }
231
232 async fn fetch(
233 &self,
234 ref_: &Ref,
235 _profile: &CapabilityProfile,
236 ctx: &FetchContext,
237 ) -> Result<FetchResult, FetchError> {
238 let doi = match ref_ {
239 Ref::Doi(d) => d,
240 Ref::Arxiv(_) => {
241 return Err(FetchError::NotEligible {
242 source_key: "crossref".into(),
243 });
244 }
245 };
246
247 let _permit = ctx.rate_limiter.acquire(self.name()).await;
249
250 let url = self.request_url(doi)?;
254 let (body, final_url) = ctx.http.fetch_bytes(self.name(), url).await?;
255
256 let envelope: CrossrefEnvelope =
260 serde_json::from_slice(&body).map_err(|e| FetchError::SourceSchema {
261 hint: format!("crossref returned non-JSON: {e}"),
262 })?;
263 if envelope.status != "ok" {
264 return Err(FetchError::SourceSchema {
265 hint: format!("crossref status = {}", envelope.status),
266 });
267 }
268
269 let canonical = ref_.promote(self.name(), None).digest_hex();
274 ctx.log.append(RowInput {
275 event: LogEvent::Fetch,
276 result: LogResult::Ok,
277 capability: Capability::Oa,
278 ref_: Some(doi.as_str()),
279 source: Some(self.name()),
280 error_code: None,
281 size_bytes: Some(body.len() as u64),
282 license: None,
283 store_path: None,
284 canonical_digest: Some(&canonical),
285 })?;
286
287 Ok(FetchResult {
288 source: self.name().to_string(),
289 license: "unknown".into(),
290 pdf_bytes: None,
293 final_url: Some(final_url),
294 metadata_json: Some(envelope.message),
295 })
296 }
297}
298
299#[derive(Debug, Deserialize)]
302struct CrossrefEnvelope {
303 status: String,
304 message: serde_json::Value,
305}
306
307#[cfg(test)]
312#[allow(clippy::expect_used, clippy::unwrap_used, clippy::panic)]
313mod tests {
314 use super::*;
315
316 use std::sync::Arc;
317
318 use camino::Utf8PathBuf;
319 use tempfile::TempDir;
320 use wiremock::matchers::{method, path};
321 use wiremock::{Mock, MockServer, ResponseTemplate};
322
323 use crate::http::HttpClient;
324 use crate::provenance::ProvenanceLog;
325 use crate::rate_limiter::RateLimiter;
326 use crate::{ArxivId, CapabilityProfile, Doi, RateLimits, Ref};
327
328 fn build_test_context(wiremock_host: &str) -> (TempDir, FetchContext) {
333 let td = TempDir::new().expect("tempdir");
334 let log_dir =
336 Utf8PathBuf::try_from(td.path().to_path_buf()).expect("temp dir path must be UTF-8");
337 let log_path = log_dir.join("test.jsonl");
338
339 let http = Arc::new(HttpClient::new_for_tests_allow_http(
344 "crossref",
345 wiremock_host,
346 ));
347 let rate_limiter = Arc::new(RateLimiter::new(RateLimits::HARD_CODED));
348 let session_id = "01J0000000000000000000TEST".to_string();
349 let log = Arc::new(
350 ProvenanceLog::open(log_path, session_id.clone()).expect("provenance log opens"),
351 );
352
353 (
354 td,
355 FetchContext {
356 http,
357 rate_limiter,
358 log,
359 session_id,
360 cache_root: None,
361 },
362 )
363 }
364
365 fn server_host(server: &MockServer) -> String {
367 server
368 .uri()
369 .parse::<Url>()
370 .expect("wiremock uri parses")
371 .host_str()
372 .expect("wiremock uri has host")
373 .to_string()
374 }
375
376 fn crossref_for(server: &MockServer) -> CrossrefSource {
378 let base = server.uri().parse::<Url>().expect("wiremock uri parses");
379 CrossrefSource::with_base(base, "test@example.org".to_string())
380 }
381
382 #[test]
383 fn crossref_can_serve_returns_true_for_doi() {
384 let s = CrossrefSource::new("test@example.org".into());
385 let profile = CapabilityProfile::for_tests();
386 let r = Ref::Doi(Doi::parse("10.1234/example").unwrap());
387 assert!(s.can_serve(&profile, &r));
388 }
389
390 #[test]
391 fn crossref_can_serve_returns_false_for_arxiv() {
392 let s = CrossrefSource::new("test@example.org".into());
393 let profile = CapabilityProfile::for_tests();
394 let r = Ref::Arxiv(ArxivId::parse("2401.12345").unwrap());
395 assert!(!s.can_serve(&profile, &r));
396 }
397
398 #[tokio::test]
399 async fn crossref_fetch_returns_envelope_message() {
400 let server = MockServer::start().await;
401 Mock::given(method("GET"))
402 .and(path("/works/10.1234/example"))
403 .respond_with(
404 ResponseTemplate::new(200)
405 .set_body_string(r#"{"status":"ok","message":{"title":["Example"]}}"#),
406 )
407 .mount(&server)
408 .await;
409
410 let host = server_host(&server);
411 let s = crossref_for(&server);
412 let (_td, ctx) = build_test_context(&host);
413 let profile = CapabilityProfile::for_tests();
414 let r = Ref::Doi(Doi::parse("10.1234/example").unwrap());
415
416 let res = s.fetch(&r, &profile, &ctx).await.expect("fetch ok");
417 assert_eq!(res.source, "crossref");
418 assert_eq!(
419 res.metadata_json,
420 Some(serde_json::json!({ "title": ["Example"] })),
421 );
422 assert!(res.pdf_bytes.is_none());
423 assert!(res.final_url.is_some());
424 }
425
426 #[tokio::test]
427 async fn crossref_fetch_with_arxiv_ref_errors_not_eligible() {
428 let s = CrossrefSource::with_base(
432 Url::parse("http://127.0.0.1:1/").unwrap(),
433 "test@example.org".into(),
434 );
435 let (_td, ctx) = build_test_context("127.0.0.1");
436 let profile = CapabilityProfile::for_tests();
437 let r = Ref::Arxiv(ArxivId::parse("2401.12345").unwrap());
438
439 let err = s.fetch(&r, &profile, &ctx).await.expect_err("not eligible");
440 match err {
441 FetchError::NotEligible { source_key } => {
442 assert_eq!(source_key, "crossref");
443 }
444 other => panic!("expected NotEligible, got {:?}", other),
445 }
446 }
447
448 #[tokio::test]
449 async fn crossref_fetch_writes_log_row() {
450 let server = MockServer::start().await;
451 Mock::given(method("GET"))
452 .and(path("/works/10.1234/example"))
453 .respond_with(
454 ResponseTemplate::new(200)
455 .set_body_string(r#"{"status":"ok","message":{"title":["Example"]}}"#),
456 )
457 .mount(&server)
458 .await;
459
460 let host = server_host(&server);
461 let s = crossref_for(&server);
462 let (_td, ctx) = build_test_context(&host);
463 let profile = CapabilityProfile::for_tests();
464 let r = Ref::Doi(Doi::parse("10.1234/example").unwrap());
465
466 let _res = s.fetch(&r, &profile, &ctx).await.expect("fetch ok");
467
468 let log_path = _td.path().join("test.jsonl");
472 let raw = std::fs::read_to_string(&log_path).expect("log file readable");
473 let lines: Vec<&str> = raw.lines().filter(|l| !l.is_empty()).collect();
474 assert_eq!(lines.len(), 1, "expected exactly one row, got {:?}", lines);
475 let row: serde_json::Value = serde_json::from_str(lines[0]).expect("row is valid JSON");
476 assert_eq!(row["event"], "fetch");
477 assert_eq!(row["result"], "ok");
478 assert_eq!(row["source"], "crossref");
479 assert_eq!(row["ref"], "10.1234/example");
480 }
481
482 #[tokio::test]
483 async fn crossref_404_maps_to_http_error() {
484 let server = MockServer::start().await;
485 Mock::given(method("GET"))
486 .and(path("/works/10.1234/example"))
487 .respond_with(ResponseTemplate::new(404))
488 .mount(&server)
489 .await;
490
491 let host = server_host(&server);
492 let s = crossref_for(&server);
493 let (_td, ctx) = build_test_context(&host);
494 let profile = CapabilityProfile::for_tests();
495 let r = Ref::Doi(Doi::parse("10.1234/example").unwrap());
496
497 let err = s.fetch(&r, &profile, &ctx).await.expect_err("404 errors");
498 match err {
499 FetchError::Http(_) => {}
500 other => panic!("expected Http(_) on 404, got {:?}", other),
501 }
502 }
503
504 #[tokio::test]
505 async fn crossref_non_ok_status_field_errors_source_schema() {
506 let server = MockServer::start().await;
507 Mock::given(method("GET"))
508 .and(path("/works/10.1234/example"))
509 .respond_with(
510 ResponseTemplate::new(200).set_body_string(r#"{"status":"error","message":{}}"#),
511 )
512 .mount(&server)
513 .await;
514
515 let host = server_host(&server);
516 let s = crossref_for(&server);
517 let (_td, ctx) = build_test_context(&host);
518 let profile = CapabilityProfile::for_tests();
519 let r = Ref::Doi(Doi::parse("10.1234/example").unwrap());
520
521 let err = s
522 .fetch(&r, &profile, &ctx)
523 .await
524 .expect_err("non-ok status errors");
525 match err {
526 FetchError::SourceSchema { hint } => {
527 assert!(
528 hint.contains("status"),
529 "expected status mention in hint, got {hint}"
530 );
531 }
532 other => panic!("expected SourceSchema, got {:?}", other),
533 }
534 }
535
536 #[tokio::test]
537 async fn test_resolve_citation_success() {
538 let server = MockServer::start().await;
539 let mock_body = serde_json::json!({
540 "status": "ok",
541 "message": {
542 "items": [
543 {
544 "DOI": "10.1000/xyz123",
545 "title": ["Lars Onsager, Crystal Statistics. I. A Two-Dimensional Model with an Order-Disorder Transition"],
546 "author": [
547 {"family": "Onsager", "given": "Lars"}
548 ],
549 "issued": {
550 "date-parts": [[1944, 2, 1]]
551 },
552 "container-title": ["Physical Review"]
553 },
554 {
555 "DOI": "10.1000/unrelated",
556 "title": ["Some Unrelated Paper"],
557 "author": [
558 {"family": "Smith", "given": "John"}
559 ],
560 "issued": {
561 "date-parts": [[2020]]
562 }
563 }
564 ]
565 }
566 });
567
568 Mock::given(method("GET"))
569 .and(path("/works"))
570 .respond_with(ResponseTemplate::new(200).set_body_json(mock_body))
571 .mount(&server)
572 .await;
573
574 let host = server_host(&server);
575 let s = crossref_for(&server);
576 let (_td, ctx) = build_test_context(&host);
577
578 let candidates = s
579 .resolve_citation("Onsager 1944", 2, &ctx)
580 .await
581 .expect("resolve ok");
582
583 assert_eq!(candidates.len(), 1);
587 let cand = &candidates[0];
588 assert_eq!(cand.doi, "10.1000/xyz123");
589 assert_eq!(cand.title, "Lars Onsager, Crystal Statistics. I. A Two-Dimensional Model with an Order-Disorder Transition");
590 assert_eq!(cand.author, "Onsager, Lars");
591 assert_eq!(cand.year, Some(1944));
592 assert_eq!(cand.score, 1.0);
593 assert_eq!(cand.confidence, crate::Confidence::Exact);
596 let mut got = cand.matched.clone();
597 got.sort();
598 assert_eq!(
599 got,
600 vec!["1944".to_string(), "onsager".to_string()],
601 "the evidence behind the score, not just the score"
602 );
603 }
604
605 #[test]
614 fn the_floor_bands_as_weak_and_a_full_match_bands_as_exact() {
615 use crate::Confidence;
616 assert_eq!(
617 Confidence::from_score(MIN_CITATION_SCORE),
618 Confidence::Weak,
619 "the worst candidate the tool can emit must not read as a match"
620 );
621 assert_eq!(Confidence::from_score(1.0), Confidence::Exact);
622
623 assert_eq!(Confidence::from_score(0.8), Confidence::Probable);
625 assert_eq!(Confidence::from_score(0.79), Confidence::Weak);
626
627 assert_eq!(Confidence::from_score(7.0 / 7.0), Confidence::Exact);
631 assert_eq!(Confidence::from_score(0.9999), Confidence::Exact);
632 }
633
634 #[test]
639 fn a_score_outside_the_ratio_range_is_never_confident() {
640 use crate::Confidence;
641 for bad in [-5.0, 1.5, 50.0, f64::NAN, f64::INFINITY, f64::NEG_INFINITY] {
642 assert_eq!(
643 Confidence::from_score(bad),
644 Confidence::Weak,
645 "{bad} is not a ratio and must not band as a match"
646 );
647 }
648 }
649
650 #[test]
653 fn confidence_is_monotonic_in_the_score() {
654 use crate::Confidence;
655 let rank = |c| match c {
656 Confidence::Weak => 0,
657 Confidence::Probable => 1,
658 Confidence::Exact => 2,
661 };
662 let mut prev = 0;
663 for i in 50..=100 {
664 let r = rank(Confidence::from_score(f64::from(i) / 100.0));
665 assert!(r >= prev, "score {i}/100 banded below a lower score");
666 prev = r;
667 }
668 assert_eq!(prev, 2, "the top of the range must reach Exact");
669 }
670
671 #[tokio::test]
672 async fn resolve_citation_matches_non_first_authors() {
673 let server = MockServer::start().await;
678 let mock_body = serde_json::json!({
679 "status": "ok",
680 "message": {
681 "items": [
682 {
683 "DOI": "10.1103/RevModPhys.80.395",
684 "title": ["Numerical renormalization group method for quantum impurity systems"],
685 "author": [
686 {"family": "Bulla", "given": "Ralf"},
687 {"family": "Costi", "given": "Theo A."},
688 {"family": "Pruschke", "given": "Thomas"}
689 ],
690 "issued": { "date-parts": [[2008, 4, 2]] },
691 "container-title": ["Reviews of Modern Physics"]
692 }
693 ]
694 }
695 });
696
697 Mock::given(method("GET"))
698 .and(path("/works"))
699 .respond_with(ResponseTemplate::new(200).set_body_json(mock_body))
700 .mount(&server)
701 .await;
702
703 let host = server_host(&server);
704 let s = crossref_for(&server);
705 let (_td, ctx) = build_test_context(&host);
706
707 let candidates = s
708 .resolve_citation("Costi Pruschke 2008", 5, &ctx)
709 .await
710 .expect("resolve ok");
711
712 assert_eq!(candidates.len(), 1);
713 assert_eq!(candidates[0].doi, "10.1103/RevModPhys.80.395");
714 assert_eq!(candidates[0].score, 1.0);
715 }
716}