aspero_openlibrary/
lib.rs

1//! Open Library (Internet Archive) provider: books.
2//!
3//! Books mostly lack DOIs, so this fills the gap the article-centric
4//! providers leave. Free and keyless; Open Library asks clients to identify
5//! themselves in the User-Agent (the engine's carries the contact address).
6//!
7//! - Lookup by ISBN in batches: `/api/books?bibkeys=ISBN:a,ISBN:b&jscmd=data`
8//!   (one edition per ISBN; ISBNs it doesn't know are simply absent).
9//! - Search: `/search.json` (Solr, relevance-ordered), returning works with
10//!   every edition's ISBNs; year ranges via `first_publish_year:[A TO B]`.
11//!
12//! Data quality varies: some ISBNs carry unrelated records, so scoring (not
13//! the identifier alone) decides matches.
14
15use std::sync::Arc;
16use std::time::Duration;
17
18use aspero_core::http::escape_lucene;
19use aspero_core::{
20    Capabilities, Cursor, Error, FieldSet, Filter, Hit, HttpClient, HttpRequest, HttpResponse,
21    IdKind, Isbn, Lookup, Page, PartialDate, Person, Provider, ProviderId, Raw, ResiliencePolicy,
22    Response, Result, SearchMode, SearchQuery, SourceRecord, Work, WorkId, WorkType,
23};
24use async_trait::async_trait;
25use jiff::Timestamp;
26use serde::Deserialize;
27
28pub const ID: ProviderId = ProviderId::from_static("openlibrary");
29
30pub const DEFAULT_BASE_URL: &str = "https://openlibrary.org";
31
32/// ISBNs per batched lookup.
33const MAX_BATCH: usize = 50;
34
35/// Results per search page (Open Library allows more; works carry long ISBN
36/// lists, so pages stay modest).
37const MAX_PAGE: u32 = 100;
38
39/// ISBNs kept per searched work: enough to match editions, without
40/// hundreds of translations.
41const MAX_WORK_ISBNS: usize = 20;
42
43const SEARCH_FIELDS: &str = "key,title,subtitle,author_name,first_publish_year,publish_year,isbn,publisher,number_of_pages_median";
44
45#[derive(Debug, Clone)]
46#[non_exhaustive]
47pub struct OpenLibraryConfig {
48    pub base_url: String,
49}
50
51impl Default for OpenLibraryConfig {
52    fn default() -> Self {
53        OpenLibraryConfig {
54            base_url: DEFAULT_BASE_URL.to_owned(),
55        }
56    }
57}
58
59pub struct OpenLibrary {
60    config: OpenLibraryConfig,
61    http: Arc<dyn HttpClient>,
62    capabilities: Capabilities,
63}
64
65impl OpenLibrary {
66    pub fn new(config: OpenLibraryConfig, http: Arc<dyn HttpClient>) -> Self {
67        let mut capabilities = Capabilities::default();
68        capabilities.lookup_kinds = vec![IdKind::Isbn];
69        capabilities.max_batch = MAX_BATCH;
70        capabilities.fields = FieldSet::of(&[
71            aspero_core::Field::Title,
72            aspero_core::Field::Authors,
73            aspero_core::Field::Issued,
74            aspero_core::Field::Type,
75            aspero_core::Field::Publication,
76            aspero_core::Field::Url,
77        ]);
78        capabilities.search = true;
79        capabilities.search_filters = vec![Filter::Years];
80        OpenLibrary {
81            config,
82            http,
83            capabilities,
84        }
85    }
86
87    /// A volunteer-run service: keep it gentle.
88    pub fn default_policy() -> ResiliencePolicy {
89        let mut policy = ResiliencePolicy::default();
90        policy.min_interval = Duration::from_millis(600);
91        policy
92    }
93
94    async fn get(&self, path: &str, params: &[(&str, String)]) -> Result<HttpResponse> {
95        let base = format!("{}{path}", self.config.base_url.trim_end_matches('/'));
96        let url = url::Url::parse_with_params(&base, params)
97            .map_err(|e| Error::BadRequest(format!("invalid Open Library base URL: {e}")))?;
98        let resp = self
99            .http
100            .send(HttpRequest::get(String::from(url)).header("accept", "application/json"))
101            .await?;
102        check(&resp)?;
103        Ok(resp)
104    }
105}
106
107fn check(resp: &HttpResponse) -> Result<()> {
108    if resp.is_success() {
109        return Ok(());
110    }
111    Err(match resp.status {
112        400 | 422 => Error::BadRequest("rejected by Open Library".into()),
113        404 => Error::NotFound,
114        429 => Error::RateLimited {
115            retry_after: resp
116                .header("retry-after")
117                .and_then(|v| v.trim().parse().ok())
118                .map(Duration::from_secs),
119        },
120        status => Error::Server {
121            status,
122            message: None,
123        },
124    })
125}
126
127/// `/api/books?jscmd=data` record for one edition.
128#[derive(Debug, Default, Deserialize)]
129#[serde(default)]
130struct Edition {
131    title: Option<String>,
132    subtitle: Option<String>,
133    authors: Vec<Named>,
134    publish_date: Option<String>,
135    publishers: Vec<Named>,
136    identifiers: Identifiers,
137    url: Option<String>,
138}
139
140#[derive(Debug, Default, Deserialize)]
141#[serde(default)]
142struct Named {
143    name: Option<String>,
144}
145
146#[derive(Debug, Default, Deserialize)]
147#[serde(default)]
148struct Identifiers {
149    isbn_13: Vec<String>,
150    isbn_10: Vec<String>,
151}
152
153/// `/search.json` document for one work.
154#[derive(Debug, Default, Deserialize)]
155#[serde(default)]
156struct SearchDoc {
157    key: Option<String>,
158    title: Option<String>,
159    subtitle: Option<String>,
160    author_name: Vec<String>,
161    first_publish_year: Option<i16>,
162    /// Every edition's year.
163    publish_year: Vec<i16>,
164    isbn: Vec<String>,
165    publisher: Vec<String>,
166}
167
168#[derive(Debug, Default, Deserialize)]
169#[serde(default, rename_all = "camelCase")]
170struct SearchResults {
171    num_found: Option<u64>,
172    docs: Vec<serde_json::Value>,
173}
174
175/// Open Library keeps subtitles apart ("Reinforcement Learning" + "An
176/// Introduction"); citations give the full title.
177fn full_title(title: Option<String>, subtitle: Option<String>) -> Option<String> {
178    match (title, subtitle.filter(|s| !s.trim().is_empty())) {
179        (Some(t), Some(s)) => Some(format!("{t}: {s}")),
180        (t, _) => t,
181    }
182}
183
184fn isbns<'a>(raw: impl IntoIterator<Item = &'a String>) -> Vec<WorkId> {
185    let mut ids = Vec::new();
186    for isbn in raw.into_iter().filter_map(|s| s.parse::<Isbn>().ok()) {
187        let id = WorkId::Isbn(isbn);
188        if !ids.contains(&id) {
189            ids.push(id);
190        }
191    }
192    ids
193}
194
195fn book(
196    title: Option<String>,
197    authors: Vec<String>,
198    year: Option<i16>,
199    publisher: Option<String>,
200    url: Option<String>,
201) -> Work {
202    let mut work = Work::default();
203    work.title = title;
204    work.authors = authors.into_iter().map(Person::literal).collect();
205    work.issued = year.map(PartialDate::from_year);
206    work.publisher = publisher;
207    work.work_type = Some(WorkType::Book);
208    work.url = url;
209    work
210}
211
212/// The first four-digit year in a free-form date ("3 January 2017").
213fn year_in(date: &str) -> Option<i16> {
214    date.as_bytes()
215        .windows(4)
216        .find(|w| w.iter().all(u8::is_ascii_digit))
217        .and_then(|w| std::str::from_utf8(w).ok()?.parse().ok())
218}
219
220fn edition_record(
221    requested: &Isbn,
222    value: &serde_json::Value,
223    at: Timestamp,
224) -> serde_json::Result<SourceRecord> {
225    let e = Edition::deserialize(value)?;
226    let mut ids = vec![WorkId::Isbn(requested.clone())];
227    for id in isbns(e.identifiers.isbn_13.iter().chain(&e.identifiers.isbn_10)) {
228        if !ids.contains(&id) {
229            ids.push(id);
230        }
231    }
232    let work = book(
233        full_title(e.title, e.subtitle),
234        e.authors.into_iter().filter_map(|a| a.name).collect(),
235        e.publish_date.as_deref().and_then(year_in),
236        e.publishers.into_iter().find_map(|p| p.name),
237        e.url,
238    );
239    Ok(SourceRecord::new(
240        ID,
241        ids,
242        work,
243        FieldSet::ALL,
244        Raw::json(value),
245        at,
246    ))
247}
248
249fn search_record(
250    base: &str,
251    value: &serde_json::Value,
252    at: Timestamp,
253) -> serde_json::Result<SourceRecord> {
254    let d = SearchDoc::deserialize(value)?;
255    let mut ids = isbns(&d.isbn);
256    ids.truncate(MAX_WORK_ISBNS);
257    let url = d.key.as_ref().map(|k| format!("{base}{k}"));
258    let mut work = book(
259        full_title(d.title, d.subtitle),
260        d.author_name,
261        d.first_publish_year,
262        d.publisher.into_iter().next(),
263        url,
264    );
265    // Later editions: a citation of the 2018 second edition is this work.
266    let mut editions: Vec<i16> = d
267        .publish_year
268        .into_iter()
269        .filter(|y| Some(*y) != d.first_publish_year)
270        .collect();
271    editions.sort_unstable();
272    editions.dedup();
273    work.other_years = editions;
274    Ok(SourceRecord::new(
275        ID,
276        ids,
277        work,
278        FieldSet::ALL,
279        Raw::json(value),
280        at,
281    ))
282}
283
284#[async_trait]
285impl Provider for OpenLibrary {
286    fn id(&self) -> ProviderId {
287        ID
288    }
289
290    fn capabilities(&self) -> &Capabilities {
291        &self.capabilities
292    }
293
294    async fn lookup(&self, ids: &[WorkId], _fields: FieldSet) -> Result<Response<Vec<Lookup>>> {
295        let isbns: Vec<&Isbn> = ids
296            .iter()
297            .map(|id| match id {
298                WorkId::Isbn(isbn) => Ok(isbn),
299                _ => Err(Error::Unsupported(
300                    "Open Library lookup of non-ISBN identifiers",
301                )),
302            })
303            .collect::<Result<_>>()?;
304        let mut lookups = Vec::with_capacity(ids.len());
305        for (chunk_ids, chunk) in ids.chunks(MAX_BATCH).zip(isbns.chunks(MAX_BATCH)) {
306            let keys: Vec<String> = chunk.iter().map(|i| format!("ISBN:{i}")).collect();
307            let params = [
308                ("bibkeys", keys.join(",")),
309                ("format", "json".to_owned()),
310                ("jscmd", "data".to_owned()),
311            ];
312            let resp = self.get("/api/books", &params).await?;
313            let body: serde_json::Map<String, serde_json::Value> =
314                serde_json::from_slice(&resp.body).map_err(Error::decode)?;
315            for ((id, isbn), key) in chunk_ids.iter().zip(chunk).zip(&keys) {
316                let record = body
317                    .get(key)
318                    .map(|v| edition_record(isbn, v, resp.received_at))
319                    .transpose()
320                    .map_err(Error::decode)?;
321                lookups.push(Lookup {
322                    id: id.clone(),
323                    record,
324                });
325            }
326        }
327        Ok(Response::new(lookups))
328    }
329
330    async fn search(&self, query: &SearchQuery) -> Result<Response<Page>> {
331        if query.mode != SearchMode::Keywords {
332            return Err(Error::Unsupported("Open Library has no citation matcher"));
333        }
334        if let Some((key, _)) = query.extra.iter().next() {
335            return Err(Error::BadRequest(format!(
336                "unknown Open Library parameter {key:?}"
337            )));
338        }
339        let mut clauses = Vec::new();
340        if !query.text.trim().is_empty() {
341            clauses.push(format!("({})", escape_lucene(query.text.trim())));
342        }
343        if let Some(years) = query.years {
344            let end = |y: Option<i16>| y.map_or("*".to_owned(), |y| y.to_string());
345            clauses.push(format!(
346                "first_publish_year:[{} TO {}]",
347                end(years.from),
348                end(years.to)
349            ));
350        }
351        let page_number: u32 = match &query.page.cursor {
352            None => 1,
353            Some(c) => c
354                .0
355                .parse()
356                .map_err(|_| Error::BadRequest(format!("invalid Open Library cursor {:?}", c.0)))?,
357        };
358        let limit = query.page.limit.clamp(1, MAX_PAGE);
359        let params = [
360            ("q", clauses.join(" AND ")),
361            ("fields", SEARCH_FIELDS.to_owned()),
362            ("limit", limit.to_string()),
363            ("page", page_number.to_string()),
364        ];
365        let resp = self.get("/search.json", &params).await?;
366        let results: SearchResults = serde_json::from_slice(&resp.body).map_err(Error::decode)?;
367        let base = self.config.base_url.trim_end_matches('/');
368        let hits: Vec<Hit> = results
369            .docs
370            .iter()
371            .map(|v| search_record(base, v, resp.received_at).map(|r| Hit::new(r, None)))
372            .collect::<serde_json::Result<_>>()
373            .map_err(Error::decode)?;
374        let seen = u64::from(page_number) * u64::from(limit);
375        let more = !hits.is_empty() && results.num_found.is_some_and(|n| n > seen);
376        let next = more.then(|| Cursor((page_number + 1).to_string()));
377        Ok(Response::new(Page::new(hits, results.num_found, next)))
378    }
379}
aspero_openlibrary/lib.rs

aspero_openlibrary/
lib.rs
