use std::path::PathBuf;
use std::time::Duration;
use reqwest::multipart::{Form, Part};
use reqwest::{Client, RequestBuilder, Response, StatusCode};
use url::Url;
use crate::tei::{parse_citation, parse_citation_list, parse_document, Citation, Document};
use crate::Error;
pub const DEFAULT_GROBID_URL: &str = "http://localhost:8070";
pub const SERVICE_FULLTEXT: &str = "processFulltextDocument";
pub const SERVICE_HEADER: &str = "processHeaderDocument";
pub const SERVICE_REFERENCES: &str = "processReferences";
pub const SERVICE_CITATION: &str = "processCitation";
pub const SERVICE_CITATION_LIST: &str = "processCitationList";
#[derive(Debug, Clone, PartialEq)]
pub struct RetryPolicy {
pub max_attempts: usize,
pub initial_delay: Duration,
pub max_delay: Duration,
pub multiplier: f64,
}
impl Default for RetryPolicy {
fn default() -> Self {
Self {
max_attempts: 5,
initial_delay: Duration::from_secs(2),
max_delay: Duration::from_secs(30),
multiplier: 2.0,
}
}
}
impl RetryPolicy {
fn delay(&self, attempt: usize) -> Duration {
let factor = self.multiplier.powi(attempt.saturating_sub(1) as i32);
let delay = self.initial_delay.mul_f64(factor);
delay.min(self.max_delay)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, serde::Serialize, serde::Deserialize)]
pub enum HeaderConsolidation {
None,
#[default]
Metadata,
Doi,
DoiOnly,
}
impl HeaderConsolidation {
fn as_param(self) -> &'static str {
match self {
HeaderConsolidation::None => "0",
HeaderConsolidation::Metadata => "1",
HeaderConsolidation::Doi => "2",
HeaderConsolidation::DoiOnly => "3",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, serde::Serialize, serde::Deserialize)]
pub enum CitationConsolidation {
#[default]
None,
Metadata,
Doi,
}
impl CitationConsolidation {
fn as_param(self) -> &'static str {
match self {
CitationConsolidation::None => "0",
CitationConsolidation::Metadata => "1",
CitationConsolidation::Doi => "2",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]
#[serde(rename_all = "camelCase")]
pub enum CoordinateElement {
Ref,
BiblStruct,
PersName,
Figure,
Formula,
Head,
Sentence,
Paragraph,
Note,
Title,
Affiliation,
}
impl CoordinateElement {
pub const COMMON: &'static [CoordinateElement] = &[
CoordinateElement::Ref,
CoordinateElement::Figure,
CoordinateElement::PersName,
CoordinateElement::Formula,
CoordinateElement::BiblStruct,
];
fn as_param(self) -> &'static str {
match self {
CoordinateElement::Ref => "ref",
CoordinateElement::BiblStruct => "biblStruct",
CoordinateElement::PersName => "persName",
CoordinateElement::Figure => "figure",
CoordinateElement::Formula => "formula",
CoordinateElement::Head => "head",
CoordinateElement::Sentence => "s",
CoordinateElement::Paragraph => "p",
CoordinateElement::Note => "note",
CoordinateElement::Title => "title",
CoordinateElement::Affiliation => "affiliation",
}
}
}
#[derive(Debug, Clone, PartialEq, Default)]
pub struct ProcessOptions {
pub generate_ids: bool,
pub consolidate_header: HeaderConsolidation,
pub consolidate_citations: CitationConsolidation,
pub include_raw_citations: bool,
pub include_raw_affiliations: bool,
pub include_raw_copyrights: bool,
pub tei_coordinates: Vec<CoordinateElement>,
pub segment_sentences: bool,
pub flavor: Option<String>,
pub start_page: Option<u32>,
pub end_page: Option<u32>,
pub debug_mode: bool,
pub models: Vec<String>,
}
impl ProcessOptions {
fn apply_to(&self, mut form: Form) -> Form {
form = form.text("consolidateHeader", self.consolidate_header.as_param());
form = form.text(
"consolidateCitations",
self.consolidate_citations.as_param(),
);
if self.generate_ids {
form = form.text("generateIDs", "1");
}
if self.include_raw_citations {
form = form.text("includeRawCitations", "1");
}
if self.include_raw_affiliations {
form = form.text("includeRawAffiliations", "1");
}
if self.include_raw_copyrights {
form = form.text("includeRawCopyrights", "1");
}
for element in &self.tei_coordinates {
form = form.text("teiCoordinates", element.as_param());
}
if self.segment_sentences {
form = form.text("segmentSentences", "1");
}
if let Some(flavor) = &self.flavor {
form = form.text("flavor", flavor.clone());
}
if let Some(start) = self.start_page {
form = form.text("start", start.to_string());
}
if let Some(end) = self.end_page {
form = form.text("end", end.to_string());
}
if self.debug_mode {
form = form.text("debugMode", "1");
}
if !self.models.is_empty() {
form = form.text("models", self.models.join(","));
}
form
}
fn apply_to_urlencoded(&self, fields: &mut Vec<(String, String)>) {
if self.consolidate_citations != CitationConsolidation::None {
fields.push((
"consolidateCitations".to_string(),
self.consolidate_citations.as_param().to_string(),
));
}
if self.include_raw_citations {
fields.push(("includeRawCitations".to_string(), "1".to_string()));
}
}
}
#[derive(Debug, Clone)]
pub enum PdfInput {
Path(PathBuf),
Data {
filename: String,
data: Vec<u8>,
},
}
impl From<&str> for PdfInput {
fn from(path: &str) -> Self {
PdfInput::Path(PathBuf::from(path))
}
}
impl From<String> for PdfInput {
fn from(path: String) -> Self {
PdfInput::Path(PathBuf::from(path))
}
}
impl From<&std::path::Path> for PdfInput {
fn from(path: &std::path::Path) -> Self {
PdfInput::Path(path.to_path_buf())
}
}
impl From<PathBuf> for PdfInput {
fn from(path: PathBuf) -> Self {
PdfInput::Path(path)
}
}
impl From<(&str, Vec<u8>)> for PdfInput {
fn from((filename, data): (&str, Vec<u8>)) -> Self {
PdfInput::Data {
filename: filename.to_string(),
data,
}
}
}
impl From<(String, Vec<u8>)> for PdfInput {
fn from((filename, data): (String, Vec<u8>)) -> Self {
PdfInput::Data { filename, data }
}
}
impl PdfInput {
async fn into_parts(self) -> Result<(String, Vec<u8>), Error> {
match self {
PdfInput::Path(path) => {
let filename = path
.file_name()
.map(|name| name.to_string_lossy().into_owned())
.unwrap_or_else(|| "document.pdf".to_string());
let data = tokio::fs::read(&path).await?;
Ok((filename, data))
}
PdfInput::Data { filename, data } => Ok((filename, data)),
}
}
}
#[derive(Debug, Clone)]
pub struct GrobidClientBuilder {
base_url: Url,
http: Option<Client>,
retry: RetryPolicy,
}
impl GrobidClientBuilder {
pub fn http_client(mut self, client: Client) -> Self {
self.http = Some(client);
self
}
pub fn retry(mut self, policy: RetryPolicy) -> Self {
self.retry = policy;
self
}
pub fn build(self) -> GrobidClient {
GrobidClient {
base_url: self.base_url,
http: self.http.unwrap_or_default(),
retry: self.retry,
}
}
}
#[derive(Debug, Clone)]
pub struct GrobidClient {
base_url: Url,
http: Client,
retry: RetryPolicy,
}
impl GrobidClient {
pub fn new(base_url: impl AsRef<str>) -> Result<Self, Error> {
Ok(Self::builder(base_url)?.build())
}
pub fn builder(base_url: impl AsRef<str>) -> Result<GrobidClientBuilder, Error> {
let base_url = normalize_base_url(base_url.as_ref())?;
Ok(GrobidClientBuilder {
base_url,
http: None,
retry: RetryPolicy::default(),
})
}
pub fn base_url(&self) -> &Url {
&self.base_url
}
pub fn http_client(&self) -> &Client {
&self.http
}
pub fn retry_policy(&self) -> &RetryPolicy {
&self.retry
}
pub async fn ping(&self) -> Result<bool, Error> {
let url = self.api_url("isalive");
let http = self.http.clone();
let request_url = url.clone();
let response = self.request(move || http.get(url.clone())).await?;
let status = response.status();
let body = response.text().await?;
match status {
StatusCode::OK => Ok(body.trim() == "true"),
status => Err(Error::http_status(
status.as_u16(),
request_url.to_string(),
body,
)),
}
}
pub async fn wait_until_ready(&self, attempts: usize, delay: Duration) -> Result<(), Error> {
let attempts = attempts.max(1);
for _ in 1..attempts {
match self.ping().await {
Ok(true) => return Ok(()),
Ok(false) => {
return Err(Error::ServerNotReady {
url: self.base_url.to_string(),
});
}
Err(err @ Error::HttpStatus { .. }) => return Err(err),
Err(_) => {}
}
tokio::time::sleep(delay).await;
}
match self.ping().await {
Ok(true) => Ok(()),
Ok(false) => Err(Error::ServerNotReady {
url: self.base_url.to_string(),
}),
Err(err @ Error::HttpStatus { .. }) => Err(err),
Err(Error::Http(source)) => Err(Error::ServerUnreachable {
url: self.base_url.to_string(),
attempts,
source,
}),
Err(err) => Err(err),
}
}
pub async fn version(&self) -> Result<String, Error> {
let url = self.api_url("version");
let http = self.http.clone();
let request_url = url.clone();
let response = self.request(move || http.get(url.clone())).await?;
let status = response.status();
let body = response.text().await?;
match status {
StatusCode::OK => Ok(body.trim().to_string()),
status => Err(Error::http_status(
status.as_u16(),
request_url.to_string(),
body,
)),
}
}
pub async fn process_fulltext_document(
&self,
pdf: impl Into<PdfInput>,
options: &ProcessOptions,
) -> Result<Document, Error> {
let xml = self
.process_pdf(SERVICE_FULLTEXT, pdf.into(), options)
.await?
.ok_or(Error::NoContent)?;
parse_document(&xml)
}
pub async fn process_header_document(
&self,
pdf: impl Into<PdfInput>,
options: &ProcessOptions,
) -> Result<Document, Error> {
let xml = self
.process_pdf(SERVICE_HEADER, pdf.into(), options)
.await?
.ok_or(Error::NoContent)?;
parse_document(&xml)
}
pub async fn process_references(
&self,
pdf: impl Into<PdfInput>,
options: &ProcessOptions,
) -> Result<Vec<Citation>, Error> {
match self
.process_pdf(SERVICE_REFERENCES, pdf.into(), options)
.await?
{
Some(xml) => parse_citation_list(&xml),
None => Ok(Vec::new()),
}
}
pub async fn process_citation(
&self,
citation: &str,
options: &ProcessOptions,
) -> Result<Option<Citation>, Error> {
let mut fields = vec![("citations".to_string(), citation.to_string())];
options.apply_to_urlencoded(&mut fields);
match self.post_urlencoded(SERVICE_CITATION, fields).await? {
Some(xml) if !xml.trim().is_empty() => parse_citation(&xml),
_ => Ok(None),
}
}
pub async fn process_citation_list(
&self,
citations: impl IntoIterator<Item = impl AsRef<str>>,
options: &ProcessOptions,
) -> Result<Vec<Citation>, Error> {
let mut fields = Vec::new();
for citation in citations {
fields.push(("citations".to_string(), citation.as_ref().to_string()));
}
options.apply_to_urlencoded(&mut fields);
match self.post_urlencoded(SERVICE_CITATION_LIST, fields).await? {
Some(xml) => parse_citation_list(&xml),
None => Ok(Vec::new()),
}
}
pub async fn process_pdf_raw(
&self,
service: &str,
pdf: impl Into<PdfInput>,
options: &ProcessOptions,
) -> Result<String, Error> {
match self.process_pdf(service, pdf.into(), options).await? {
Some(body) => Ok(body),
None => Err(Error::NoContent),
}
}
async fn process_pdf(
&self,
service: &str,
pdf: PdfInput,
options: &ProcessOptions,
) -> Result<Option<String>, Error> {
let (filename, data) = pdf.into_parts().await?;
let url = self.api_url(service);
let http = self.http.clone();
let options = options.clone();
let request_url = url.clone();
self.send_for_text(request_url, move || {
let part = Part::bytes(data.clone())
.file_name(filename.clone())
.mime_str("application/pdf")
.expect("fixed MIME type is always valid");
let form = options.apply_to(Form::new()).part("input", part);
http.post(url.clone()).multipart(form)
})
.await
}
async fn post_urlencoded(
&self,
service: &str,
fields: Vec<(String, String)>,
) -> Result<Option<String>, Error> {
let url = self.api_url(service);
let http = self.http.clone();
let request_url = url.clone();
self.send_for_text(request_url, move || http.post(url.clone()).form(&fields))
.await
}
async fn send_for_text<F>(&self, url: Url, build: F) -> Result<Option<String>, Error>
where
F: Fn() -> RequestBuilder,
{
let response = self.request(build).await?;
let status = response.status();
let body = response.text().await?;
match status {
StatusCode::OK => Ok(Some(body)),
StatusCode::NO_CONTENT => Ok(None),
StatusCode::SERVICE_UNAVAILABLE | StatusCode::TOO_MANY_REQUESTS => {
Err(Error::ServerBusy {
attempts: self.retry.max_attempts,
})
}
status => Err(Error::http_status(status.as_u16(), url.to_string(), body)),
}
}
async fn request<F>(&self, build: F) -> Result<Response, Error>
where
F: Fn() -> RequestBuilder,
{
let mut attempt = 0usize;
loop {
attempt += 1;
let response = build().send().await?;
let busy = matches!(
response.status(),
StatusCode::SERVICE_UNAVAILABLE | StatusCode::TOO_MANY_REQUESTS
);
if !busy || attempt >= self.retry.max_attempts {
return Ok(response);
}
tokio::time::sleep(self.retry.delay(attempt)).await;
}
}
fn api_url(&self, service: &str) -> Url {
self.base_url
.join(&format!("api/{service}"))
.expect("base URL always supports relative paths")
}
}
fn normalize_base_url(base_url: &str) -> Result<Url, Error> {
let trimmed = base_url.trim().trim_end_matches('/');
let with_scheme = if trimmed.contains("://") {
trimmed.to_string()
} else {
format!("http://{trimmed}")
};
let mut url = Url::parse(&with_scheme).map_err(|source| Error::InvalidBaseUrl {
base_url: base_url.to_string(),
source,
})?;
if !matches!(url.scheme(), "http" | "https") {
return Err(Error::UnsupportedScheme {
base_url: base_url.to_string(),
scheme: url.scheme().to_string(),
});
}
if !url.path().ends_with('/') {
let path = format!("{}/", url.path());
url.set_path(&path);
}
Ok(url)
}