use std::sync::Arc;
use std::time::Duration;
use async_trait::async_trait;
use url::Url;
use super::remote::{
multipart_file, status_error, ConverterClient, ConverterRequest, ReqwestConverterClient,
SecretString, DEFAULT_REMOTE_TIMEOUT, MAX_REMOTE_RESPONSE_BYTES,
};
use super::{
ConvertError, ConvertedDocument, DocumentConverter, DocumentFormat, DocumentInput, Locality,
};
#[derive(Clone)]
pub struct HttpConverter {
name: String,
endpoint: Url,
field: String,
headers: Vec<(String, SecretString)>,
formats: Vec<DocumentFormat>,
json_pointer: Option<String>,
locality: Locality,
timeout: Duration,
max_input_bytes: Option<usize>,
client: Arc<dyn ConverterClient>,
}
impl std::fmt::Debug for HttpConverter {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("HttpConverter")
.field("name", &self.name)
.field("endpoint", &self.endpoint.as_str())
.field("field", &self.field)
.field(
"headers",
&self
.headers
.iter()
.map(|(name, _)| name.as_str())
.collect::<Vec<_>>(),
)
.field("formats", &self.formats)
.field("json_pointer", &self.json_pointer)
.field("locality", &self.locality)
.finish()
}
}
impl HttpConverter {
pub fn new(name: impl Into<String>, endpoint: Url) -> Self {
Self {
name: name.into(),
endpoint,
field: "file".to_string(),
headers: Vec::new(),
formats: DocumentFormat::ALL.to_vec(),
json_pointer: None,
locality: Locality::Remote,
timeout: DEFAULT_REMOTE_TIMEOUT,
max_input_bytes: None,
client: Arc::new(ReqwestConverterClient::new()),
}
}
pub fn field(mut self, field: impl Into<String>) -> Self {
self.field = field.into();
self
}
pub fn header(mut self, name: impl Into<String>, value: impl Into<SecretString>) -> Self {
self.headers.push((name.into(), value.into()));
self
}
pub fn formats(mut self, formats: impl IntoIterator<Item = DocumentFormat>) -> Self {
self.formats = formats.into_iter().collect();
self
}
pub fn json_pointer(mut self, pointer: impl Into<String>) -> Self {
self.json_pointer = Some(pointer.into());
self
}
pub fn locality(mut self, locality: Locality) -> Self {
self.locality = locality;
self
}
pub fn timeout(mut self, timeout: Duration) -> Self {
self.timeout = timeout;
self
}
pub fn max_input_bytes(mut self, bytes: usize) -> Self {
self.max_input_bytes = Some(bytes);
self
}
pub fn client(mut self, client: Arc<dyn ConverterClient>) -> Self {
self.client = client;
self
}
}
#[async_trait]
impl DocumentConverter for HttpConverter {
fn name(&self) -> &str {
&self.name
}
fn locality(&self) -> Locality {
self.locality
}
fn supports(&self, format: DocumentFormat) -> bool {
self.formats.contains(&format)
}
fn max_input_bytes(&self) -> Option<usize> {
self.max_input_bytes
}
async fn convert(&self, input: DocumentInput) -> Result<ConvertedDocument, ConvertError> {
let (content_type, body) = multipart_file(
&self.field,
&input.file_name,
input.format.media_type(),
&input.body,
);
let mut headers = vec![("Content-Type".to_string(), content_type)];
headers.extend(
self.headers
.iter()
.map(|(name, value)| (name.clone(), value.expose().to_string())),
);
let response = self
.client
.post(ConverterRequest {
url: self.endpoint.clone(),
headers,
body,
timeout: self.timeout,
max_response_bytes: MAX_REMOTE_RESPONSE_BYTES,
})
.await?;
if let Some(error) = status_error(&self.name, response.status, &response.body) {
return Err(error);
}
let markdown = match &self.json_pointer {
None => String::from_utf8_lossy(&response.body).into_owned(),
Some(pointer) => {
let json: serde_json::Value =
serde_json::from_slice(&response.body).map_err(|error| {
ConvertError::Failed(format!("{}: invalid JSON: {error}", self.name))
})?;
json.pointer(pointer)
.and_then(serde_json::Value::as_str)
.ok_or_else(|| {
ConvertError::Failed(format!("{}: no string at {pointer}", self.name))
})?
.to_string()
}
};
Ok(ConvertedDocument::markdown(markdown))
}
}
#[cfg(test)]
mod tests {
use bytes::Bytes;
use wiremock::matchers::{header, method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
use super::*;
fn input() -> DocumentInput {
DocumentInput {
url: Url::parse("https://example.com/deck.pptx").unwrap(),
format: DocumentFormat::Pptx,
content_type: None,
file_name: "deck.pptx".to_string(),
body: Bytes::from_static(b"PK\x03\x04pptx"),
}
}
#[tokio::test]
async fn reads_markdown_from_json_pointer() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/convert"))
.and(header("x-api-key", "k"))
.respond_with(
ResponseTemplate::new(200)
.set_body_json(serde_json::json!({"document": {"md": "# Slides"}})),
)
.mount(&server)
.await;
let converter = HttpConverter::new(
"docling",
Url::parse(&format!("{}/convert", server.uri())).unwrap(),
)
.header("x-api-key", "k")
.json_pointer("/document/md");
let result = converter.convert(input()).await.unwrap();
assert_eq!(result.markdown, "# Slides");
assert!(!format!("{converter:?}").contains("\"k\""));
let requests = server.received_requests().await.unwrap();
assert!(String::from_utf8_lossy(&requests[0].body).contains("name=\"file\""));
}
#[tokio::test]
async fn raw_body_and_http_errors() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/ok"))
.respond_with(ResponseTemplate::new(200).set_body_string("plain md"))
.mount(&server)
.await;
Mock::given(method("POST"))
.and(path("/fail"))
.respond_with(ResponseTemplate::new(500).set_body_string("kaput"))
.mount(&server)
.await;
let ok = HttpConverter::new("svc", Url::parse(&format!("{}/ok", server.uri())).unwrap());
assert_eq!(ok.convert(input()).await.unwrap().markdown, "plain md");
let fail = HttpConverter::new(
"svc",
Url::parse(&format!("{}/fail", server.uri())).unwrap(),
);
let error = fail.convert(input()).await.unwrap_err().to_string();
assert!(error.contains("HTTP 500: kaput"));
}
}