use serde::{Deserialize, Serialize};
use serde_json::Value;
#[cfg(windows)]
use std::os::windows::process::CommandExt;
use std::{
path::{Path, PathBuf},
sync::atomic::{AtomicU64, Ordering},
time::{Duration, Instant},
};
use tempfile::TempDir;
use tokio::{
io::{AsyncRead, AsyncReadExt, AsyncWriteExt},
process::Command,
sync::OwnedSemaphorePermit,
task::JoinHandle,
};
#[cfg(test)]
mod per_document_tests;
mod persistent;
pub(crate) mod process;
mod python_shim;
mod staging;
pub(crate) use persistent::{OfficialPersistentWorker, OfficialSessionConfig};
use python_shim::PythonShim;
#[cfg(test)]
pub(crate) use python_shim::{PYTHON_SHIM, PYTHON_WORKER_FILES};
use staging::stage_input;
const PROTOCOL: &str = "mineru-rs-official-worker/1";
pub(crate) const PACKAGE_VERSION: &str = "4.2.3";
const SCHEMA_VERSION: &str = "2.0";
const PERSISTENT_PROTOCOL: &str = "mineru-rs-official-worker/2";
const STDOUT_CAP: usize = 64 * 1024;
const STDERR_CAP: usize = 64 * 1024;
const REQUEST_CAP: usize = 64 * 1024;
const REAP_GRACE: Duration = Duration::from_millis(250);
const PERSISTENT_TIERS: &[&str] = &["flash", "basic", "standard", "advanced"];
const PERSISTENT_OCR_MODES: &[&str] = &["auto", "txt", "ocr"];
const PERSISTENT_INPUT_FORMATS: &[&str] = &[
"pdf", "png", "jpeg", "jpg", "jp2", "webp", "gif", "bmp", "tiff",
];
#[cfg(unix)]
use process::ProcessGroup;
#[cfg(windows)]
use process::WindowsJob;
#[cfg(target_os = "linux")]
use process::install_parent_death_signal;
use process::{
copy_runtime_environment, official_executable, terminate, with_truncated_diagnostic,
};
#[derive(Clone, Serialize)]
pub(crate) struct OfficialRequest {
pub(crate) protocol: &'static str,
pub(crate) request_id: String,
pub(crate) tier: String,
pub(crate) ocr_mode: String,
pub(crate) image_analysis: bool,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) page_range: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) vlm_server_url: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) vlm_api_key: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) vlm_model: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) model_base_dir: Option<PathBuf>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) config: Option<PathBuf>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) source_context: Option<Value>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) official_vlm_config: Option<Value>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) operation: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) operation_assets: Option<PathBuf>,
pub(crate) max_bundle_bytes: u64,
pub(crate) bundle_name: &'static str,
pub(crate) input_path: PathBuf,
pub(crate) bundle_path: PathBuf,
}
impl OfficialRequest {
#[allow(clippy::too_many_arguments)]
pub(crate) fn new(
tier: String,
ocr_mode: String,
image_analysis: bool,
page_range: Option<String>,
vlm_server_url: Option<String>,
vlm_api_key: Option<String>,
vlm_model: Option<String>,
model_base_dir: Option<PathBuf>,
config: Option<PathBuf>,
max_bundle_bytes: u64,
input_path: PathBuf,
bundle_path: PathBuf,
) -> Self {
Self {
protocol: PROTOCOL,
request_id: next_request_id(),
tier,
ocr_mode,
image_analysis,
page_range,
vlm_server_url,
vlm_api_key,
vlm_model,
model_base_dir,
config,
source_context: None,
official_vlm_config: None,
operation: None,
operation_assets: None,
max_bundle_bytes,
bundle_name: crate::hybrid_v4_output::BUNDLE_NAME,
input_path,
bundle_path,
}
}
}
pub(crate) struct OfficialWorker {
executable: PathBuf,
}
pub(crate) struct OfficialBundle {
_temporary: TempDir,
bundle: PathBuf,
_permit: Option<OwnedSemaphorePermit>,
}
impl OfficialBundle {
pub(crate) fn path(&self) -> &Path {
&self.bundle
}
}
impl OfficialWorker {
pub(crate) fn new(executable: Option<PathBuf>) -> Result<Self, String> {
Ok(Self {
executable: official_executable(executable)?,
})
}
pub(crate) async fn run(
&self,
input: &[u8],
suffix: &str,
request: OfficialRequest,
deadline: Instant,
) -> Result<OfficialBundle, String> {
self.run_owned(
bytes::Bytes::copy_from_slice(input),
suffix,
request,
deadline,
)
.await
}
pub(crate) async fn run_owned(
&self,
input: bytes::Bytes,
suffix: &str,
request: OfficialRequest,
deadline: Instant,
) -> Result<OfficialBundle, String> {
self.run_with_files_owned(input, suffix, request, deadline, Vec::new())
.await
}
pub(crate) async fn run_with_files(
&self,
input: &[u8],
suffix: &str,
request: OfficialRequest,
deadline: Instant,
files: &[(&str, &[u8])],
) -> Result<OfficialBundle, String> {
let files = files
.iter()
.map(|(name, bytes)| ((*name).to_owned(), bytes.to_vec()))
.collect();
self.run_with_files_owned(
bytes::Bytes::copy_from_slice(input),
suffix,
request,
deadline,
files,
)
.await
}
async fn run_with_files_owned(
&self,
input: bytes::Bytes,
suffix: &str,
mut request: OfficialRequest,
deadline: Instant,
files: Vec<(String, Vec<u8>)>,
) -> Result<OfficialBundle, String> {
if input.is_empty() {
return Err("official Hybrid input is empty".into());
}
if !files.is_empty() && request.operation.is_none() {
return Err("private operation files require an operation".into());
}
let stage =
stage_input(input, suffix, files, request.operation.is_some(), deadline).await?;
let temporary = stage.temporary;
request.operation_assets = stage.operation_assets;
let bundle = temporary.path().join(crate::hybrid_v4_output::BUNDLE_NAME);
request.input_path = stage.input_path;
request.bundle_path = bundle.clone();
let mut encoded = serde_json::to_vec(&request).map_err(|e| e.to_string())?;
encoded.push(b'\n');
if encoded.len() > REQUEST_CAP {
return Err("official worker request exceeds protocol limit".into());
}
let shim = PythonShim::new()?;
let mut command = Command::new(&self.executable);
command
.arg(shim.path())
.current_dir(temporary.path())
.env_clear()
.stdin(std::process::Stdio::piped())
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped())
.kill_on_drop(true);
copy_runtime_environment(&mut command);
#[cfg(unix)]
command.process_group(0);
#[cfg(target_os = "linux")]
install_parent_death_signal(&mut command);
#[cfg(windows)]
command.creation_flags(0x0000_0200);
let mut child = command
.spawn()
.map_err(|e| format!("official Python worker unavailable: {e}"))?;
#[cfg(unix)]
let mut process_group = ProcessGroup::new(child.id());
#[cfg(windows)]
let _job = WindowsJob::attach(&child).map_err(|error| {
let _ = child.start_kill();
error
})?;
let stdin = child
.stdin
.take()
.ok_or_else(|| "official worker stdin pipe unavailable".to_owned())?;
let stdout = child
.stdout
.take()
.ok_or_else(|| "official worker stdout pipe unavailable".to_owned())?;
let stderr = child
.stderr
.take()
.ok_or_else(|| "official worker stderr pipe unavailable".to_owned())?;
let mut input_task = Some(tokio::spawn(write_request(stdin, encoded)));
let mut stdout_task = Some(tokio::spawn(read_capped(stdout, STDOUT_CAP)));
let mut stderr_task = Some(tokio::spawn(read_diagnostic(stderr, STDERR_CAP)));
let mut status = None;
let mut stdout_bytes = None;
let mut stderr_bytes = None;
let mut failure = None;
let sleep = tokio::time::sleep_until(tokio::time::Instant::from_std(deadline));
tokio::pin!(sleep);
loop {
tokio::select! {
result = child.wait(), if status.is_none() => {
match result {
Ok(value) => {
status = Some(value);
#[cfg(unix)] process_group.kill();
}
Err(error) => failure = Some(format!("official worker wait failed: {error}")),
}
}
result = async { input_task.as_mut().expect("input task exists").await }, if input_task.is_some() => {
input_task = None;
if !matches!(result, Ok(Ok(()))) {
failure = Some("official worker stdin failed".into());
}
}
result = async { stdout_task.as_mut().expect("stdout task exists").await }, if stdout_task.is_some() => {
stdout_task = None;
match result {
Ok(Ok(bytes)) => stdout_bytes = Some(bytes),
Ok(Err(ReadError::TooLarge)) => failure = Some("official worker stdout exceeded its cap".into()),
_ => failure = Some("official worker stdout failed".into()),
}
}
result = async { stderr_task.as_mut().expect("stderr task exists").await }, if stderr_task.is_some() => {
stderr_task = None;
match result {
Ok(Ok(bytes)) => stderr_bytes = Some(bytes),
_ => failure = Some("official worker stderr failed".into()),
}
}
_ = &mut sleep => failure = Some("official worker deadline expired".into()),
}
if failure.is_some()
|| (status.is_some()
&& input_task.is_none()
&& stdout_task.is_none()
&& stderr_task.is_none())
{
break;
}
}
if let Some(error) = failure {
abort(&mut input_task, &mut stdout_task, &mut stderr_task);
#[cfg(unix)]
process_group.kill();
terminate(&mut child).await;
return Err(with_stderr_diagnostic(error, stderr_bytes.as_ref()));
}
if !status.is_some_and(|status| status.success()) {
return Err(with_stderr_diagnostic(
"official worker exited unsuccessfully".into(),
stderr_bytes.as_ref(),
));
}
let response: Response = serde_json::from_slice(
stdout_bytes
.as_deref()
.ok_or_else(|| "official worker returned no protocol response".to_owned())?,
)
.map_err(|e| format!("official worker protocol JSON is invalid: {e}"))?;
if response.protocol != PROTOCOL {
return Err("official worker protocol version mismatch".into());
}
if response.request_id != request.request_id {
return Err("official worker request id mismatch".into());
}
if response.package_version != PACKAGE_VERSION {
return Err(format!(
"official worker MinerU package version is not {PACKAGE_VERSION}"
));
}
if response.schema_version != SCHEMA_VERSION {
return Err("official worker result schema version mismatch".into());
}
if response.bundle_name != crate::hybrid_v4_output::BUNDLE_NAME {
return Err("official worker bundle name mismatch".into());
}
if response.status != "ok" {
return Err(with_stderr_diagnostic(
response
.error
.unwrap_or_else(|| "official worker failed".into()),
stderr_bytes.as_ref(),
));
}
#[cfg(unix)]
process_group.disarm();
Ok(OfficialBundle {
_temporary: temporary,
bundle,
_permit: None,
})
}
}
#[derive(Debug, Deserialize)]
struct Response {
protocol: String,
request_id: String,
status: String,
package_version: String,
schema_version: String,
bundle_name: String,
error: Option<String>,
}
async fn write_request(mut stdin: tokio::process::ChildStdin, request: Vec<u8>) -> Result<(), ()> {
stdin.write_all(&request).await.map_err(|_| ())?;
stdin.shutdown().await.map_err(|_| ())
}
#[derive(Debug)]
enum ReadError {
TooLarge,
Io,
}
struct Diagnostic {
bytes: Vec<u8>,
truncated: bool,
}
enum ReadMode {
Prefix,
HardCap,
}
async fn read_bounded(
mut reader: impl AsyncRead + Unpin,
cap: usize,
mode: ReadMode,
) -> Result<Diagnostic, ReadError> {
let mut bytes = Vec::with_capacity(cap);
let mut truncated = false;
let mut buffer = [0u8; 8192];
loop {
let read = reader.read(&mut buffer).await.map_err(|_| ReadError::Io)?;
if read == 0 {
return Ok(Diagnostic { bytes, truncated });
}
match mode {
ReadMode::Prefix => {
let retained = read.min(cap.saturating_sub(bytes.len()));
bytes.extend_from_slice(&buffer[..retained]);
truncated |= retained < read;
}
ReadMode::HardCap => {
if bytes.len().checked_add(read).is_none_or(|size| size > cap) {
return Err(ReadError::TooLarge);
}
bytes.extend_from_slice(&buffer[..read]);
}
}
}
}
async fn read_capped(reader: impl AsyncRead + Unpin, cap: usize) -> Result<Vec<u8>, ReadError> {
Ok(read_bounded(reader, cap, ReadMode::HardCap).await?.bytes)
}
async fn read_diagnostic(
reader: impl AsyncRead + Unpin,
cap: usize,
) -> Result<Diagnostic, ReadError> {
read_bounded(reader, cap, ReadMode::Prefix).await
}
fn with_stderr_diagnostic(message: String, diagnostic: Option<&Diagnostic>) -> String {
let Some(diagnostic) = diagnostic else {
return message;
};
with_truncated_diagnostic(message, Some(&diagnostic.bytes), diagnostic.truncated)
}
fn abort(
input: &mut Option<JoinHandle<Result<(), ()>>>,
stdout: &mut Option<JoinHandle<Result<Vec<u8>, ReadError>>>,
stderr: &mut Option<JoinHandle<Result<Diagnostic, ReadError>>>,
) {
if let Some(task) = input.take() {
task.abort();
}
if let Some(task) = stdout.take() {
task.abort();
}
if let Some(task) = stderr.take() {
task.abort();
}
}
fn next_request_id() -> String {
static NEXT: AtomicU64 = AtomicU64::new(0);
format!(
"{}-{}",
std::process::id(),
NEXT.fetch_add(1, Ordering::Relaxed)
)
}