mineru 0.5.0

Rust client, CLI, and API server for OpenAI-compatible MinerU document parsing
Documentation
use std::{
    fs,
    path::{Path, PathBuf},
    process::Stdio,
    time::{Duration, Instant},
};

use tempfile::TempDir;
use tokio::process::{Child, Command};

use super::super::{
    OfficialPersistentWorker, OfficialRequest, OfficialSessionConfig, OfficialWorker,
};
use super::WindowsJob;

const POWERSHELL_SCRIPT: &str = r#"
$ready = $env:MINERU_RS_JOB_READY
$go = $env:MINERU_RS_JOB_GO
$pids = $env:MINERU_RS_JOB_PIDS
[System.IO.File]::WriteAllText($ready, [string]$PID)
while (-not [System.IO.File]::Exists($go)) {
    Start-Sleep -Milliseconds 10
}
$descendant = Start-Process -FilePath $env:MINERU_RS_JOB_PING `
    -ArgumentList '127.0.0.1 -n 31' -WindowStyle Hidden -PassThru
[System.IO.File]::WriteAllText($pids, "$PID $($descendant.Id)")
Wait-Process -Id $descendant.Id
"#;

struct FixturePaths {
    ready: PathBuf,
    go: PathBuf,
    pids: PathBuf,
}

fn powershell_path() -> PathBuf {
    let root = std::env::var_os("SystemRoot").expect("SystemRoot is required on Windows");
    let path = PathBuf::from(root)
        .join("System32")
        .join("WindowsPowerShell")
        .join("v1.0")
        .join("powershell.exe");
    assert!(
        path.is_file(),
        "missing Windows PowerShell: {}",
        path.display()
    );
    path
}

fn python_path() -> PathBuf {
    let path = std::env::var_os("PATH")
        .map(|value| {
            std::env::split_paths(&value).find_map(|directory| {
                ["python.exe", "python3.exe"]
                    .into_iter()
                    .map(|name| directory.join(name))
                    .find(|path| path.is_file())
            })
        })
        .flatten()
        .expect("Windows CI must provide a Python executable for the official worker");
    assert!(path.is_absolute(), "Python executable path is not absolute");
    path
}

struct AttachFailureReset;

impl Drop for AttachFailureReset {
    fn drop(&mut self) {
        super::set_attach_failure_for_test(false);
    }
}

fn force_attach_failure() -> AttachFailureReset {
    super::set_attach_failure_for_test(true);
    AttachFailureReset
}

fn attach_failure_pid(error: &str) -> u32 {
    error
        .rsplit_once("pid=")
        .and_then(|(_, pid)| pid.parse().ok())
        .unwrap_or_else(|| panic!("attach failure did not publish a PID: {error}"))
}

fn spawn_fixture() -> (TempDir, FixturePaths, Child) {
    let temp = tempfile::tempdir().unwrap();
    let paths = FixturePaths {
        ready: temp.path().join("ready"),
        go: temp.path().join("go"),
        pids: temp.path().join("pids"),
    };
    let system_root = std::env::var_os("SystemRoot").unwrap();
    let mut command = Command::new(powershell_path());
    command
        .args([
            "-NoLogo",
            "-NoProfile",
            "-NonInteractive",
            "-ExecutionPolicy",
            "Bypass",
            "-Command",
            POWERSHELL_SCRIPT,
        ])
        .env("MINERU_RS_JOB_READY", &paths.ready)
        .env("MINERU_RS_JOB_GO", &paths.go)
        .env("MINERU_RS_JOB_PIDS", &paths.pids)
        .env(
            "MINERU_RS_JOB_PING",
            PathBuf::from(&system_root)
                .join("System32")
                .join("ping.exe"),
        )
        .stdin(Stdio::null())
        .stdout(Stdio::null())
        .stderr(Stdio::null())
        .kill_on_drop(true)
        .creation_flags(windows_sys::Win32::System::Threading::CREATE_NO_WINDOW);
    let child = command
        .spawn()
        .expect("Windows PowerShell fixture failed to spawn");
    (temp, paths, child)
}

async fn wait_for_file(path: &Path) {
    let deadline = Instant::now() + Duration::from_secs(60);
    while !path.is_file() && Instant::now() < deadline {
        tokio::time::sleep(Duration::from_millis(10)).await;
    }
    assert!(path.is_file(), "fixture did not create {}", path.display());
}

async fn attached_fixture() -> (TempDir, FixturePaths, Child, WindowsJob) {
    let (temp, paths, child) = spawn_fixture();
    wait_for_file(&paths.ready).await;
    let job = WindowsJob::attach(&child).expect("worker Job Object attach failed");
    (temp, paths, child, job)
}

async fn release_fixture(paths: &FixturePaths) -> Vec<u32> {
    fs::write(&paths.go, b"go").unwrap();
    wait_for_file(&paths.pids).await;
    let pids = fs::read_to_string(&paths.pids)
        .unwrap()
        .split_whitespace()
        .map(|pid| pid.parse::<u32>().unwrap())
        .collect::<Vec<_>>();
    assert_eq!(
        pids.len(),
        2,
        "fixture did not publish worker and descendant PIDs"
    );
    assert!(pids.iter().all(|pid| *pid != 0));
    pids
}

fn process_is_running(pid: u32) -> bool {
    use windows_sys::Win32::{
        Foundation::{CloseHandle, INVALID_HANDLE_VALUE},
        System::Threading::{
            GetExitCodeProcess, OpenProcess, PROCESS_QUERY_LIMITED_INFORMATION, PROCESS_SYNCHRONIZE,
        },
    };

    const STILL_ACTIVE: u32 = 259;
    unsafe {
        let handle = OpenProcess(
            PROCESS_QUERY_LIMITED_INFORMATION | PROCESS_SYNCHRONIZE,
            0,
            pid,
        );
        if handle.is_null() || handle == INVALID_HANDLE_VALUE {
            return false;
        }
        let mut exit_code = 0;
        let running = GetExitCodeProcess(handle, &mut exit_code) != 0 && exit_code == STILL_ACTIVE;
        CloseHandle(handle);
        running
    }
}

async fn assert_processes_dead(pids: &[u32]) {
    let deadline = Instant::now() + Duration::from_secs(10);
    while pids.iter().any(|pid| process_is_running(*pid)) && Instant::now() < deadline {
        tokio::time::sleep(Duration::from_millis(20)).await;
    }
    assert!(
        pids.iter().all(|pid| !process_is_running(*pid)),
        "worker or descendant survived Job Object cleanup: {pids:?}"
    );
}

async fn wait_for_child_exit(child: &mut Child) {
    let deadline = Instant::now() + Duration::from_secs(10);
    while child.try_wait().unwrap().is_none() && Instant::now() < deadline {
        tokio::time::sleep(Duration::from_millis(20)).await;
    }
    assert!(child.try_wait().unwrap().is_some(), "worker did not exit");
}

fn assert_kill_on_close(job: &WindowsJob) {
    use std::mem::size_of;
    use windows_sys::Win32::System::JobObjects::{
        JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, JOBOBJECT_EXTENDED_LIMIT_INFORMATION,
        JobObjectExtendedLimitInformation, QueryInformationJobObject,
    };

    let mut limits = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default();
    let result = unsafe {
        QueryInformationJobObject(
            job.0,
            JobObjectExtendedLimitInformation,
            (&mut limits as *mut JOBOBJECT_EXTENDED_LIMIT_INFORMATION).cast(),
            size_of::<JOBOBJECT_EXTENDED_LIMIT_INFORMATION>() as u32,
            std::ptr::null_mut(),
        )
    };
    assert_ne!(result, 0, "Job Object limit query failed");
    assert_ne!(
        limits.BasicLimitInformation.LimitFlags & JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE,
        0,
        "worker Job Object is not configured to kill on close"
    );
}

fn process_is_in_any_job(handle: windows_sys::Win32::Foundation::HANDLE) -> bool {
    use windows_sys::Win32::System::{JobObjects::IsProcessInJob, Threading::GetCurrentProcess};

    let process = if handle.is_null() {
        unsafe { GetCurrentProcess() }
    } else {
        handle
    };
    let mut in_job = 0;
    let result = unsafe { IsProcessInJob(process, std::ptr::null_mut(), &mut in_job) };
    assert_ne!(result, 0, "IsProcessInJob failed for process");
    in_job != 0
}

#[tokio::test]
async fn windows_job_attach_sets_kill_on_close_for_descendants() {
    let (_temp, paths, mut child, job) = attached_fixture().await;
    assert_kill_on_close(&job);
    let pids = release_fixture(&paths).await;
    assert!(pids.iter().all(|pid| process_is_running(*pid)));

    drop(job);
    wait_for_child_exit(&mut child).await;
    assert_processes_dead(&pids).await;
}

async fn hold_process_until_dropped(_child: Child, _job: WindowsJob) {
    std::future::pending::<()>().await;
}

#[tokio::test]
async fn windows_job_cleanup_survives_caller_cancellation() {
    let (_temp, paths, child, job) = attached_fixture().await;
    let pids = release_fixture(&paths).await;
    let task = tokio::spawn(hold_process_until_dropped(child, job));
    task.abort();
    assert!(matches!(task.await, Err(error) if error.is_cancelled()));
    assert_processes_dead(&pids).await;
}

#[tokio::test]
async fn windows_job_cleanup_survives_a_deadline() {
    let (_temp, paths, child, job) = attached_fixture().await;
    let pids = release_fixture(&paths).await;
    let result = tokio::time::timeout(
        Duration::from_millis(100),
        hold_process_until_dropped(child, job),
    )
    .await;
    assert!(
        result.is_err(),
        "fixture unexpectedly completed before deadline"
    );
    assert_processes_dead(&pids).await;
}

#[tokio::test]
async fn windows_job_drop_reaps_an_idle_worker_tree() {
    let (_temp, paths, child, job) = attached_fixture().await;
    let pids = release_fixture(&paths).await;
    {
        let _job = job;
        let _child = child;
    }
    assert_processes_dead(&pids).await;
}

fn test_request() -> OfficialRequest {
    OfficialRequest::new(
        "standard".into(),
        "auto".into(),
        true,
        None,
        None,
        None,
        None,
        None,
        None,
        1024,
        PathBuf::new(),
        PathBuf::new(),
    )
}

fn persistent_config() -> OfficialSessionConfig {
    OfficialSessionConfig::new(None, None, None, None).unwrap()
}

#[tokio::test]
async fn windows_attach_failure_cleans_live_per_document_worker() {
    let _failure = force_attach_failure();
    let worker = OfficialWorker::new(Some(python_path())).unwrap();
    let error = match worker
        .run(
            b"input",
            "pdf",
            test_request(),
            Instant::now() + Duration::from_secs(60),
        )
        .await
    {
        Ok(_) => panic!("per-document worker unexpectedly attached"),
        Err(error) => error,
    };
    assert!(
        error.contains("test attach failure"),
        "unexpected error: {error}"
    );
    assert_processes_dead(&[attach_failure_pid(&error)]).await;
}

#[tokio::test]
async fn windows_attach_failure_cleans_live_persistent_worker() {
    let _failure = force_attach_failure();
    let worker = OfficialPersistentWorker::new(Some(python_path()), persistent_config()).unwrap();
    let error = match worker
        .run(
            b"input",
            "pdf",
            test_request(),
            Instant::now() + Duration::from_secs(60),
        )
        .await
    {
        Ok(_) => panic!("persistent worker unexpectedly attached"),
        Err(error) => error,
    };
    worker.shutdown().await.unwrap();
    assert!(
        error.contains("test attach failure"),
        "unexpected error: {error}"
    );
    assert_processes_dead(&[attach_failure_pid(&error)]).await;
}

#[tokio::test]
async fn windows_job_attach_handles_a_process_already_in_a_job() {
    let (_temp, paths, mut child) = spawn_fixture();
    wait_for_file(&paths.ready).await;
    let child_pid = child.id().unwrap();
    let child_handle = child.raw_handle().unwrap();
    let test_process_in_job = process_is_in_any_job(std::ptr::null_mut());
    assert_eq!(
        process_is_in_any_job(child_handle),
        test_process_in_job,
        "child did not inherit the test process job membership"
    );

    match WindowsJob::attach(&child) {
        Ok(job) => {
            assert_kill_on_close(&job);
            let pids = release_fixture(&paths).await;
            drop(job);
            wait_for_child_exit(&mut child).await;
            assert_processes_dead(&pids).await;
        }
        Err(error) if test_process_in_job => {
            let _ = child.start_kill();
            wait_for_child_exit(&mut child).await;
            assert!(!process_is_running(child_pid));
            assert!(
                !error.is_empty(),
                "existing-job attach failure lost its diagnostic"
            );
        }
        Err(error) => panic!("uncontained test process could not attach worker: {error}"),
    }
}