use serde::de::DeserializeOwned;
use crate::segments::Segment;
use super::{ConvertError, Converter, Outcome, normalize_text};
pub const PDF_SUBPROCESS_ID: &str = "pdf-subprocess";
pub const PDF_SUBPROCESS_VERSION: &str = "1.0.0";
pub const PDF_RECOVERED_ID: &str = "pdf-text-recovered";
pub const PDF_RECOVERED_VERSION: &str = PDF_SUBPROCESS_VERSION;
#[cfg_attr(not(unix), allow(dead_code))]
pub(crate) fn pdf_converter_identity(recovered: bool) -> (&'static str, &'static str) {
if recovered {
(PDF_RECOVERED_ID, PDF_RECOVERED_VERSION)
} else {
(PDF_SUBPROCESS_ID, PDF_SUBPROCESS_VERSION)
}
}
pub const RECORDS_SUBPROCESS_ID: &str = "records-worker";
pub const RECORDS_SUBPROCESS_VERSION: &str = "1.0.0";
pub const IMAGE_PIXEL_OCR_ID: &str = "image-pixel-ocr";
pub const IMAGE_PIXEL_OCR_VERSION: &str = "1.1.0";
pub const OCR_SEGMENT_SOURCE_DOCUMENT: &str = "document";
pub const OCR_SEGMENT_SOURCE_OCR: &str = "ocr";
pub const OCR_ADAPTER_ID: &str = "ocr-adapter";
pub const OCR_ADAPTER_VERSION: &str = "1.0.0";
pub const ASR_ADAPTER_ID: &str = "asr-adapter";
pub const ASR_ADAPTER_VERSION: &str = "2.0.0";
pub const ASR_DECODE_POLICY: &str = "-l en -t 1 -p 1 -bo 1 -bs 1 -nf -tp 0 -tpi 0 -nfa -mc 0";
pub const ASR_ROLE_CLI: &str = "asr-cli";
pub const ASR_ROLE_WEIGHTS: &str = "asr-weights";
pub const ASR_ROLE_DECODE_POLICY: &str = "asr-decode-policy";
pub const ASR_ROLE_PROBE: &str = "asr-probe";
pub const ASR_FILE_ROLE_LABELS: [&str; 3] = [ASR_ROLE_CLI, ASR_ROLE_WEIGHTS, ASR_ROLE_PROBE];
pub const VIDEO_ADAPTER_ID: &str = "video-adapter";
pub const VIDEO_ADAPTER_VERSION: &str = "1.0.0";
pub const OCR_CONFIDENCE_WARNING: f64 = 0.5;
pub fn format_timecode(seconds: f64) -> Result<String, String> {
if !seconds.is_finite() || seconds < 0.0 {
return Err(format!("timestamp {seconds} is not a non-negative time"));
}
let whole = seconds as u64;
Ok(format!(
"{:02}:{:02}:{:02}",
whole / 3600,
(whole % 3600) / 60,
whole % 60
))
}
fn single_line(text: &str) -> String {
text.replace(['\n', '\r'], " ").trim().to_string()
}
#[cfg(unix)]
mod imp {
use std::sync::OnceLock;
use super::*;
use crate::convert::RecordsLimits;
use crate::manifest::ArtifactKind;
use crate::runner::protocol::bodies::{
OcrInput, OcrOk, OcrRequest, PdfOk, PdfRequest, RecordsOk, RecordsRequest, VideoOk,
VideoRequest,
};
use crate::runner::{Runner, RunnerError};
fn static_code(code: &str) -> &'static str {
const KNOWN: &[&str] = &[
"pdf_no_text_layer",
"malformed",
"encrypted",
"resourceLimit",
"missingPart",
"unsupported",
"io",
"empty_output",
"invalid_utf8",
"converter_panic",
"unclaimed_format",
"resource_limit",
"record-limit-exceeded",
"records_read_error",
"not_sqlite",
"image-ocr-decode-failed",
"image-ocr-area-exceeded",
"image-ocr-runtime-missing",
"image-ocr-runtime-mismatch",
"image-ocr-no-accelerator",
"ocr-protocol-error",
"image-metadata-malformed",
"image-metadata-decompress-exceeded",
"image-metadata-xml-exceeded",
"image-metadata-limit-exceeded",
"image-metadata-unsupported",
"rasterizer-missing",
"rasterizer-hash-drift",
"rasterizer-version-drift",
"raster-exec-failed",
"raster-geometry-mismatch",
"encoder-area-cap",
"asr-decode-failed",
"asr-codec-unsupported",
"asr-duration-exceeded",
"asr-decoded-too-large",
"asr-no-speech",
"asr-runtime-missing",
"asr-runtime-mismatch",
"asr-backend-unavailable",
"asr-protocol-error",
];
KNOWN
.iter()
.find(|known| **known == code)
.copied()
.unwrap_or("adapter_error")
}
fn runner_failure(error: RunnerError) -> ConvertError {
ConvertError {
code: error.code,
message: error.message(),
}
}
fn call<Req: serde::Serialize, Ok: DeserializeOwned>(
runner: &Runner,
mode: &str,
request: &Req,
files: &[(&str, &[u8])],
) -> Result<Ok, ConvertError> {
let payload = serde_json::to_value(request).map_err(|e| ConvertError {
code: "adapter_error",
message: format!("cannot encode the adapter request: {e}"),
})?;
let response = runner.run(mode, payload, files).map_err(runner_failure)?;
if let Some(error) = response.error {
return Err(ConvertError {
code: static_code(&error.code),
message: error.message,
});
}
let ok = response.ok.expect("validated response has a body");
serde_json::from_value(ok).map_err(|e| ConvertError {
code: "adapter_error",
message: format!("adapter success body did not parse: {e}"),
})
}
fn shared_runner() -> Result<&'static Runner, ConvertError> {
static RUNNER: OnceLock<Result<Runner, RunnerError>> = OnceLock::new();
match RUNNER.get_or_init(Runner::with_platform_defaults) {
Ok(runner) => Ok(runner),
Err(error) => Err(runner_failure(error.clone())),
}
}
pub struct PdfSubprocess;
impl Converter for PdfSubprocess {
fn id(&self) -> &'static str {
PDF_SUBPROCESS_ID
}
fn version(&self) -> &'static str {
PDF_SUBPROCESS_VERSION
}
fn convert(
&self,
source: &[u8],
detected_format: &str,
) -> std::result::Result<Outcome, ConvertError> {
if !matches!(detected_format, "pdf" | "ai") {
return Err(ConvertError {
code: "unclaimed_format",
message: format!(
"the subprocess PDF adapter does not handle {detected_format}"
),
});
}
let body: PdfOk = call(
shared_runner()?,
"pdf",
&PdfRequest {
input: "input.pdf".to_string(),
},
&[("input.pdf", source)],
)?;
let (converter_id, converter_version) = pdf_converter_identity(body.recovered);
Ok(Outcome {
artifact_kind: ArtifactKind::Text,
converter_id: converter_id.to_string(),
converter_version: converter_version.to_string(),
detected_format: detected_format.to_string(),
text: body.text,
warnings: body.warnings,
segments: body.segments,
media: None,
})
}
}
pub struct RecordsSubprocess {
limits: RecordsLimits,
}
impl RecordsSubprocess {
pub fn new(limits: RecordsLimits) -> RecordsSubprocess {
RecordsSubprocess { limits }
}
}
impl Converter for RecordsSubprocess {
fn id(&self) -> &'static str {
RECORDS_SUBPROCESS_ID
}
fn version(&self) -> &'static str {
RECORDS_SUBPROCESS_VERSION
}
fn convert(
&self,
source: &[u8],
detected_format: &str,
) -> std::result::Result<Outcome, ConvertError> {
if !matches!(detected_format, "parquet" | "avro" | "sqlite") {
return Err(ConvertError {
code: "unclaimed_format",
message: format!("the records worker does not handle {detected_format}"),
});
}
let input = format!("input.{detected_format}");
let body: RecordsOk = call(
shared_runner()?,
"records",
&RecordsRequest {
input: input.clone(),
format: detected_format.to_string(),
max_records: self.limits.max_records,
max_tables: self.limits.max_tables,
max_output_bytes: self.limits.max_output_bytes,
},
&[(&input, source)],
)?;
Ok(Outcome {
artifact_kind: ArtifactKind::Text,
converter_id: RECORDS_SUBPROCESS_ID.to_string(),
converter_version: RECORDS_SUBPROCESS_VERSION.to_string(),
detected_format: detected_format.to_string(),
text: body.text,
warnings: body.warnings,
segments: body.segments,
media: None,
})
}
}
#[cfg(feature = "image-metadata")]
pub struct ImageMetadata {
limits: crate::convert::ImageMetadataLimits,
}
#[cfg(feature = "image-metadata")]
impl ImageMetadata {
pub fn new(limits: crate::convert::ImageMetadataLimits) -> ImageMetadata {
ImageMetadata { limits }
}
}
#[cfg(feature = "image-metadata")]
impl Converter for ImageMetadata {
fn id(&self) -> &'static str {
crate::convert::image_metadata::IMAGE_METADATA_ID
}
fn version(&self) -> &'static str {
crate::convert::image_metadata::IMAGE_METADATA_VERSION
}
fn convert(
&self,
source: &[u8],
detected_format: &str,
) -> std::result::Result<Outcome, ConvertError> {
use crate::runner::protocol::bodies::{ImageMetadataOk, ImageMetadataRequest};
if !crate::convert::image_metadata::image_metadata_applies(detected_format) {
return Err(ConvertError {
code: "unclaimed_format",
message: format!("the image-metadata worker does not read {detected_format}"),
});
}
let input = format!("input.{detected_format}");
let body: ImageMetadataOk = call(
shared_runner()?,
"image-metadata",
&ImageMetadataRequest {
input: input.clone(),
format: detected_format.to_string(),
max_decompressed_bytes: self.limits.max_decompressed_bytes,
max_xml_depth: self.limits.max_xml_depth,
max_xml_events: self.limits.max_xml_events,
max_boxes: self.limits.max_boxes,
max_rows: self.limits.max_rows,
max_output_bytes: self.limits.max_output_bytes,
},
&[(&input, source)],
)?;
if body.rows.is_empty() {
return Err(ConvertError {
code: crate::convert::image_metadata::IMAGE_METADATA_NOT_APPLICABLE,
message: "the image carries no textual metadata".to_string(),
});
}
let (text, segments) = crate::convert::image_metadata::render_rows(&body.rows);
if text.is_empty() {
return Err(empty_output(source.len()));
}
Ok(Outcome {
artifact_kind: ArtifactKind::Text,
converter_id: crate::convert::image_metadata::IMAGE_METADATA_ID.to_string(),
converter_version: crate::convert::image_metadata::IMAGE_METADATA_VERSION
.to_string(),
detected_format: detected_format.to_string(),
text,
warnings: Vec::new(),
segments,
media: None,
})
}
}
#[cfg(feature = "image-ocr")]
pub struct ImagePixelOcr {
mode: &'static str,
#[cfg(feature = "svg-provider")]
svg: Option<SvgProviderLeg>,
inventory: Vec<crate::runner::protocol::bodies::InventoryEntry>,
runner: Result<Runner, RunnerError>,
}
#[cfg(all(feature = "image-ocr", feature = "svg-provider"))]
struct SvgProviderLeg {
roles: Vec<crate::runner::protocol::bodies::SvgRoleEntry>,
runner: Result<Runner, RunnerError>,
#[cfg(feature = "test-adapters")]
replace_after_preflight: Option<std::path::PathBuf>,
}
#[cfg(all(feature = "image-ocr", feature = "svg-provider"))]
const PROVIDER_MAX_RESPONSE_BYTES: u32 = 64 * 1024 * 1024;
#[cfg(feature = "image-ocr")]
impl ImagePixelOcr {
pub fn new(
limits: crate::convert::ImageOcrLimits,
inventory: &crate::convert::RuntimeInventory,
) -> ImagePixelOcr {
let entries: Vec<crate::runner::protocol::bodies::InventoryEntry> =
crate::convert::IMAGE_OCR_ROLE_LABELS
.iter()
.filter_map(|role| {
let path = inventory.path(role)?.to_str()?.to_string();
let expected = limits.inventory.get(*role)?.clone();
Some(crate::runner::protocol::bodies::InventoryEntry {
role: (*role).to_string(),
path,
expected_blake3: expected,
})
})
.collect();
let mut exec_grants = Vec::new();
let mut read_grants = Vec::new();
for entry in &entries {
let path = std::path::PathBuf::from(&entry.path);
if entry.role == "engine-cli" {
exec_grants.push(path);
} else {
read_grants.push(path);
}
}
ImagePixelOcr {
mode: "image-ocr",
#[cfg(feature = "svg-provider")]
svg: None,
inventory: entries,
runner: build_image_runner(image_runner_limits(&limits), exec_grants, read_grants),
}
}
#[cfg(feature = "svg-provider")]
pub fn with_svg_provider(
limits: crate::convert::ImageOcrLimits,
inventory: &crate::convert::RuntimeInventory,
provider: &crate::convert::provider::SvgProvider,
) -> ImagePixelOcr {
use crate::runner::protocol::bodies::SvgRoleEntry;
let entry = |label: &str, role: &crate::convert::provider::ProviderRole| {
let pin = limits.svg_provider.get(label);
SvgRoleEntry {
role: label.to_string(),
path: role.path.display().to_string(),
expected_blake3: pin.map(|pin| pin.blake3.clone()).unwrap_or_default(),
version: pin.map(|pin| pin.version.clone()).unwrap_or_default(),
closure_roots: role
.closure_roots
.iter()
.map(|path| path.display().to_string())
.collect(),
closure_blake3: pin.and_then(|pin| pin.closure_blake3.clone()),
jail_temp_env: role.jail_temp_env.clone(),
}
};
let roles = vec![
entry(
crate::convert::provider::PROVIDER_ROLE_RASTER,
&provider.raster,
),
entry(
crate::convert::provider::PROVIDER_ROLE_ENCODER,
&provider.encoder,
),
];
let exec_grants = vec![provider.raster.path.clone(), provider.encoder.path.clone()];
let mut closures = Vec::new();
for role in [&provider.raster, &provider.encoder] {
closures.extend(role.closure_roots.iter().cloned());
}
let mut provider_limits = image_runner_limits(&limits);
provider_limits.max_response_bytes = PROVIDER_MAX_RESPONSE_BYTES;
let runner = crate::runner::jail::ProviderJail::new(
closures,
provider.raster.jail_service_prefix.clone(),
provider.raster.jail_temp_env.clone(),
)
.and_then(|jail| {
Ok(Runner::with_provider_jail(
crate::runner::jail::platform_backend()?,
crate::runner::locate_worker()?,
provider_limits,
exec_grants,
Vec::new(),
jail,
))
});
let mut converter = ImagePixelOcr::new(limits, inventory);
converter.svg = Some(SvgProviderLeg {
roles,
runner,
#[cfg(feature = "test-adapters")]
replace_after_preflight: None,
});
converter
}
#[cfg(all(feature = "svg-provider", feature = "test-adapters"))]
pub fn new_fake_with_svg_provider(
limits: crate::convert::ImageOcrLimits,
provider: &crate::convert::provider::SvgProvider,
) -> ImagePixelOcr {
let wired = ImagePixelOcr::with_svg_provider(
limits.clone(),
&crate::convert::RuntimeInventory::empty(),
provider,
);
let mut converter = ImagePixelOcr::new_fake(limits);
converter.svg = wired.svg;
converter
}
#[cfg(all(feature = "svg-provider", feature = "test-adapters"))]
pub fn new_fake_with_svg_provider_spawn_recheck(
limits: crate::convert::ImageOcrLimits,
provider: &crate::convert::provider::SvgProvider,
replace_after_preflight: std::path::PathBuf,
) -> ImagePixelOcr {
let mut converter = ImagePixelOcr::new_fake_with_svg_provider(limits, provider);
converter
.svg
.as_mut()
.expect("the provider leg is wired")
.replace_after_preflight = Some(replace_after_preflight);
converter
}
#[cfg(feature = "svg-provider")]
fn rasterize(&self, source: &[u8]) -> std::result::Result<Vec<u8>, ConvertError> {
use crate::runner::protocol::bodies::{SvgRasterOk, SvgRasterRequest};
let leg = self.svg.as_ref().ok_or_else(|| ConvertError {
code: "unclaimed_format",
message: "no svg provider is configured for this converter".to_string(),
})?;
for entry in &leg.roles {
let present = std::fs::symlink_metadata(&entry.path)
.is_ok_and(|metadata| metadata.file_type().is_file());
if !present {
return Err(ConvertError {
code: "rasterizer-missing",
message: format!("provider component {} is not present", entry.role),
});
}
}
#[cfg(feature = "test-adapters")]
if let Some(path) = &leg.replace_after_preflight {
std::fs::remove_file(path).map_err(|e| ConvertError {
code: "adapter_spawn_error",
message: format!(
"the provider spawn-check test seam could not remove its executable: {:?}",
e.kind()
),
})?;
std::fs::create_dir_all(path).map_err(|e| ConvertError {
code: "adapter_spawn_error",
message: format!(
"the provider spawn-check test seam could not place its directory: {:?}",
e.kind()
),
})?;
}
let runner = leg.runner.as_ref().map_err(|e| runner_failure(e.clone()))?;
let body: SvgRasterOk = call(
runner,
"svg-raster",
&SvgRasterRequest {
input: crate::convert::svg_raster::JAIL_SOURCE.to_string(),
roles: leg.roles.clone(),
},
&[(crate::convert::svg_raster::JAIL_SOURCE, source)],
)?;
crate::convert::svg_raster::hex_decode(&body.raster_png_hex).ok_or_else(|| {
ConvertError {
code: "raster-exec-failed",
message: "the provider returned a raster that does not decode".to_string(),
}
})
}
#[cfg(feature = "test-adapters")]
pub fn new_fake(limits: crate::convert::ImageOcrLimits) -> ImagePixelOcr {
ImagePixelOcr {
mode: "image-ocr-fake",
#[cfg(feature = "svg-provider")]
svg: None,
inventory: Vec::new(),
runner: build_image_runner(image_runner_limits(&limits), Vec::new(), Vec::new()),
}
}
}
#[cfg(feature = "image-ocr")]
fn image_runner_limits(limits: &crate::convert::ImageOcrLimits) -> crate::runner::Limits {
crate::runner::Limits {
wall_timeout: std::time::Duration::from_secs(limits.wall_timeout_secs),
max_response_bytes: limits.max_response_bytes,
max_stderr_bytes: limits.max_stderr_bytes,
address_space_bytes: Some(limits.address_space_bytes),
cpu_seconds: limits.cpu_seconds,
file_size_bytes: limits.file_size_bytes,
max_processes: limits.max_processes,
max_resident_bytes: None,
}
}
#[cfg(feature = "image-ocr")]
fn build_image_runner(
limits: crate::runner::Limits,
exec_grants: Vec<std::path::PathBuf>,
read_grants: Vec<std::path::PathBuf>,
) -> Result<Runner, RunnerError> {
Ok(Runner::with_grants(
crate::runner::jail::platform_backend()?,
crate::runner::locate_worker()?,
limits,
exec_grants,
read_grants,
crate::runner::jail::RuntimeProfile::Plain,
))
}
#[cfg(feature = "image-ocr")]
impl Converter for ImagePixelOcr {
fn id(&self) -> &'static str {
IMAGE_PIXEL_OCR_ID
}
fn version(&self) -> &'static str {
IMAGE_PIXEL_OCR_VERSION
}
fn convert(
&self,
source: &[u8],
detected_format: &str,
) -> std::result::Result<Outcome, ConvertError> {
#[cfg(feature = "svg-provider")]
let is_svg = detected_format == "svg" && self.svg.is_some();
#[cfg(not(feature = "svg-provider"))]
let is_svg = false;
if !is_svg && !matches!(detected_format, "png" | "jpeg" | "webp") {
return Err(ConvertError {
code: "unclaimed_format",
message: format!(
"the image-pixel-ocr converter does not handle {detected_format}"
),
});
}
#[cfg(feature = "svg-provider")]
let rendered = if is_svg {
Some(self.rasterize(source)?)
} else {
None
};
#[cfg(feature = "svg-provider")]
let (input, staged): (String, &[u8]) = match &rendered {
Some(raster) => ("input.png".to_string(), raster.as_slice()),
None => (format!("input.{detected_format}"), source),
};
#[cfg(not(feature = "svg-provider"))]
let (input, staged): (String, &[u8]) = (format!("input.{detected_format}"), source);
let runner = self
.runner
.as_ref()
.map_err(|e| runner_failure(e.clone()))?;
let body: OcrOk = call(
runner,
self.mode,
&OcrRequest {
input: input.clone(),
kind: OcrInput::Image,
inventory: self.inventory.clone(),
},
&[(&input, staged)],
)?;
let mut warnings = body.warnings;
let (text, low_confidence) = render_ocr_spans(&body.spans, source.len())?;
warnings.extend(low_confidence);
let segment_source = if is_svg {
OCR_SEGMENT_SOURCE_OCR
} else {
OCR_SEGMENT_SOURCE_DOCUMENT
};
Ok(Outcome {
converter_id: IMAGE_PIXEL_OCR_ID.to_string(),
converter_version: IMAGE_PIXEL_OCR_VERSION.to_string(),
detected_format: detected_format.to_string(),
artifact_kind: ArtifactKind::Ocr,
segments: vec![Segment::span(0, text.len(), segment_source)],
text,
warnings,
media: None,
})
}
}
pub struct OcrAdapter {
runner: Runner,
}
impl OcrAdapter {
pub fn new(runner: Runner) -> OcrAdapter {
OcrAdapter { runner }
}
}
impl Converter for OcrAdapter {
fn id(&self) -> &'static str {
OCR_ADAPTER_ID
}
fn version(&self) -> &'static str {
OCR_ADAPTER_VERSION
}
fn convert(
&self,
source: &[u8],
detected_format: &str,
) -> std::result::Result<Outcome, ConvertError> {
if !matches!(detected_format, "png" | "jpeg" | "tiff") {
return Err(ConvertError {
code: "unclaimed_format",
message: format!("the OCR adapter does not handle {detected_format}"),
});
}
let input = format!("input.{detected_format}");
let body: OcrOk = call(
&self.runner,
"ocr",
&OcrRequest {
input: input.clone(),
kind: OcrInput::Image,
inventory: Vec::new(),
},
&[(&input, source)],
)?;
let mut warnings = body.warnings;
let mut lines = Vec::new();
for (index, span) in body.spans.iter().enumerate() {
if !(0.0..=1.0).contains(&span.confidence) {
return Err(ConvertError {
code: "adapter_error",
message: format!(
"span {index} confidence {} is outside 0 to 1",
span.confidence
),
});
}
if span.confidence < OCR_CONFIDENCE_WARNING {
warnings.push(format!(
"ocr_low_confidence: span {index} at {:.2}",
span.confidence
));
}
lines.push(single_line(&span.text));
}
let text = normalize_text(&finish_lines(lines));
if !source.is_empty() && text.is_empty() {
return Err(empty_output(source.len()));
}
Ok(Outcome {
artifact_kind: ArtifactKind::Ocr,
converter_id: OCR_ADAPTER_ID.to_string(),
converter_version: OCR_ADAPTER_VERSION.to_string(),
detected_format: detected_format.to_string(),
segments: vec![Segment::span(0, text.len(), "document")],
text,
warnings,
media: None,
})
}
}
#[cfg(feature = "audio-asr")]
pub struct AsrAdapter {
mode: &'static str,
profile: crate::convert::AsrProfile,
inventory: Vec<(String, std::path::PathBuf)>,
runner: Result<Runner, RunnerError>,
}
#[cfg(feature = "audio-asr")]
impl AsrAdapter {
pub fn new(
profile: crate::convert::AsrProfile,
inventory: &crate::convert::RuntimeInventory,
) -> AsrAdapter {
let files = pinned_asr_runtime_files(&profile, inventory);
let (exec_grants, read_grants) = asr_grant_lists(&files);
AsrAdapter {
mode: "asr",
runner: build_asr_runner(
&profile,
exec_grants,
read_grants,
crate::runner::jail::RuntimeProfile::Accelerator,
),
profile,
inventory: files,
}
}
#[cfg(feature = "test-adapters")]
pub fn new_fake(profile: crate::convert::AsrProfile) -> AsrAdapter {
AsrAdapter {
mode: "asr-fake",
runner: build_asr_runner(
&profile,
Vec::new(),
Vec::new(),
crate::runner::jail::RuntimeProfile::Plain,
),
profile,
inventory: Vec::new(),
}
}
}
#[cfg(feature = "audio-asr")]
fn pinned_asr_runtime_files(
profile: &crate::convert::AsrProfile,
inventory: &crate::convert::RuntimeInventory,
) -> Vec<(String, std::path::PathBuf)> {
let mut files = Vec::new();
for role in ASR_FILE_ROLE_LABELS {
if !profile.inventory.contains_key(role) {
continue;
}
if let Some(path) = inventory.path(role) {
files.push((role.to_string(), path.to_path_buf()));
}
}
files
}
#[cfg(feature = "audio-asr")]
fn asr_grant_lists(
files: &[(String, std::path::PathBuf)],
) -> (Vec<std::path::PathBuf>, Vec<std::path::PathBuf>) {
let mut exec_grants = Vec::new();
let mut read_grants = Vec::new();
for (role, path) in files {
if role == ASR_ROLE_WEIGHTS {
read_grants.push(path.clone());
} else {
exec_grants.push(path.clone());
}
}
(exec_grants, read_grants)
}
#[cfg(feature = "audio-asr")]
fn asr_runner_limits(profile: &crate::convert::AsrProfile) -> crate::runner::Limits {
crate::runner::Limits {
wall_timeout: std::time::Duration::from_secs(profile.wall_timeout_secs),
max_response_bytes: profile.max_response_bytes,
max_stderr_bytes: profile.max_stderr_bytes,
address_space_bytes: None,
cpu_seconds: profile.cpu_seconds,
file_size_bytes: profile.file_size_bytes,
max_processes: profile.max_processes,
max_resident_bytes: Some(profile.max_resident_bytes),
}
}
#[cfg(feature = "audio-asr")]
fn build_asr_runner(
profile: &crate::convert::AsrProfile,
exec_grants: Vec<std::path::PathBuf>,
read_grants: Vec<std::path::PathBuf>,
runtime_profile: crate::runner::jail::RuntimeProfile,
) -> Result<Runner, RunnerError> {
Ok(Runner::with_grants(
crate::runner::jail::platform_backend()?,
crate::runner::locate_worker()?,
asr_runner_limits(profile),
exec_grants,
read_grants,
runtime_profile,
))
}
#[cfg(feature = "audio-asr")]
impl Converter for AsrAdapter {
fn id(&self) -> &'static str {
ASR_ADAPTER_ID
}
fn version(&self) -> &'static str {
ASR_ADAPTER_VERSION
}
fn convert(
&self,
source: &[u8],
detected_format: &str,
) -> std::result::Result<Outcome, ConvertError> {
use crate::runner::protocol::bodies::{AsrOk, AsrRequest, InventoryEntry};
if !matches!(detected_format, "wav" | "mp3" | "flac" | "m4a") {
return Err(ConvertError {
code: "unclaimed_format",
message: format!("the ASR adapter does not handle {detected_format}"),
});
}
let runner = self
.runner
.as_ref()
.map_err(|e| runner_failure(e.clone()))?;
let input = format!("input.{detected_format}");
let mut wire_inventory = Vec::new();
for (role, path) in &self.inventory {
let (Some(path), Some(expected)) =
(path.to_str(), self.profile.inventory.get(role))
else {
continue;
};
wire_inventory.push(InventoryEntry {
role: role.clone(),
path: path.to_string(),
expected_blake3: expected.clone(),
});
}
let body: AsrOk = call(
runner,
self.mode,
&AsrRequest {
input: input.clone(),
language: Some(self.profile.language.clone()),
max_duration_seconds: self.profile.max_duration_seconds,
inventory: wire_inventory,
},
&[(&input, source)],
)?;
if body.segments.iter().any(|segment| segment.speaker != 1) {
return Err(ConvertError {
code: "asr-protocol-error",
message: "a transcription segment carries an unexpected speaker label"
.to_string(),
});
}
let text = normalize_text(&render_transcript(&body.segments).map_err(|detail| {
ConvertError {
code: "adapter_error",
message: detail,
}
})?);
if !source.is_empty() && text.is_empty() {
return Err(empty_output(source.len()));
}
let media = crate::manifest::Media {
duration_seconds: body.duration_seconds,
language: body
.language
.clone()
.or_else(|| Some(self.profile.language.clone())),
model_id: Some(ASR_ROLE_WEIGHTS.to_string()),
model_hash: self.profile.inventory.get(ASR_ROLE_WEIGHTS).cloned(),
decode_options_hash: self.profile.inventory.get(ASR_ROLE_DECODE_POLICY).cloned(),
speaker_count: None,
};
Ok(Outcome {
artifact_kind: ArtifactKind::Transcript,
converter_id: ASR_ADAPTER_ID.to_string(),
converter_version: ASR_ADAPTER_VERSION.to_string(),
detected_format: detected_format.to_string(),
segments: vec![Segment::span(0, text.len(), "document")],
text,
warnings: Vec::new(),
media: Some(media),
})
}
}
pub struct VideoAdapter {
runner: Runner,
}
impl VideoAdapter {
pub fn new(runner: Runner) -> VideoAdapter {
VideoAdapter { runner }
}
}
impl Converter for VideoAdapter {
fn id(&self) -> &'static str {
VIDEO_ADAPTER_ID
}
fn version(&self) -> &'static str {
VIDEO_ADAPTER_VERSION
}
fn convert(
&self,
source: &[u8],
detected_format: &str,
) -> std::result::Result<Outcome, ConvertError> {
if !matches!(detected_format, "mp4" | "mov") {
return Err(ConvertError {
code: "unclaimed_format",
message: format!("the video adapter does not handle {detected_format}"),
});
}
let input = format!("input.{detected_format}");
let body: VideoOk = call(
&self.runner,
"video",
&VideoRequest {
input: input.clone(),
sample_interval_seconds: 1.0,
max_frames: 10_000,
},
&[(&input, source)],
)?;
let text = normalize_text(&render_screen_states(&body.states).map_err(|detail| {
ConvertError {
code: "adapter_error",
message: detail,
}
})?);
if !source.is_empty() && text.is_empty() {
return Err(empty_output(source.len()));
}
Ok(Outcome {
artifact_kind: ArtifactKind::Text,
converter_id: VIDEO_ADAPTER_ID.to_string(),
converter_version: VIDEO_ADAPTER_VERSION.to_string(),
detected_format: detected_format.to_string(),
segments: vec![Segment::span(0, text.len(), "document")],
text,
warnings: Vec::new(),
media: None,
})
}
}
fn empty_output(source_len: usize) -> ConvertError {
ConvertError {
code: "empty_output",
message: format!("source is {source_len} bytes but conversion produced no text"),
}
}
#[cfg(all(test, feature = "audio-asr"))]
mod audio_grant_tests {
use super::*;
#[test]
fn a_supplied_but_unpinned_role_earns_no_grant_and_no_wire_entry() {
let dir = tempfile::tempdir().unwrap();
let make = |name: &str| {
let path = dir.path().join(name);
std::fs::write(&path, name).unwrap();
path.canonicalize().unwrap()
};
let cli = make("cli");
let weights = make("weights");
let probe = make("probe");
let mut pins = std::collections::BTreeMap::new();
pins.insert(ASR_ROLE_CLI.to_string(), "0".repeat(64));
pins.insert(ASR_ROLE_WEIGHTS.to_string(), "0".repeat(64));
pins.insert(
ASR_ROLE_DECODE_POLICY.to_string(),
crate::hash::hash_bytes(ASR_DECODE_POLICY.as_bytes()),
);
let profile = crate::convert::AsrProfile {
max_duration_seconds: 3600,
wall_timeout_secs: 900,
cpu_seconds: 600,
max_resident_bytes: 6_442_450_944,
max_response_bytes: 4_194_304,
max_stderr_bytes: 4_194_304,
file_size_bytes: 1_207_959_552,
max_processes: 64,
language: "en".to_string(),
inventory: pins,
};
let mut supplied = crate::convert::RuntimeInventory::empty();
supplied.set(ASR_ROLE_CLI, cli.clone()).unwrap();
supplied.set(ASR_ROLE_WEIGHTS, weights.clone()).unwrap();
supplied.set(ASR_ROLE_PROBE, probe.clone()).unwrap();
let files = pinned_asr_runtime_files(&profile, &supplied);
assert_eq!(files.len(), 2, "{files:?}");
assert!(files.iter().all(|(role, _)| role != ASR_ROLE_PROBE));
let (exec_grants, read_grants) = asr_grant_lists(&files);
assert_eq!(exec_grants, vec![cli.clone()]);
assert_eq!(read_grants, vec![weights.clone()]);
let adapter = AsrAdapter::new(profile, &supplied);
let runner = adapter
.runner
.as_ref()
.expect("this platform has a jail backend");
let (spec_exec, spec_read) = runner.grant_lists();
assert_eq!(spec_exec, std::slice::from_ref(&cli));
assert_eq!(spec_read, std::slice::from_ref(&weights));
assert!(!spec_exec.contains(&probe) && !spec_read.contains(&probe));
assert_eq!(
runner.runtime_profile(),
crate::runner::jail::RuntimeProfile::Accelerator
);
}
#[cfg(feature = "test-adapters")]
#[test]
fn the_fake_transcription_mode_keeps_the_base_profile() {
let profile = crate::convert::AsrProfile {
max_duration_seconds: 3600,
wall_timeout_secs: 900,
cpu_seconds: 600,
max_resident_bytes: 6_442_450_944,
max_response_bytes: 4_194_304,
max_stderr_bytes: 4_194_304,
file_size_bytes: 1_207_959_552,
max_processes: 64,
language: "en".to_string(),
inventory: std::collections::BTreeMap::new(),
};
let adapter = AsrAdapter::new_fake(profile);
let runner = adapter
.runner
.as_ref()
.expect("this platform has a jail backend");
assert_eq!(
runner.runtime_profile(),
crate::runner::jail::RuntimeProfile::Plain
);
}
}
#[cfg(all(test, feature = "svg-provider"))]
mod svg_provider_grant_tests {
use super::*;
#[test]
fn the_provider_leg_grants_exactly_what_the_configuration_named() {
let dir = tempfile::tempdir().unwrap();
let root = dir.path().canonicalize().unwrap();
let make = |name: &str| {
let path = root.join(name);
std::fs::write(&path, name).unwrap();
path
};
let raster = make("raster");
let encoder = make("encoder");
let closure = root.clone();
let text = format!(
"schema = \"{schema}\"\n\n\
[providers.svg.\"{raster_role}\"]\n\
path = \"{raster}\"\n\
closure_roots = [\"{closure}\"]\n\
jail_service_prefix = \"^ex\\\\.ample\\\\.\"\n\
jail_temp_env = \"EXAMPLE_TMPDIR\"\n\n\
[providers.svg.\"{encoder_role}\"]\n\
path = \"{encoder}\"\n",
schema = crate::convert::provider::PROVIDER_SCHEMA,
raster_role = crate::convert::provider::PROVIDER_ROLE_RASTER,
encoder_role = crate::convert::provider::PROVIDER_ROLE_ENCODER,
raster = raster.display(),
encoder = encoder.display(),
closure = closure.display(),
);
let mut limits = crate::convert::ImageOcrLimits::default();
for (label, version) in [
(crate::convert::provider::PROVIDER_ROLE_RASTER, "1.2.3"),
(crate::convert::provider::PROVIDER_ROLE_ENCODER, "4.5.6"),
] {
limits.svg_provider.insert(
label.to_string(),
crate::convert::provider::ProviderPin {
blake3: "0".repeat(64),
version: version.to_string(),
closure_blake3: None,
},
);
}
let config = crate::convert::provider::ProviderConfig::parse(&text).unwrap();
let svg = config.svg().expect("both roles");
let adapter = ImagePixelOcr::with_svg_provider(
limits,
&crate::convert::RuntimeInventory::empty(),
svg,
);
let leg = adapter.svg.as_ref().expect("the leg is wired");
let runner = leg
.runner
.as_ref()
.expect("this platform has a jail backend");
let (exec, read) = runner.grant_lists();
assert_eq!(exec, [raster.clone(), encoder.clone()]);
assert!(read.is_empty(), "{read:?}");
assert_eq!(
runner.runtime_profile(),
crate::runner::jail::RuntimeProfile::Provider
);
let jail = runner.provider_jail().expect("the provider parameters");
assert_eq!(jail.closures(), std::slice::from_ref(&closure));
let _ = &encoder;
assert_eq!(jail.service_prefix(), Some(r"^ex\.ample\."));
assert_eq!(jail.temp_env(), Some("EXAMPLE_TMPDIR"));
let roles: Vec<&str> = leg.roles.iter().map(|entry| entry.role.as_str()).collect();
assert_eq!(
roles,
vec![
crate::convert::provider::PROVIDER_ROLE_RASTER,
crate::convert::provider::PROVIDER_ROLE_ENCODER
]
);
assert_eq!(leg.roles[0].version, "1.2.3");
assert_eq!(leg.roles[1].version, "4.5.6");
}
#[test]
fn a_configured_provider_never_moves_the_raster_path_off_the_base_class() {
let adapter = ImagePixelOcr::new(
crate::convert::ImageOcrLimits::default(),
&crate::convert::RuntimeInventory::empty(),
);
let runner = adapter
.runner
.as_ref()
.expect("this platform has a jail backend");
assert_eq!(
runner.runtime_profile(),
crate::runner::jail::RuntimeProfile::Plain
);
assert!(runner.provider_jail().is_none());
}
}
#[cfg(all(test, feature = "image-ocr"))]
mod image_grant_tests {
use super::*;
#[test]
fn an_image_worker_with_an_engine_grant_never_asks_for_accelerator_allowances() {
let dir = tempfile::tempdir().unwrap();
let engine = dir.path().join("engine");
std::fs::write(&engine, b"engine bytes").unwrap();
let engine = engine.canonicalize().unwrap();
let mut limits = crate::convert::ImageOcrLimits::default();
limits
.inventory
.insert("engine-cli".to_string(), "0".repeat(64));
let mut supplied = crate::convert::RuntimeInventory::empty();
supplied.set("engine-cli", engine.clone()).unwrap();
let adapter = ImagePixelOcr::new(limits, &supplied);
let runner = adapter
.runner
.as_ref()
.expect("this platform has a jail backend");
let (spec_exec, _) = runner.grant_lists();
assert_eq!(spec_exec, std::slice::from_ref(&engine));
assert_eq!(
runner.runtime_profile(),
crate::runner::jail::RuntimeProfile::Plain
);
}
}
}
#[cfg(all(unix, feature = "audio-asr"))]
pub use imp::AsrAdapter;
#[cfg(all(unix, feature = "image-metadata"))]
pub use imp::ImageMetadata;
#[cfg(all(unix, feature = "image-ocr"))]
pub use imp::ImagePixelOcr;
#[cfg(unix)]
pub use imp::{OcrAdapter, PdfSubprocess, RecordsSubprocess, VideoAdapter};
#[cfg(not(unix))]
mod imp {
use super::*;
use crate::convert::RecordsLimits;
fn no_backend() -> ConvertError {
ConvertError {
code: "sandbox_unavailable",
message: "no jail backend exists for this platform, refusing to run adapters"
.to_string(),
}
}
pub struct PdfSubprocess;
impl Converter for PdfSubprocess {
fn id(&self) -> &'static str {
PDF_SUBPROCESS_ID
}
fn version(&self) -> &'static str {
PDF_SUBPROCESS_VERSION
}
fn convert(
&self,
_source: &[u8],
_detected_format: &str,
) -> std::result::Result<Outcome, ConvertError> {
Err(no_backend())
}
}
pub struct RecordsSubprocess;
impl RecordsSubprocess {
pub fn new(_limits: RecordsLimits) -> RecordsSubprocess {
RecordsSubprocess
}
}
impl Converter for RecordsSubprocess {
fn id(&self) -> &'static str {
RECORDS_SUBPROCESS_ID
}
fn version(&self) -> &'static str {
RECORDS_SUBPROCESS_VERSION
}
fn convert(
&self,
_source: &[u8],
_detected_format: &str,
) -> std::result::Result<Outcome, ConvertError> {
Err(no_backend())
}
}
}
#[cfg(not(unix))]
pub use imp::{PdfSubprocess, RecordsSubprocess};
pub fn render_transcript(segments: &[crate::runner_bodies::AsrSegment]) -> Result<String, String> {
let mut ordered: Vec<_> = segments.iter().collect();
ordered.sort_by(|a, b| {
a.start_seconds
.total_cmp(&b.start_seconds)
.then(a.end_seconds.total_cmp(&b.end_seconds))
.then(a.speaker.cmp(&b.speaker))
});
let mut lines = Vec::new();
for segment in ordered {
if segment.end_seconds < segment.start_seconds {
return Err(format!(
"segment ends at {} before it starts at {}",
segment.end_seconds, segment.start_seconds
));
}
lines.push(format!(
"[{} -> {}] Speaker {}: {}",
format_timecode(segment.start_seconds)?,
format_timecode(segment.end_seconds)?,
segment.speaker,
single_line(&segment.text)
));
}
Ok(finish_lines(lines))
}
pub fn render_screen_states(
states: &[crate::runner_bodies::ScreenState],
) -> Result<String, String> {
let mut ordered: Vec<_> = states.iter().collect();
ordered.sort_by(|a, b| {
a.first_seen_seconds
.total_cmp(&b.first_seen_seconds)
.then(a.text.cmp(&b.text))
});
let mut lines = Vec::new();
let mut previous: Option<String> = None;
for state in ordered {
let text = single_line(&state.text);
if previous.as_deref() == Some(text.as_str()) {
continue;
}
lines.push(format!(
"[on-screen @ {}] {}",
format_timecode(state.first_seen_seconds)?,
text
));
previous = Some(text);
}
Ok(finish_lines(lines))
}
fn finish_lines(lines: Vec<String>) -> String {
if lines.is_empty() {
String::new()
} else {
let mut text = lines.join("\n");
text.push('\n');
text
}
}
#[cfg(all(unix, feature = "image-ocr"))]
pub(crate) fn render_ocr_spans(
spans: &[crate::runner::protocol::bodies::OcrSpan],
source_len: usize,
) -> Result<(String, Vec<String>), ConvertError> {
let mut warnings = Vec::new();
let mut lines = Vec::new();
for (index, span) in spans.iter().enumerate() {
if !(0.0..=1.0).contains(&span.confidence) {
return Err(ConvertError {
code: "ocr-protocol-error",
message: format!(
"span {index} confidence {} is outside 0 to 1",
span.confidence
),
});
}
if span.confidence < OCR_CONFIDENCE_WARNING {
warnings.push(format!(
"ocr_low_confidence: span {index} at {:.2}",
span.confidence
));
}
lines.push(single_line(&span.text));
}
let text = normalize_text(&finish_lines(lines));
if source_len > 0 && !text.chars().any(crate::convert::is_meaningful) {
return Err(ConvertError {
code: "empty_output",
message: format!("source is {source_len} bytes but conversion produced no text"),
});
}
Ok((text, warnings))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::runner_bodies::{AsrSegment, ScreenState};
#[test]
fn recovered_pdf_text_is_recorded_under_the_recovery_id() {
assert_eq!(
pdf_converter_identity(false),
(PDF_SUBPROCESS_ID, PDF_SUBPROCESS_VERSION)
);
assert_eq!(
pdf_converter_identity(true),
(PDF_RECOVERED_ID, PDF_RECOVERED_VERSION)
);
assert_ne!(PDF_RECOVERED_ID, PDF_SUBPROCESS_ID);
assert_eq!(PDF_RECOVERED_VERSION, PDF_SUBPROCESS_VERSION);
}
#[test]
fn timecodes_render_as_hours_minutes_seconds() {
assert_eq!(format_timecode(0.0).unwrap(), "00:00:00");
assert_eq!(format_timecode(12.9).unwrap(), "00:00:12");
assert_eq!(format_timecode(182.0).unwrap(), "00:03:02");
assert_eq!(format_timecode(3661.0).unwrap(), "01:01:01");
assert!(format_timecode(-1.0).is_err());
assert!(format_timecode(f64::NAN).is_err());
}
#[test]
fn transcripts_render_one_ordered_line_per_segment() {
let segments = vec![
AsrSegment {
start_seconds: 19.0,
end_seconds: 24.0,
speaker: 2,
text: "second\nline".to_string(),
},
AsrSegment {
start_seconds: 12.0,
end_seconds: 19.0,
speaker: 1,
text: "first remark".to_string(),
},
];
assert_eq!(
render_transcript(&segments).unwrap(),
"[00:00:12 -> 00:00:19] Speaker 1: first remark\n\
[00:00:19 -> 00:00:24] Speaker 2: second line\n"
);
}
#[test]
fn transcripts_reject_a_segment_that_ends_before_it_starts() {
let segments = vec![AsrSegment {
start_seconds: 5.0,
end_seconds: 4.0,
speaker: 1,
text: "impossible".to_string(),
}];
assert!(render_transcript(&segments).is_err());
}
#[test]
fn screen_states_render_deduplicated_in_order() {
let states = vec![
ScreenState {
first_seen_seconds: 182.0,
text: "Quarterly revenue".to_string(),
},
ScreenState {
first_seen_seconds: 10.0,
text: "Agenda".to_string(),
},
ScreenState {
first_seen_seconds: 200.0,
text: "Quarterly revenue".to_string(),
},
];
assert_eq!(
render_screen_states(&states).unwrap(),
"[on-screen @ 00:00:10] Agenda\n[on-screen @ 00:03:02] Quarterly revenue\n"
);
}
}