Skip to main content

convert_rs/
lib.rs

1//! File conversion through a policy-controlled graph of independent adapters.
2//!
3//! Use [`Converter::new`] for compiled-in pure-Rust adapters. With the `external`
4//! feature, `Converter::discover().await` also probes installed native tools.
5//! Discovery does not enable external routes; set [`Policy::allow_external`]
6//! explicitly. Register additional implementations through [`Converter::register`].
7//!
8//! # Convert a file
9//!
10//! Conversion requires a Tokio runtime. This example uses only the byte/text API
11//! and no native tools. A same-format request stages a byte-for-byte copy without
12//! decoding, validating the claimed format, or choosing a codec.
13//!
14//! ```rust
15//! use convert_rs::{ConversionOptions, Converter};
16//!
17//! # #[tokio::main(flavor = "current_thread")]
18//! # async fn main() -> Result<(), Box<dyn std::error::Error>> {
19//! let inputs = tempfile::tempdir()?;
20//! let source = inputs.path().join("note.txt");
21//! std::fs::write(&source, "Conversion example\n")?;
22//! let converter = Converter::new();
23//! let report = converter.convert(&[source], "txt", &ConversionOptions::default()).await?;
24//! assert!(report.plan.steps.is_empty());
25//! assert!(report.warnings.is_empty());
26//! assert_eq!(std::fs::read_to_string(&report.artifacts[0].path)?, "Conversion example\n");
27//! // Persist output outside the report-owned workspace before dropping it.
28//! let saved = inputs.path().join("saved.txt");
29//! std::fs::copy(&report.artifacts[0].path, &saved)?;
30//! let temporary = report.artifacts[0].path.clone();
31//! drop(report);
32//! assert!(!temporary.exists());
33//! assert!(saved.exists());
34//! # Ok(())
35//! # }
36//! ```
37//!
38//! # Features and fidelity
39//!
40//! Default features are `data` (JSON, TOML, CSV), `media` (image and audio
41//! operations), `archives` (archive and compression operations), and `external`
42//! (native-tool discovery). Without default features, the built-in byte and text
43//! operations remain. Registered formats and available routes depend on features
44//! and adapters; listing a format does not promise an encoder for every input.
45//!
46//! Plans use adapter-declared costs and semantic classifications. The default
47//! policy permits lossy conversions. There is no lossless-only policy flag:
48//! inspect [`Plan::steps`] and, if appropriate, execute a reviewed forced plan.
49//! [`ConversionReport::warnings`] describes non-lossless steps of the successful
50//! route, not measured quality loss or every possible metadata change. An empty
51//! warning list is not proof of complete fidelity. No public option selects
52//! codecs, bitrates, or encoder quality settings.
53//!
54//! # Execution boundaries
55//!
56//! Inputs are staged separately for each attempted route. Final paths belong to
57//! the returned [`ConversionReport`] and are removed when it is dropped. Cloning
58//! an [`Artifact`] does not keep its file alive. Returned output paths, declared
59//! formats, counts, and byte totals are checked, but file contents are not
60//! independently decoded by the engine to verify their declared format.
61//!
62//! [`Limits`] provides staging checks and adapter-specific budgets, not a hard
63//! memory, CPU, disk, or wall-clock sandbox. Custom adapters are trusted code.
64//! Native tools require operating-system isolation when processing hostile data;
65//! this library does not provide that isolation.
66#![warn(missing_docs)]
67
68#[cfg(feature = "archives")]
69mod archives;
70mod builtin;
71mod engine;
72#[cfg(feature = "external")]
73mod external;
74#[cfg(feature = "media")]
75mod media;
76#[cfg(feature = "external")]
77mod process;
78
79pub use convert_rs_core::{
80    ContentKind, ContentProfile, Conversion, ConversionCategory, ConversionKind, Format, Graph,
81    MediaKind, Plan, Policy, Stream,
82};
83pub use engine::{
84    CapabilityReason, ConversionOptions, ConversionReport, Converter, Identification,
85    IdentificationMethod, InputReport, ProfileReport, ProfileSource, RouteAttempt, StepCost,
86    TargetCapability, Warning,
87};
88
89use std::collections::BTreeMap;
90use std::future::Future;
91use std::path::PathBuf;
92use std::pin::Pin;
93use std::time::Duration;
94
95/// A path and declared format for a staged input or adapter output.
96///
97/// Artifact vectors preserve order. This value neither owns the file nor verifies
98/// its contents. Cloning it clones only the path and format; report output files
99/// remain owned by the [`ConversionReport`]'s temporary directory.
100///
101/// Construct through [`Artifact::new`]; the struct is `#[non_exhaustive]` so new
102/// fields can be added without breaking semver. Direct struct literals only
103/// compile inside this crate.
104#[derive(Clone, Debug)]
105#[non_exhaustive]
106pub struct Artifact {
107    /// Path to the artifact in the report-owned temporary workspace.
108    pub path: PathBuf,
109    /// Canonical format ID for the artifact.
110    pub format: String,
111    /// Logical name this file had (or should have) outside the workspace.
112    ///
113    /// Set by extraction adapters to the archive member path and preserved
114    /// unchanged through steps that carry files forward. Packaging adapters use
115    /// it as the member name when present. It is a naming hint, not a trusted
116    /// path: adapters must sanitize it before using it, and the engine never
117    /// resolves it against the filesystem.
118    pub logical_name: Option<String>,
119}
120
121impl Artifact {
122    /// Construct an artifact from a path and its declared canonical format ID.
123    #[must_use]
124    pub fn new(path: impl Into<PathBuf>, format: impl Into<String>) -> Self {
125        Self {
126            path: path.into(),
127            format: format.into(),
128            logical_name: None,
129        }
130    }
131
132    /// Attach a logical name; see [`Self::logical_name`].
133    #[must_use]
134    pub fn with_logical_name(mut self, name: impl Into<String>) -> Self {
135        self.logical_name = Some(name.into());
136        self
137    }
138}
139
140/// Resource budgets used during staging, output checks, and adapter execution.
141///
142/// The engine enforces input count, input bytes per staged file, and returned
143/// output count and aggregate bytes per step, plus a cumulative output byte
144/// budget across every step of one attempt. Other checks depend on the adapter.
145/// The engine does not wrap arbitrary adapter futures in a timeout, preempt
146/// blocking work, or enforce process-wide memory and disk quotas. Budgets do
147/// not bound unreported files that adapters create but do not return.
148/// Identity copies do not pass through adapter output validation.
149///
150/// Defaults are 120 seconds, 256 MiB input bytes, 512 MiB output bytes per step,
151/// 4 GiB cumulative output bytes per attempt, 256 artifacts, 16 million pixels
152/// and audio samples, 256 archive entries, a 100:1 expansion ratio, and a
153/// 256 MiB adapter memory budget. Values are not validated
154/// by [`Converter::limits`]; zero and small budgets can make operations fail.
155#[derive(Clone, Debug)]
156pub struct Limits {
157    /// Per-invocation time budget used by adapters, not a universal hard deadline.
158    ///
159    /// Pure-Rust adapters check elapsed time at operation-specific points; native
160    /// process execution uses a timeout. Staging and arbitrary custom adapters
161    /// are not interrupted by the engine.
162    pub timeout: Duration,
163    /// Maximum bytes per file during engine staging.
164    ///
165    /// Some adapters additionally apply this as an aggregate input budget.
166    pub max_input_bytes: u64,
167    /// Maximum aggregate bytes of returned output files per step.
168    ///
169    /// Engine validation occurs after the adapter returns; this is not a disk quota.
170    pub max_output_bytes: u64,
171    /// Maximum bytes summed across every step of a single conversion attempt.
172    ///
173    /// The engine tracks each validated step's byte total as it executes a route
174    /// and exposes the running sum through [`StepRequest::spent_output_bytes`],
175    /// so adapters can budget before writing. Unlike [`Self::max_output_bytes`],
176    /// which resets per step, this budget covers intermediates: a 6-step route
177    /// that emits 1 GiB per step trips a 4 GiB total budget. Zero disables.
178    /// Default: 4 GiB.
179    pub max_total_output_bytes: u64,
180    /// Maximum input count and maximum returned output count per step.
181    pub max_artifacts: usize,
182    /// Decoded image pixel budget used by media operations, not by engine validation.
183    pub max_pixels: u64,
184    /// Decoded audio sample budget used by media operations.
185    pub max_audio_samples: u64,
186    /// Archive entry budget used by archive operations, including directory entries.
187    pub max_archive_entries: usize,
188    /// Decompression byte expansion ratio used by archive operations.
189    pub max_expansion_ratio: u64,
190    /// Memory budget for adapter-specific allocation checks, not process resident memory.
191    pub max_memory_bytes: u64,
192}
193
194impl Default for Limits {
195    fn default() -> Self {
196        Self {
197            timeout: Duration::from_secs(120),
198            max_input_bytes: 256 * 1024 * 1024,
199            max_output_bytes: 512 * 1024 * 1024,
200            max_total_output_bytes: 4 * 1024 * 1024 * 1024,
201            max_artifacts: 256,
202            max_pixels: 16_000_000,
203            max_audio_samples: 16_000_000,
204            max_archive_entries: 256,
205            max_expansion_ratio: 100,
206            max_memory_bytes: 256 * 1024 * 1024,
207        }
208    }
209}
210
211/// One adapter invocation. Adapters must write only inside `output_dir`, must
212/// not change inputs, and must return at least one regular file on success.
213///
214/// The engine creates the output directory before calling the adapter. The
215/// adapter is responsible for validating operation-specific inputs, observing
216/// limits, and keeping blocking work off the asynchronous executor. The engine
217/// checks returned files but does not enforce every part of this contract; in
218/// particular, it currently accepts an empty output vector.
219#[derive(Clone, Debug)]
220pub struct StepRequest {
221    /// Ordered staged inputs or outputs from the preceding step; do not mutate them.
222    pub inputs: Vec<Artifact>,
223    /// Existing private step directory where the adapter must create outputs.
224    pub output_dir: PathBuf,
225    /// Concrete source format ID, not a filename extension or wildcard.
226    pub from: String,
227    /// Concrete target format ID that every returned artifact must declare.
228    pub to: String,
229    /// Adapter-specific operation ID copied from the selected conversion edge.
230    pub operation: String,
231    /// Resource budgets for this invocation; see [`Limits`] for enforcement scope.
232    pub limits: Limits,
233    /// Quality knobs copied from [`ConversionOptions::params`](engine::ConversionOptions::params).
234    ///
235    /// Keys are adapter-defined names such as `quality`; the reserved key
236    /// `quality` carries a 0-100 integer string and should be mapped onto the
237    /// closest native encoder control. Adapters must ignore unknown keys and
238    /// must never treat these values as paths or shell input.
239    pub params: BTreeMap<String, String>,
240    /// Cumulative bytes the engine has already validated across earlier steps
241    /// of the current route, used with [`Limits::max_total_output_bytes`].
242    pub spent_output_bytes: u64,
243}
244
245impl StepRequest {
246    /// Build a request with no params and no prior output spend.
247    #[must_use]
248    pub fn new(
249        inputs: Vec<Artifact>,
250        output_dir: impl Into<PathBuf>,
251        from: impl Into<String>,
252        to: impl Into<String>,
253        operation: impl Into<String>,
254        limits: Limits,
255    ) -> Self {
256        Self {
257            inputs,
258            output_dir: output_dir.into(),
259            from: from.into(),
260            to: to.into(),
261            operation: operation.into(),
262            limits,
263            params: BTreeMap::new(),
264            spent_output_bytes: 0,
265        }
266    }
267
268    /// Attach quality params; see [`Self::params`].
269    #[must_use]
270    pub fn with_params(mut self, params: &BTreeMap<String, String>) -> Self {
271        self.params = params.clone();
272        self
273    }
274
275    /// Record cumulative earlier-step output bytes; see [`Self::spent_output_bytes`].
276    #[must_use]
277    pub fn with_spent_output_bytes(mut self, spent: u64) -> Self {
278        self.spent_output_bytes = spent;
279        self
280    }
281}
282
283/// Read the reserved `quality` param as a 1-100 encoder value.
284///
285/// Unparseable or absent values return `None`; out-of-range values clamp to the
286/// supported range. Adapters ignore every other key of [`StepRequest::params`].
287#[cfg(any(feature = "media", feature = "external"))]
288pub(crate) fn requested_quality(params: &BTreeMap<String, String>) -> Option<u8> {
289    let value = params.get("quality")?.trim().parse::<u32>().ok()?;
290    Some(value.clamp(1, 100) as u8)
291}
292
293/// Result of a library operation, with [`Error`] describing failure.
294pub type Result<T> = std::result::Result<T, Error>;
295/// A sendable, boxed adapter future yielding ordered artifacts or an [`Error`].
296///
297/// The lifetime allows the future to borrow both the adapter and its request.
298pub type ConversionFuture<'a> = Pin<Box<dyn Future<Output = Result<Vec<Artifact>>> + Send + 'a>>;
299
300/// Boxed future returned by [`Adapter::inspect`].
301pub type InspectFuture<'a> =
302    Pin<Box<dyn Future<Output = Option<Result<ContentProfile>>> + Send + 'a>>;
303
304/// Output port for conversion backends. Implement this to add formats without
305/// changing the planner or orchestrator. Backend names identify registrations;
306/// registering the same name replaces its previous implementation.
307///
308/// Implementations are trusted code, not sandboxed plugins. Advertised metadata
309/// should remain stable between calls. Each edge's `backend` must match the
310/// adapter's name, and the implementation must reject unsupported operations.
311/// Registration validates graph metadata, not this correspondence or fidelity.
312///
313/// # Register a custom adapter
314///
315/// This adapter copies one text file into a custom text representation. It uses
316/// asynchronous file I/O and checks its input size and timeout explicitly.
317///
318/// ```rust
319/// use std::sync::Arc;
320/// use convert_rs::{Adapter, Artifact, Conversion, ConversionFuture, ConversionKind,
321///     ConversionOptions, Converter, Error, Format, MediaKind, StepRequest};
322///
323/// struct NoteAdapter;
324/// impl Adapter for NoteAdapter {
325///     fn name(&self) -> &str { "note-example" }
326///     fn formats(&self) -> Vec<Format> {
327///         vec![Format::new("example-note", "Example note", MediaKind::Text)]
328///     }
329///     fn conversions(&self) -> Vec<Conversion> {
330///         vec![Conversion::new("txt", "example-note", self.name(), "copy-note",
331///             ConversionKind::Lossless, 1)]
332///     }
333///     fn convert<'a>(&'a self, request: &'a StepRequest) -> ConversionFuture<'a> {
334///         Box::pin(async move {
335///             if request.operation != "copy-note" || request.from != "txt"
336///                 || request.to != "example-note" || request.inputs.len() != 1 {
337///                 return Err(Error::Invalid("unsupported note request".into()));
338///             }
339///             let job = async {
340///                 let size = tokio::fs::metadata(&request.inputs[0].path).await?.len();
341///                 if size > request.limits.max_input_bytes
342///                     || size > request.limits.max_output_bytes {
343///                     return Err(Error::Limit("note bytes".into()));
344///                 }
345///                 let path = request.output_dir.join("note.example-note");
346///                 tokio::fs::copy(&request.inputs[0].path, &path).await?;
347///                 Ok(vec![Artifact::new(path, request.to.clone())])
348///             };
349///             tokio::time::timeout(request.limits.timeout, job).await
350///                 .map_err(|_| Error::Timeout { backend: self.name().into() })?
351///         })
352///     }
353/// }
354///
355/// # #[tokio::main(flavor = "current_thread")]
356/// # async fn main() -> Result<(), Box<dyn std::error::Error>> {
357/// let mut converter = Converter::new();
358/// converter.register(Arc::new(NoteAdapter))?;
359/// let inputs = tempfile::tempdir()?;
360/// let source = inputs.path().join("note.txt");
361/// std::fs::write(&source, "custom adapter")?;
362/// let report = converter.convert(&[source], "example-note",
363///     &ConversionOptions::default()).await?;
364/// assert_eq!(report.plan.steps[0].backend, "note-example");
365/// assert_eq!(std::fs::read_to_string(&report.artifacts[0].path)?, "custom adapter");
366/// # Ok(())
367/// # }
368/// ```
369pub trait Adapter: Send + Sync {
370    /// Stable registration name, nonempty and at most 64 bytes.
371    fn name(&self) -> &str;
372    /// Format declarations contributed to the graph.
373    ///
374    /// Registration requires 1 through 8,192 declarations. Edges can also refer
375    /// to formats supplied by other registered adapters.
376    fn formats(&self) -> Vec<Format>;
377    /// Supported directed edges, including their semantic kind and relative cost.
378    ///
379    /// Registration accepts at most 1,000,000 edges from the incoming adapter.
380    fn conversions(&self) -> Vec<Conversion>;
381    /// Execute one operation using the [`StepRequest`] contract.
382    ///
383    /// # Errors
384    ///
385    /// Return an appropriate [`Error`] for invalid requests, unsupported content,
386    /// I/O failure, or exceeded limits. A failed route may be retried through
387    /// another adapter; do not rely on side effects outside the output directory.
388    fn convert<'a>(&'a self, request: &'a StepRequest) -> ConversionFuture<'a>;
389
390    /// Return availability information for this adapter.
391    ///
392    /// The default reports availability unconditionally with an empty detail.
393    /// An implementation may return cached discovery results rather than probe
394    /// again. Availability is informational and does not guarantee execution.
395    fn status(&self) -> BackendStatus {
396        BackendStatus {
397            name: self.name().to_owned(),
398            available: true,
399            detail: String::new(),
400        }
401    }
402
403    /// List the [`StepRequest::params`] keys this operation consumes.
404    ///
405    /// The engine uses the union over an executed route's steps to report params
406    /// that no step honored as [`ConversionReport::ignored_params`](engine::ConversionReport::ignored_params),
407    /// surfacing typos instead of silently discarding knobs. The `engine` key
408    /// prefix is reserved. The default declares no supported keys.
409    fn param_keys(&self, _operation: &str) -> Vec<String> {
410        Vec::new()
411    }
412
413    /// Deep-check whether `path` really decodes as `format`.
414    ///
415    /// [`engine::ConversionOptions::verify`](engine::ConversionOptions::verify)
416    /// first re-sniffs magic signatures, then asks available adapters that own
417    /// the format. `Some(true)` confirms, `Some(false)` rejects the artifact and
418    /// fails the route, and `None` (the default) expresses no opinion and never
419    /// blocks verification. Implementations must not allocate more than
420    /// `limits.max_input_bytes` and must treat the file as hostile input.
421    fn accepts_input(
422        &self,
423        _path: &std::path::Path,
424        _format: &str,
425        _limits: &Limits,
426    ) -> Option<bool> {
427        None
428    }
429
430    /// Report the content streams this adapter observes inside `path`.
431    ///
432    /// [`Converter::profile`](crate::Converter::profile) and profile-driven
433    /// planning call this on adapters available for the identified format. The
434    /// default returns `None`, meaning this adapter has no content view and the
435    /// engine falls back to another adapter or to declared [`Format::payloads`].
436    ///
437    /// Implementations must not allocate more than `limits.max_input_bytes`,
438    /// must stay within `limits.timeout`, and must treat the file as hostile
439    /// input. A returned profile's `format` field must equal `format`; an
440    /// adapter that inspects the file but finds no describable streams returns
441    /// `Some` with an empty stream list, and returns `Err` when inspection of
442    /// this file fails.
443    fn inspect<'a>(
444        &'a self,
445        _path: &'a std::path::Path,
446        _format: &'a str,
447        _limits: &'a Limits,
448    ) -> InspectFuture<'a> {
449        Box::pin(async { None })
450    }
451}
452
453/// An adapter's reported availability and diagnostic detail.
454#[derive(Clone, Debug)]
455pub struct BackendStatus {
456    /// Registered adapter name.
457    pub name: String,
458    /// Whether the adapter reports itself available; not a per-file capability check.
459    pub available: bool,
460    /// Human-readable probe details.
461    pub detail: String,
462}
463
464/// Failures from registration, identification, planning, and execution.
465///
466/// Errors raised while attempting routes are rendered into [`Self::RoutesFailed`]
467/// by [`Converter::convert`], rather than returned as their original variants.
468#[derive(Debug, thiserror::Error)]
469pub enum Error {
470    /// A filesystem or other I/O operation failed.
471    #[error(transparent)]
472    Io(
473        /// Underlying I/O failure.
474        #[from]
475        std::io::Error,
476    ),
477    /// Graph registration or route search failed.
478    #[error(transparent)]
479    Plan(
480        /// Underlying graph or search failure.
481        #[from]
482        convert_rs_core::PlanError,
483    ),
484    /// Invalid caller input or unsupported request parameters.
485    #[error("invalid request: {0}")]
486    Invalid(
487        /// Explanation of the rejected request.
488        String,
489    ),
490    /// An adapter operation or artifact format check failed.
491    #[error("conversion failed: {0}")]
492    Conversion(
493        /// Adapter or output-validation failure details.
494        String,
495    ),
496    /// An adapter reported exceeding its time budget.
497    #[error("{backend} exceeded its timeout")]
498    Timeout {
499        /// Name of the adapter whose time budget was exceeded.
500        backend: String,
501    },
502    /// A checked resource budget was exceeded.
503    #[error("resource limit exceeded: {0}")]
504    Limit(
505        /// Description of the exceeded budget.
506        String,
507    ),
508    /// A policy or input/output safety check rejected the operation.
509    #[error("security policy rejected input: {0}")]
510    Security(
511        /// Description of the rejected condition.
512        String,
513    ),
514    /// Every attempted route failed, with route descriptions and rendered errors.
515    ///
516    /// Only routes returned within the planner's budgets are attempted. This
517    /// does not mean every possible graph route was tried.
518    #[error("all attempted routes failed: {0}")]
519    RoutesFailed(
520        /// Semicolon-separated route descriptions and rendered failure messages.
521        String,
522    ),
523}