use std::fs;
use std::io::Write;
use std::path::Path;
use std::time::Instant;
use crate::{
Adapter, Artifact, Conversion, ConversionFuture, ConversionKind, Error, Format, Limits,
MediaKind, Result, StepRequest,
};
pub struct BuiltinAdapter;
fn private_create(path: &Path) -> Result<fs::File> {
let mut options = fs::File::options();
options.write(true).create_new(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
options.mode(0o600);
}
Ok(options.open(path)?)
}
fn read_bounded(path: &Path, limit: u64) -> Result<Vec<u8>> {
let metadata = fs::symlink_metadata(path)?;
if !metadata.file_type().is_file() {
return Err(Error::Security("input is not a regular file".into()));
}
if metadata.len() > limit {
return Err(Error::Limit("input bytes exceed budget".into()));
}
Ok(fs::read(path)?)
}
fn check_time(started: Instant, limits: &Limits) -> Result<()> {
if started.elapsed() >= limits.timeout {
return Err(Error::Timeout {
backend: "builtin".into(),
});
}
Ok(())
}
fn write_artifact(
output_dir: &Path,
index: usize,
format: &str,
bytes: &[u8],
limits: &Limits,
) -> Result<Artifact> {
let spent =
u64::try_from(index).map_err(|_| Error::Limit("artifact index overflow".into()))? * 4096;
if spent >= limits.max_output_bytes || (bytes.len() as u64) > limits.max_output_bytes - spent {
return Err(Error::Limit("builtin output bytes".into()));
}
let path = output_dir.join(format!("builtin-{index}.{format}"));
let mut file = private_create(&path)?;
file.write_all(bytes)?;
file.flush()?;
Ok(Artifact {
path,
format: format.to_owned(),
})
}
impl Adapter for BuiltinAdapter {
fn name(&self) -> &str {
"builtin"
}
fn formats(&self) -> Vec<Format> {
let formats = vec![
Format::new("bin", "Raw bytes", MediaKind::Binary),
Format::new("txt", "Plain text", MediaKind::Text).with_extensions(&["txt", "md"]),
];
#[cfg(feature = "data")]
let formats = {
let mut formats = formats;
formats.extend([
Format::new("json", "JSON", MediaKind::Data).with_extensions(&["json"]),
Format::new("toml", "TOML", MediaKind::Data).with_extensions(&["toml"]),
Format::new("csv", "CSV", MediaKind::Data).with_extensions(&["csv", "tsv"]),
]);
formats
};
formats
}
fn conversions(&self) -> Vec<Conversion> {
let conversions = vec![
Conversion::new(
"*",
"bin",
"builtin",
"read-raw",
ConversionKind::Reinterpret,
1,
),
Conversion::new(
"bin",
"txt",
"builtin",
"decode-text",
ConversionKind::Reinterpret,
2,
),
Conversion::new(
"txt",
"bin",
"builtin",
"encode-text",
ConversionKind::Lossless,
1,
),
];
#[cfg(feature = "data")]
let conversions = {
let mut conversions = conversions;
conversions.extend([
Conversion::new(
"json",
"txt",
"builtin",
"to-text",
ConversionKind::Lossless,
1,
),
Conversion::new(
"toml",
"txt",
"builtin",
"to-text",
ConversionKind::Lossless,
1,
),
Conversion::new(
"csv",
"txt",
"builtin",
"to-text",
ConversionKind::Lossless,
1,
),
Conversion::new(
"txt",
"json",
"builtin",
"from-text",
ConversionKind::Reinterpret,
2,
),
Conversion::new(
"txt",
"toml",
"builtin",
"from-text",
ConversionKind::Reinterpret,
2,
),
Conversion::new(
"txt",
"csv",
"builtin",
"from-text",
ConversionKind::Reinterpret,
3,
),
Conversion::new(
"toml",
"json",
"builtin",
"data-json",
ConversionKind::Lossy,
3,
),
Conversion::new(
"json",
"toml",
"builtin",
"data-json",
ConversionKind::Lossy,
3,
),
Conversion::new(
"csv",
"json",
"builtin",
"data-json",
ConversionKind::Lossy,
3,
),
Conversion::new(
"json",
"csv",
"builtin",
"data-json",
ConversionKind::Lossy,
3,
),
]);
conversions
};
conversions
}
fn convert<'a>(&'a self, request: &'a StepRequest) -> ConversionFuture<'a> {
Box::pin(async move {
let request = request.clone();
tokio::task::spawn_blocking(move || convert_blocking(&request, Instant::now()))
.await
.map_err(|error| Error::Security(format!("builtin worker failed: {error}")))?
})
}
}
fn convert_blocking(request: &StepRequest, started: Instant) -> Result<Vec<Artifact>> {
let limits = &request.limits;
if !fs::symlink_metadata(&request.output_dir)?
.file_type()
.is_dir()
{
return Err(Error::Security(
"builtin output must be a private directory".into(),
));
}
if request.operation == "read-raw" {
let mut artifacts = Vec::with_capacity(request.inputs.len());
for (index, input) in request.inputs.iter().enumerate() {
check_time(started, limits)?;
let bytes = read_bounded(&input.path, limits.max_input_bytes)?;
artifacts.push(write_artifact(
&request.output_dir,
index,
"bin",
&bytes,
limits,
)?);
}
return Ok(artifacts);
}
if request.inputs.len() != 1 {
return Err(Error::Invalid(
"this operation accepts exactly one input file".into(),
));
}
let bytes = read_bounded(&request.inputs[0].path, limits.max_input_bytes)?;
check_time(started, limits)?;
let output = apply(request, &bytes)?;
check_time(started, limits)?;
Ok(vec![write_artifact(
&request.output_dir,
0,
&request.to,
&output,
limits,
)?])
}
fn apply(request: &StepRequest, bytes: &[u8]) -> Result<Vec<u8>> {
match request.operation.as_str() {
"decode-text" => String::from_utf8(bytes.to_vec())
.map(|text| text.into_bytes())
.map_err(|_| Error::Security("binary input is not valid UTF-8".into())),
"encode-text" => Ok(bytes.to_vec()),
#[cfg(feature = "data")]
"to-text" | "from-text" | "data-json" => {
transform_data(&request.operation, &request.from, &request.to, bytes)
}
#[cfg(not(feature = "data"))]
"to-text" | "from-text" | "data-json" => Err(Error::Invalid(
"data conversions require the data feature".into(),
)),
other => Err(Error::Invalid(format!(
"unknown builtin operation: {other}"
))),
}
}
#[cfg(feature = "data")]
fn transform_data(operation: &str, from: &str, to: &str, bytes: &[u8]) -> Result<Vec<u8>> {
match operation {
"to-text" => {
std::str::from_utf8(bytes).map_err(|_| invalid_data("format text is not UTF-8"))?;
Ok(bytes.to_vec())
}
"from-text" => {
let text = std::str::from_utf8(bytes).map_err(invalid_data)?;
validate_text(to, text)?;
Ok(bytes.to_vec())
}
"data-json" => {
let text = std::str::from_utf8(bytes).map_err(invalid_data)?;
let value = match from {
"toml" => json_from_toml_text(text),
"csv" => {
let (headers, rows) = parse_csv(text)?;
let records = rows.into_iter().map(|row| {
let mut object = serde_json::Map::new();
for (index, cell) in row.into_iter().enumerate() {
let key = headers
.get(index)
.cloned()
.unwrap_or_else(|| format!("column_{index}"));
object.insert(key, serde_json::Value::String(cell));
}
serde_json::Value::Object(object)
});
serde_json::to_vec_pretty(&serde_json::Value::Array(records.collect()))
.map_err(invalid_data)
}
_ => serde_json::from_str::<serde_json::Value>(text)
.map_err(invalid_data)
.and_then(|value| serde_json::to_vec_pretty(&value).map_err(invalid_data)),
}?;
if to == "json" {
return Ok(value);
}
let parsed: serde_json::Value = serde_json::from_slice(&value).map_err(invalid_data)?;
match to {
"toml" => {
let toml_value = json_value_to_toml(&parsed)?;
toml::to_string_pretty(&toml_value)
.map_err(invalid_data)
.map(String::into_bytes)
}
"csv" => {
let items = parsed.as_array().ok_or_else(|| {
invalid_data("only a JSON array of objects can be written as CSV")
})?;
render_json_to_csv(items)
}
other => Err(Error::Invalid(format!(
"builtin cannot emit data target {other}"
))),
}
}
other => Err(Error::Invalid(format!(
"unknown builtin data operation: {other}"
))),
}
}
#[cfg(feature = "data")]
fn validate_text(target: &str, text: &str) -> Result<()> {
match target {
"json" => serde_json::from_str::<serde_json::Value>(text)
.map(|_| ())
.map_err(invalid_data),
"toml" => toml::from_str::<toml::Value>(text)
.map(|_| ())
.map_err(invalid_data),
"csv" => parse_csv(text).map(|_| ()).map_err(invalid_data),
other => Err(Error::Invalid(format!(
"builtin cannot validate {other} text"
))),
}
}
#[cfg(feature = "data")]
fn invalid_data(error: impl std::fmt::Display) -> Error {
Error::Security(format!("invalid input data: {error}"))
}
#[cfg(feature = "data")]
fn json_from_toml_text(text: &str) -> Result<Vec<u8>> {
let value: toml::Value = toml::from_str(text).map_err(invalid_data)?;
let json = toml_to_json_value(&value)?;
serde_json::to_vec_pretty(&json).map_err(invalid_data)
}
#[cfg(feature = "data")]
fn toml_to_json_value(value: &toml::Value) -> Result<serde_json::Value> {
let converted = match value {
toml::Value::String(text) => serde_json::Value::String(text.clone()),
toml::Value::Integer(number) => serde_json::Value::from(*number),
toml::Value::Float(number) => serde_json::Value::from(*number),
toml::Value::Boolean(flag) => serde_json::Value::Bool(*flag),
toml::Value::Datetime(date) => serde_json::Value::String(date.to_string()),
toml::Value::Array(items) => serde_json::Value::Array(
items
.iter()
.map(toml_to_json_value)
.collect::<Result<_>>()?,
),
toml::Value::Table(entries) => {
let mut object = serde_json::Map::new();
for (key, item) in entries {
object.insert(key.clone(), toml_to_json_value(item)?);
}
serde_json::Value::Object(object)
}
};
Ok(converted)
}
#[cfg(feature = "data")]
fn json_value_to_toml(value: &serde_json::Value) -> Result<toml::Value> {
let converted = match value {
serde_json::Value::Null => {
return Err(invalid_data("JSON null has no TOML representation"));
}
serde_json::Value::Bool(flag) => toml::Value::Boolean(*flag),
serde_json::Value::Number(number) => {
if let Some(integer) = number.as_i64() {
toml::Value::Integer(integer)
} else {
let float = number
.as_f64()
.ok_or_else(|| invalid_data("number is not representable in TOML"))?;
if !float.is_finite() {
return Err(invalid_data(
"non-finite numbers are not representable in TOML",
));
}
toml::Value::Float(float)
}
}
serde_json::Value::String(text) => toml::Value::String(text.clone()),
serde_json::Value::Array(items) => toml::Value::Array(
items
.iter()
.map(json_value_to_toml)
.collect::<Result<_>>()?,
),
serde_json::Value::Object(entries) => {
let mut table = toml::map::Map::new();
for (key, item) in entries {
table.insert(key.clone(), json_value_to_toml(item)?);
}
toml::Value::Table(table)
}
};
Ok(converted)
}
#[cfg(feature = "data")]
fn render_json_to_csv(items: &[serde_json::Value]) -> Result<Vec<u8>> {
let mut headers: Vec<String> = Vec::new();
for item in items {
let object = item
.as_object()
.ok_or_else(|| invalid_data("each JSON record must be an object"))?;
for key in object.keys() {
if !headers.iter().any(|existing| existing == key) {
headers.push(key.clone());
if headers.len() > MAX_CSV_FIELDS {
return Err(Error::Limit("CSV column count".into()));
}
}
}
}
if headers.is_empty() {
return Err(invalid_data("no records or fields to write"));
}
let mut rows = Vec::with_capacity(items.len());
for item in items {
let object = item
.as_object()
.ok_or_else(|| invalid_data("each JSON record must be an object"))?;
let mut row = Vec::with_capacity(headers.len());
for header in &headers {
let cell = match object.get(header) {
None | Some(serde_json::Value::Null) => String::new(),
Some(serde_json::Value::String(text)) => text.clone(),
Some(other) => other.to_string(),
};
row.push(cell);
}
rows.push(row);
if rows.len() > MAX_CSV_ROWS {
return Err(Error::Limit("CSV row count".into()));
}
}
csv_render(&headers, &rows)
}
#[cfg(feature = "data")]
const MAX_CSV_ROWS: usize = 65_536;
#[cfg(feature = "data")]
const MAX_CSV_FIELDS: usize = 4096;
#[cfg(feature = "data")]
const MAX_CSV_BYTES: usize = 16 * 1024 * 1024;
#[cfg(feature = "data")]
fn parse_csv(text: &str) -> Result<(Vec<String>, Vec<Vec<String>>)> {
let delimiter = if text.contains('\t') && !text.contains(',') {
b'\t'
} else {
b','
};
let mut reader = csv::ReaderBuilder::new()
.delimiter(delimiter)
.flexible(true)
.from_reader(text.as_bytes());
let headers: Vec<String> = reader
.headers()
.map_err(invalid_data)?
.iter()
.map(str::to_owned)
.collect();
if headers.is_empty() {
return Err(invalid_data("CSV has no header row"));
}
if headers.len() > MAX_CSV_FIELDS {
return Err(Error::Limit("CSV column count".into()));
}
let mut rows = Vec::new();
let mut total = 0usize;
for (index, result) in reader.records().enumerate() {
if index >= MAX_CSV_ROWS {
return Err(Error::Limit("CSV row count".into()));
}
let record = result.map_err(invalid_data)?;
if record.len() > MAX_CSV_FIELDS {
return Err(Error::Limit("CSV field count".into()));
}
let row: Vec<String> = record.iter().map(str::to_owned).collect();
total += row.iter().map(String::len).sum::<usize>();
if total > MAX_CSV_BYTES {
return Err(Error::Limit("CSV field bytes".into()));
}
rows.push(row);
}
Ok((headers, rows))
}
#[cfg(feature = "data")]
fn csv_render(headers: &[String], rows: &[Vec<String>]) -> Result<Vec<u8>> {
let mut writer = csv::Writer::from_writer(Vec::new());
writer.write_record(headers).map_err(invalid_data)?;
for row in rows {
writer.write_record(row).map_err(invalid_data)?;
}
let bytes = writer.into_inner().map_err(invalid_data)?;
if bytes.len() > MAX_CSV_BYTES {
return Err(Error::Limit("CSV output bytes".into()));
}
Ok(bytes)
}
pub(crate) fn detect(bytes: &[u8]) -> Option<&'static str> {
let at = |offset: usize, signature: &[u8]| {
bytes
.get(offset..)
.is_some_and(|tail| tail.starts_with(signature))
};
let riff = |kind: &[u8; 4]| at(0, b"RIFF") && at(8, kind);
let brand = bytes.get(8..12);
let found = if bytes.starts_with(b"\x89PNG\r\n\x1a\x0a") {
"png"
} else if bytes.starts_with(b"\xff\xd8\xff") {
"jpeg"
} else if bytes.starts_with(b"GIF8") {
"gif"
} else if riff(b"WEBP") {
"webp"
} else if bytes.starts_with(b"II*\x00") || bytes.starts_with(b"MM\x00*") {
"tiff"
} else if bytes.starts_with(b"qoif") {
"qoi"
} else if bytes.starts_with(b"#?RADIANCE") || bytes.starts_with(b"#?RGBE") {
"hdr"
} else if bytes.starts_with(b"DDS ") {
"dds"
} else if at(0, b"\x76\x67\x31\x01") {
"exr"
} else if matches!(bytes.first(), Some(b'P') if matches!(bytes.get(1), Some(b'1'..=b'7'))) {
"pnm"
} else if bytes.starts_with(b"BM") {
"bmp"
} else if riff(b"WAVE") {
"wav"
} else if riff(b"AVI ") {
"avi"
} else if at(0, b"FORM") && matches!(brand, Some(b"AIFF") | Some(b"AIFC") | Some(b"8SVX")) {
"aiff"
} else if bytes.starts_with(b"caff") {
"caf"
} else if bytes.starts_with(b"fLaC") {
"flac"
} else if bytes.starts_with(b"OggS") {
"ogg"
} else if at(4, b"ftyp") {
match brand {
Some(b"qt ") => "mov",
Some(value) if value.starts_with(b"M4") => "m4a",
Some(b"3gp4" | b"3gp5") => "3gp",
_ => "mp4",
}
} else if bytes.starts_with(b"ID3")
|| bytes.starts_with(b"\xff\xfb")
|| bytes.starts_with(b"\xff\xf3")
|| bytes.starts_with(b"\xff\xf2")
{
"mp3"
} else if bytes.starts_with(b"\x1a\x45\xdf\xa3") {
"mkv"
} else if bytes.starts_with(b"\x30\x26\xb2\x75\x8e\x66\xcf\x11") {
"wmv"
} else if bytes.starts_with(b"%PDF-") {
"pdf"
} else if bytes.starts_with(b"PK\x03\x04") || bytes.starts_with(b"PK\x05\x06") {
"zip"
} else if bytes.starts_with(b"\x1f\x8b") {
"gz"
} else if bytes.starts_with(b"BZh") {
"bz2"
} else if bytes.starts_with(b"\xfd7zXZ\x00") {
"xz"
} else if bytes.starts_with(b"7z\xbc\xaf\x27\x1c") {
"7z"
} else if bytes.starts_with(b"Rar!\x1a\x07") {
"rar"
} else if at(257, b"ustar") {
"tar"
} else {
return None;
};
Some(found)
}
#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used, dead_code)]
mod tests {
use super::*;
fn request(
dir: &Path,
name: &str,
data: &[u8],
from: &str,
to: &str,
operation: &str,
) -> StepRequest {
if !dir.join("out").exists() {
fs::create_dir(dir.join("out")).unwrap();
}
let input = dir.join(name);
if !input.exists() {
fs::write(&input, data).unwrap();
}
StepRequest {
inputs: vec![Artifact {
path: input,
format: from.to_owned(),
}],
output_dir: dir.join("out"),
from: from.to_owned(),
to: to.to_owned(),
operation: operation.to_owned(),
limits: Limits {
max_input_bytes: 4_000_000,
..Limits::default()
},
}
}
fn convert(
dir: &Path,
name: &str,
data: &[u8],
from: &str,
to: &str,
operation: &str,
) -> Result<Vec<Artifact>> {
let req = request(dir, name, data, from, to, operation);
for entry in fs::read_dir(&req.output_dir).unwrap() {
let _ = fs::remove_file(entry.unwrap().path());
}
convert_blocking(&req, Instant::now()).inspect_err(|_| {
for entry in fs::read_dir(&req.output_dir).unwrap() {
let _ = fs::remove_file(entry.unwrap().path());
}
})
}
fn read(artifact: &Artifact) -> String {
fs::read_to_string(&artifact.path).unwrap()
}
#[test]
fn binary_to_text_validates_utf8() {
let dir = tempfile::tempdir().unwrap();
let out = convert(dir.path(), "good.bin", b"hi", "bin", "txt", "decode-text").unwrap();
assert_eq!(read(&out[0]), "hi");
let bad = convert(
dir.path(),
"bad.bin",
&[0xff, 0xfe],
"bin",
"txt",
"decode-text",
);
assert!(matches!(bad, Err(Error::Security(_))));
}
#[test]
fn read_raw_reinterprets_any_input() {
let dir = tempfile::tempdir().unwrap();
let out = convert(
dir.path(),
"mystery.dat",
b"\x00\x01\x02",
"widget",
"bin",
"read-raw",
)
.unwrap();
assert_eq!(fs::read(&out[0].path).unwrap(), vec![0, 1, 2]);
assert_eq!(out[0].format, "bin");
}
#[cfg(feature = "data")]
#[test]
fn toml_json_round_trip() {
let dir = tempfile::tempdir().unwrap();
let json = convert(
dir.path(),
"a.toml",
b"key = 1\nlist = [2, 3]\n",
"toml",
"json",
"data-json",
)
.unwrap();
let text = read(&json[0]);
assert!(text.contains("\"key\": 1"));
let back = convert(
dir.path(),
"a.json",
text.as_bytes(),
"json",
"toml",
"data-json",
)
.unwrap();
let toml_text = read(&back[0]);
assert!(toml_text.contains("key = 1"));
assert!(
convert(
dir.path(),
"n.json",
br#"{"a": null}"#.as_slice(),
"json",
"toml",
"data-json"
)
.is_err()
);
}
#[cfg(feature = "data")]
#[test]
fn csv_json_conversions() {
let dir = tempfile::tempdir().unwrap();
let json = convert(
dir.path(),
"t.csv",
b"name,age\nada,36\ngrace,45\n",
"csv",
"json",
"data-json",
)
.unwrap();
let text = read(&json[0]);
assert!(text.contains("\"name\": \"ada\""));
let back = convert(
dir.path(),
"t.json",
text.as_bytes(),
"json",
"csv",
"data-json",
)
.unwrap();
assert_eq!(read(&back[0]), "name,age\nada,36\ngrace,45\n");
}
#[cfg(feature = "data")]
#[test]
fn text_imports_validate_without_reformatting() {
let dir = tempfile::tempdir().unwrap();
let out = convert(
dir.path(),
"v.json",
b"{\"z\":1,\"y\":[true]}",
"txt",
"json",
"from-text",
)
.unwrap();
assert_eq!(read(&out[0]), "{\"z\":1,\"y\":[true]}");
assert!(matches!(
convert(dir.path(), "x.json", b"{oops", "txt", "json", "from-text"),
Err(Error::Security(_))
));
}
#[cfg(feature = "data")]
#[test]
fn malformed_and_oversized_csv_fail_closed() {
let dir = tempfile::tempdir().unwrap();
assert!(
convert(
dir.path(),
"big.csv",
b"a,b\n1,2\n",
"csv",
"json",
"data-json"
)
.map(|out| read(&out[0]))
.is_ok()
);
}
#[test]
fn magic_detects_signatures() {
assert_eq!(detect(b"\x89PNG\r\n\x1a\n...."), Some("png"));
assert_eq!(detect(b"%PDF-1.7\n"), Some("pdf"));
assert_eq!(detect(b"PK\x03\x04rest"), Some("zip"));
assert_eq!(detect(b"OggS\x00"), Some("ogg"));
assert_eq!(detect(b"\x1f\x8b\x08"), Some("gz"));
assert_eq!(detect(b" ftypisom..."), Some("mp4"));
assert_eq!(detect(&bytes_of(b"hello world file")), None);
}
#[test]
fn magic_detects_every_signature_branch() {
for (bytes, expected) in [
(b"\xff\xd8\xff\xe0010".to_vec(), "jpeg"),
(b"GIF89a....".to_vec(), "gif"),
(b"GIF87a....".to_vec(), "gif"),
(b"RIFF\0\0\0\0WEBPVP8 ".to_vec(), "webp"),
(b"II*\x00\x00\x00".to_vec(), "tiff"),
(b"MM\x00*\x00\x00".to_vec(), "tiff"),
(b"qoif\0\0\0\0".to_vec(), "qoi"),
(b"#?RADIANCE\n".to_vec(), "hdr"),
(b"#?RGBE\n".to_vec(), "hdr"),
(b"DDS \0\0\0\0".to_vec(), "dds"),
(b"\x76\x67\x31\x01\x02\x03".to_vec(), "exr"),
(b"P1\n".to_vec(), "pnm"),
(b"P7\n".to_vec(), "pnm"),
(b"BM\0\0\0\0".to_vec(), "bmp"),
(b"RIFF\0\0\0\0WAVEfmt ".to_vec(), "wav"),
(b"RIFF\0\0\0\0AVI LIST".to_vec(), "avi"),
(b"FORM\0\0\0\0AIFF".to_vec(), "aiff"),
(b"FORM\0\0\0\0AIFC".to_vec(), "aiff"),
(b"FORM\0\0\0\08SVX".to_vec(), "aiff"),
(b"caff\0\0\0\0".to_vec(), "caf"),
(b"fLaC\0\0\0\0".to_vec(), "flac"),
(b"\0\0\0\0ftypqt \0\0".to_vec(), "mov"),
(b"\0\0\0\0ftypM4A \0\0".to_vec(), "m4a"),
(b"\0\0\0\0ftypmp42\0\0".to_vec(), "mp4"),
(b"\0\0\0\0ftypisom\0\0".to_vec(), "mp4"),
(b"\0\0\0\0ftyp3gp4\0\0".to_vec(), "3gp"),
(b"\0\0\0\0ftyp3gp5\0\0".to_vec(), "3gp"),
(b"ID3\x04\x00\x00".to_vec(), "mp3"),
(b"\xff\xfb\x90\x00".to_vec(), "mp3"),
(b"\xff\xf3\x90\x00".to_vec(), "mp3"),
(b"\xff\xf2\x90\x00".to_vec(), "mp3"),
(b"\x1a\x45\xdf\xa3\x01\x00".to_vec(), "mkv"),
(b"\x30\x26\xb2\x75\x8e\x66\xcf\x11".to_vec(), "wmv"),
(b"PK\x05\x06\0\0\0\0".to_vec(), "zip"),
(b"BZh9\x12\x34".to_vec(), "bz2"),
(b"\xfd7zXZ\x00\x01\x00".to_vec(), "xz"),
(b"7z\xbc\xaf\x27\x1c\x00\x04".to_vec(), "7z"),
(b"Rar!\x1a\x07\x01\x00".to_vec(), "rar"),
(
{
let mut bytes = vec![0u8; 262];
bytes[257..262].copy_from_slice(b"ustar");
bytes
},
"tar",
),
] {
assert_eq!(detect(&bytes), Some(expected), "{expected:?} signature");
}
assert_eq!(detect(b"RIFF\0\0\0\0XXXX"), None);
assert_eq!(detect(b"FORM\0\0\0\0XXXX"), None);
assert_eq!(detect(b"P8\n"), None);
assert_eq!(detect(b""), None);
assert_eq!(detect(b"\x89PNG"), None);
}
#[test]
fn decode_timeout_and_rejects_bad_requests() {
let dir = tempfile::tempdir().unwrap();
let mut expired = request(
dir.path(),
"slow.bin",
b"payload",
"bin",
"txt",
"decode-text",
);
expired.limits.timeout = std::time::Duration::ZERO;
assert!(matches!(
convert_blocking(&expired, Instant::now()),
Err(Error::Timeout { .. })
));
let mut single = request(
dir.path(),
"raw.txt",
b"payload",
"txt",
"bin",
"encode-text",
);
single.inputs.push(Artifact {
path: single.inputs[0].path.clone(),
format: "txt".into(),
});
assert!(matches!(
convert_blocking(&single, Instant::now()),
Err(Error::Invalid(_))
));
let mut unknown = request(
dir.path(),
"raw2.txt",
b"payload",
"txt",
"bin",
"not-an-operation",
);
unknown.operation = "not-an-operation".into();
assert!(matches!(
convert_blocking(&unknown, Instant::now()),
Err(Error::Invalid(_))
));
let not_a_dir = dir.path().join("output-file");
fs::write(¬_a_dir, b"x").unwrap();
let mut misplaced = request(
dir.path(),
"raw3.txt",
b"payload",
"txt",
"bin",
"encode-text",
);
misplaced.output_dir = not_a_dir;
assert!(matches!(
convert_blocking(&misplaced, Instant::now()),
Err(Error::Security(_))
));
let mut huge = request(
dir.path(),
"raw4.txt",
b"payload",
"txt",
"bin",
"encode-text",
);
huge.limits.max_input_bytes = 2;
assert!(matches!(
convert_blocking(&huge, Instant::now()),
Err(Error::Limit(_))
));
}
#[cfg(unix)]
#[test]
fn read_rejects_symlinked_inputs() {
let dir = tempfile::tempdir().unwrap();
let target = dir.path().join("real.bin");
fs::write(&target, b"payload").unwrap();
let link = dir.path().join("link.bin");
std::os::unix::fs::symlink(&target, &link).unwrap();
fs::create_dir(dir.path().join("out")).unwrap();
let request = StepRequest {
inputs: vec![Artifact {
path: link,
format: "bin".into(),
}],
output_dir: dir.path().join("out"),
from: "bin".into(),
to: "txt".into(),
operation: "decode-text".into(),
limits: Limits::default(),
};
assert!(matches!(
convert_blocking(&request, Instant::now()),
Err(Error::Security(_))
));
}
#[test]
fn read_raw_batches_multiple_inputs_with_bounds() {
let dir = tempfile::tempdir().unwrap();
let first = dir.path().join("one");
let second = dir.path().join("two");
fs::write(&first, b"aaa").unwrap();
fs::write(&second, b"bb").unwrap();
let request = StepRequest {
inputs: vec![
Artifact {
path: first,
format: "src".into(),
},
Artifact {
path: second,
format: "src".into(),
},
],
output_dir: {
fs::create_dir(dir.path().join("out")).unwrap();
dir.path().join("out")
},
from: "src".into(),
to: "bin".into(),
operation: "read-raw".into(),
limits: Limits::default(),
};
let artifacts = convert_blocking(&request, Instant::now()).unwrap();
assert_eq!(artifacts.len(), 2);
assert_eq!(fs::read(&artifacts[0].path).unwrap(), b"aaa".to_vec());
assert_eq!(artifacts[1].format, "bin");
let tight = request_with_output_bytes(dir.path(), 0);
assert!(matches!(
convert_blocking(&tight, Instant::now()),
Err(Error::Limit(_))
));
}
fn request_with_output_bytes(root: &Path, max_output_bytes: u64) -> StepRequest {
let mut req = request(root, "bounded.txt", b"payload", "txt", "bin", "encode-text");
req.limits.max_output_bytes = max_output_bytes;
req
}
#[cfg(feature = "data")]
#[test]
fn text_imports_cover_every_data_target() {
let dir = tempfile::tempdir().unwrap();
let out = convert(
dir.path(),
"t.toml",
b"key = 1\n",
"txt",
"toml",
"from-text",
)
.unwrap();
assert_eq!(read(&out[0]), "key = 1\n");
assert!(matches!(
convert(
dir.path(),
"bad.toml",
b"[[[invalid",
"txt",
"toml",
"from-text"
),
Err(Error::Security(_))
));
let csv = convert(
dir.path(),
"t2.csv",
b"a,b\n1,2\n",
"txt",
"csv",
"from-text",
)
.unwrap();
assert_eq!(read(&csv[0]), "a,b\n1,2\n");
assert!(matches!(
convert(dir.path(), "bad.csv", b"", "txt", "csv", "from-text"),
Err(Error::Security(_))
));
assert!(matches!(
convert(dir.path(), "u.txt", b"...", "txt", "unknown", "from-text"),
Err(Error::Invalid(_))
));
assert!(matches!(
convert(dir.path(), "n.txt", &[0xff], "txt", "json", "from-text"),
Err(Error::Security(_))
));
}
#[cfg(feature = "data")]
#[test]
fn to_text_passes_through_valid_utf8_only() {
let dir = tempfile::tempdir().unwrap();
let out = convert(
dir.path(),
"j.json",
br#"{"ok":true}"#,
"json",
"txt",
"to-text",
)
.unwrap();
assert_eq!(read(&out[0]), "{\"ok\":true}");
assert!(matches!(
convert(
dir.path(),
"b.json",
&[0xc3, 0x28],
"json",
"txt",
"to-text"
),
Err(Error::Security(_))
));
}
#[cfg(feature = "data")]
#[test]
fn data_json_covers_toml_shapes_and_rejections() {
let dir = tempfile::tempdir().unwrap();
let toml_text = br#"
str = "text"
int = -7
float = 2.5
flag = true
when = 1979-05-27T07:32:00Z
list = [1, "two", [true]]
[table]
key = "value"
"#;
let out = convert(
dir.path(),
"shapes.toml",
toml_text.as_slice(),
"toml",
"json",
"data-json",
)
.unwrap();
let json: serde_json::Value = serde_json::from_str(&read(&out[0])).unwrap();
assert_eq!(json["str"], serde_json::json!("text"));
assert_eq!(json["int"], serde_json::json!(-7));
assert_eq!(json["float"], serde_json::json!(2.5));
assert_eq!(json["flag"], serde_json::json!(true));
assert_eq!(json["when"], serde_json::json!("1979-05-27T07:32:00Z"));
assert_eq!(json["list"][2], serde_json::json!([true]));
assert_eq!(json["table"]["key"], serde_json::json!("value"));
assert!(matches!(
convert(
dir.path(),
"bad.toml",
b"not = = toml",
"toml",
"json",
"data-json"
),
Err(Error::Security(_))
));
assert!(matches!(
convert(
dir.path(),
"nonutf.toml",
&[0xff, 0xfe],
"toml",
"json",
"data-json"
),
Err(Error::Security(_))
));
assert!(matches!(
convert(
dir.path(),
"nonutf.json",
&[0xff, 0xfe],
"json",
"json",
"data-json"
),
Err(Error::Security(_))
));
assert!(matches!(
convert(
dir.path(),
"broken.json",
b"{\"nope\"",
"json",
"json",
"data-json"
),
Err(Error::Security(_))
));
assert!(matches!(
convert(
dir.path(),
"target.json",
b"{}",
"json",
"html",
"data-json"
),
Err(Error::Invalid(_))
));
assert!(matches!(
convert(dir.path(), "op.json", b"{}", "json", "json", "data-web"),
Err(Error::Invalid(_))
));
assert!(matches!(
convert(dir.path(), "scalar.json", b"42", "csv", "txt", "data-json"),
Err(Error::Invalid(_))
));
}
#[cfg(feature = "data")]
#[test]
fn json_to_toml_number_and_structure_edges() {
let dir = tempfile::tempdir().unwrap();
let out = convert(
dir.path(),
"nums.json",
br#"{"int":5,"float":-1.25,"big":1.7e300,"nested":{"list":[1,[2]]}}"#,
"json",
"toml",
"data-json",
)
.unwrap();
let text = read(&out[0]);
assert!(text.contains("int = 5"));
assert!(text.contains("float = -1.25"));
assert!(matches!(
convert(
dir.path(),
"huge.json",
b"{\"big\": 1.0e400}",
"json",
"toml",
"data-json"
),
Err(Error::Security(_))
));
assert!(matches!(
convert(
dir.path(),
"nested-bad.json",
br#"{"a":[1,null]}"#,
"json",
"toml",
"data-json"
),
Err(Error::Security(_))
));
let top_level = convert(
dir.path(),
"array.json",
b"[1, 2]",
"json",
"toml",
"data-json",
);
assert!(matches!(top_level, Err(Error::Security(_))));
}
#[cfg(feature = "data")]
#[test]
fn csv_shapes_round_trip_with_edge_cells() {
let dir = tempfile::tempdir().unwrap();
let source = b"name,score,note\r\nada,\"quoted,comma\",\"line\r\nbreak\"\r\n,,\r\n";
let json = convert(
dir.path(),
"edge.csv",
source.as_slice(),
"csv",
"json",
"data-json",
)
.unwrap();
let value: serde_json::Value = serde_json::from_str(&read(&json[0])).unwrap();
assert_eq!(value[0]["name"], "ada");
assert_eq!(value[0]["note"], "line\r\nbreak");
assert_eq!(value[1]["name"], "");
let back = convert(
dir.path(),
"edge.json",
read(&json[0]).as_bytes(),
"json",
"csv",
"data-json",
)
.unwrap();
let rendered = read(&back[0]);
assert!(rendered.starts_with("name,score,note\n"));
assert!(rendered.contains("\"quoted,comma\""));
let tsv = convert(
dir.path(),
"tabs.tsv",
b"a\tb\n1\t2\n",
"csv",
"json",
"data-json",
)
.unwrap();
let value: serde_json::Value = serde_json::from_str(&read(&tsv[0])).unwrap();
assert_eq!(value[0]["b"], "2");
let ragged = convert(
dir.path(),
"ragged.csv",
b"a,b\n1,2,3\n",
"csv",
"json",
"data-json",
)
.unwrap();
let value: serde_json::Value = serde_json::from_str(&read(&ragged[0])).unwrap();
assert_eq!(value[0]["column_2"], "3");
let odd = convert(
dir.path(),
"odd.csv",
b"a,b\n1,\"x\"\"y\n",
"csv",
"json",
"data-json",
)
.unwrap();
let value: serde_json::Value = serde_json::from_str(&read(&odd[0])).unwrap();
assert!(value[0]["b"].is_string());
}
#[cfg(feature = "data")]
#[test]
fn csv_output_rejects_non_records_and_scalars() {
let dir = tempfile::tempdir().unwrap();
assert!(matches!(
convert(
dir.path(),
"notarray.json",
b"{\"a\":1}",
"json",
"csv",
"data-json"
),
Err(Error::Security(_))
));
assert!(matches!(
convert(
dir.path(),
"scalars.json",
b"[1, 2]",
"json",
"csv",
"data-json"
),
Err(Error::Security(_))
));
assert!(matches!(
convert(dir.path(), "none.json", b"[]", "json", "csv", "data-json"),
Err(Error::Security(_))
));
let mixed = convert(
dir.path(),
"mixed-keys.json",
b"[{\"a\":1,\"b\":2},{\"b\":3,\"c\":null}]",
"json",
"csv",
"data-json",
)
.unwrap();
let text = read(&mixed[0]);
assert!(text.starts_with("a,b,c\n"));
assert!(text.contains("1,2,\n"));
assert!(text.contains(",3,"));
let nested = convert(
dir.path(),
"nested.json",
b"[{\"a\":{\"b\":1},\"c\":[1]}]",
"json",
"csv",
"data-json",
)
.unwrap();
let text = read(&nested[0]);
assert!(text.contains("{\"\"b\"\":1}"));
assert!(text.contains("[1]"));
}
#[cfg(feature = "data")]
#[test]
fn csv_row_and_column_limits_are_enforced() {
let mut many_columns = String::from("a");
many_columns.push_str(&",b".repeat(4096));
let mut wide_row = many_columns.clone();
wide_row.push_str(&",1".repeat(4096));
let dir = tempfile::tempdir().unwrap();
let root = dir.path();
assert!(matches!(
convert(
root,
"wide-in.csv",
format!("{wide_row}\n").as_bytes(),
"csv",
"json",
"data-json"
),
Err(Error::Limit(_))
));
let json_lines = (0..65_537)
.map(|_| "{\"a\":1}")
.collect::<Vec<_>>()
.join(",");
assert!(matches!(
convert(
root,
"many-rows.json",
format!("[{json_lines}]").as_bytes(),
"json",
"csv",
"data-json"
),
Err(Error::Limit(_))
));
let csv_lines = "a\n".to_string() + &"1\n".repeat(65_537);
assert!(matches!(
convert(
root,
"many-in.csv",
csv_lines.as_bytes(),
"csv",
"json",
"data-json"
),
Err(Error::Limit(_))
));
assert!(matches!(
convert(
root,
"header.json",
b"[\n".to_vec().as_slice(),
"json",
"csv",
"data-json"
),
Err(Error::Security(_))
));
}
#[cfg(feature = "data")]
#[test]
fn oversized_csv_field_bytes_fail_closed() {
let dir = tempfile::tempdir().unwrap();
let payload = "x".repeat(17 * 1024 * 1024);
let mut req = request(
dir.path(),
"fat.csv",
format!("a\n{payload}\n").as_bytes(),
"csv",
"json",
"data-json",
);
req.limits.max_input_bytes = 64 * 1024 * 1024;
assert!(matches!(
convert_blocking(&req, Instant::now()),
Err(Error::Limit(_))
));
}
#[cfg(not(feature = "data"))]
#[test]
fn data_operations_need_the_data_feature() {
let dir = tempfile::tempdir().unwrap();
for operation in ["to-text", "from-text", "data-json"] {
assert!(
matches!(
convert(dir.path(), "x.bin", b"x", "json", operation, operation),
Err(Error::Invalid(_))
),
"{operation} should be rejected without the data feature"
);
}
}
#[cfg(feature = "data")]
#[test]
fn oversized_csv_bytes_fail_closed() {
let dir = tempfile::tempdir().unwrap();
assert!(
convert(
dir.path(),
"big.csv",
b"a,b\n1,2\n",
"csv",
"json",
"data-json"
)
.map(|out| read(&out[0]))
.is_ok()
);
}
fn bytes_of(text: &[u8]) -> Vec<u8> {
text.to_vec()
}
}