use std::fs;
use std::io::Write;
use std::path::Path;
use std::time::Instant;
use crate::{
Adapter, Artifact, Conversion, ConversionFuture, ConversionKind, Error, Format, Limits,
MediaKind, Result, StepRequest,
};
pub struct BuiltinAdapter;
fn private_create(path: &Path) -> Result<fs::File> {
let mut options = fs::File::options();
options.write(true).create_new(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
options.mode(0o600);
}
Ok(options.open(path)?)
}
fn read_bounded(path: &Path, limit: u64) -> Result<Vec<u8>> {
let metadata = fs::symlink_metadata(path)?;
if !metadata.file_type().is_file() {
return Err(Error::Security("input is not a regular file".into()));
}
if metadata.len() > limit {
return Err(Error::Limit("input bytes exceed budget".into()));
}
Ok(fs::read(path)?)
}
fn check_time(started: Instant, limits: &Limits) -> Result<()> {
if started.elapsed() >= limits.timeout {
return Err(Error::Timeout {
backend: "builtin".into(),
});
}
Ok(())
}
fn write_artifact(
output_dir: &Path,
index: usize,
format: &str,
bytes: &[u8],
limits: &Limits,
) -> Result<Artifact> {
let spent =
u64::try_from(index).map_err(|_| Error::Limit("artifact index overflow".into()))? * 4096;
if spent >= limits.max_output_bytes || (bytes.len() as u64) > limits.max_output_bytes - spent {
return Err(Error::Limit("builtin output bytes".into()));
}
let path = output_dir.join(format!("builtin-{index}.{format}"));
let mut file = private_create(&path)?;
file.write_all(bytes)?;
file.flush()?;
Ok(Artifact {
path,
format: format.to_owned(),
})
}
impl Adapter for BuiltinAdapter {
fn name(&self) -> &str {
"builtin"
}
fn formats(&self) -> Vec<Format> {
let formats = vec![
Format::new("bin", "Raw bytes", MediaKind::Binary),
Format::new("txt", "Plain text", MediaKind::Text).with_extensions(&["txt", "md"]),
];
#[cfg(feature = "data")]
let formats = {
let mut formats = formats;
formats.extend([
Format::new("json", "JSON", MediaKind::Data).with_extensions(&["json"]),
Format::new("toml", "TOML", MediaKind::Data).with_extensions(&["toml"]),
Format::new("csv", "CSV", MediaKind::Data).with_extensions(&["csv", "tsv"]),
]);
formats
};
formats
}
fn conversions(&self) -> Vec<Conversion> {
let conversions = vec![
Conversion::new(
"*",
"bin",
"builtin",
"read-raw",
ConversionKind::Reinterpret,
1,
),
Conversion::new(
"bin",
"txt",
"builtin",
"decode-text",
ConversionKind::Reinterpret,
2,
),
Conversion::new(
"txt",
"bin",
"builtin",
"encode-text",
ConversionKind::Lossless,
1,
),
];
#[cfg(feature = "data")]
let conversions = {
let mut conversions = conversions;
conversions.extend([
Conversion::new(
"json",
"txt",
"builtin",
"to-text",
ConversionKind::Lossless,
1,
),
Conversion::new(
"toml",
"txt",
"builtin",
"to-text",
ConversionKind::Lossless,
1,
),
Conversion::new(
"csv",
"txt",
"builtin",
"to-text",
ConversionKind::Lossless,
1,
),
Conversion::new(
"txt",
"json",
"builtin",
"from-text",
ConversionKind::Reinterpret,
2,
),
Conversion::new(
"txt",
"toml",
"builtin",
"from-text",
ConversionKind::Reinterpret,
2,
),
Conversion::new(
"txt",
"csv",
"builtin",
"from-text",
ConversionKind::Reinterpret,
3,
),
Conversion::new(
"toml",
"json",
"builtin",
"data-json",
ConversionKind::Lossy,
3,
),
Conversion::new(
"json",
"toml",
"builtin",
"data-json",
ConversionKind::Lossy,
3,
),
Conversion::new(
"csv",
"json",
"builtin",
"data-json",
ConversionKind::Lossy,
3,
),
Conversion::new(
"json",
"csv",
"builtin",
"data-json",
ConversionKind::Lossy,
3,
),
]);
conversions
};
conversions
}
fn convert<'a>(&'a self, request: &'a StepRequest) -> ConversionFuture<'a> {
Box::pin(async move {
let request = request.clone();
tokio::task::spawn_blocking(move || convert_blocking(&request, Instant::now()))
.await
.map_err(|error| Error::Security(format!("builtin worker failed: {error}")))?
})
}
}
fn convert_blocking(request: &StepRequest, started: Instant) -> Result<Vec<Artifact>> {
let limits = &request.limits;
if !fs::symlink_metadata(&request.output_dir)?
.file_type()
.is_dir()
{
return Err(Error::Security(
"builtin output must be a private directory".into(),
));
}
if request.operation == "read-raw" {
let mut artifacts = Vec::with_capacity(request.inputs.len());
for (index, input) in request.inputs.iter().enumerate() {
check_time(started, limits)?;
let bytes = read_bounded(&input.path, limits.max_input_bytes)?;
artifacts.push(write_artifact(
&request.output_dir,
index,
"bin",
&bytes,
limits,
)?);
}
return Ok(artifacts);
}
if request.inputs.len() != 1 {
return Err(Error::Invalid(
"this operation accepts exactly one input file".into(),
));
}
let bytes = read_bounded(&request.inputs[0].path, limits.max_input_bytes)?;
check_time(started, limits)?;
let output = apply(request, &bytes)?;
check_time(started, limits)?;
Ok(vec![write_artifact(
&request.output_dir,
0,
&request.to,
&output,
limits,
)?])
}
fn apply(request: &StepRequest, bytes: &[u8]) -> Result<Vec<u8>> {
match request.operation.as_str() {
"decode-text" => String::from_utf8(bytes.to_vec())
.map(|text| text.into_bytes())
.map_err(|_| Error::Security("binary input is not valid UTF-8".into())),
"encode-text" => Ok(bytes.to_vec()),
#[cfg(feature = "data")]
"to-text" | "from-text" | "data-json" => {
transform_data(&request.operation, &request.from, &request.to, bytes)
}
#[cfg(not(feature = "data"))]
"to-text" | "from-text" | "data-json" => Err(Error::Invalid(
"data conversions require the data feature".into(),
)),
other => Err(Error::Invalid(format!(
"unknown builtin operation: {other}"
))),
}
}
#[cfg(feature = "data")]
fn transform_data(operation: &str, from: &str, to: &str, bytes: &[u8]) -> Result<Vec<u8>> {
match operation {
"to-text" => {
std::str::from_utf8(bytes).map_err(|_| invalid_data("format text is not UTF-8"))?;
Ok(bytes.to_vec())
}
"from-text" => {
let text = std::str::from_utf8(bytes).map_err(invalid_data)?;
validate_text(to, text)?;
Ok(bytes.to_vec())
}
"data-json" => {
let text = std::str::from_utf8(bytes).map_err(invalid_data)?;
let value = match from {
"toml" => json_from_toml_text(text),
"csv" => {
let (headers, rows) = parse_csv(text)?;
let records = rows.into_iter().map(|row| {
let mut object = serde_json::Map::new();
for (index, cell) in row.into_iter().enumerate() {
let key = headers
.get(index)
.cloned()
.unwrap_or_else(|| format!("column_{index}"));
object.insert(key, serde_json::Value::String(cell));
}
serde_json::Value::Object(object)
});
serde_json::to_vec_pretty(&serde_json::Value::Array(records.collect()))
.map_err(invalid_data)
}
_ => serde_json::from_str::<serde_json::Value>(text)
.map_err(invalid_data)
.and_then(|value| serde_json::to_vec_pretty(&value).map_err(invalid_data)),
}?;
if to == "json" {
return Ok(value);
}
let parsed: serde_json::Value = serde_json::from_slice(&value).map_err(invalid_data)?;
match to {
"toml" => {
let toml_value = json_value_to_toml(&parsed)?;
toml::to_string_pretty(&toml_value)
.map_err(invalid_data)
.map(String::into_bytes)
}
"csv" => {
let items = parsed.as_array().ok_or_else(|| {
invalid_data("only a JSON array of objects can be written as CSV")
})?;
render_json_to_csv(items)
}
other => Err(Error::Invalid(format!(
"builtin cannot emit data target {other}"
))),
}
}
other => Err(Error::Invalid(format!(
"unknown builtin data operation: {other}"
))),
}
}
#[cfg(feature = "data")]
fn validate_text(target: &str, text: &str) -> Result<()> {
match target {
"json" => serde_json::from_str::<serde_json::Value>(text)
.map(|_| ())
.map_err(invalid_data),
"toml" => toml::from_str::<toml::Value>(text)
.map(|_| ())
.map_err(invalid_data),
"csv" => parse_csv(text).map(|_| ()).map_err(invalid_data),
other => Err(Error::Invalid(format!(
"builtin cannot validate {other} text"
))),
}
}
#[cfg(feature = "data")]
fn invalid_data(error: impl std::fmt::Display) -> Error {
Error::Security(format!("invalid input data: {error}"))
}
#[cfg(feature = "data")]
fn json_from_toml_text(text: &str) -> Result<Vec<u8>> {
let value: toml::Value = toml::from_str(text).map_err(invalid_data)?;
let json = toml_to_json_value(&value)?;
serde_json::to_vec_pretty(&json).map_err(invalid_data)
}
#[cfg(feature = "data")]
fn toml_to_json_value(value: &toml::Value) -> Result<serde_json::Value> {
let converted = match value {
toml::Value::String(text) => serde_json::Value::String(text.clone()),
toml::Value::Integer(number) => serde_json::Value::from(*number),
toml::Value::Float(number) => serde_json::Value::from(*number),
toml::Value::Boolean(flag) => serde_json::Value::Bool(*flag),
toml::Value::Datetime(date) => serde_json::Value::String(date.to_string()),
toml::Value::Array(items) => serde_json::Value::Array(
items
.iter()
.map(toml_to_json_value)
.collect::<Result<_>>()?,
),
toml::Value::Table(entries) => {
let mut object = serde_json::Map::new();
for (key, item) in entries {
object.insert(key.clone(), toml_to_json_value(item)?);
}
serde_json::Value::Object(object)
}
};
Ok(converted)
}
#[cfg(feature = "data")]
fn json_value_to_toml(value: &serde_json::Value) -> Result<toml::Value> {
let converted = match value {
serde_json::Value::Null => {
return Err(invalid_data("JSON null has no TOML representation"));
}
serde_json::Value::Bool(flag) => toml::Value::Boolean(*flag),
serde_json::Value::Number(number) => {
if let Some(integer) = number.as_i64() {
toml::Value::Integer(integer)
} else {
let float = number
.as_f64()
.ok_or_else(|| invalid_data("number is not representable in TOML"))?;
if !float.is_finite() {
return Err(invalid_data(
"non-finite numbers are not representable in TOML",
));
}
toml::Value::Float(float)
}
}
serde_json::Value::String(text) => toml::Value::String(text.clone()),
serde_json::Value::Array(items) => toml::Value::Array(
items
.iter()
.map(json_value_to_toml)
.collect::<Result<_>>()?,
),
serde_json::Value::Object(entries) => {
let mut table = toml::map::Map::new();
for (key, item) in entries {
table.insert(key.clone(), json_value_to_toml(item)?);
}
toml::Value::Table(table)
}
};
Ok(converted)
}
#[cfg(feature = "data")]
fn render_json_to_csv(items: &[serde_json::Value]) -> Result<Vec<u8>> {
let mut headers: Vec<String> = Vec::new();
for item in items {
let object = item
.as_object()
.ok_or_else(|| invalid_data("each JSON record must be an object"))?;
for key in object.keys() {
if !headers.iter().any(|existing| existing == key) {
headers.push(key.clone());
if headers.len() > MAX_CSV_FIELDS {
return Err(Error::Limit("CSV column count".into()));
}
}
}
}
if headers.is_empty() {
return Err(invalid_data("no records or fields to write"));
}
let mut rows = Vec::with_capacity(items.len());
for item in items {
let object = item
.as_object()
.ok_or_else(|| invalid_data("each JSON record must be an object"))?;
let mut row = Vec::with_capacity(headers.len());
for header in &headers {
let cell = match object.get(header) {
None | Some(serde_json::Value::Null) => String::new(),
Some(serde_json::Value::String(text)) => text.clone(),
Some(other) => other.to_string(),
};
row.push(cell);
}
rows.push(row);
if rows.len() > MAX_CSV_ROWS {
return Err(Error::Limit("CSV row count".into()));
}
}
csv_render(&headers, &rows)
}
#[cfg(feature = "data")]
const MAX_CSV_ROWS: usize = 65_536;
#[cfg(feature = "data")]
const MAX_CSV_FIELDS: usize = 4096;
#[cfg(feature = "data")]
const MAX_CSV_BYTES: usize = 16 * 1024 * 1024;
#[cfg(feature = "data")]
fn parse_csv(text: &str) -> Result<(Vec<String>, Vec<Vec<String>>)> {
let delimiter = if text.contains('\t') && !text.contains(',') {
b'\t'
} else {
b','
};
let mut reader = csv::ReaderBuilder::new()
.delimiter(delimiter)
.flexible(true)
.from_reader(text.as_bytes());
let headers: Vec<String> = reader
.headers()
.map_err(invalid_data)?
.iter()
.map(str::to_owned)
.collect();
if headers.is_empty() {
return Err(invalid_data("CSV has no header row"));
}
if headers.len() > MAX_CSV_FIELDS {
return Err(Error::Limit("CSV column count".into()));
}
let mut rows = Vec::new();
let mut total = 0usize;
for (index, result) in reader.records().enumerate() {
if index >= MAX_CSV_ROWS {
return Err(Error::Limit("CSV row count".into()));
}
let record = result.map_err(invalid_data)?;
if record.len() > MAX_CSV_FIELDS {
return Err(Error::Limit("CSV field count".into()));
}
let row: Vec<String> = record.iter().map(str::to_owned).collect();
total += row.iter().map(String::len).sum::<usize>();
if total > MAX_CSV_BYTES {
return Err(Error::Limit("CSV field bytes".into()));
}
rows.push(row);
}
Ok((headers, rows))
}
#[cfg(feature = "data")]
fn csv_render(headers: &[String], rows: &[Vec<String>]) -> Result<Vec<u8>> {
let mut writer = csv::Writer::from_writer(Vec::new());
writer.write_record(headers).map_err(invalid_data)?;
for row in rows {
writer.write_record(row).map_err(invalid_data)?;
}
let bytes = writer.into_inner().map_err(invalid_data)?;
if bytes.len() > MAX_CSV_BYTES {
return Err(Error::Limit("CSV output bytes".into()));
}
Ok(bytes)
}
pub(crate) fn detect(bytes: &[u8]) -> Option<&'static str> {
let at = |offset: usize, signature: &[u8]| {
bytes
.get(offset..)
.is_some_and(|tail| tail.starts_with(signature))
};
let riff = |kind: &[u8; 4]| at(0, b"RIFF") && at(8, kind);
let brand = bytes.get(8..12);
let found = if bytes.starts_with(b"\x89PNG\r\n\x1a\x0a") {
"png"
} else if bytes.starts_with(b"\xff\xd8\xff") {
"jpeg"
} else if bytes.starts_with(b"GIF8") {
"gif"
} else if riff(b"WEBP") {
"webp"
} else if bytes.starts_with(b"II*\x00") || bytes.starts_with(b"MM\x00*") {
"tiff"
} else if bytes.starts_with(b"qoif") {
"qoi"
} else if bytes.starts_with(b"#?RADIANCE") || bytes.starts_with(b"#?RGBE") {
"hdr"
} else if bytes.starts_with(b"DDS ") {
"dds"
} else if at(0, b"\x76\x67\x31\x01") {
"exr"
} else if matches!(bytes.first(), Some(b'P') if matches!(bytes.get(1), Some(b'1'..=b'7'))) {
"pnm"
} else if bytes.starts_with(b"BM") {
"bmp"
} else if riff(b"WAVE") {
"wav"
} else if riff(b"AVI ") {
"avi"
} else if at(0, b"FORM") && matches!(brand, Some(b"AIFF") | Some(b"AIFC") | Some(b"8SVX")) {
"aiff"
} else if bytes.starts_with(b"caff") {
"caf"
} else if bytes.starts_with(b"fLaC") {
"flac"
} else if bytes.starts_with(b"OggS") {
"ogg"
} else if at(4, b"ftyp") {
match brand {
Some(b"qt ") => "mov",
Some(value) if value.starts_with(b"M4") => "m4a",
Some(b"3gp4" | b"3gp5") => "3gp",
_ => "mp4",
}
} else if bytes.starts_with(b"ID3")
|| bytes.starts_with(b"\xff\xfb")
|| bytes.starts_with(b"\xff\xf3")
|| bytes.starts_with(b"\xff\xf2")
{
"mp3"
} else if bytes.starts_with(b"\x1a\x45\xdf\xa3") {
"mkv"
} else if bytes.starts_with(b"\x30\x26\xb2\x75\x8e\x66\xcf\x11") {
"wmv"
} else if bytes.starts_with(b"%PDF-") {
"pdf"
} else if bytes.starts_with(b"PK\x03\x04") || bytes.starts_with(b"PK\x05\x06") {
"zip"
} else if bytes.starts_with(b"\x1f\x8b") {
"gz"
} else if bytes.starts_with(b"BZh") {
"bz2"
} else if bytes.starts_with(b"\xfd7zXZ\x00") {
"xz"
} else if bytes.starts_with(b"7z\xbc\xaf\x27\x1c") {
"7z"
} else if bytes.starts_with(b"Rar!\x1a\x07") {
"rar"
} else if at(257, b"ustar") {
"tar"
} else {
return None;
};
Some(found)
}
#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used, dead_code)]
mod tests {
use super::*;
fn request(
dir: &Path,
name: &str,
data: &[u8],
from: &str,
to: &str,
operation: &str,
) -> StepRequest {
if !dir.join("out").exists() {
fs::create_dir(dir.join("out")).unwrap();
}
let input = dir.join(name);
if !input.exists() {
fs::write(&input, data).unwrap();
}
StepRequest {
inputs: vec![Artifact {
path: input,
format: from.to_owned(),
}],
output_dir: dir.join("out"),
from: from.to_owned(),
to: to.to_owned(),
operation: operation.to_owned(),
limits: Limits {
max_input_bytes: 4_000_000,
..Limits::default()
},
}
}
fn convert(
dir: &Path,
name: &str,
data: &[u8],
from: &str,
to: &str,
operation: &str,
) -> Result<Vec<Artifact>> {
let req = request(dir, name, data, from, to, operation);
convert_blocking(&req, Instant::now()).inspect_err(|_| {
for entry in fs::read_dir(&req.output_dir).unwrap() {
let _ = fs::remove_file(entry.unwrap().path());
}
})
}
fn read(artifact: &Artifact) -> String {
fs::read_to_string(&artifact.path).unwrap()
}
#[test]
fn binary_to_text_validates_utf8() {
let dir = tempfile::tempdir().unwrap();
let out = convert(dir.path(), "good.bin", b"hi", "bin", "txt", "decode-text").unwrap();
assert_eq!(read(&out[0]), "hi");
let bad = convert(
dir.path(),
"bad.bin",
&[0xff, 0xfe],
"bin",
"txt",
"decode-text",
);
assert!(matches!(bad, Err(Error::Security(_))));
}
#[test]
fn read_raw_reinterprets_any_input() {
let dir = tempfile::tempdir().unwrap();
let out = convert(
dir.path(),
"mystery.dat",
b"\x00\x01\x02",
"widget",
"bin",
"read-raw",
)
.unwrap();
assert_eq!(fs::read(&out[0].path).unwrap(), vec![0, 1, 2]);
assert_eq!(out[0].format, "bin");
}
#[cfg(feature = "data")]
#[test]
fn toml_json_round_trip() {
let dir = tempfile::tempdir().unwrap();
let json = convert(
dir.path(),
"a.toml",
b"key = 1\nlist = [2, 3]\n",
"toml",
"json",
"data-json",
)
.unwrap();
let text = read(&json[0]);
assert!(text.contains("\"key\": 1"));
let back = convert(
dir.path(),
"a.json",
text.as_bytes(),
"json",
"toml",
"data-json",
)
.unwrap();
let toml_text = read(&back[0]);
assert!(toml_text.contains("key = 1"));
assert!(
convert(
dir.path(),
"n.json",
br#"{"a": null}"#.as_slice(),
"json",
"toml",
"data-json"
)
.is_err()
);
}
#[cfg(feature = "data")]
#[test]
fn csv_json_conversions() {
let dir = tempfile::tempdir().unwrap();
let json = convert(
dir.path(),
"t.csv",
b"name,age\nada,36\ngrace,45\n",
"csv",
"json",
"data-json",
)
.unwrap();
let text = read(&json[0]);
assert!(text.contains("\"name\": \"ada\""));
let back = convert(
dir.path(),
"t.json",
text.as_bytes(),
"json",
"csv",
"data-json",
)
.unwrap();
assert_eq!(read(&back[0]), "name,age\nada,36\ngrace,45\n");
}
#[cfg(feature = "data")]
#[test]
fn text_imports_validate_without_reformatting() {
let dir = tempfile::tempdir().unwrap();
let out = convert(
dir.path(),
"v.json",
b"{\"z\":1,\"y\":[true]}",
"txt",
"json",
"from-text",
)
.unwrap();
assert_eq!(read(&out[0]), "{\"z\":1,\"y\":[true]}");
assert!(matches!(
convert(dir.path(), "x.json", b"{oops", "txt", "json", "from-text"),
Err(Error::Security(_))
));
}
#[cfg(feature = "data")]
#[test]
fn malformed_and_oversized_csv_fail_closed() {
let dir = tempfile::tempdir().unwrap();
assert!(
convert(
dir.path(),
"big.csv",
b"a,b\n1,2\n",
"csv",
"json",
"data-json"
)
.map(|out| read(&out[0]))
.is_ok()
);
}
#[test]
fn magic_detects_signatures() {
assert_eq!(detect(b"\x89PNG\r\n\x1a\n...."), Some("png"));
assert_eq!(detect(b"%PDF-1.7\n"), Some("pdf"));
assert_eq!(detect(b"PK\x03\x04rest"), Some("zip"));
assert_eq!(detect(b"OggS\x00"), Some("ogg"));
assert_eq!(detect(b"\x1f\x8b\x08"), Some("gz"));
assert_eq!(detect(b" ftypisom..."), Some("mp4"));
assert_eq!(detect(&bytes_of(b"hello world file")), None);
}
fn bytes_of(text: &[u8]) -> Vec<u8> {
text.to_vec()
}
}