#![cfg(target_os = "linux")]
use std::collections::BTreeMap;
use std::io::{self, Read, Write};
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::time::{Duration, Instant};
use serde_json::Value;
use sha2::{Digest, Sha256};
use tempfile::TempDir;
#[path = "support/portable_package.rs"]
mod portable_package;
use portable_package::{
Component, DeclaredFile, MANIFEST_PATH, controls, graph_parquet, hex, staged_entry_length,
tar_header, write_large_package,
};
const GIB: u64 = 1024 * 1024 * 1024;
const MIB: u64 = 1024 * 1024;
const EVIDENCE_PATH: &str = "data/components/evidence/scan-0001/scan.bin";
const GRAPH_PATH: &str = "data/components/graph-data/graph-tree/topology/nodes.parquet";
const HUB_BUNDLE: &str = concat!(
env!("CARGO_MANIFEST_DIR"),
"/../../tests/fixtures/hub/generated/v1/objects/opendata-openalex.gfpb"
);
const EVIDENCE_LENGTH: u64 = 5 * GIB;
const EVIDENCE_DIGEST: &str = "7f06c62352aebd8125b2a1841e2b9e1ffcbed602f381c3dcb3200200e383d1d5";
const DEADLINE: Duration = Duration::from_secs(120);
const PEAK_RSS_LIMIT: u64 = 768 * MIB;
const GRAPH_ROWS: usize = 64;
const GRAPH_ROW_BYTES: usize = 1_150_000;
#[derive(Clone, Copy)]
struct Limits {
address_space: u64,
file_size: u64,
}
const OPAQUE: Limits = Limits {
address_space: GIB,
file_size: 64 * MIB,
};
const STAGED: Limits = Limits {
address_space: GIB,
file_size: 128 * MIB,
};
impl Limits {
fn script(self) -> String {
format!(
"ulimit -v {}; ulimit -f {}; exec \"$0\" portable verify --input - --json",
self.address_space / 1024,
self.file_size / 512
)
}
}
fn gf_bin() -> PathBuf {
std::fs::canonicalize(env!("CARGO_BIN_EXE_gf")).expect("resolve same-build gf binary")
}
enum Body {
Bytes(Vec<u8>),
Zeros(u64),
}
impl Body {
fn length(&self) -> u64 {
match self {
Body::Bytes(bytes) => bytes.len() as u64,
Body::Zeros(length) => *length,
}
}
}
struct Artifact {
kind: &'static str,
participant: &'static str,
path: &'static str,
media_type: &'static str,
body: Body,
sha256: String,
}
fn evidence(sha256: &str, body: Body) -> Artifact {
Artifact {
kind: "evidence",
participant: "scan-0001",
path: EVIDENCE_PATH,
media_type: "application/octet-stream",
body,
sha256: sha256.into(),
}
}
fn package(artifacts: Vec<Artifact>, manifest: Option<Vec<u8>>) -> (Vec<(String, Body)>, String) {
let components: Vec<_> = artifacts
.iter()
.map(|artifact| Component {
kind: artifact.kind,
participant: artifact.participant,
files: vec![DeclaredFile {
path: artifact.path.to_owned(),
media_type: artifact.media_type,
length: artifact.body.length(),
sha256: artifact.sha256.clone(),
}],
})
.collect();
let controls = controls(&components, manifest);
let mut entries: Vec<_> = controls
.entries
.into_iter()
.map(|(path, bytes)| (path, Body::Bytes(bytes)))
.collect();
entries.extend(
artifacts
.into_iter()
.map(|artifact| (artifact.path.to_owned(), artifact.body)),
);
entries.sort_by(|left, right| left.0.cmp(&right.0));
(entries, controls.package_digest)
}
fn large_graph_parquet() -> Vec<u8> {
graph_parquet(GRAPH_ROWS, GRAPH_ROW_BYTES, "")
}
fn graph_data(file: Vec<u8>) -> Artifact {
let sha256 = hex(&Sha256::digest(&file));
Artifact {
kind: "graph-data",
participant: "graph-tree",
path: GRAPH_PATH,
media_type: "application/vnd.apache.parquet",
body: Body::Bytes(file),
sha256,
}
}
struct Bundle {
entries: std::vec::IntoIter<(String, Body)>,
current: Vec<u8>,
at: usize,
zeros: u64,
padding: usize,
ended: bool,
limit: u64,
}
impl Bundle {
fn new(entries: Vec<(String, Body)>, limit: u64) -> Self {
Self {
entries: entries.into_iter(),
current: Vec::new(),
at: 0,
zeros: 0,
padding: 0,
ended: false,
limit,
}
}
fn length(entries: &[(String, Body)]) -> u64 {
entries
.iter()
.map(|(_, body)| 512 + body.length().div_ceil(512) * 512)
.sum::<u64>()
+ 1024
}
}
impl Read for Bundle {
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
let room = buf
.len()
.min(usize::try_from(self.limit).unwrap_or(usize::MAX));
let buf = &mut buf[..room];
if buf.is_empty() {
return Ok(0);
}
loop {
let count = if self.at < self.current.len() {
let count = buf.len().min(self.current.len() - self.at);
buf[..count].copy_from_slice(&self.current[self.at..self.at + count]);
self.at += count;
count
} else if self.zeros > 0 {
let count = buf
.len()
.min(usize::try_from(self.zeros).unwrap_or(usize::MAX));
buf[..count].fill(0);
self.zeros -= count as u64;
count
} else if self.padding > 0 {
let count = buf.len().min(self.padding);
buf[..count].fill(0);
self.padding -= count;
count
} else if let Some((path, body)) = self.entries.next() {
self.current = tar_header(&path, body.length());
self.at = 0;
self.padding = (body.length().div_ceil(512) * 512 - body.length()) as usize;
match body {
Body::Bytes(bytes) => self.current.extend(bytes),
Body::Zeros(length) => self.zeros = length,
}
continue;
} else if !self.ended {
self.ended = true;
self.current = vec![0u8; 1024];
self.at = 0;
continue;
} else {
return Ok(0);
};
self.limit -= count as u64;
return Ok(count);
}
}
}
struct Effective {
address_space: u64,
file_size: u64,
}
fn limit_field(text: &str, name: &str) -> Option<u64> {
let line = text.lines().find(|line| line.starts_with(name))?;
line.split_whitespace().nth(3)?.parse().ok()
}
fn effective_limits(pid: u32) -> Result<Effective, String> {
let give_up = Instant::now() + Duration::from_secs(10);
let gf = gf_bin();
loop {
let read = |name: &str| std::fs::read_to_string(format!("/proc/{pid}/{name}"));
if let (Ok(cmdline), Ok(limits)) = (read("cmdline"), read("limits"))
&& cmdline.split('\0').next() == gf.to_str()
{
return Ok(Effective {
address_space: limit_field(&limits, "Max address space")
.ok_or("/proc limits: no address-space limit")?,
file_size: limit_field(&limits, "Max file size")
.ok_or("/proc limits: no file-size limit")?,
});
}
if Instant::now() >= give_up {
return Err(format!("/proc/{pid}: gf never showed its command line"));
}
std::thread::sleep(Duration::from_millis(5));
}
}
fn children_max_rss() -> u64 {
let (status, usage) = unsafe {
let mut usage = std::mem::zeroed::<libc::rusage>();
(libc::getrusage(libc::RUSAGE_CHILDREN, &mut usage), usage)
};
assert_eq!(status, 0, "getrusage failed");
u64::try_from(usage.ru_maxrss).expect("non-negative") * 1024
}
struct Verified {
code: Option<i32>,
stdout: Vec<u8>,
stderr: String,
effective: Option<Effective>,
max_rss: u64,
fed: u64,
}
impl Verified {
fn json(&self) -> Value {
serde_json::from_slice(&self.stdout).unwrap_or_else(|error| {
panic!("stdout is one JSON document ({error}): {}", self.stderr)
})
}
fn error(&self) -> Value {
assert!(self.stdout.is_empty(), "a refusal prints no report");
serde_json::from_str::<Value>(&self.stderr).unwrap_or_else(|error| {
panic!("stderr is one JSON document ({error}): {}", self.stderr)
})["error"]
.clone()
}
fn assert_ran_within(&self, limits: Limits) {
let effective = self
.effective
.as_ref()
.expect("the run did not read the child's limits");
assert_eq!(effective.address_space, limits.address_space);
assert_eq!(effective.file_size, limits.file_size);
println!("ru_maxrss_bytes={}", self.max_rss);
assert!(self.max_rss > 0, "the child's peak memory was not recorded");
assert!(
self.max_rss < PEAK_RSS_LIMIT,
"peak resident memory was {} bytes",
self.max_rss
);
}
}
#[derive(Clone, Copy, PartialEq)]
enum Observe {
No,
Limits,
}
fn verify_stream(
mut stdin: impl Read + Send + 'static,
tmpdir: &Path,
limits: Limits,
observe: Observe,
) -> Verified {
let started = Instant::now();
let mut child = Command::new("sh")
.args(["-c", &limits.script()])
.arg(gf_bin())
.env("TMPDIR", tmpdir)
.current_dir(tmpdir)
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.expect("spawn gf under ulimit");
let effective = (observe == Observe::Limits).then(|| {
effective_limits(child.id()).unwrap_or_else(|error| {
child.kill().ok();
child.wait().ok();
panic!("{error}")
})
});
let mut pipe = child.stdin.take().unwrap();
let feeder = std::thread::spawn(move || {
let mut fed = 0u64;
let mut block = vec![0u8; MIB as usize];
loop {
let count = stdin.read(&mut block).expect("generate the stream");
if count == 0 || pipe.write_all(&block[..count]).is_err() {
return fed;
}
fed += count as u64;
}
});
let mut stdout = child.stdout.take().unwrap();
let mut stderr = child.stderr.take().unwrap();
let out = std::thread::spawn(move || {
let mut bytes = Vec::new();
stdout.read_to_end(&mut bytes).unwrap();
bytes
});
let err = std::thread::spawn(move || {
let mut text = String::new();
stderr.read_to_string(&mut text).unwrap();
text
});
let status = loop {
if let Some(status) = child.try_wait().expect("poll gf") {
break status;
}
if started.elapsed() >= DEADLINE {
child.kill().expect("kill the stalled verifier");
child.wait().expect("reap the stalled verifier");
let fed = feeder.join().unwrap();
let stderr = err.join().unwrap();
panic!(
"gf did not finish within {DEADLINE:?}; killed after {fed} bytes were fed: {stderr}"
);
}
std::thread::sleep(Duration::from_millis(25));
};
Verified {
code: status.code(),
stdout: out.join().unwrap(),
stderr: err.join().unwrap(),
effective,
max_rss: children_max_rss(),
fed: feeder.join().unwrap(),
}
}
fn assert_empty(dir: &Path) {
let left: Vec<_> = std::fs::read_dir(dir)
.unwrap()
.map(|entry| entry.unwrap().file_name())
.collect();
assert!(left.is_empty(), "TMPDIR holds {left:?}");
}
fn zeros_digest(length: u64) -> String {
let mut hash = Sha256::new();
let block = vec![0u8; MIB as usize];
let mut left = length;
while left > 0 {
let count = left.min(MIB);
hash.update(&block[..count as usize]);
left -= count;
}
hex(&hash.finalize())
}
fn bytes_by_kind(report: &Value) -> BTreeMap<String, u64> {
report["bytes_by_kind"]
.as_object()
.unwrap()
.iter()
.map(|(kind, bytes)| (kind.clone(), bytes.as_u64().unwrap()))
.collect()
}
#[test]
fn a_five_gib_package_streams_from_stdin_in_bounded_memory() {
let (entries, package_digest) = package(
vec![evidence(EVIDENCE_DIGEST, Body::Zeros(EVIDENCE_LENGTH))],
None,
);
let length = Bundle::length(&entries);
assert!(length > 5 * GIB);
let tmp = TempDir::new().unwrap();
let run = verify_stream(
Bundle::new(entries, u64::MAX),
tmp.path(),
OPAQUE,
Observe::Limits,
);
run.assert_ran_within(OPAQUE);
assert_eq!(run.code, Some(0), "{}", run.stderr);
assert_eq!(run.fed, length, "the whole stream was read");
let report = run.json();
assert_eq!(report["integrity"], "verified");
assert_eq!(report["source"], "stream");
assert_eq!(report["package_digest"], format!("sha256:{package_digest}"));
let kinds = bytes_by_kind(&report);
assert_eq!(kinds["evidence"], EVIDENCE_LENGTH);
assert_eq!(
kinds.values().sum::<u64>(),
report["payload_bytes"].as_u64().unwrap()
);
assert_empty(tmp.path());
}
#[test]
fn a_graph_entry_over_the_walk_threshold_verifies_from_stdin() {
let file = large_graph_parquet();
let entry = file.len() as u64;
assert!(
entry > 64 * MIB,
"{entry} bytes does not exceed the walk threshold"
);
let (entries, package_digest) = package(vec![graph_data(file)], None);
let length = Bundle::length(&entries);
let tmp = TempDir::new().unwrap();
let run = verify_stream(
Bundle::new(entries, u64::MAX),
tmp.path(),
STAGED,
Observe::Limits,
);
run.assert_ran_within(STAGED);
assert_eq!(run.code, Some(0), "{}", run.stderr);
assert_eq!(run.fed, length);
let report = run.json();
assert_eq!(report["integrity"], "verified");
assert_eq!(report["package_digest"], format!("sha256:{package_digest}"));
assert_eq!(bytes_by_kind(&report)["graph-data"], entry);
assert_eq!(report["plausibility"]["parquet"]["files_with_footer"], 1);
assert_empty(tmp.path());
}
#[test]
fn a_stream_truncated_at_sixty_percent_is_refused() {
let (entries, _) = package(
vec![evidence(EVIDENCE_DIGEST, Body::Zeros(EVIDENCE_LENGTH))],
None,
);
let cut = Bundle::length(&entries) / 10 * 6;
let tmp = TempDir::new().unwrap();
let run = verify_stream(Bundle::new(entries, cut), tmp.path(), OPAQUE, Observe::No);
assert_eq!(run.code, Some(2), "{}", run.stderr);
let error = run.error();
assert_eq!(error["code"], "GF_VALIDATION");
assert!(
error["diagnostics"][0]["message"]
.as_str()
.unwrap()
.contains("truncated payload"),
"{error}"
);
assert_empty(tmp.path());
}
#[test]
fn impostor_and_corrupted_bundles_are_refused() {
let digest = zeros_digest(MIB);
let impostor = package(
vec![evidence(&digest, Body::Zeros(MIB))],
Some(b"this is not a manifest".to_vec()),
)
.0;
let corrupted = package(
vec![evidence(
&hex(&Sha256::digest(b"other bytes")),
Body::Zeros(MIB),
)],
None,
)
.0;
for entries in [impostor, corrupted] {
let tmp = TempDir::new().unwrap();
let run = verify_stream(
Bundle::new(entries, u64::MAX),
tmp.path(),
OPAQUE,
Observe::No,
);
assert_eq!(run.code, Some(2), "{}", run.stderr);
assert_eq!(run.error()["code"], "GF_VALIDATION");
assert_empty(tmp.path());
}
}
#[test]
fn a_limit_refusal_reports_its_stable_code_in_json() {
let (mut entries, _) = package(vec![evidence(&zeros_digest(MIB), Body::Zeros(MIB))], None);
entries
.iter_mut()
.find(|(path, _)| path == MANIFEST_PATH)
.unwrap()
.1 = Body::Zeros(32 * MIB);
let tmp = TempDir::new().unwrap();
let run = verify_stream(
Bundle::new(entries, u64::MAX),
tmp.path(),
OPAQUE,
Observe::No,
);
assert_eq!(run.code, Some(1), "{}", run.stderr);
assert_eq!(run.error()["code"], "GF_LIMIT_EXCEEDED");
assert_empty(tmp.path());
}
struct Run {
code: Option<i32>,
stdout: Vec<u8>,
stderr: String,
}
fn gf(args: &[&str], stdin: &[u8]) -> Run {
let mut child = Command::new(gf_bin())
.args(args)
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.expect("spawn gf");
let mut pipe = child.stdin.take().unwrap();
let bytes = stdin.to_vec();
let feeder = std::thread::spawn(move || {
let _ = pipe.write_all(&bytes);
});
let output = child.wait_with_output().expect("run gf");
feeder.join().unwrap();
Run {
code: output.status.code(),
stdout: output.stdout,
stderr: String::from_utf8_lossy(&output.stderr).into_owned(),
}
}
fn one_document(text: &[u8]) -> Value {
let mut documents = serde_json::Deserializer::from_slice(text).into_iter::<Value>();
let document = documents.next().expect("a document").expect("valid JSON");
assert!(documents.next().is_none(), "more than one document");
document
}
#[test]
fn stdin_verifies_the_hub_bundle_with_one_json_document() {
let bytes = std::fs::read(HUB_BUNDLE).unwrap();
let stdin = gf(&["portable", "verify", "--input", "-", "--json"], &bytes);
assert_eq!(stdin.code, Some(0), "{}", stdin.stderr);
let mut stream = one_document(&stdin.stdout);
let file = gf(
&["portable", "verify", "--input", HUB_BUNDLE, "--json"],
&[],
);
assert_eq!(file.code, Some(0), "{}", file.stderr);
let mut file = one_document(&file.stdout);
for report in [&mut file, &mut stream] {
report.as_object_mut().unwrap().remove("source");
}
assert_eq!(file, stream);
}
#[test]
fn stdin_input_requires_full_mode() {
let bytes = std::fs::read(HUB_BUNDLE).unwrap();
let run = gf(
&[
"portable", "verify", "--input", "-", "--mode", "inspect", "--json",
],
&bytes,
);
assert_eq!(run.code, Some(2));
assert!(run.stdout.is_empty());
assert_eq!(
one_document(run.stderr.as_bytes())["error"]["code"],
"GF_VALIDATION"
);
}
#[test]
fn url_input_requires_full_mode() {
let run = gf(
&[
"portable",
"verify",
"--url",
"https://example.invalid/object",
"--mode",
"inspect",
"--json",
],
b"",
);
assert_eq!(run.code, Some(2));
assert!(run.stdout.is_empty());
let error = one_document(run.stderr.as_bytes());
assert_eq!(error["error"]["code"], "GF_VALIDATION");
assert!(
error["error"]["message"]
.as_str()
.unwrap()
.contains("--mode full"),
"{error}"
);
}
#[test]
fn an_argument_error_under_json_is_one_json_document() {
for args in [
&[
"portable", "verify", "--input", "-", "--mode", "invalid", "--json",
][..],
&["--json", "portable", "verify", "--no-such-flag"][..],
&["--json", "no-such-command"][..],
] {
let run = gf(args, &[]);
assert_eq!(run.code, Some(2), "{args:?}: {}", run.stderr);
assert!(run.stdout.is_empty(), "{args:?}");
let error = &one_document(run.stderr.as_bytes())["error"];
assert_eq!(error["code"], "GF_VALIDATION", "{args:?}");
assert_eq!(error["details"]["source"], "argument_parser", "{args:?}");
}
let run = gf(&["portable", "verify", "--mode", "invalid"], &[]);
assert_eq!(run.code, Some(2));
assert!(run.stdout.is_empty());
assert!(run.stderr.contains("invalid value"), "{}", run.stderr);
}
#[test]
fn help_and_version_print_normally() {
for args in [
&["--help"][..],
&["--version"][..],
&["portable", "verify", "--help"][..],
&["--json", "--help"][..],
&["--json", "--version"][..],
] {
let run = gf(args, &[]);
assert_eq!(run.code, Some(0), "{args:?}: {}", run.stderr);
assert!(run.stderr.is_empty(), "{args:?}: {}", run.stderr);
let text = String::from_utf8_lossy(&run.stdout);
assert!(!text.trim().is_empty(), "{args:?}");
assert!(
serde_json::from_str::<Value>(&text).is_err(),
"{args:?}: {text}"
);
}
}
#[test]
fn a_generated_package_is_never_short_of_the_requested_size() {
let staged = staged_entry_length();
for size in [
staged - 1,
staged,
staged + 1,
staged + 4 * MIB - 1,
staged + 4 * MIB,
staged + 4 * MIB + 1,
] {
let mut bundle = Vec::new();
let package = write_large_package(size, 7, &mut bundle, |_, _, _| {}).unwrap();
assert_eq!(package.written, bundle.len() as u64);
assert!(
package.written >= size,
"{size} requested, {} written",
package.written
);
let run = gf(&["portable", "verify", "--input", "-", "--json"], &bundle);
assert_eq!(run.code, Some(0), "{size}: {}", run.stderr);
let report = one_document(&run.stdout);
assert_eq!(report["integrity"], "verified", "{size}");
assert_eq!(
report["package_digest"],
format!("sha256:{}", package.package_digest)
);
}
}