use std::collections::{BTreeMap, BTreeSet};
use std::fs::File;
use std::io::{self, BufRead as _, BufReader, Read};
use std::path::Path;
use flate2::bufread::GzDecoder;
use ring::digest;
use super::pins::is_action_file;
#[cfg(test)]
mod tests;
const BLOCK: usize = 512;
const RECORDS: u64 = 64 << 10;
#[derive(Debug, Clone, Copy)]
pub(crate) struct Budget {
pub(crate) expanded: u64,
pub(crate) entries: u64,
pub(crate) action_file: u64,
}
pub(crate) const BUDGET: Budget = Budget {
expanded: 1 << 30,
entries: 200_000,
action_file: 1 << 20,
};
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct Verified {
pub(crate) sha256: String,
pub(crate) size: u64,
pub(crate) actions: BTreeMap<String, ActionFile>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct ActionFile {
pub(crate) yml: bool,
pub(crate) bytes: Option<Vec<u8>>,
}
#[derive(Debug, thiserror::Error)]
pub(crate) enum Rejected {
#[error("cannot be read: {0}")]
Unreadable(io::Error),
#[error("not a whole gzip stream of a whole tar: {0}")]
Tar(io::Error),
#[error("past its budget of {0}")]
Budget(&'static str),
#[error("its PAX comment is not the commit asked for")]
Comment,
#[error("a header is not one git archive writes: {0}")]
Header(&'static str),
#[error("entry {0:?} is outside a directory listed before it")]
Outside(String),
#[error("entry {0:?} is of a kind it does not hold")]
Kind(String),
#[error("entry {0:?} is in it twice")]
Duplicate(String),
#[error("its tar does not end as one does")]
End,
#[error("data follows its gzip stream")]
Trailing,
}
pub(crate) fn verify(path: &Path, sha: &str, budget: Budget) -> Result<Verified, Rejected> {
let file = File::open(path).map_err(Rejected::Unreadable)?;
let mut hashed = Hashed::new(BufReader::new(file));
let verified = read_archive(&mut hashed, sha, budget)?;
let (sha256, size) = hashed.finish();
Ok(Verified {
sha256,
size,
actions: verified,
})
}
pub(crate) fn hash_file(path: &Path) -> io::Result<(String, u64)> {
let mut hashed = Hashed::new(BufReader::new(File::open(path)?));
io::copy(&mut hashed, &mut io::sink())?;
Ok(hashed.finish())
}
#[cfg(test)]
pub(crate) fn listing(path: &Path, sha: &str) -> Result<Vec<String>, Rejected> {
let file = File::open(path).map_err(Rejected::Unreadable)?;
let mut hashed = Hashed::new(BufReader::new(file));
let checker = read(&mut hashed, sha, BUDGET)?;
let mut names: Vec<String> = checker
.seen
.iter()
.map(|name| {
if checker.dirs.contains(name) {
format!("{name}/")
} else {
name.clone()
}
})
.collect();
names.sort();
Ok(names)
}
fn read_archive<R: Read>(
hashed: &mut Hashed<R>,
sha: &str,
budget: Budget,
) -> Result<BTreeMap<String, ActionFile>, Rejected> {
read(hashed, sha, budget).map(|checker| checker.actions)
}
fn read<'sha, R: Read>(
hashed: &mut Hashed<R>,
sha: &'sha str,
budget: Budget,
) -> Result<Checker<'sha>, Rejected> {
let mut gzip = GzDecoder::new(BufReader::new(hashed));
let mut tar = Capped {
inner: &mut gzip,
left: budget.expanded,
};
let mut checker = Checker {
sha,
budget,
top: None,
seen: BTreeSet::new(),
dirs: BTreeSet::new(),
entries: 0,
commented: false,
path: None,
actions: BTreeMap::new(),
};
loop {
let block = read_block(&mut tar)?;
if block.iter().all(|&byte| byte == 0) {
break;
}
checker.check(&block, &mut tar)?;
}
if !checker.commented || checker.top.is_none() || checker.path.is_some() {
return Err(if checker.commented {
Rejected::End
} else {
Rejected::Comment
});
}
if read_block(&mut tar)?.iter().any(|&byte| byte != 0) {
return Err(Rejected::End);
}
let mut buffer = [0_u8; 8192];
loop {
let read = tar.read(&mut buffer).map_err(tar_error)?;
if read == 0 {
break;
}
if buffer[..read].iter().any(|&byte| byte != 0) {
return Err(Rejected::End);
}
}
let mut buffered = gzip.into_inner();
match buffered.fill_buf() {
Ok([]) => Ok(checker),
Ok(_) => Err(Rejected::Trailing),
Err(error) => Err(Rejected::Tar(error)),
}
}
fn read_block<R: Read>(reader: &mut R) -> Result<[u8; BLOCK], Rejected> {
let mut block = [0_u8; BLOCK];
reader.read_exact(&mut block).map_err(tar_error)?;
Ok(block)
}
fn read_data<R: Read>(reader: &mut R, size: u64, keep: bool) -> Result<Option<Vec<u8>>, Rejected> {
let padded = size.next_multiple_of(BLOCK as u64);
let data = if keep {
let mut data = Vec::new();
reader
.by_ref()
.take(size)
.read_to_end(&mut data)
.map_err(tar_error)?;
if data.len() as u64 != size {
return Err(Rejected::Tar(io::ErrorKind::UnexpectedEof.into()));
}
Some(data)
} else {
skip(reader, size)?;
None
};
skip(reader, padded - size)?;
Ok(data)
}
fn skip<R: Read>(reader: &mut R, count: u64) -> Result<(), Rejected> {
let skipped = io::copy(&mut reader.by_ref().take(count), &mut io::sink()).map_err(tar_error)?;
if skipped == count {
Ok(())
} else {
Err(Rejected::Tar(io::ErrorKind::UnexpectedEof.into()))
}
}
fn tar_error(error: io::Error) -> Rejected {
if error.kind() == io::ErrorKind::FileTooLarge {
Rejected::Budget("expanded bytes")
} else {
Rejected::Tar(error)
}
}
struct Header<'block> {
name: String,
kind: u8,
size: u64,
block: &'block [u8; BLOCK],
}
impl Header<'_> {
fn parse(block: &[u8; BLOCK]) -> Result<Header<'_>, Rejected> {
if &block[257..263] != b"ustar\0" || &block[263..265] != b"00" {
return Err(Rejected::Header("its magic is not POSIX ustar's"));
}
let stored =
octal(&block[148..156]).ok_or(Rejected::Header("its checksum is not octal"))?;
let sum: u64 = block
.iter()
.enumerate()
.map(|(at, &byte)| {
if (148..156).contains(&at) {
u64::from(b' ')
} else {
u64::from(byte)
}
})
.sum();
if stored != sum {
return Err(Rejected::Header("its checksum does not match"));
}
let size = octal(&block[124..136]).ok_or(Rejected::Header("its size is not octal"))?;
let mode = octal(&block[100..108]).ok_or(Rejected::Header("its mode is not octal"))?;
if mode & !0o777 != 0 {
return Err(Rejected::Header("its mode has more than permission bits"));
}
let name = field(&block[..100])?;
let prefix = field(&block[345..500])?;
let name = if prefix.is_empty() {
name.to_owned()
} else {
format!("{prefix}/{name}")
};
Ok(Header {
name,
kind: block[156],
size,
block,
})
}
fn unlinked(&self) -> bool {
self.block[157..257].iter().all(|&byte| byte == 0)
}
}
fn field(bytes: &[u8]) -> Result<&str, Rejected> {
let end = bytes
.iter()
.position(|&byte| byte == 0)
.unwrap_or(bytes.len());
std::str::from_utf8(&bytes[..end]).map_err(|_| Rejected::Header("a name is not UTF-8"))
}
fn octal(bytes: &[u8]) -> Option<u64> {
let end = bytes
.iter()
.position(|&byte| byte == 0 || byte == b' ')
.unwrap_or(bytes.len());
let digits = &bytes[..end];
if digits.is_empty() || bytes[end..].iter().any(|&byte| byte != 0 && byte != b' ') {
return None;
}
digits.iter().try_fold(0_u64, |value, &digit| {
(b'0'..=b'7')
.contains(&digit)
.then(|| value.checked_mul(8)?.checked_add(u64::from(digit - b'0')))
.flatten()
})
}
fn records(data: &[u8]) -> Result<Vec<(&str, &str)>, Rejected> {
let malformed = Rejected::Header("its PAX records are malformed");
let mut found = Vec::new();
let mut rest = data;
while !rest.is_empty() {
let space = rest
.iter()
.position(|&byte| byte == b' ')
.ok_or(Rejected::Header("its PAX records are malformed"))?;
let length: usize = std::str::from_utf8(&rest[..space])
.ok()
.filter(|digits| !digits.is_empty() && digits.bytes().all(|byte| byte.is_ascii_digit()))
.and_then(|digits| digits.parse().ok())
.ok_or(Rejected::Header("its PAX records are malformed"))?;
if length <= space + 1 || length > rest.len() || rest[length - 1] != b'\n' {
return Err(malformed);
}
let record = &rest[space + 1..length - 1];
if record.iter().any(|&byte| byte.is_ascii_control()) {
return Err(Rejected::Header("a PAX record holds a control character"));
}
let record = std::str::from_utf8(record)
.map_err(|_| Rejected::Header("its PAX records are malformed"))?;
let (key, value) = record
.split_once('=')
.ok_or(Rejected::Header("its PAX records are malformed"))?;
found.push((key, value));
rest = &rest[length..];
}
Ok(found)
}
struct Checker<'sha> {
sha: &'sha str,
budget: Budget,
top: Option<String>,
seen: BTreeSet<String>,
dirs: BTreeSet<String>,
entries: u64,
commented: bool,
path: Option<String>,
actions: BTreeMap<String, ActionFile>,
}
impl Checker<'_> {
fn check<R: Read>(&mut self, block: &[u8; BLOCK], tar: &mut R) -> Result<(), Rejected> {
let header = Header::parse(block)?;
if !header.unlinked() {
return Err(Rejected::Kind(header.name));
}
match header.kind {
b'g' => return self.global(&header, tar),
b'x' => return self.extended(&header, tar),
_ => {}
}
if !self.commented {
return Err(Rejected::Comment);
}
self.entries += 1;
if self.entries > self.budget.entries {
return Err(Rejected::Budget("entries"));
}
let name = self.path.take().unwrap_or(header.name);
match header.kind {
b'5' => {
let path = name.strip_suffix('/').unwrap_or(&name);
self.place(path, &name)?;
if header.size != 0 {
return Err(Rejected::Header("a directory has data"));
}
self.dirs.insert(path.to_owned());
Ok(())
}
b'0' => {
self.place(&name, &name)?;
let Some((_, inside)) = name.split_once('/') else {
return Err(Rejected::Outside(name));
};
if !is_action_file(inside) {
read_data(tar, header.size, false)?;
return Ok(());
}
let keep = header.size <= self.budget.action_file;
let bytes = read_data(tar, header.size, keep)?;
let file = ActionFile {
yml: inside.rsplit('/').next() == Some("action.yml"),
bytes,
};
let dir = super::pins::action_dir(inside).to_owned();
match self.actions.get(&dir) {
Some(kept) if kept.yml => {}
_ => {
self.actions.insert(dir, file);
}
}
Ok(())
}
_ => Err(Rejected::Kind(name)),
}
}
fn global<R: Read>(&mut self, header: &Header<'_>, tar: &mut R) -> Result<(), Rejected> {
if self.commented || self.entries != 0 || header.size > RECORDS {
return Err(Rejected::Comment);
}
let data = read_data(tar, header.size, true)?.unwrap_or_default();
match records(&data)?.as_slice() {
[("comment", comment)] if *comment == self.sha => {
self.commented = true;
Ok(())
}
_ => Err(Rejected::Comment),
}
}
fn extended<R: Read>(&mut self, header: &Header<'_>, tar: &mut R) -> Result<(), Rejected> {
if !self.commented {
return Err(Rejected::Comment);
}
if self.path.is_some() || header.size > RECORDS {
return Err(Rejected::Header("an extended header is not one path"));
}
let data = read_data(tar, header.size, true)?.unwrap_or_default();
match records(&data)?.as_slice() {
[("path", path)] if !path.is_empty() => {
self.path = Some((*path).to_owned());
Ok(())
}
_ => Err(Rejected::Header("an extended header is not one path")),
}
}
fn place(&mut self, path: &str, name: &str) -> Result<(), Rejected> {
let parts: Vec<&str> = path.split('/').collect();
if parts
.iter()
.any(|part| part.is_empty() || *part == "." || *part == "..")
{
return Err(Rejected::Outside(name.to_owned()));
}
match parts.split_last() {
Some((_, [])) if self.top.is_none() && name.ends_with('/') => {
self.top = Some(path.to_owned());
}
Some((_, parents)) if !parents.is_empty() => {
if !self.dirs.contains(&parents.join("/")) {
return Err(Rejected::Outside(name.to_owned()));
}
}
_ => return Err(Rejected::Outside(name.to_owned())),
}
if !self.seen.insert(path.to_owned()) {
return Err(Rejected::Duplicate(name.to_owned()));
}
Ok(())
}
}
struct Hashed<R> {
inner: R,
context: digest::Context,
size: u64,
}
impl<R: Read> Hashed<R> {
fn new(inner: R) -> Hashed<R> {
Hashed {
inner,
context: digest::Context::new(&digest::SHA256),
size: 0,
}
}
fn finish(self) -> (String, u64) {
(hex(self.context.finish().as_ref()), self.size)
}
}
pub(crate) fn hex(bytes: &[u8]) -> String {
use std::fmt::Write as _;
bytes.iter().fold(String::new(), |mut out, byte| {
let _ = write!(out, "{byte:02x}");
out
})
}
impl<R: Read> Read for Hashed<R> {
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
let read = self.inner.read(buf)?;
self.context.update(&buf[..read]);
self.size += read as u64;
Ok(read)
}
}
struct Capped<R> {
inner: R,
left: u64,
}
impl<R: Read> Read for Capped<R> {
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
let read = self.inner.read(buf)?;
self.left = self
.left
.checked_sub(read as u64)
.ok_or_else(|| io::Error::new(io::ErrorKind::FileTooLarge, "past the budget"))?;
Ok(read)
}
}