use std::collections::{BTreeMap, BTreeSet};
use std::fs::{self, File, OpenOptions};
use std::io::{self, Write as _};
use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt, PermissionsExt};
use std::path::{Path, PathBuf};
use std::time::Duration;
use ring::rand::{SecureRandom, SystemRandom};
use rustix::fs::{CWD, FlockOperation, RenameFlags, flock, renameat_with};
use serde::{Deserialize, Serialize};
use tracing::warn;
use super::pins::Pin;
use super::verify::hash_file;
pub(crate) const ARCHIVES: &str = "archives";
const STAGING: &str = "staging";
const QUARANTINE: &str = "quarantine";
const INDEX: &str = "index.json";
const LOCK: &str = "lock";
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub(crate) struct Key {
pub(crate) repository: String,
pub(crate) sha: String,
}
impl Key {
pub(crate) fn valid(&self) -> bool {
super::pins::repository(&self.repository) && super::pins::commit(&self.sha)
}
fn file(&self) -> PathBuf {
Path::new(&self.repository.replace('/', "_")).join(format!("{}.tar.gz", self.sha))
}
pub(crate) fn label(&self) -> String {
format!("{}@{}", self.repository, self.sha)
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
struct Entry {
#[serde(flatten)]
key: Key,
sha256: String,
size: u64,
fetched_at: u64,
pinned_at: u64,
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
uses: BTreeMap<String, Vec<Pin>>,
#[serde(default, skip_serializing_if = "BTreeSet::is_empty")]
unread: BTreeSet<String>,
#[serde(default, skip_serializing_if = "BTreeSet::is_empty")]
actions: BTreeSet<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub(crate) struct Scan {
pub(crate) head: String,
pub(crate) pins: BTreeSet<Pin>,
}
#[derive(Debug, Default, Serialize, Deserialize)]
struct Index {
entries: Vec<Entry>,
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
scans: BTreeMap<String, Scan>,
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
reached: BTreeMap<String, BTreeSet<Key>>,
#[serde(default, skip_serializing_if = "BTreeSet::is_empty")]
refused: BTreeSet<Key>,
#[serde(default, skip_serializing_if = "BTreeSet::is_empty")]
packages: BTreeSet<String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum Action {
Missing,
NoFile,
Unread,
Pins(Vec<Pin>),
}
#[derive(Debug)]
pub(crate) struct Staged {
pub(crate) file: PathBuf,
pub(crate) sha256: String,
pub(crate) size: u64,
pub(crate) uses: BTreeMap<String, Vec<Pin>>,
pub(crate) unread: BTreeSet<String>,
pub(crate) actions: BTreeSet<String>,
}
#[derive(Debug, Default, PartialEq, Eq)]
pub(crate) struct Checked {
pub(crate) changed: Vec<Key>,
pub(crate) strangers: Vec<String>,
pub(crate) errors: Vec<String>,
}
impl Checked {
pub(crate) fn clean(&self) -> bool {
self.errors.is_empty()
}
}
enum Standing {
Matches,
Changed,
Gone,
}
#[derive(Debug, Default, PartialEq, Eq)]
pub(crate) struct Packaged {
pub(crate) new: bool,
pub(crate) removed: Vec<Key>,
}
#[derive(Debug)]
pub(crate) struct ArchiveStore {
root: PathBuf,
entries: BTreeMap<Key, Entry>,
scans: BTreeMap<String, Scan>,
reached: BTreeMap<String, BTreeSet<Key>>,
refused: BTreeSet<Key>,
packages: BTreeSet<String>,
_lock: File,
}
impl ArchiveStore {
pub(crate) fn open(root: &Path) -> io::Result<ArchiveStore> {
fs::DirBuilder::new()
.recursive(true)
.mode(0o755)
.create(root)?;
let lock = OpenOptions::new()
.create(true)
.truncate(false)
.write(true)
.mode(0o600)
.open(root.join(LOCK))?;
lock_exclusive(&lock)?;
for dir in [root.to_owned(), root.join(ARCHIVES), root.join(QUARANTINE)] {
make_dir(&dir)?;
}
let staging = root.join(STAGING);
if staging.exists() {
fs::remove_dir_all(&staging)?;
}
make_dir(&staging)?;
let index = match fs::read(root.join(INDEX)) {
Ok(bytes) => serde_json::from_slice::<Index>(&bytes).unwrap_or_else(|error| {
warn!("the action archives' index does not parse; every archive is checked again: {error}");
Index::default()
}),
Err(error) if error.kind() == io::ErrorKind::NotFound => Index::default(),
Err(error) => return Err(error),
};
let (valid, invalid): (Vec<Entry>, Vec<Entry>) = index
.entries
.into_iter()
.partition(|entry| entry.key.valid());
if !invalid.is_empty() {
warn!(
"the action archives' index names {} archives that are not GitHub's names; they are dropped",
invalid.len()
);
}
Ok(ArchiveStore {
root: root.to_owned(),
entries: valid
.into_iter()
.map(|entry| (entry.key.clone(), entry))
.collect(),
scans: index
.scans
.into_iter()
.filter(|(repository, _)| super::pins::repository(repository))
.collect(),
reached: index
.reached
.into_iter()
.filter(|(repository, keys)| {
super::pins::repository(repository) && keys.iter().all(Key::valid)
})
.collect(),
refused: index.refused.into_iter().filter(Key::valid).collect(),
packages: index.packages,
_lock: lock,
})
}
pub(crate) fn archives(&self) -> PathBuf {
self.root.join(ARCHIVES)
}
pub(crate) fn action(&self, key: &Key, path: &str) -> Action {
let Some(entry) = self.entries.get(key) else {
return Action::Missing;
};
if entry.unread.contains(path) {
Action::Unread
} else if !entry.actions.contains(path) && !entry.uses.contains_key(path) {
Action::NoFile
} else {
Action::Pins(entry.uses.get(path).cloned().unwrap_or_default())
}
}
pub(crate) fn refused(&self, key: &Key) -> bool {
self.refused.contains(key)
}
pub(crate) fn refuse(&mut self, key: Key) -> io::Result<()> {
if self.refused.insert(key) {
self.save()?;
}
Ok(())
}
pub(crate) fn package(&self, repository: &str) -> bool {
self.packages.contains(&repository.to_ascii_lowercase())
}
pub(crate) fn packaged(&mut self, repository: &str) -> io::Result<Packaged> {
let lower = repository.to_ascii_lowercase();
let new = self.packages.insert(lower.clone());
let held: Vec<String> = self
.repositories()
.into_iter()
.filter(|held| held.to_ascii_lowercase() == lower)
.collect();
let removed = self.withdraw(&held)?;
if new {
self.save()?;
}
Ok(Packaged { new, removed })
}
pub(crate) fn packages(&self) -> Vec<String> {
self.packages.iter().cloned().collect()
}
pub(crate) fn scan(&self, repository: &str) -> Option<&Scan> {
self.scans.get(repository)
}
pub(crate) fn scanned(&mut self, repository: &str, scan: Scan) -> io::Result<()> {
if self.scans.get(repository) != Some(&scan) {
self.scans.insert(repository.to_owned(), scan);
self.save()?;
}
Ok(())
}
pub(crate) fn reached(&self, repository: &str) -> BTreeSet<Key> {
self.reached.get(repository).cloned().unwrap_or_default()
}
pub(crate) fn record(
&mut self,
whole: BTreeMap<String, BTreeSet<Key>>,
served: &[String],
) -> io::Result<()> {
self.reached.extend(whole);
self.reached
.retain(|repository, _| served.contains(repository));
self.scans
.retain(|repository, _| served.contains(repository));
self.save()
}
pub(crate) fn repositories(&self) -> Vec<String> {
let mut repositories: Vec<String> = self
.entries
.keys()
.map(|key| key.repository.clone())
.collect();
repositories.dedup();
repositories
}
pub(crate) fn withdraw(&mut self, repositories: &[String]) -> io::Result<Vec<Key>> {
let removed: Vec<Key> = self
.entries
.keys()
.filter(|key| repositories.contains(&key.repository))
.cloned()
.collect();
let archives = self.archives();
for key in &removed {
match fs::remove_file(archives.join(key.file())) {
Err(error) if error.kind() != io::ErrorKind::NotFound => return Err(error),
_ => {}
}
self.entries.remove(key);
}
self.save()?;
Ok(removed)
}
pub(crate) fn size(&self) -> (u64, u64) {
(
self.entries.len() as u64,
self.entries.values().map(|entry| entry.size).sum(),
)
}
pub(crate) fn staging_file(&self) -> io::Result<PathBuf> {
let mut bytes = [0_u8; 12];
SystemRandom::new()
.fill(&mut bytes)
.map_err(|_| io::Error::other("no random bytes"))?;
let name = super::verify::hex(&bytes);
Ok(self.root.join(STAGING).join(name))
}
pub(crate) fn check(&mut self, now: u64) -> io::Result<Checked> {
let mut checked = Checked::default();
let archives = self.archives();
let keys: Vec<Key> = self.entries.keys().cloned().collect();
let mut dropped = false;
for key in keys {
let path = archives.join(key.file());
let standing = match self.standing(&key, &path) {
Ok(standing) => standing,
Err(error) => {
checked.errors.push(format!("{}: {error}", key.label()));
continue;
}
};
match standing {
Standing::Matches => {}
Standing::Gone => {
self.entries.remove(&key);
dropped = true;
}
Standing::Changed => match quarantine(&self.root, &path, now) {
Ok(()) => {
self.entries.remove(&key);
dropped = true;
checked.changed.push(key);
}
Err(error) => checked.errors.push(format!("{}: {error}", key.label())),
},
}
}
let known: BTreeSet<PathBuf> = self.entries.keys().map(Key::file).collect();
for (relative, file) in strangers(&archives, &known, &mut checked.errors)? {
match quarantine(&self.root, &file, now) {
Ok(()) => checked.strangers.push(relative),
Err(error) => checked.errors.push(format!("{relative}: {error}")),
}
}
if dropped {
self.save()?;
}
Ok(checked)
}
fn standing(&self, key: &Key, path: &Path) -> io::Result<Standing> {
let metadata = match fs::symlink_metadata(path) {
Ok(metadata) => metadata,
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(Standing::Gone),
Err(error) => return Err(error),
};
if !metadata.file_type().is_file() {
return Ok(Standing::Changed);
}
let entry = &self.entries[key];
let (sha256, size) = hash_file(path)?;
Ok(if sha256 == entry.sha256 && size == entry.size {
Standing::Matches
} else {
Standing::Changed
})
}
pub(crate) fn publish(&mut self, key: Key, staged: Staged, now: u64) -> io::Result<()> {
if self.entries.contains_key(&key) {
return fs::remove_file(&staged.file);
}
fs::set_permissions(&staged.file, fs::Permissions::from_mode(0o644))?;
File::open(&staged.file)?.sync_all()?;
let target = self.archives().join(key.file());
let dir = target
.parent()
.map_or_else(|| self.archives(), Path::to_owned);
if !dir.exists() {
make_dir(&dir)?;
sync_dir(&self.archives())?;
}
renameat_with(CWD, &staged.file, CWD, &target, RenameFlags::NOREPLACE)?;
sync_dir(&dir)?;
self.entries.insert(
key.clone(),
Entry {
key,
sha256: staged.sha256,
size: staged.size,
fetched_at: now,
pinned_at: now,
uses: staged.uses,
unread: staged.unread,
actions: staged.actions,
},
);
self.save()
}
pub(crate) fn retain(
&mut self,
pinned: &BTreeSet<Key>,
keep: Duration,
now: u64,
) -> io::Result<Vec<Key>> {
let mut removed = Vec::new();
self.refused.retain(|key| pinned.contains(key));
for (key, entry) in &mut self.entries {
if pinned.contains(key) {
entry.pinned_at = now;
} else if now.saturating_sub(entry.pinned_at) > keep.as_secs() {
removed.push(key.clone());
}
}
let archives = self.archives();
for key in &removed {
match fs::remove_file(archives.join(key.file())) {
Err(error) if error.kind() != io::ErrorKind::NotFound => return Err(error),
_ => {}
}
self.entries.remove(key);
}
for dir in fs::read_dir(&archives)? {
let dir = dir?;
if dir.file_type()?.is_dir() && fs::read_dir(dir.path())?.next().is_none() {
fs::remove_dir(dir.path())?;
}
}
prune_quarantine(&self.root, keep, now)?;
self.save()?;
Ok(removed)
}
fn save(&self) -> io::Result<()> {
let index = Index {
entries: self.entries.values().cloned().collect(),
scans: self.scans.clone(),
reached: self.reached.clone(),
refused: self.refused.clone(),
packages: self.packages.clone(),
};
let bytes = serde_json::to_vec_pretty(&index).map_err(io::Error::other)?;
let temporary = self.root.join(format!("{INDEX}.tmp"));
let mut file = OpenOptions::new()
.write(true)
.create(true)
.truncate(true)
.open(&temporary)?;
file.write_all(&bytes)?;
file.sync_all()?;
fs::rename(&temporary, self.root.join(INDEX))?;
sync_dir(&self.root)
}
}
const LOCK_TRIES: u32 = 40;
const LOCK_WAIT: Duration = Duration::from_millis(25);
fn lock_exclusive(lock: &File) -> io::Result<()> {
for _ in 1..LOCK_TRIES {
match flock(lock, FlockOperation::NonBlockingLockExclusive) {
Err(error) if error == rustix::io::Errno::WOULDBLOCK => std::thread::sleep(LOCK_WAIT),
other => return other.map_err(io::Error::from),
}
}
flock(lock, FlockOperation::NonBlockingLockExclusive).map_err(|error| {
if error == rustix::io::Errno::WOULDBLOCK {
io::Error::new(
io::ErrorKind::WouldBlock,
"another runnerd keeps its action archives here",
)
} else {
error.into()
}
})
}
fn strangers(
archives: &Path,
known: &BTreeSet<PathBuf>,
errors: &mut Vec<String>,
) -> io::Result<Vec<(String, PathBuf)>> {
let mut found = Vec::new();
for top in fs::read_dir(archives)? {
let top = top?;
let inside = match top.file_type() {
Ok(kind) if kind.is_dir() => match fs::read_dir(top.path()) {
Ok(inside) => inside,
Err(error) => {
errors.push(format!("{}: {error}", top.path().display()));
continue;
}
},
Ok(_) => {
found.push((top.file_name().to_string_lossy().into_owned(), top.path()));
continue;
}
Err(error) => {
errors.push(format!("{}: {error}", top.path().display()));
continue;
}
};
for file in inside {
let file = match file {
Ok(file) => file,
Err(error) => {
errors.push(format!("{}: {error}", top.path().display()));
continue;
}
};
let path = file.path();
let relative = path.strip_prefix(archives).unwrap_or(&path).to_owned();
let regular = file.file_type().is_ok_and(|kind| kind.is_file());
if !regular || !known.contains(&relative) {
found.push((relative.to_string_lossy().into_owned(), path));
}
}
}
Ok(found)
}
fn make_dir(dir: &Path) -> io::Result<()> {
match fs::DirBuilder::new().mode(0o755).create(dir) {
Err(error) if error.kind() != io::ErrorKind::AlreadyExists => return Err(error),
_ => {}
}
fs::set_permissions(dir, fs::Permissions::from_mode(0o755))
}
fn sync_dir(dir: &Path) -> io::Result<()> {
File::open(dir)?.sync_all()
}
fn quarantine(root: &Path, file: &Path, now: u64) -> io::Result<()> {
let parent = file
.parent()
.and_then(Path::file_name)
.map(|name| name.to_string_lossy().into_owned())
.unwrap_or_default();
let name = file
.file_name()
.map(|name| name.to_string_lossy().into_owned())
.unwrap_or_default();
for attempt in 0_u32.. {
let suffix = if attempt == 0 {
String::new()
} else {
format!(".{attempt}")
};
let target = root
.join(QUARANTINE)
.join(format!("{now}-{parent}-{name}{suffix}"));
match renameat_with(CWD, file, CWD, &target, RenameFlags::NOREPLACE) {
Ok(()) => {
warn!(
"action archive {parent}/{name} does not match what was verified; moved to {}",
target.display()
);
return Ok(());
}
Err(error) if error == rustix::io::Errno::EXIST && attempt < 1000 => {}
Err(error) => return Err(error.into()),
}
}
Ok(())
}
fn prune_quarantine(root: &Path, keep: Duration, now: u64) -> io::Result<()> {
for file in fs::read_dir(root.join(QUARANTINE))? {
let file = file?;
let name = file.file_name().to_string_lossy().into_owned();
let at: Option<u64> = name.split('-').next().and_then(|at| at.parse().ok());
if at.is_none_or(|at| now.saturating_sub(at) > keep.as_secs()) {
let path = file.path();
if file.file_type()?.is_dir() {
fs::remove_dir_all(path)?;
} else {
fs::remove_file(path)?;
}
}
}
Ok(())
}