use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
use std::io::{BufRead, Write};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use datumesh_core::json::{self, Value};
use datumesh_core::{
sha256_hex, Arg, Args, Cell, ClassDescriptor, FunctionDescriptor, Receipt, ReceiptBuilder, Scalar, SigningKey, Type,
};
use datumesh_engine::{Clock, SystemClock};
use datumesh_formula::definition::{Callee, Callees, ClassCallee};
use datumesh_store::{Binding, CellWrite, Namespace, Store};
use crate::testrun::{self, run_test, Overlay, Ran, RowWrite, TestSpec};
use crate::config::ConfigError;
const END: &str = "9999-12-31T23:59:59.999999Z";
const CHUNK: usize = 1_000;
const LANES_VAR: &str = "DATUMESH_PUBLISH_LANES";
fn publish_lanes() -> Result<usize, PublishError> {
let bad = |detail: String| PublishError::at(0, ConfigError::new(LANES_VAR, detail).to_string());
match std::env::var(LANES_VAR) {
Err(std::env::VarError::NotPresent) => Ok(std::thread::available_parallelism().map_or(1, |n| n.get()).min(4)),
Ok(v) => match v.parse::<usize>() {
Ok(n) if n > 0 => Ok(n),
_ => Err(bad(format!("must be a positive integer, not {v:?}"))),
},
Err(e) => Err(bad(e.to_string())),
}
}
#[non_exhaustive]
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct Published {
pub functions: usize,
pub classes: usize,
pub cells: usize,
pub receipts: usize,
pub versions: usize,
pub blobs: usize,
pub blob_bytes: u64,
pub descriptors: u64,
}
#[derive(Clone)]
pub struct PublishOptions {
pub clock: Arc<dyn Clock + Send + Sync>,
pub backfill: Option<PathBuf>,
}
impl PublishOptions {
pub fn system() -> PublishOptions {
PublishOptions { clock: Arc::new(SystemClock), backfill: None }
}
}
#[non_exhaustive]
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum PublishErrorKind {
Line,
StartBeforeNow {
name: String,
start: String,
now: String,
},
Backfill(String),
OwnedVectorName(String),
TestsMissing(String),
TestsFailing(String),
}
impl std::fmt::Display for PublishErrorKind {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
PublishErrorKind::Line => f.write_str("refused"),
PublishErrorKind::StartBeforeNow { name, start, now } => {
write!(f, "binding start {start} of {name} is earlier than the mesh clock {now} (rule A)")
}
PublishErrorKind::Backfill(why) => write!(f, "--backfill refused: {why}"),
PublishErrorKind::OwnedVectorName(name) => {
write!(f, "{name} is under a namespace the mesh owns: import-vectors does not bind it")
}
PublishErrorKind::TestsMissing(text) => write!(f, "E_TESTS_MISSING: {text}"),
PublishErrorKind::TestsFailing(text) => write!(f, "E_TESTS_FAILING: {text}"),
}
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct PublishError {
pub line: usize,
pub detail: String,
pub kind: PublishErrorKind,
}
impl PublishError {
fn at(line: usize, detail: impl Into<String>) -> PublishError {
PublishError { line, detail: detail.into(), kind: PublishErrorKind::Line }
}
pub(crate) fn typed(line: usize, kind: PublishErrorKind) -> PublishError {
PublishError { line, detail: kind.to_string(), kind }
}
}
impl std::fmt::Display for PublishError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "line {}: {}", self.line, self.detail)
}
}
impl std::error::Error for PublishError {}
fn s(v: &str) -> Value {
Value::String(v.to_string())
}
fn namespace_of(name: &str) -> &str {
name.rsplit_once('/').map(|(ns, _)| ns).unwrap_or("")
}
fn host_of(name: &str) -> &str {
name.split('/').next().unwrap_or("")
}
fn definer<'a>(name: &'a str, host: &'a str, owned: &[String]) -> &'a str {
let host_of_name = name.split('/').next().unwrap_or("");
if owned.iter().any(|o| o == host_of_name) {
namespace_of(name)
} else {
host
}
}
fn bound_of(v: &Value, line: usize, now: &str) -> Result<String, PublishError> {
match v.get("bound") {
None => Ok(now.to_string()),
Some(b) => b.as_str().map(str::to_string).ok_or_else(|| PublishError::at(line, "bound is not an instant")),
}
}
#[derive(Default)]
struct Simulation {
functions: BTreeMap<String, Vec<(String, String)>>,
classes: BTreeMap<String, Vec<(String, String)>>,
descriptors: BTreeMap<String, Vec<u8>>,
class_bytes: BTreeMap<String, Vec<u8>>,
versions: BTreeMap<String, Vec<(String, String)>>,
}
impl Simulation {
fn latest(binds: &BTreeMap<String, Vec<(String, String)>>, name: &str, at: &str) -> Option<(String, String)> {
binds.get(name)?.iter().filter(|(t, _)| t.as_str() <= at).max_by(|a, b| a.0.cmp(&b.0)).cloned()
}
}
struct Callers<'a, S: Store + ?Sized> {
store: &'a S,
sim: Option<&'a Simulation>,
}
impl<S: Store + ?Sized> Callers<'_, S> {
fn function_bytes(&self, hash: &str) -> Result<Option<Vec<u8>>, String> {
if let Some(b) = self.sim.and_then(|s| s.descriptors.get(hash)) {
return Ok(Some(b.clone()));
}
Ok(self.store.descriptor(hash).map(|b| b.into_owned()))
}
fn class_bytes(&self, hash: &str) -> Result<Option<Vec<u8>>, String> {
if let Some(b) = self.sim.and_then(|s| s.class_bytes.get(hash)) {
return Ok(Some(b.clone()));
}
self.store.class(hash).map_err(|e| e.to_string())
}
}
impl<S: Store + ?Sized> Callees for Callers<'_, S> {
fn callee(&self, name: &str, at: &str) -> Result<Option<Callee>, String> {
let stored = self.store.resolve_binding(name, at).map_err(|e| e.to_string())?;
let simmed = self.sim.and_then(|s| Simulation::latest(&s.functions, name, at));
let (fn_hash, start) = match (stored, simmed) {
(Some(b), Some((t, h))) if t >= b.tx => (h, t),
(Some(b), _) => (b.fn_hash, b.tx),
(None, Some((t, h))) => (h, t),
(None, None) => return Ok(None),
};
let bytes = self.function_bytes(&fn_hash)?.ok_or_else(|| format!("descriptor {fn_hash} is not stored"))?;
let descriptor = json::parse(&bytes).map_err(|e| e.to_string())?;
Ok(Some(Callee { fn_hash, start, descriptor }))
}
fn bound_class(&self, name: &str, at: &str) -> Result<Option<ClassCallee>, String> {
let stored = self.store.resolve_class_binding(name, at).map_err(|e| e.to_string())?;
let simmed = self.sim.and_then(|s| Simulation::latest(&s.classes, name, at));
let (class_hash, start) = match (stored, simmed) {
(Some(b), Some((t, h))) if t >= b.tx => (h, t),
(Some(b), _) => (b.class_hash, b.tx),
(None, Some((t, h))) => (h, t),
(None, None) => return Ok(None),
};
let bytes = self.class_bytes(&class_hash)?.ok_or_else(|| format!("class {class_hash} is not stored"))?;
let descriptor = ClassDescriptor::from_json(&bytes).map_err(|e| e.to_string())?;
Ok(Some(ClassCallee { class_hash, start, descriptor }))
}
fn class_descriptor(&self, class_hash: &str) -> Result<Option<ClassDescriptor>, String> {
let Some(bytes) = self.class_bytes(class_hash)? else { return Ok(None) };
ClassDescriptor::from_json(&bytes).map(Some).map_err(|e| e.to_string())
}
fn function_descriptor(&self, fn_hash: &str) -> Result<Option<Value>, String> {
let Some(bytes) = self.function_bytes(fn_hash)? else { return Ok(None) };
json::parse(&bytes).map(Some).map_err(|e| e.to_string())
}
}
#[derive(Default)]
struct Reads {
bindings: HashMap<(String, String), Result<Option<Binding>, String>>,
descriptors: HashMap<String, Option<Vec<u8>>>,
namespaces: HashMap<String, Result<Option<Namespace>, String>>,
latest_of: HashMap<String, Option<(String, String)>>,
receipt_of: HashMap<String, Option<Vec<u8>>>,
}
impl Reads {
fn binding<S: Store + ?Sized>(&mut self, store: &S, name: &str, at: &str) -> Result<Option<Binding>, String> {
let key = (name.to_string(), at.to_string());
if let Some(r) = self.bindings.get(&key) {
return r.clone();
}
let r = store.resolve_binding(name, at).map_err(|e| e.to_string());
self.bindings.insert(key, r.clone());
r
}
fn descriptor<S: Store + ?Sized>(&mut self, store: &S, hash: &str) -> Option<Vec<u8>> {
if let Some(b) = self.descriptors.get(hash) {
return b.clone();
}
let b = store.descriptor(hash).map(|b| b.into_owned());
self.descriptors.insert(hash.to_string(), b.clone());
b
}
fn namespace<S: Store + ?Sized>(&mut self, store: &S, host: &str) -> Result<Option<Namespace>, String> {
if let Some(r) = self.namespaces.get(host) {
return r.clone();
}
let r = store.namespace(host).map_err(|e| e.to_string());
self.namespaces.insert(host.to_string(), r.clone());
r
}
fn latest<S: Store + ?Sized>(&mut self, store: &S, id: &str) -> Result<Option<(String, String)>, String> {
if let Some(l) = self.latest_of.get(id) {
return Ok(l.clone());
}
store.latest(id).map(|v| v.map(|v| (v.tx, v.receipt_hash))).map_err(|e| e.to_string())
}
fn receipt<S: Store + ?Sized>(&mut self, store: &S, hash: &str) -> Option<Vec<u8>> {
if let Some(b) = self.receipt_of.get(hash) {
return b.clone();
}
store.receipt(hash).map(|b| b.into_owned())
}
fn prefetch_latest<S: Store + ?Sized>(&mut self, store: &S, ids: &[&str]) {
let mut seen = HashSet::new();
let missing: Vec<&str> =
ids.iter().copied().filter(|id| !self.latest_of.contains_key(*id) && seen.insert(*id)).collect();
if missing.is_empty() {
return;
}
if let Ok(found) = store.latest_many(&missing) {
for (id, v) in missing.iter().zip(found) {
self.latest_of.insert(id.to_string(), v.map(|v| (v.tx, v.receipt_hash)));
}
}
}
fn prefetch_receipts<S: Store + ?Sized>(&mut self, store: &S, hashes: &[&str]) {
let mut seen = HashSet::new();
let missing: Vec<&str> =
hashes.iter().copied().filter(|h| !self.receipt_of.contains_key(*h) && seen.insert(*h)).collect();
if missing.is_empty() {
return;
}
if let Ok(found) = store.receipts_many(&missing) {
for (h, b) in missing.iter().zip(found) {
self.receipt_of.insert(h.to_string(), b);
}
}
}
}
fn parallel_owned<T: Send, R: Send>(items: Vec<T>, f: impl Fn(T) -> R + Sync) -> Vec<R> {
let threads = std::thread::available_parallelism().map_or(1, |n| n.get()).min(items.len());
if threads <= 1 {
return items.into_iter().map(f).collect();
}
let size = items.len().div_ceil(threads);
let mut parts: Vec<Vec<T>> = Vec::with_capacity(threads);
let mut rest = items.into_iter();
loop {
let part: Vec<T> = rest.by_ref().take(size).collect();
if part.is_empty() {
break;
}
parts.push(part);
}
std::thread::scope(|s| {
let handles: Vec<_> = parts
.into_iter()
.map(|part| {
let f = &f;
s.spawn(move || part.into_iter().map(f).collect::<Vec<R>>())
})
.collect();
handles.into_iter().flat_map(|h| h.join().unwrap_or_else(|e| std::panic::resume_unwind(e))).collect()
})
}
fn current_function<S: Store + ?Sized>(store: &S, sim: &Simulation, name: &str) -> Result<Option<String>, String> {
if let Some((_, h)) = Simulation::latest(&sim.functions, name, END) {
return Ok(Some(h));
}
store.resolve_name(name, END).map_err(|e| e.to_string())
}
fn current_class<S: Store + ?Sized>(store: &S, sim: &Simulation, name: &str) -> Result<Option<String>, String> {
if let Some((_, h)) = Simulation::latest(&sim.classes, name, END) {
return Ok(Some(h));
}
store.resolve_class(name, END).map_err(|e| e.to_string())
}
fn function_at<S: Store + ?Sized>(
reads: &mut Reads,
store: &S,
sim: &Simulation,
name: &str,
at: &str,
) -> Result<Option<String>, String> {
let stored = reads.binding(store, name, at)?;
let simmed = Simulation::latest(&sim.functions, name, at);
Ok(match (stored, simmed) {
(Some(b), Some((t, h))) if t >= b.tx => Some(h),
(Some(b), _) => Some(b.fn_hash),
(None, Some((_, h))) => Some(h),
(None, None) => None,
})
}
fn latest_bind<S: Store + ?Sized>(
store: &S,
sim: &Simulation,
name: &str,
class: bool,
) -> Result<Option<String>, String> {
let stored: Option<String> = if class {
store.resolve_class_binding(name, END).map_err(|e| e.to_string())?.map(|b| b.tx)
} else {
store.resolve_binding(name, END).map_err(|e| e.to_string())?.map(|b| b.tx)
};
let binds = if class { &sim.classes } else { &sim.functions };
let planned = binds.get(name).and_then(|v| v.iter().map(|(t, _)| t.clone()).max());
Ok(match (stored, planned) {
(Some(a), Some(b)) => Some(a.max(b)),
(a, b) => a.or(b),
})
}
fn bound_as_other<S: Store + ?Sized>(store: &S, sim: &Simulation, name: &str, class: bool) -> Result<bool, String> {
if class {
Ok(store.resolve_binding(name, END).map_err(|e| e.to_string())?.is_some() || sim.functions.contains_key(name))
} else {
Ok(store.resolve_class_binding(name, END).map_err(|e| e.to_string())?.is_some()
|| sim.classes.contains_key(name))
}
}
fn cell_latest<S: Store + ?Sized>(
reads: &mut Reads,
store: &S,
sim: &Simulation,
id: &str,
) -> Result<Option<(String, String)>, String> {
let stored = reads.latest(store, id)?;
let planned = sim.versions.get(id).and_then(|v| v.last().cloned());
Ok(match (stored, planned) {
(Some(a), Some(b)) => Some(if b.0 > a.0 { b } else { a }),
(a, b) => a.or(b),
})
}
fn cell_version_asof<S: Store + ?Sized>(
store: &S,
sim: &Simulation,
id: &str,
at: &str,
) -> Result<Option<String>, String> {
let stored = store.version_asof(id, at).map_err(|e| e.to_string())?.map(|v| (v.tx, v.receipt_hash));
let planned = sim.versions.get(id).and_then(|v| v.iter().rev().find(|(t, _)| t.as_str() <= at).cloned());
let latest = match (stored, planned) {
(Some(a), Some(b)) => Some(if b.0 > a.0 { b } else { a }),
(a, b) => a.or(b),
};
Ok(latest.map(|x| x.1))
}
fn function_bytes_of<S: Store + ?Sized>(
reads: &mut Reads,
store: &S,
sim: &Simulation,
hash: &str,
) -> Result<Option<Vec<u8>>, String> {
if let Some(b) = sim.descriptors.get(hash) {
return Ok(Some(b.clone()));
}
Ok(reads.descriptor(store, hash))
}
fn suspended<S: Store + ?Sized>(reads: &mut Reads, store: &S, host: &str, line: usize) -> Result<(), PublishError> {
match reads.namespace(store, host) {
Ok(Some(n)) if n.suspended_at.is_some() => {
Err(PublishError::at(line, format!("{host} is suspended: no new bindings or classes under it")))
}
Ok(_) => Ok(()),
Err(e) => Err(PublishError::at(line, e)),
}
}
struct Bind {
line: usize,
name: String,
}
#[derive(Default)]
struct Written {
namespaces: BTreeSet<String>,
starts: BTreeSet<String>,
}
impl Written {
fn bound(&mut self, name: &str, start: &str) {
self.namespaces.insert(namespace_of(name).to_string());
self.starts.insert(start.to_string());
}
}
struct Ctx<'a> {
host: &'a str,
owned: &'a [String],
key_from: &'a str,
now: &'a str,
base: &'a Path,
}
enum Op {
Descriptor { hash: String, bytes: Vec<u8>, counted: bool },
Function { name: String, start: String, hash: String, bytes: Vec<u8>, bind: bool },
Class { name: String, start: String, hash: String, bytes: Vec<u8>, bind: bool, methods: Vec<String> },
Cell { tx: String, id: String, receipt: Vec<u8>, blob: Option<Vec<u8>>, version: bool },
Catalog { entry: datumesh_store::CatalogEntry, dictionaries: Vec<(String, Vec<datumesh_store::KeyValue>)> },
}
fn read_lines(path: &Path) -> Result<Vec<(usize, String)>, PublishError> {
let file = std::fs::File::open(path).map_err(|e| PublishError::at(0, format!("{}: {e}", path.display())))?;
let mut out = Vec::new();
for (i, line) in std::io::BufReader::new(file).lines().enumerate() {
let n = i + 1;
let line = line.map_err(|e| PublishError::at(n, e.to_string()))?;
if !line.trim().is_empty() {
out.push((n, line));
}
}
Ok(out)
}
fn index_cell_values<S: Store + ?Sized>(store: &S, ops: &[Op]) -> Result<(), String> {
let mut seen = BTreeSet::new();
for op in ops {
if let Op::Cell { receipt, .. } = op {
if let Ok(r) = datumesh_core::Receipt::from_json(receipt) {
cell_string_values(&r, &mut seen);
}
}
}
store_cell_values(store, seen)
}
pub(crate) fn cell_string_values(r: &datumesh_core::Receipt, out: &mut BTreeSet<(String, String, String)>) {
let cell = r.cell();
for (key, arg) in &cell.args {
if let datumesh_core::Arg::Value(datumesh_core::Scalar::String(v)) = arg {
if !v.is_empty() && v.len() <= 1024 {
out.insert((cell.function_name(), key.clone(), v.clone()));
}
}
}
}
pub(crate) fn store_cell_values<S: Store + ?Sized>(
store: &S,
seen: BTreeSet<(String, String, String)>,
) -> Result<(), String> {
for (owner, key, value) in seen {
if crate::find::has_value(store, &owner, &key, &value) {
continue;
}
let kv = datumesh_store::KeyValue { value, label: String::new(), aliases: Vec::new() };
store.upsert_key_values(&owner, &key, std::slice::from_ref(&kv)).map_err(|e| e.to_string())?;
}
Ok(())
}
fn is_cell_line(v: &Value) -> bool {
v.get("class").and_then(Value::as_str).is_none()
&& v.get("descriptor").is_none()
&& v.get("function").and_then(Value::as_str).is_none()
&& v.get("formula").and_then(Value::as_str).is_none()
&& v.get("cell").is_some()
}
struct Pending {
line: usize,
ns: String,
cell: Cell,
value: Scalar,
valid: Args,
tx: String,
asof: String,
issued: String,
has_tx: bool,
licence: String,
summary: String,
blob: Option<Vec<u8>>,
}
impl Pending {
fn sign(self, key: &SigningKey, host: &str) -> Result<Signed, PublishError> {
let line = self.line;
let err = move |d: String| PublishError::at(line, d);
let r = ReceiptBuilder::new()
.cell(self.cell.clone())
.value(self.value)
.valid(self.valid)
.asof(self.asof.as_str())
.tx(self.tx.as_str())
.issued(self.issued.as_str())
.publisher(self.ns.as_str())
.signer_host(host)
.licence(self.licence.as_str())
.summary(self.summary.as_str())
.sign(key)
.map_err(|e| err(e.to_string()))?;
let receipt = receipt_json(&r);
let hash = Receipt::from_json(&receipt).map_err(|e| err(e.to_string()))?.hash().to_string();
let id = self.cell.id();
Ok(Signed { line, has_tx: self.has_tx, tx: self.tx, id, hash, receipt, blob: self.blob })
}
}
struct Signed {
line: usize,
has_tx: bool,
tx: String,
id: String,
hash: String,
receipt: Vec<u8>,
blob: Option<Vec<u8>>,
}
struct Planner<'a, S: Store + ?Sized> {
store: &'a S,
key: &'a SigningKey,
ctx: Ctx<'a>,
backfill: bool,
sim: Simulation,
reads: Reads,
ops: Vec<Op>,
binds: Vec<Bind>,
tests: Vec<(usize, TestSpec)>,
formulas: Vec<(usize, String, String)>,
}
impl<S: Store + ?Sized> Planner<'_, S> {
fn callers(&self) -> Callers<'_, S> {
Callers { store: self.store, sim: Some(&self.sim) }
}
fn run(&mut self, lines: &[(usize, String)]) -> Result<(), PublishError> {
let mut cells: Vec<(usize, Value)> = Vec::new();
for (n, text) in lines {
let n = *n;
match json::parse(text.as_bytes()).map_err(|e| PublishError::at(n, e.to_string())) {
Ok(v) if is_cell_line(&v) => {
cells.push((n, v));
if cells.len() == CHUNK {
self.plan_cells(std::mem::take(&mut cells))?;
}
}
other => {
self.plan_cells(std::mem::take(&mut cells))?;
self.plan_line(n, &other?)?;
}
}
}
self.plan_cells(cells)
}
fn plan_line(&mut self, n: usize, v: &Value) -> Result<(), PublishError> {
let err = |d: String| PublishError::at(n, d);
if let Some(name) = v.get("class").and_then(Value::as_str) {
self.class_line(n, name, v)?;
} else if let Some(d) = v.get("descriptor") {
self.descriptor_line(n, d)?;
} else if let Some(name) = v.get("function").and_then(Value::as_str) {
let sig = v.get("signature").ok_or_else(|| err("a function without a signature".into()))?;
let start = bound_of(v, n, self.ctx.now)?;
let fd = FunctionDescriptor::from_value(&stored_value(sig)).map_err(|e| err(e.to_string()))?;
self.function(n, name, start, fd.hash(), fd.canonical())?;
} else if v.get("catalog").is_some() {
self.catalog_line(n, v)?;
} else if let Some(name) = v.get("formula").and_then(Value::as_str) {
let source =
v.get("source").and_then(Value::as_str).ok_or_else(|| err("a formula without source".into()))?;
let sig = v.get("signature").ok_or_else(|| err("a formula without a signature".into()))?;
let start = bound_of(v, n, self.ctx.now)?;
let def = datumesh_formula::definition::define_function(
name,
source,
sig,
&Default::default(),
&self.callers(),
&start,
)
.map_err(|e| err(format!("define {name}: {e}")))?;
let hash = def.fn_hash.clone();
self.function(n, name, start, def.fn_hash, def.bytes)?;
self.formulas.push((n, name.to_string(), hash));
} else if v.get("test").is_some() {
let spec = testrun::parse_test(v).map_err(err)?;
self.tests.push((n, spec));
} else {
return Err(err("neither a function nor a cell".into()));
}
Ok(())
}
fn catalog_line(&mut self, n: usize, v: &Value) -> Result<(), PublishError> {
let err = |d: String| PublishError::at(n, d);
let d = crate::catalog::parse(v, "catalog").map_err(err)?;
let host = host_of(&d.name);
if host != self.ctx.host && !self.ctx.owned.iter().any(|o| o == host) {
return Err(err(format!("{} is not under a host this mesh owns", d.name)));
}
let hash = match d.kind {
datumesh_store::CatalogKind::Class => current_class(self.store, &self.sim, &d.name).map_err(err)?,
_ => current_function(self.store, &self.sim, &d.name).map_err(err)?,
};
if hash.is_none() {
return Err(err(format!("{} is bound nowhere", d.name)));
}
let entry = d.entry(hash, datumesh_store::CatalogSource::Publisher, Some(self.key.key_id().to_string()));
self.ops.push(Op::Catalog { entry, dictionaries: d.dictionary });
Ok(())
}
fn descriptor_line(&mut self, n: usize, d: &Value) -> Result<(), PublishError> {
let fd = FunctionDescriptor::from_value(d).map_err(|e| PublishError::at(n, e.to_string()))?;
if fd.lang != "stored" || !fd.body.is_empty() || !fd.inputs.is_empty() {
return Err(PublishError::at(
n,
"a descriptor line takes a stored function only: code.lang stored, code.body empty, no inputs",
));
}
let (hash, bytes) = (fd.hash(), fd.canonical());
self.sim.descriptors.insert(hash.clone(), bytes.clone());
self.ops.push(Op::Descriptor { hash, bytes, counted: true });
Ok(())
}
fn function(
&mut self,
n: usize,
name: &str,
start: String,
hash: String,
bytes: Vec<u8>,
) -> Result<(), PublishError> {
self.sim.descriptors.insert(hash.clone(), bytes.clone());
let current = current_function(self.store, &self.sim, name).map_err(|e| PublishError::at(n, e))?;
let bind = current.as_deref() != Some(hash.as_str());
if bind {
self.check_bind(n, name, &start, false)?;
self.sim.functions.entry(name.to_string()).or_default().push((start.clone(), hash.clone()));
}
self.ops.push(Op::Function { name: name.to_string(), start, hash, bytes, bind });
Ok(())
}
fn class_line(&mut self, n: usize, name: &str, v: &Value) -> Result<(), PublishError> {
let err = |d: String| PublishError::at(n, d);
let descriptor = v.get("descriptor").ok_or_else(|| err("a class without descriptor".into()))?;
let start = bound_of(v, n, self.ctx.now)?;
let def = datumesh_formula::definition::define_class(
name,
&descriptor.to_jcs(),
definer(name, self.ctx.host, self.ctx.owned),
&self.callers(),
&start,
)
.map_err(|e| err(format!("define {name}: {e}")))?;
suspended(&mut self.reads, self.store, host_of(&def.descriptor.steward.namespace), n)?;
let hash = def.class_hash.clone();
let methods: Vec<(String, String)> =
def.descriptor.methods.iter().map(|(m, fh)| (format!("{name}/{m}"), fh.clone())).collect();
let bind = current_class(self.store, &self.sim, name).map_err(err)?.as_deref() != Some(hash.as_str());
if bind {
self.check_bind(n, name, &start, true)?;
for (mname, _) in &methods {
self.check_bind(n, mname, &start, false)?;
}
self.sim.classes.entry(name.to_string()).or_default().push((start.clone(), hash.clone()));
for (mname, fh) in &methods {
self.sim.functions.entry(mname.clone()).or_default().push((start.clone(), fh.clone()));
}
}
self.sim.class_bytes.insert(hash.clone(), def.bytes.clone());
self.ops.push(Op::Class {
name: name.to_string(),
start,
hash,
bytes: def.bytes,
bind,
methods: methods.into_iter().map(|(m, _)| m).collect(),
});
Ok(())
}
fn check_bind(&mut self, line: usize, name: &str, start: &str, class: bool) -> Result<(), PublishError> {
if !self.backfill && start < self.ctx.now {
return Err(PublishError::typed(
line,
PublishErrorKind::StartBeforeNow { name: name.into(), start: start.into(), now: self.ctx.now.into() },
));
}
let err = |d: String| PublishError::at(line, d);
if bound_as_other(self.store, &self.sim, name, class).map_err(err)? {
let kind = if class { "function" } else { "class" };
return Err(err(format!("{name} is bound as a {kind}: one namespace of names")));
}
suspended(&mut self.reads, self.store, host_of(name), line)?;
if let Some(latest) = latest_bind(self.store, &self.sim, name, class).map_err(err)? {
if start <= latest.as_str() {
return Err(err(format!("{name} is bound from {latest}: a bind at {start} is not later")));
}
}
self.binds.push(Bind { line, name: name.into() });
Ok(())
}
fn plan_cells(&mut self, lines: Vec<(usize, Value)>) -> Result<(), PublishError> {
if lines.is_empty() {
return Ok(());
}
let mut pending = Vec::with_capacity(lines.len());
let mut failed = None;
for (n, v) in &lines {
match self.prepare_cell(*n, v) {
Ok(p) => pending.push(p),
Err(e) => {
failed = Some(e);
break;
}
}
}
let (key, host) = (self.key, self.ctx.host);
let signed = parallel_owned(pending, |p| p.sign(key, host));
let ids: Vec<&str> = signed.iter().flatten().map(|s| s.id.as_str()).collect();
self.reads.prefetch_latest(self.store, &ids);
let hashes: Vec<String> = signed
.iter()
.flatten()
.filter(|s| !s.has_tx)
.filter_map(|s| self.reads.latest_of.get(&s.id).cloned().flatten())
.map(|(_, h)| h)
.collect();
let hashes: Vec<&str> = hashes.iter().map(String::as_str).collect();
self.reads.prefetch_receipts(self.store, &hashes);
for s in signed {
self.check_cell(s?)?;
}
match failed {
Some(e) => Err(e),
None => Ok(()),
}
}
fn prepare_cell(&mut self, n: usize, v: &Value) -> Result<Pending, PublishError> {
let err = |d: String| PublishError::at(n, d);
let cell_v = v.get("cell").ok_or_else(|| err("a cell without fn".into()))?;
let name = cell_v.get("fn").and_then(Value::as_str).ok_or_else(|| err("a cell without fn".into()))?;
let (ns, f) = name.rsplit_once('/').ok_or_else(|| err(format!("{name}: not a function name")))?;
let field = |k: &str| v.get(k).and_then(Value::as_str).map(str::to_string);
let required = |k: &str| field(k).ok_or_else(|| err(format!("no {k}")));
let (licence, summary) = (required("licence")?, required("summary")?);
let asof = field("asof").unwrap_or_else(|| self.ctx.now.to_string());
let has_tx = field("tx").is_some();
let tx = field("tx").unwrap_or_else(|| self.ctx.now.to_string());
if tx > asof {
return Err(err(format!("tx {tx} is after asof {asof} (SPEC 8)")));
}
let issued = field("issued").unwrap_or_else(|| std::cmp::max(asof.clone(), self.ctx.key_from.to_string()));
if asof > issued {
return Err(err(format!("asof {asof} is after issued {issued} (SPEC 8)")));
}
let fn_hash = function_at(&mut self.reads, self.store, &self.sim, name, &tx)
.map_err(err)?
.ok_or_else(|| err(format!("{name} is not bound as of {tx}")))?;
let d = function_bytes_of(&mut self.reads, self.store, &self.sim, &fn_hash)
.map_err(err)?
.ok_or_else(|| err(format!("no descriptor {fn_hash}")))?;
let d = FunctionDescriptor::from_json(&d).map_err(|e| err(e.to_string()))?;
let mut args = Args::new();
for (k, a) in cell_v.get("args").and_then(Value::as_object).ok_or_else(|| err("a cell without args".into()))? {
args.insert(k.clone(), Arg::Value(Scalar::from_json(&a.to_jcs()).map_err(|e| err(format!("{k}: {e}")))?));
}
let valid: Args = args
.iter()
.filter(|(k, _)| d.signature.args.get(*k).is_some_and(|x| x.valid))
.map(|(k, a)| (k.clone(), a.clone()))
.collect();
let value_v = v.get("value").ok_or_else(|| err("a cell without value".into()))?;
let (value, blob) = match value_v.get("blob").and_then(Value::as_str) {
Some(file) => {
let media = value_v.get("media_type").and_then(Value::as_str).unwrap_or("application/octet-stream");
let bytes = std::fs::read(self.ctx.base.join(file)).map_err(|e| err(format!("{file}: {e}")))?;
let sha = sha256_hex(&bytes);
let scalar = Scalar::from_lexical(Type::Blob, &format!("{sha}:{}:{media}", bytes.len()))
.map_err(|e| err(e.to_string()))?;
(scalar, Some(bytes))
}
None => (Scalar::from_json(&value_v.to_jcs()).map_err(|e| err(e.to_string()))?, None),
};
let cell = Cell { ns: ns.to_string(), fn_name: f.to_string(), fn_hash, args };
Ok(Pending {
line: n,
ns: ns.to_string(),
cell,
value,
valid,
tx,
asof,
issued,
has_tx,
licence,
summary,
blob,
})
}
fn check_cell(&mut self, s: Signed) -> Result<(), PublishError> {
let line = s.line;
let err = |d: String| PublishError::at(line, d);
if !s.has_tx {
if let Some((_, latest)) = cell_latest(&mut self.reads, self.store, &self.sim, &s.id).map_err(err)? {
if let Some(old) = self.reads.receipt(self.store, &latest) {
let old = json::parse(&old).map_err(|e| err(e.to_string()))?;
let new = json::parse(&s.receipt).map_err(|e| err(e.to_string()))?;
if ["type", "value", "valid", "licence", "summary"].iter().all(|k| old.get(k) == new.get(k)) {
return Ok(());
}
}
}
}
let version = match cell_latest(&mut self.reads, self.store, &self.sim, &s.id).map_err(err)? {
Some((latest_tx, _)) if latest_tx.as_str() >= s.tx.as_str() => {
if cell_version_asof(self.store, &self.sim, &s.id, &s.tx).map_err(err)?.as_deref()
!= Some(s.hash.as_str())
{
return Err(err(format!("{} has a version at or after {} that is not this receipt", s.id, s.tx)));
}
false
}
_ => {
self.sim.versions.entry(s.id.clone()).or_default().push((s.tx.clone(), s.hash.clone()));
true
}
};
self.ops.push(Op::Cell { tx: s.tx, id: s.id, receipt: s.receipt, blob: s.blob, version });
Ok(())
}
}
fn stored_value(signature: &Value) -> Value {
Value::Object(vec![
("code".into(), Value::Object(vec![("body".into(), s("")), ("lang".into(), s("stored"))])),
("inputs".into(), Value::Array(vec![])),
("signature".into(), signature.clone()),
])
}
fn backfill_check<S: Store + ?Sized>(store: &S, binds: &[Bind]) -> Result<(), PublishError> {
for b in binds {
let refuse = |why: String| PublishError::typed(b.line, PublishErrorKind::Backfill(why));
if store.resolve_binding(&b.name, END).map_err(|e| PublishError::at(b.line, e.to_string()))?.is_some() {
return Err(refuse(format!("{} has a binding", b.name)));
}
let ns = namespace_of(&b.name);
if store.has_bindings_under(ns).map_err(|e| PublishError::at(b.line, e.to_string()))? {
return Err(refuse(format!("{ns} has bindings")));
}
}
Ok(())
}
fn append_log(log: &Path, written: &Written) -> Result<(), String> {
let line = Value::Object(vec![
("at".into(), s(&SystemClock.now())),
("namespaces".into(), Value::Array(written.namespaces.iter().map(|n| s(n)).collect())),
("starts".into(), Value::Array(written.starts.iter().map(|t| s(t)).collect())),
])
.to_jcs();
let mut opts = std::fs::OpenOptions::new();
opts.create(true).append(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
opts.mode(0o600);
}
let mut f = opts.open(log).map_err(|e| format!("{}: {e}", log.display()))?;
let mut bytes = line;
bytes.push(b'\n');
f.write_all(&bytes).map_err(|e| format!("{}: {e}", log.display()))
}
fn write<S: Store + ?Sized>(
store: &S,
ops: &[Op],
cells: impl Fn(&S, &[Op], &mut Published) -> Result<(), String>,
) -> Result<(Published, Written), PublishError> {
let mut out = Published::default();
let mut written = Written::default();
let err = |d: String| PublishError::at(0, d);
let mut i = 0;
while i < ops.len() {
match &ops[i] {
Op::Descriptor { hash, bytes, counted } => {
let existed = store.descriptor(hash).is_some();
store.put_descriptor(bytes).map_err(|e| err(e.to_string()))?;
if *counted && !existed {
out.descriptors += 1;
}
}
Op::Function { name, start, hash, bytes, bind } => {
store.put_descriptor(bytes).map_err(|e| err(e.to_string()))?;
if *bind {
store.bind_name(name, hash, start).map_err(|e| err(format!("bind {name}: {e}")))?;
out.functions += 1;
written.bound(name, start);
}
}
Op::Class { name, start, hash, bytes, bind, methods } => {
store.put_class(bytes).map_err(|e| err(e.to_string()))?;
if *bind {
store.bind_class(name, hash, start).map_err(|e| err(format!("bind {name}: {e}")))?;
out.classes += 1;
written.bound(name, start);
for m in methods {
written.bound(m, start);
}
}
}
Op::Catalog { entry, dictionaries } => {
crate::catalog::store_publisher(store, entry, dictionaries).map_err(err)?;
}
Op::Cell { .. } => {
let mut j = i;
while j < ops.len() && j - i < CHUNK && matches!(ops[j], Op::Cell { .. }) {
j += 1;
}
cells(store, &ops[i..j], &mut out).map_err(err)?;
i = j;
continue;
}
}
i += 1;
}
Ok((out, written))
}
fn write_cells_one_lane<S: Store + ?Sized>(store: &S, group: &[Op], out: &mut Published) -> Result<(), String> {
let items = put_blobs_and_items(store, group, out)?;
let failed = AtomicBool::new(false);
let ks: Vec<usize> = (0..items.len()).collect();
if let Some((_, detail)) = put_lane(store, &items, &ks, &failed) {
return Err(detail);
}
count_cells(group, &items, out);
Ok(())
}
fn write_cells<S: Store + Sync + ?Sized>(
store: &S,
group: &[Op],
lanes: usize,
out: &mut Published,
) -> Result<(), String> {
if lanes <= 1 {
return write_cells_one_lane(store, group, out);
}
let items = put_blobs_and_items(store, group, out)?;
let mut by_lane: Vec<Vec<usize>> = vec![Vec::new(); lanes];
for (k, op) in group.iter().enumerate() {
if let Op::Cell { id, .. } = op {
by_lane[lane_of(id, lanes)].push(k);
}
}
let failed = AtomicBool::new(false);
let failures: Vec<(usize, String)> = std::thread::scope(|scope| {
let handles: Vec<_> = by_lane
.iter()
.map(|ks| {
let (items, failed) = (&items, &failed);
scope.spawn(move || put_lane(store, items, ks, failed))
})
.collect();
handles.into_iter().filter_map(|h| h.join().unwrap_or_else(|p| std::panic::resume_unwind(p))).collect()
});
if let Some((_, detail)) = failures.into_iter().min_by_key(|(k, _)| *k) {
return Err(detail);
}
count_cells(group, &items, out);
Ok(())
}
fn put_blobs_and_items<'a, S: Store + ?Sized>(
store: &S,
group: &'a [Op],
out: &mut Published,
) -> Result<Vec<CellItem<'a>>, String> {
for op in group {
if let Op::Cell { blob: Some(bytes), .. } = op {
store.put_blob(bytes).map_err(|e| e.to_string())?;
out.blobs += 1;
out.blob_bytes += bytes.len() as u64;
}
}
Ok(group
.iter()
.filter_map(|op| match op {
Op::Cell { tx, id, receipt, version, .. } => {
Some((receipt.as_slice(), if *version { Some((id.as_str(), tx.as_str())) } else { None }))
}
_ => None,
})
.collect())
}
fn count_cells(group: &[Op], items: &[CellItem<'_>], out: &mut Published) {
out.cells += group.len();
out.receipts += group.len();
out.versions += items.iter().filter(|(_, v)| v.is_some()).count();
}
fn lane_of(id: &str, lanes: usize) -> usize {
id.bytes().next().and_then(|b| (b as char).to_digit(16)).unwrap_or(0) as usize % lanes
}
type CellItem<'a> = (&'a [u8], Option<(&'a str, &'a str)>);
fn put_lane<S: Store + ?Sized>(
store: &S,
items: &[CellItem<'_>],
ks: &[usize],
failed: &AtomicBool,
) -> Option<(usize, String)> {
for call in ks.chunks(CHUNK) {
if failed.load(Ordering::SeqCst) {
return None;
}
let writes: Vec<CellWrite<'_>> = call.iter().map(|&k| CellWrite::new(items[k].0, items[k].1)).collect();
if let Err(e) = store.put_cells(&writes) {
failed.store(true, Ordering::SeqCst);
return Some((call[0], e.to_string()));
}
}
None
}
pub fn receipt_json(r: &datumesh_core::Receipt) -> Vec<u8> {
let mut members = vec![("sig".to_string(), Value::String(r.sig().to_string()))];
if let Ok(Value::Object(m)) = json::parse(r.payload()) {
members.extend(m);
}
Value::Object(members).to_jcs()
}
pub fn publish_stored<S: Store + ?Sized>(
store: &S,
key: &SigningKey,
host: &str,
path: &Path,
) -> Result<Published, PublishError> {
publish_stored_in(store, key, host, &[host.to_string()], path)
}
pub fn publish_stored_in<S: Store + ?Sized>(
store: &S,
key: &SigningKey,
host: &str,
owned: &[String],
path: &Path,
) -> Result<Published, PublishError> {
publish_stored_with(store, key, host, owned, path, &PublishOptions::system())
}
pub fn publish_stored_with<S: Store + ?Sized>(
store: &S,
key: &SigningKey,
host: &str,
owned: &[String],
path: &Path,
opts: &PublishOptions,
) -> Result<Published, PublishError> {
publish_planned(store, key, host, owned, path, opts, false, |s, ops| {
write(s, ops, |s, g, out| write_cells_one_lane(s, g, out))
})
}
pub fn publish_stored_checked<S: Store + ?Sized>(
store: &S,
key: &SigningKey,
host: &str,
owned: &[String],
path: &Path,
opts: &PublishOptions,
) -> Result<Published, PublishError> {
publish_planned(store, key, host, owned, path, opts, true, |s, ops| {
write(s, ops, |s, g, out| write_cells_one_lane(s, g, out))
})
}
pub fn publish_stored_lanes<S: Store + Sync + ?Sized>(
store: &S,
key: &SigningKey,
host: &str,
owned: &[String],
path: &Path,
opts: &PublishOptions,
) -> Result<Published, PublishError> {
let lanes = publish_lanes()?;
publish_planned(store, key, host, owned, path, opts, false, |s, ops| {
write(s, ops, |s, g, out| write_cells(s, g, lanes, out))
})
}
fn overlay_of(ops: &[Op]) -> Result<Overlay, PublishError> {
let mut overlay = Overlay::default();
for op in ops {
match op {
Op::Function { name, start, hash, bytes, bind } => {
overlay.descriptors.insert(hash.clone(), bytes.clone());
if *bind {
overlay.functions.entry(name.clone()).or_default().push((start.clone(), hash.clone()));
}
}
Op::Descriptor { hash, bytes, .. } => {
overlay.descriptors.insert(hash.clone(), bytes.clone());
}
Op::Cell { receipt, .. } => {
overlay.cell(receipt).map_err(|e| PublishError::at(0, format!("tests: {e}")))?;
}
Op::Class { .. } | Op::Catalog { .. } => {}
}
}
Ok(overlay)
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn check_tests<S: Store + ?Sized>(
store: &S,
overlay: &Overlay,
tests: &[(usize, TestSpec)],
formulas: &[(usize, String, String)],
host: &str,
now: &str,
checked: bool,
) -> Result<Vec<RowWrite>, PublishError> {
if formulas.is_empty() {
return Ok(Vec::new());
}
let mut file_tests: Vec<(usize, TestSpec, String)> = Vec::new();
for (n, spec) in tests {
let at = spec.asof.clone().unwrap_or_else(|| now.to_string());
match testrun::binding_of(store, overlay, &spec.function, &at) {
Ok(Some((hash, _))) => file_tests.push((*n, spec.clone(), hash)),
Ok(None) => {
return Err(PublishError::at(
*n,
format!("a test of {}: no function of that name is bound", spec.function),
))
}
Err(e) => return Err(PublishError::at(*n, e)),
}
}
let mut rows = Vec::new();
for (line, name, hash) in formulas {
let own: Vec<(usize, TestSpec)> =
file_tests.iter().filter(|t| &t.2 == hash).map(|t| (t.0, t.1.clone())).collect();
let held = testrun::held_tests(store, hash).map_err(|e| PublishError::at(*line, e))?;
if !checked && own.is_empty() && held.is_empty() {
continue;
}
let mut specs: Vec<(Option<usize>, TestSpec)> = own.into_iter().map(|(n, s)| (Some(n), s)).collect();
for (_, spec) in held {
if !specs.iter().any(|(_, s)| s.case == spec.case) {
specs.push((None, spec));
}
}
let ran: Vec<Ran> = specs.iter().map(|(_, s)| run_test(store, overlay, now, s)).collect();
let mut run = testrun::Run::new(store, overlay, now);
let built = run.built(name, hash).map_err(|e| PublishError::at(*line, format!("{name}: {}", e.0)))?;
let Some(compiled) = built.compiled.as_ref() else { continue };
if checked {
let mut failing: Vec<(usize, String)> = Vec::new();
for ((n, spec), r) in specs.iter().zip(&ran) {
if let Some(n) = n {
let (result, detail) = testrun::result_of(spec, r);
if result != "pass" {
failing.push((*n, format!("{name}: test {} {detail}", spec.case)));
}
}
}
failing.sort_by_key(|f| f.0);
if let Some((n, detail)) = failing.into_iter().next() {
return Err(PublishError::typed(n, PublishErrorKind::TestsFailing(detail)));
}
let refs: Vec<&Ran> = ran.iter().collect();
let cov = testrun::coverage_of(compiled, &built.function.signature, &refs);
if !cov.uncovered.is_empty() || !cov.unexercised.is_empty() {
let mut text = format!("{name} is missing tests:");
if !cov.uncovered.is_empty() {
let cases: Vec<&str> = cov.uncovered.iter().map(|c| c.0.as_str()).collect();
text += &format!(" uncovered cases {};", cases.join(", "));
}
if !cov.unexercised.is_empty() {
text += &format!(" unexercised branches {}", cov.unexercised.join(", "));
}
return Err(PublishError::typed(
*line,
PublishErrorKind::TestsMissing(text.trim_end_matches(';').to_string()),
));
}
}
let refs: Vec<&Ran> = ran.iter().collect();
let cov = testrun::coverage_of(compiled, &built.function.signature, &refs);
for ((_, spec), r) in specs.iter().zip(&ran) {
let (result, detail) = testrun::result_of(spec, r);
rows.push(RowWrite {
fn_hash: hash.clone(),
case: spec.case.clone(),
jcs: spec.jcs.clone(),
signed_by: host.to_string(),
result,
detail,
branches_total: cov.branches,
branches_covered: cov.covered,
});
}
}
Ok(rows)
}
pub fn publish_stored_checked_lanes<S: Store + Sync + ?Sized>(
store: &S,
key: &SigningKey,
host: &str,
owned: &[String],
path: &Path,
opts: &PublishOptions,
) -> Result<Published, PublishError> {
let lanes = publish_lanes()?;
publish_planned(store, key, host, owned, path, opts, true, |s, ops| {
write(s, ops, |s, g, out| write_cells(s, g, lanes, out))
})
}
#[allow(clippy::too_many_arguments)]
fn publish_planned<S: Store + ?Sized>(
store: &S,
key: &SigningKey,
host: &str,
owned: &[String],
path: &Path,
opts: &PublishOptions,
checked: bool,
write_ops: impl FnOnce(&S, &[Op]) -> Result<(Published, Written), PublishError>,
) -> Result<Published, PublishError> {
let lines = read_lines(path)?;
let key_from = store
.namespace(host)
.ok()
.flatten()
.and_then(|n| json::parse(&n.keydoc).ok())
.and_then(|d| match d.get("keys") {
Some(Value::Array(ks)) => ks
.iter()
.find(|k| k.get("id").and_then(Value::as_str) == Some(key.key_id()))
.and_then(|k| k.get("valid_from").and_then(Value::as_str).map(str::to_string)),
_ => None,
})
.ok_or_else(|| {
PublishError::at(0, format!("the key {} is not in {host}'s key document in the store", key.key_id()))
})?;
let now = opts.clock.now();
let ctx = Ctx { host, owned, key_from: &key_from, now: &now, base: path.parent().unwrap_or(Path::new(".")) };
let mut planner = Planner {
store,
key,
ctx,
backfill: opts.backfill.is_some(),
sim: Simulation::default(),
reads: Reads::default(),
ops: Vec::new(),
binds: Vec::new(),
tests: Vec::new(),
formulas: Vec::new(),
};
planner.run(&lines)?;
let Planner { ops, binds, tests, formulas, .. } = planner;
if opts.backfill.is_some() {
backfill_check(store, &binds)?;
}
let rows = check_tests(store, &overlay_of(&ops)?, &tests, &formulas, host, &now, checked)?;
let (out, written) = write_ops(store, &ops)?;
index_cell_values(store, &ops).map_err(|e| PublishError::at(0, format!("catalogue: {e}")))?;
testrun::apply_rows(store, &rows, &now).map_err(|e| PublishError::at(0, format!("tests: {e}")))?;
let rebound: Vec<String> = ops
.iter()
.filter_map(|op| match op {
Op::Function { name, bind: true, .. } => Some(name.clone()),
_ => None,
})
.collect();
testrun::rerun_readers(store, &rebound, &now).map_err(|e| PublishError::at(0, format!("tests: {e}")))?;
if let Some(log) = &opts.backfill {
if out.functions + out.classes + out.versions > 0 {
append_log(log, &written).map_err(|e| PublishError::at(0, format!("backfill log: {e}")))?;
}
}
Ok(out)
}