use std::collections::BTreeSet;
use std::io::{BufRead, Write};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use datumesh_core::json::{self, Value};
use datumesh_core::{Arg, Args, Cell, ClassDescriptor, FunctionDescriptor, ReceiptBuilder, Scalar, SigningKey, Type};
use datumesh_engine::{Clock, SystemClock};
use datumesh_formula::definition::{Callee, Callees, ClassCallee};
use datumesh_store::Store;
const END: &str = "9999-12-31T23:59:59.999999Z";
#[non_exhaustive]
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct Published {
pub functions: usize,
pub classes: usize,
pub cells: usize,
pub receipts: usize,
pub versions: usize,
pub blobs: usize,
pub blob_bytes: u64,
}
#[derive(Clone)]
pub struct PublishOptions {
pub clock: Arc<dyn Clock + Send + Sync>,
pub backfill: Option<PathBuf>,
}
impl PublishOptions {
pub fn system() -> PublishOptions {
PublishOptions { clock: Arc::new(SystemClock), backfill: None }
}
}
#[non_exhaustive]
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum PublishErrorKind {
Line,
StartBeforeNow {
name: String,
start: String,
now: String,
},
Backfill(String),
OwnedVectorName(String),
}
impl std::fmt::Display for PublishErrorKind {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
PublishErrorKind::Line => f.write_str("refused"),
PublishErrorKind::StartBeforeNow { name, start, now } => {
write!(f, "binding start {start} of {name} is earlier than the mesh clock {now} (rule A)")
}
PublishErrorKind::Backfill(why) => write!(f, "--backfill refused: {why}"),
PublishErrorKind::OwnedVectorName(name) => {
write!(f, "{name} is under a namespace the mesh owns: import-vectors does not bind it")
}
}
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct PublishError {
pub line: usize,
pub detail: String,
pub kind: PublishErrorKind,
}
impl PublishError {
fn at(line: usize, detail: impl Into<String>) -> PublishError {
PublishError { line, detail: detail.into(), kind: PublishErrorKind::Line }
}
pub(crate) fn typed(line: usize, kind: PublishErrorKind) -> PublishError {
PublishError { line, detail: kind.to_string(), kind }
}
}
impl std::fmt::Display for PublishError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "line {}: {}", self.line, self.detail)
}
}
impl std::error::Error for PublishError {}
fn s(v: &str) -> Value {
Value::String(v.to_string())
}
fn namespace_of(name: &str) -> &str {
name.rsplit_once('/').map(|(ns, _)| ns).unwrap_or("")
}
fn definer<'a>(name: &'a str, host: &'a str, owned: &[String]) -> &'a str {
let host_of_name = name.split('/').next().unwrap_or("");
if owned.iter().any(|o| o == host_of_name) {
namespace_of(name)
} else {
host
}
}
fn bound_of(v: &Value, line: usize, now: &str) -> Result<String, PublishError> {
match v.get("bound") {
None => Ok(now.to_string()),
Some(b) => b.as_str().map(str::to_string).ok_or_else(|| PublishError::at(line, "bound is not an instant")),
}
}
#[derive(Default)]
struct Simulation {
functions: std::collections::BTreeMap<String, Vec<(String, String)>>,
classes: std::collections::BTreeMap<String, Vec<(String, String)>>,
descriptors: std::collections::BTreeMap<String, Vec<u8>>,
class_bytes: std::collections::BTreeMap<String, Vec<u8>>,
}
impl Simulation {
fn latest(
binds: &std::collections::BTreeMap<String, Vec<(String, String)>>,
name: &str,
at: &str,
) -> Option<(String, String)> {
binds.get(name)?.iter().filter(|(t, _)| t.as_str() <= at).max_by(|a, b| a.0.cmp(&b.0)).cloned()
}
}
struct Callers<'a, S: Store + ?Sized> {
store: &'a S,
sim: Option<&'a Simulation>,
}
impl<S: Store + ?Sized> Callers<'_, S> {
fn function_bytes(&self, hash: &str) -> Result<Option<Vec<u8>>, String> {
if let Some(b) = self.sim.and_then(|s| s.descriptors.get(hash)) {
return Ok(Some(b.clone()));
}
Ok(self.store.descriptor(hash).map(|b| b.into_owned()))
}
fn class_bytes(&self, hash: &str) -> Result<Option<Vec<u8>>, String> {
if let Some(b) = self.sim.and_then(|s| s.class_bytes.get(hash)) {
return Ok(Some(b.clone()));
}
self.store.class(hash).map_err(|e| e.to_string())
}
}
impl<S: Store + ?Sized> Callees for Callers<'_, S> {
fn callee(&self, name: &str, at: &str) -> Result<Option<Callee>, String> {
let stored = self.store.resolve_binding(name, at).map_err(|e| e.to_string())?;
let simmed = self.sim.and_then(|s| Simulation::latest(&s.functions, name, at));
let (fn_hash, start) = match (stored, simmed) {
(Some(b), Some((t, h))) if t >= b.tx => (h, t),
(Some(b), _) => (b.fn_hash, b.tx),
(None, Some((t, h))) => (h, t),
(None, None) => return Ok(None),
};
let bytes = self.function_bytes(&fn_hash)?.ok_or_else(|| format!("descriptor {fn_hash} is not stored"))?;
let descriptor = json::parse(&bytes).map_err(|e| e.to_string())?;
Ok(Some(Callee { fn_hash, start, descriptor }))
}
fn bound_class(&self, name: &str, at: &str) -> Result<Option<ClassCallee>, String> {
let stored = self.store.resolve_class_binding(name, at).map_err(|e| e.to_string())?;
let simmed = self.sim.and_then(|s| Simulation::latest(&s.classes, name, at));
let (class_hash, start) = match (stored, simmed) {
(Some(b), Some((t, h))) if t >= b.tx => (h, t),
(Some(b), _) => (b.class_hash, b.tx),
(None, Some((t, h))) => (h, t),
(None, None) => return Ok(None),
};
let bytes = self.class_bytes(&class_hash)?.ok_or_else(|| format!("class {class_hash} is not stored"))?;
let descriptor = ClassDescriptor::from_json(&bytes).map_err(|e| e.to_string())?;
Ok(Some(ClassCallee { class_hash, start, descriptor }))
}
fn class_descriptor(&self, class_hash: &str) -> Result<Option<ClassDescriptor>, String> {
let Some(bytes) = self.class_bytes(class_hash)? else { return Ok(None) };
ClassDescriptor::from_json(&bytes).map(Some).map_err(|e| e.to_string())
}
fn function_descriptor(&self, fn_hash: &str) -> Result<Option<Value>, String> {
let Some(bytes) = self.function_bytes(fn_hash)? else { return Ok(None) };
json::parse(&bytes).map(Some).map_err(|e| e.to_string())
}
}
fn current_function<S: Store + ?Sized>(store: &S, sim: &Simulation, name: &str) -> Result<Option<String>, String> {
if let Some((_, h)) = Simulation::latest(&sim.functions, name, END) {
return Ok(Some(h));
}
store.resolve_name(name, END).map_err(|e| e.to_string())
}
fn current_class<S: Store + ?Sized>(store: &S, sim: &Simulation, name: &str) -> Result<Option<String>, String> {
if let Some((_, h)) = Simulation::latest(&sim.classes, name, END) {
return Ok(Some(h));
}
store.resolve_class(name, END).map_err(|e| e.to_string())
}
struct Bind {
line: usize,
name: String,
}
fn record(
binds: &mut Vec<Bind>,
line: usize,
name: &str,
start: &str,
now: &str,
backfill: bool,
) -> Result<(), PublishError> {
if !backfill && start < now {
return Err(PublishError::typed(
line,
PublishErrorKind::StartBeforeNow { name: name.into(), start: start.into(), now: now.into() },
));
}
binds.push(Bind { line, name: name.into() });
Ok(())
}
#[derive(Default)]
struct Written {
namespaces: BTreeSet<String>,
starts: BTreeSet<String>,
}
impl Written {
fn bound(&mut self, name: &str, start: &str) {
self.namespaces.insert(namespace_of(name).to_string());
self.starts.insert(start.to_string());
}
}
struct Ctx<'a> {
host: &'a str,
owned: &'a [String],
key_from: &'a str,
now: &'a str,
base: &'a Path,
}
fn read_lines(path: &Path) -> Result<Vec<(usize, String)>, PublishError> {
let file = std::fs::File::open(path).map_err(|e| PublishError::at(0, format!("{}: {e}", path.display())))?;
let mut out = Vec::new();
for (i, line) in std::io::BufReader::new(file).lines().enumerate() {
let n = i + 1;
let line = line.map_err(|e| PublishError::at(n, e.to_string()))?;
if !line.trim().is_empty() {
out.push((n, line));
}
}
Ok(out)
}
fn plan<S: Store + ?Sized>(
store: &S,
ctx: &Ctx<'_>,
lines: &[(usize, String)],
backfill: bool,
) -> Result<Vec<Bind>, PublishError> {
let mut sim = Simulation::default();
let mut binds = Vec::new();
for (n, text) in lines {
let n = *n;
let err = |d: String| PublishError::at(n, d);
let v = json::parse(text.as_bytes()).map_err(|e| err(e.to_string()))?;
if let Some(name) = v.get("class").and_then(Value::as_str) {
let descriptor = v.get("descriptor").ok_or_else(|| err("a class without descriptor".into()))?;
let start = bound_of(&v, n, ctx.now)?;
let def = datumesh_formula::definition::define_class(
name,
&descriptor.to_jcs(),
definer(name, ctx.host, ctx.owned),
&Callers { store, sim: Some(&sim) },
&start,
)
.map_err(|e| err(format!("define {name}: {e}")))?;
if current_class(store, &sim, name).map_err(err)?.as_deref() != Some(def.class_hash.as_str()) {
record(&mut binds, n, name, &start, ctx.now, backfill)?;
sim.classes.entry(name.to_string()).or_default().push((start.clone(), def.class_hash.clone()));
sim.class_bytes.insert(def.class_hash.clone(), def.bytes.clone());
for (m, fh) in &def.descriptor.methods {
let mname = format!("{name}/{m}");
if current_function(store, &sim, &mname).map_err(err)?.as_deref() != Some(fh.as_str()) {
record(&mut binds, n, &mname, &start, ctx.now, backfill)?;
sim.functions.entry(mname).or_default().push((start.clone(), fh.clone()));
}
}
}
continue;
}
let function = if let Some(name) = v.get("function").and_then(Value::as_str) {
let sig = v.get("signature").ok_or_else(|| err("a function without a signature".into()))?;
let start = bound_of(&v, n, ctx.now)?;
let d = Value::Object(vec![
("code".into(), Value::Object(vec![("body".into(), s("")), ("lang".into(), s("stored"))])),
("inputs".into(), Value::Array(vec![])),
("signature".into(), sig.clone()),
]);
let fd = FunctionDescriptor::from_value(&d).map_err(|e| err(e.to_string()))?;
Some((name, start, fd.hash(), fd.canonical()))
} else if let Some(name) = v.get("formula").and_then(Value::as_str) {
let source =
v.get("source").and_then(Value::as_str).ok_or_else(|| err("a formula without source".into()))?;
let sig = v.get("signature").ok_or_else(|| err("a formula without a signature".into()))?;
let start = bound_of(&v, n, ctx.now)?;
let def = datumesh_formula::definition::define_function(
name,
source,
sig,
&Default::default(),
&Callers { store, sim: Some(&sim) },
&start,
)
.map_err(|e| err(format!("define {name}: {e}")))?;
Some((name, start, def.fn_hash, def.bytes))
} else if v.get("cell").is_none() {
return Err(err("neither a function nor a cell".into()));
} else {
None
};
if let Some((name, start, h, bytes)) = function {
if current_function(store, &sim, name).map_err(err)?.as_deref() != Some(h.as_str()) {
record(&mut binds, n, name, &start, ctx.now, backfill)?;
sim.functions.entry(name.to_string()).or_default().push((start, h.clone()));
sim.descriptors.insert(h, bytes);
}
}
}
Ok(binds)
}
fn backfill_check<S: Store + ?Sized>(store: &S, binds: &[Bind]) -> Result<(), PublishError> {
for b in binds {
let refuse = |why: String| PublishError::typed(b.line, PublishErrorKind::Backfill(why));
if store.resolve_binding(&b.name, END).map_err(|e| PublishError::at(b.line, e.to_string()))?.is_some() {
return Err(refuse(format!("{} has a binding", b.name)));
}
let ns = namespace_of(&b.name);
if store.has_bindings_under(ns).map_err(|e| PublishError::at(b.line, e.to_string()))? {
return Err(refuse(format!("{ns} has bindings")));
}
}
Ok(())
}
fn append_log(log: &Path, written: &Written) -> Result<(), String> {
let line = Value::Object(vec![
("at".into(), s(&SystemClock.now())),
("namespaces".into(), Value::Array(written.namespaces.iter().map(|n| s(n)).collect())),
("starts".into(), Value::Array(written.starts.iter().map(|t| s(t)).collect())),
])
.to_jcs();
let mut opts = std::fs::OpenOptions::new();
opts.create(true).append(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
opts.mode(0o600);
}
let mut f = opts.open(log).map_err(|e| format!("{}: {e}", log.display()))?;
let mut bytes = line;
bytes.push(b'\n');
f.write_all(&bytes).map_err(|e| format!("{}: {e}", log.display()))
}
fn write<S: Store + ?Sized>(
store: &S,
key: &SigningKey,
ctx: &Ctx<'_>,
lines: &[(usize, String)],
) -> Result<(Published, Written), PublishError> {
let mut out = Published::default();
let mut written = Written::default();
for (n, text) in lines {
let n = *n;
let err = |d: String| PublishError::at(n, d);
let v = json::parse(text.as_bytes()).map_err(|e| err(e.to_string()))?;
if let Some(name) = v.get("class").and_then(Value::as_str) {
let descriptor = v.get("descriptor").ok_or_else(|| err("a class without descriptor".into()))?;
let start = bound_of(&v, n, ctx.now)?;
let def = datumesh_formula::definition::define_class(
name,
&descriptor.to_jcs(),
definer(name, ctx.host, ctx.owned),
&Callers { store, sim: None },
&start,
)
.map_err(|e| err(format!("define {name}: {e}")))?;
let bound_hash = store.put_class(&def.bytes).map_err(|e| err(e.to_string()))?;
match store.resolve_class(name, END).map_err(|e| err(e.to_string()))? {
Some(h) if h == bound_hash => {}
_ => {
store.bind_class(name, &bound_hash, &start).map_err(|e| err(format!("bind {name}: {e}")))?;
out.classes += 1;
written.bound(name, &start);
for m in def.descriptor.methods.keys() {
written.bound(&format!("{name}/{m}"), &start);
}
}
}
continue;
}
if let Some(name) = v.get("function").and_then(Value::as_str) {
let sig = v.get("signature").ok_or_else(|| err("a function without a signature".into()))?;
let start = bound_of(&v, n, ctx.now)?;
let d = Value::Object(vec![
("code".into(), Value::Object(vec![("body".into(), s("")), ("lang".into(), s("stored"))])),
("inputs".into(), Value::Array(vec![])),
("signature".into(), sig.clone()),
]);
let fd = FunctionDescriptor::from_value(&d).map_err(|e| err(e.to_string()))?;
let h = store.put_descriptor(&fd.canonical()).map_err(|e| err(e.to_string()))?;
match store.resolve_binding(name, END).map_err(|e| err(e.to_string()))? {
Some(b) if b.fn_hash == h => {}
_ => {
store.bind_name(name, &h, &start).map_err(|e| err(format!("bind {name}: {e}")))?;
out.functions += 1;
written.bound(name, &start);
}
}
continue;
}
if let Some(name) = v.get("formula").and_then(Value::as_str) {
let source =
v.get("source").and_then(Value::as_str).ok_or_else(|| err("a formula without source".into()))?;
let sig = v.get("signature").ok_or_else(|| err("a formula without a signature".into()))?;
let start = bound_of(&v, n, ctx.now)?;
let def = datumesh_formula::definition::define_function(
name,
source,
sig,
&Default::default(),
&Callers { store, sim: None },
&start,
)
.map_err(|e| err(format!("define {name}: {e}")))?;
let h = store.put_descriptor(&def.bytes).map_err(|e| err(e.to_string()))?;
match store.resolve_binding(name, END).map_err(|e| err(e.to_string()))? {
Some(b) if b.fn_hash == h => {}
_ => {
store.bind_name(name, &h, &start).map_err(|e| err(format!("bind {name}: {e}")))?;
out.functions += 1;
written.bound(name, &start);
}
}
continue;
}
let cell_v = v.get("cell").ok_or_else(|| err("neither a function nor a cell".into()))?;
let name = cell_v.get("fn").and_then(Value::as_str).ok_or_else(|| err("a cell without fn".into()))?;
let (ns, f) = name.rsplit_once('/').ok_or_else(|| err(format!("{name}: not a function name")))?;
let field =
|k: &str| v.get(k).and_then(Value::as_str).map(str::to_string).ok_or_else(|| err(format!("no {k}")));
let (tx, asof, licence, summary) = (field("tx")?, field("asof")?, field("licence")?, field("summary")?);
let issued = v
.get("issued")
.and_then(Value::as_str)
.map(str::to_string)
.unwrap_or_else(|| std::cmp::max(asof.clone(), ctx.key_from.to_string()));
let fn_hash = store
.resolve_name(name, &tx)
.map_err(|e| err(e.to_string()))?
.ok_or_else(|| err(format!("{name} is not bound as of {tx}")))?;
let d = store.descriptor(&fn_hash).ok_or_else(|| err(format!("no descriptor {fn_hash}")))?;
let d = FunctionDescriptor::from_json(&d).map_err(|e| err(e.to_string()))?;
let mut args = Args::new();
for (k, a) in cell_v.get("args").and_then(Value::as_object).ok_or_else(|| err("a cell without args".into()))? {
args.insert(k.clone(), Arg::Value(Scalar::from_json(&a.to_jcs()).map_err(|e| err(format!("{k}: {e}")))?));
}
let valid: Args = args
.iter()
.filter(|(k, _)| d.signature.args.get(*k).is_some_and(|x| x.valid))
.map(|(k, a)| (k.clone(), a.clone()))
.collect();
let value_v = v.get("value").ok_or_else(|| err("a cell without value".into()))?;
let value = match value_v.get("blob").and_then(Value::as_str) {
Some(file) => {
let media = value_v.get("media_type").and_then(Value::as_str).unwrap_or("application/octet-stream");
let bytes = std::fs::read(ctx.base.join(file)).map_err(|e| err(format!("{file}: {e}")))?;
let sha = store.put_blob(&bytes).map_err(|e| err(e.to_string()))?;
out.blobs += 1;
out.blob_bytes += bytes.len() as u64;
Scalar::from_lexical(Type::Blob, &format!("{sha}:{}:{media}", bytes.len()))
.map_err(|e| err(e.to_string()))?
}
None => Scalar::from_json(&value_v.to_jcs()).map_err(|e| err(e.to_string()))?,
};
let cell = Cell { ns: ns.to_string(), fn_name: f.to_string(), fn_hash, args };
let r = ReceiptBuilder::new()
.cell(cell.clone())
.value(value)
.valid(valid)
.asof(asof.as_str())
.tx(tx.as_str())
.issued(issued.as_str())
.publisher(ns)
.signer_host(ctx.host)
.licence(licence.as_str())
.summary(summary.as_str())
.sign(key)
.map_err(|e| err(e.to_string()))?;
let h = store.put_receipt(&receipt_json(&r)).map_err(|e| err(e.to_string()))?;
out.cells += 1;
out.receipts += 1;
let id = cell.id();
match store.latest(&id).map_err(|e| err(e.to_string()))? {
Some(l) if l.tx.as_str() >= tx.as_str() => {
if store.version_asof(&id, &tx).ok().flatten().is_none_or(|v| v.receipt_hash != h) {
return Err(err(format!("{id} has a version at or after {tx} that is not this receipt")));
}
}
_ => {
store.put_version(&id, &tx, &h).map_err(|e| err(e.to_string()))?;
out.versions += 1;
}
}
}
Ok((out, written))
}
pub fn receipt_json(r: &datumesh_core::Receipt) -> Vec<u8> {
let mut members = vec![("sig".to_string(), Value::String(r.sig().to_string()))];
if let Ok(Value::Object(m)) = json::parse(r.payload()) {
members.extend(m);
}
Value::Object(members).to_jcs()
}
pub fn publish_stored<S: Store + ?Sized>(
store: &S,
key: &SigningKey,
host: &str,
path: &Path,
) -> Result<Published, PublishError> {
publish_stored_in(store, key, host, &[host.to_string()], path)
}
pub fn publish_stored_in<S: Store + ?Sized>(
store: &S,
key: &SigningKey,
host: &str,
owned: &[String],
path: &Path,
) -> Result<Published, PublishError> {
publish_stored_with(store, key, host, owned, path, &PublishOptions::system())
}
pub fn publish_stored_with<S: Store + ?Sized>(
store: &S,
key: &SigningKey,
host: &str,
owned: &[String],
path: &Path,
opts: &PublishOptions,
) -> Result<Published, PublishError> {
let lines = read_lines(path)?;
let key_from = store
.namespace(host)
.ok()
.flatten()
.and_then(|n| json::parse(&n.keydoc).ok())
.and_then(|d| match d.get("keys") {
Some(Value::Array(ks)) => ks
.iter()
.find(|k| k.get("id").and_then(Value::as_str) == Some(key.key_id()))
.and_then(|k| k.get("valid_from").and_then(Value::as_str).map(str::to_string)),
_ => None,
})
.ok_or_else(|| {
PublishError::at(0, format!("the key {} is not in {host}'s key document in the store", key.key_id()))
})?;
let now = opts.clock.now();
let ctx = Ctx { host, owned, key_from: &key_from, now: &now, base: path.parent().unwrap_or(Path::new(".")) };
let binds = plan(store, &ctx, &lines, opts.backfill.is_some())?;
if opts.backfill.is_some() {
backfill_check(store, &binds)?;
}
let (out, written) = write(store, key, &ctx, &lines)?;
if let Some(log) = &opts.backfill {
if out.functions + out.classes + out.versions > 0 {
append_log(log, &written).map_err(|e| PublishError::at(0, format!("backfill log: {e}")))?;
}
}
Ok(out)
}